1 /******************************************************************* 2 * This file is part of the Emulex Linux Device Driver for * 3 * Fibre Channel Host Bus Adapters. * 4 * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * 5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * 6 * Copyright (C) 2004-2016 Emulex. All rights reserved. * 7 * EMULEX and SLI are trademarks of Emulex. * 8 * www.broadcom.com * 9 * Portions Copyright (C) 2004-2005 Christoph Hellwig * 10 * * 11 * This program is free software; you can redistribute it and/or * 12 * modify it under the terms of version 2 of the GNU General * 13 * Public License as published by the Free Software Foundation. * 14 * This program is distributed in the hope that it will be useful. * 15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 19 * TO BE LEGALLY INVALID. See the GNU General Public License for * 20 * more details, a copy of which can be found in the file COPYING * 21 * included with this package. * 22 ********************************************************************/ 23 #include <linux/pci.h> 24 #include <linux/slab.h> 25 #include <linux/interrupt.h> 26 #include <linux/delay.h> 27 #include <asm/unaligned.h> 28 #include <linux/crc-t10dif.h> 29 #include <net/checksum.h> 30 31 #include <scsi/scsi.h> 32 #include <scsi/scsi_device.h> 33 #include <scsi/scsi_eh.h> 34 #include <scsi/scsi_host.h> 35 #include <scsi/scsi_tcq.h> 36 #include <scsi/scsi_transport_fc.h> 37 #include <scsi/fc/fc_fs.h> 38 39 #include "lpfc_version.h" 40 #include "lpfc_hw4.h" 41 #include "lpfc_hw.h" 42 #include "lpfc_sli.h" 43 #include "lpfc_sli4.h" 44 #include "lpfc_nl.h" 45 #include "lpfc_disc.h" 46 #include "lpfc.h" 47 #include "lpfc_nvme.h" 48 #include "lpfc_scsi.h" 49 #include "lpfc_logmsg.h" 50 #include "lpfc_crtn.h" 51 #include "lpfc_vport.h" 52 #include "lpfc_debugfs.h" 53 54 /* NVME initiator-based functions */ 55 56 static struct lpfc_io_buf * 57 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 58 int idx, int expedite); 59 60 static void 61 lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *); 62 63 static struct nvme_fc_port_template lpfc_nvme_template; 64 65 static union lpfc_wqe128 lpfc_iread_cmd_template; 66 static union lpfc_wqe128 lpfc_iwrite_cmd_template; 67 static union lpfc_wqe128 lpfc_icmnd_cmd_template; 68 69 /* Setup WQE templates for NVME IOs */ 70 void 71 lpfc_nvme_cmd_template(void) 72 { 73 union lpfc_wqe128 *wqe; 74 75 /* IREAD template */ 76 wqe = &lpfc_iread_cmd_template; 77 memset(wqe, 0, sizeof(union lpfc_wqe128)); 78 79 /* Word 0, 1, 2 - BDE is variable */ 80 81 /* Word 3 - cmd_buff_len, payload_offset_len is zero */ 82 83 /* Word 4 - total_xfer_len is variable */ 84 85 /* Word 5 - is zero */ 86 87 /* Word 6 - ctxt_tag, xri_tag is variable */ 88 89 /* Word 7 */ 90 bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE); 91 bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK); 92 bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3); 93 bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI); 94 95 /* Word 8 - abort_tag is variable */ 96 97 /* Word 9 - reqtag is variable */ 98 99 /* Word 10 - dbde, wqes is variable */ 100 bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0); 101 bf_set(wqe_nvme, &wqe->fcp_iread.wqe_com, 1); 102 bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ); 103 bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4); 104 bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0); 105 bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1); 106 107 /* Word 11 - pbde is variable */ 108 bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, NVME_READ_CMD); 109 bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 110 bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1); 111 112 /* Word 12 - is zero */ 113 114 /* Word 13, 14, 15 - PBDE is variable */ 115 116 /* IWRITE template */ 117 wqe = &lpfc_iwrite_cmd_template; 118 memset(wqe, 0, sizeof(union lpfc_wqe128)); 119 120 /* Word 0, 1, 2 - BDE is variable */ 121 122 /* Word 3 - cmd_buff_len, payload_offset_len is zero */ 123 124 /* Word 4 - total_xfer_len is variable */ 125 126 /* Word 5 - initial_xfer_len is variable */ 127 128 /* Word 6 - ctxt_tag, xri_tag is variable */ 129 130 /* Word 7 */ 131 bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE); 132 bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK); 133 bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3); 134 bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI); 135 136 /* Word 8 - abort_tag is variable */ 137 138 /* Word 9 - reqtag is variable */ 139 140 /* Word 10 - dbde, wqes is variable */ 141 bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0); 142 bf_set(wqe_nvme, &wqe->fcp_iwrite.wqe_com, 1); 143 bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE); 144 bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4); 145 bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0); 146 bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1); 147 148 /* Word 11 - pbde is variable */ 149 bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, NVME_WRITE_CMD); 150 bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 151 bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1); 152 153 /* Word 12 - is zero */ 154 155 /* Word 13, 14, 15 - PBDE is variable */ 156 157 /* ICMND template */ 158 wqe = &lpfc_icmnd_cmd_template; 159 memset(wqe, 0, sizeof(union lpfc_wqe128)); 160 161 /* Word 0, 1, 2 - BDE is variable */ 162 163 /* Word 3 - payload_offset_len is variable */ 164 165 /* Word 4, 5 - is zero */ 166 167 /* Word 6 - ctxt_tag, xri_tag is variable */ 168 169 /* Word 7 */ 170 bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE); 171 bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0); 172 bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3); 173 bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI); 174 175 /* Word 8 - abort_tag is variable */ 176 177 /* Word 9 - reqtag is variable */ 178 179 /* Word 10 - dbde, wqes is variable */ 180 bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1); 181 bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1); 182 bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE); 183 bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE); 184 bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0); 185 bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1); 186 187 /* Word 11 */ 188 bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, FCP_COMMAND); 189 bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 190 bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0); 191 192 /* Word 12, 13, 14, 15 - is zero */ 193 } 194 195 /** 196 * lpfc_nvme_prep_abort_wqe - set up 'abort' work queue entry. 197 * @pwqeq: Pointer to command iocb. 198 * @xritag: Tag that uniqely identifies the local exchange resource. 199 * @opt: Option bits - 200 * bit 0 = inhibit sending abts on the link 201 * 202 * This function is called with hbalock held. 203 **/ 204 void 205 lpfc_nvme_prep_abort_wqe(struct lpfc_iocbq *pwqeq, u16 xritag, u8 opt) 206 { 207 union lpfc_wqe128 *wqe = &pwqeq->wqe; 208 209 /* WQEs are reused. Clear stale data and set key fields to 210 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag. 211 */ 212 memset(wqe, 0, sizeof(*wqe)); 213 214 if (opt & INHIBIT_ABORT) 215 bf_set(abort_cmd_ia, &wqe->abort_cmd, 1); 216 /* Abort specified xri tag, with the mask deliberately zeroed */ 217 bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG); 218 219 bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX); 220 221 /* Abort the IO associated with this outstanding exchange ID. */ 222 wqe->abort_cmd.wqe_com.abort_tag = xritag; 223 224 /* iotag for the wqe completion. */ 225 bf_set(wqe_reqtag, &wqe->abort_cmd.wqe_com, pwqeq->iotag); 226 227 bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1); 228 bf_set(wqe_lenloc, &wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE); 229 230 bf_set(wqe_cmd_type, &wqe->abort_cmd.wqe_com, OTHER_COMMAND); 231 bf_set(wqe_wqec, &wqe->abort_cmd.wqe_com, 1); 232 bf_set(wqe_cqid, &wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 233 } 234 235 /** 236 * lpfc_nvme_create_queue - 237 * @lpfc_pnvme: Pointer to the driver's nvme instance data 238 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 239 * @handle: An opaque driver handle used in follow-up calls. 240 * 241 * Driver registers this routine to preallocate and initialize any 242 * internal data structures to bind the @qidx to its internal IO queues. 243 * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ. 244 * 245 * Return value : 246 * 0 - Success 247 * -EINVAL - Unsupported input value. 248 * -ENOMEM - Could not alloc necessary memory 249 **/ 250 static int 251 lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, 252 unsigned int qidx, u16 qsize, 253 void **handle) 254 { 255 struct lpfc_nvme_lport *lport; 256 struct lpfc_vport *vport; 257 struct lpfc_nvme_qhandle *qhandle; 258 char *str; 259 260 if (!pnvme_lport->private) 261 return -ENOMEM; 262 263 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 264 vport = lport->vport; 265 qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL); 266 if (qhandle == NULL) 267 return -ENOMEM; 268 269 qhandle->cpu_id = raw_smp_processor_id(); 270 qhandle->qidx = qidx; 271 /* 272 * NVME qidx == 0 is the admin queue, so both admin queue 273 * and first IO queue will use MSI-X vector and associated 274 * EQ/CQ/WQ at index 0. After that they are sequentially assigned. 275 */ 276 if (qidx) { 277 str = "IO "; /* IO queue */ 278 qhandle->index = ((qidx - 1) % 279 lpfc_nvme_template.max_hw_queues); 280 } else { 281 str = "ADM"; /* Admin queue */ 282 qhandle->index = qidx; 283 } 284 285 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 286 "6073 Binding %s HdwQueue %d (cpu %d) to " 287 "hdw_queue %d qhandle x%px\n", str, 288 qidx, qhandle->cpu_id, qhandle->index, qhandle); 289 *handle = (void *)qhandle; 290 return 0; 291 } 292 293 /** 294 * lpfc_nvme_delete_queue - 295 * @lpfc_pnvme: Pointer to the driver's nvme instance data 296 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 297 * @handle: An opaque driver handle from lpfc_nvme_create_queue 298 * 299 * Driver registers this routine to free 300 * any internal data structures to bind the @qidx to its internal 301 * IO queues. 302 * 303 * Return value : 304 * 0 - Success 305 * TODO: What are the failure codes. 306 **/ 307 static void 308 lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport, 309 unsigned int qidx, 310 void *handle) 311 { 312 struct lpfc_nvme_lport *lport; 313 struct lpfc_vport *vport; 314 315 if (!pnvme_lport->private) 316 return; 317 318 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 319 vport = lport->vport; 320 321 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 322 "6001 ENTER. lpfc_pnvme x%px, qidx x%x qhandle x%px\n", 323 lport, qidx, handle); 324 kfree(handle); 325 } 326 327 static void 328 lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport) 329 { 330 struct lpfc_nvme_lport *lport = localport->private; 331 332 lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME, 333 "6173 localport x%px delete complete\n", 334 lport); 335 336 /* release any threads waiting for the unreg to complete */ 337 if (lport->vport->localport) 338 complete(lport->lport_unreg_cmp); 339 } 340 341 /* lpfc_nvme_remoteport_delete 342 * 343 * @remoteport: Pointer to an nvme transport remoteport instance. 344 * 345 * This is a template downcall. NVME transport calls this function 346 * when it has completed the unregistration of a previously 347 * registered remoteport. 348 * 349 * Return value : 350 * None 351 */ 352 static void 353 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport) 354 { 355 struct lpfc_nvme_rport *rport = remoteport->private; 356 struct lpfc_vport *vport; 357 struct lpfc_nodelist *ndlp; 358 359 ndlp = rport->ndlp; 360 if (!ndlp) 361 goto rport_err; 362 363 vport = ndlp->vport; 364 if (!vport) 365 goto rport_err; 366 367 /* Remove this rport from the lport's list - memory is owned by the 368 * transport. Remove the ndlp reference for the NVME transport before 369 * calling state machine to remove the node. 370 */ 371 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 372 "6146 remoteport delete of remoteport x%px\n", 373 remoteport); 374 spin_lock_irq(&vport->phba->hbalock); 375 376 /* The register rebind might have occurred before the delete 377 * downcall. Guard against this race. 378 */ 379 if (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG) { 380 ndlp->nrport = NULL; 381 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 382 spin_unlock_irq(&vport->phba->hbalock); 383 384 /* Remove original register reference. The host transport 385 * won't reference this rport/remoteport any further. 386 */ 387 lpfc_nlp_put(ndlp); 388 } else { 389 spin_unlock_irq(&vport->phba->hbalock); 390 } 391 392 rport_err: 393 return; 394 } 395 396 /** 397 * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request 398 * @phba: pointer to lpfc hba data structure. 399 * @axchg: pointer to exchange context for the NVME LS request 400 * 401 * This routine is used for processing an asychronously received NVME LS 402 * request. Any remaining validation is done and the LS is then forwarded 403 * to the nvme-fc transport via nvme_fc_rcv_ls_req(). 404 * 405 * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing) 406 * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done. 407 * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg. 408 * 409 * Returns 0 if LS was handled and delivered to the transport 410 * Returns 1 if LS failed to be handled and should be dropped 411 */ 412 int 413 lpfc_nvme_handle_lsreq(struct lpfc_hba *phba, 414 struct lpfc_async_xchg_ctx *axchg) 415 { 416 #if (IS_ENABLED(CONFIG_NVME_FC)) 417 struct lpfc_vport *vport; 418 struct lpfc_nvme_rport *lpfc_rport; 419 struct nvme_fc_remote_port *remoteport; 420 struct lpfc_nvme_lport *lport; 421 uint32_t *payload = axchg->payload; 422 int rc; 423 424 vport = axchg->ndlp->vport; 425 lpfc_rport = axchg->ndlp->nrport; 426 if (!lpfc_rport) 427 return -EINVAL; 428 429 remoteport = lpfc_rport->remoteport; 430 if (!vport->localport) 431 return -EINVAL; 432 433 lport = vport->localport->private; 434 if (!lport) 435 return -EINVAL; 436 437 rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload, 438 axchg->size); 439 440 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 441 "6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x " 442 "%08x %08x %08x\n", 443 axchg->size, rc, 444 *payload, *(payload+1), *(payload+2), 445 *(payload+3), *(payload+4), *(payload+5)); 446 447 if (!rc) 448 return 0; 449 #endif 450 return 1; 451 } 452 453 /** 454 * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME 455 * LS request. 456 * @phba: Pointer to HBA context object 457 * @vport: The local port that issued the LS 458 * @cmdwqe: Pointer to driver command WQE object. 459 * @wcqe: Pointer to driver response CQE object. 460 * 461 * This function is the generic completion handler for NVME LS requests. 462 * The function updates any states and statistics, calls the transport 463 * ls_req done() routine, then tears down the command and buffers used 464 * for the LS request. 465 **/ 466 void 467 __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport, 468 struct lpfc_iocbq *cmdwqe, 469 struct lpfc_wcqe_complete *wcqe) 470 { 471 struct nvmefc_ls_req *pnvme_lsreq; 472 struct lpfc_dmabuf *buf_ptr; 473 struct lpfc_nodelist *ndlp; 474 uint32_t status; 475 476 pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2; 477 ndlp = (struct lpfc_nodelist *)cmdwqe->context1; 478 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 479 480 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 481 "6047 NVMEx LS REQ %px cmpl DID %x Xri: %x " 482 "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px " 483 "ndlp:x%px\n", 484 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 485 cmdwqe->sli4_xritag, status, 486 (wcqe->parameter & 0xffff), 487 cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp); 488 489 lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n", 490 cmdwqe->sli4_xritag, status, wcqe->parameter); 491 492 if (cmdwqe->context3) { 493 buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3; 494 lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys); 495 kfree(buf_ptr); 496 cmdwqe->context3 = NULL; 497 } 498 if (pnvme_lsreq->done) 499 pnvme_lsreq->done(pnvme_lsreq, status); 500 else 501 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 502 "6046 NVMEx cmpl without done call back? " 503 "Data %px DID %x Xri: %x status %x\n", 504 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 505 cmdwqe->sli4_xritag, status); 506 if (ndlp) { 507 lpfc_nlp_put(ndlp); 508 cmdwqe->context1 = NULL; 509 } 510 lpfc_sli_release_iocbq(phba, cmdwqe); 511 } 512 513 static void 514 lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 515 struct lpfc_wcqe_complete *wcqe) 516 { 517 struct lpfc_vport *vport = cmdwqe->vport; 518 struct lpfc_nvme_lport *lport; 519 uint32_t status; 520 521 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 522 523 if (vport->localport) { 524 lport = (struct lpfc_nvme_lport *)vport->localport->private; 525 if (lport) { 526 atomic_inc(&lport->fc4NvmeLsCmpls); 527 if (status) { 528 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 529 atomic_inc(&lport->cmpl_ls_xb); 530 atomic_inc(&lport->cmpl_ls_err); 531 } 532 } 533 } 534 535 __lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe); 536 } 537 538 static int 539 lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp, 540 struct lpfc_dmabuf *inp, 541 struct nvmefc_ls_req *pnvme_lsreq, 542 void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *, 543 struct lpfc_wcqe_complete *), 544 struct lpfc_nodelist *ndlp, uint32_t num_entry, 545 uint32_t tmo, uint8_t retry) 546 { 547 struct lpfc_hba *phba = vport->phba; 548 union lpfc_wqe128 *wqe; 549 struct lpfc_iocbq *genwqe; 550 struct ulp_bde64 *bpl; 551 struct ulp_bde64 bde; 552 int i, rc, xmit_len, first_len; 553 554 /* Allocate buffer for command WQE */ 555 genwqe = lpfc_sli_get_iocbq(phba); 556 if (genwqe == NULL) 557 return 1; 558 559 wqe = &genwqe->wqe; 560 /* Initialize only 64 bytes */ 561 memset(wqe, 0, sizeof(union lpfc_wqe)); 562 563 genwqe->context3 = (uint8_t *)bmp; 564 genwqe->iocb_flag |= LPFC_IO_NVME_LS; 565 566 /* Save for completion so we can release these resources */ 567 genwqe->context1 = lpfc_nlp_get(ndlp); 568 genwqe->context2 = (uint8_t *)pnvme_lsreq; 569 /* Fill in payload, bp points to frame payload */ 570 571 if (!tmo) 572 /* FC spec states we need 3 * ratov for CT requests */ 573 tmo = (3 * phba->fc_ratov); 574 575 /* For this command calculate the xmit length of the request bde. */ 576 xmit_len = 0; 577 first_len = 0; 578 bpl = (struct ulp_bde64 *)bmp->virt; 579 for (i = 0; i < num_entry; i++) { 580 bde.tus.w = bpl[i].tus.w; 581 if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64) 582 break; 583 xmit_len += bde.tus.f.bdeSize; 584 if (i == 0) 585 first_len = xmit_len; 586 } 587 588 genwqe->rsvd2 = num_entry; 589 genwqe->hba_wqidx = 0; 590 591 /* Words 0 - 2 */ 592 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 593 wqe->generic.bde.tus.f.bdeSize = first_len; 594 wqe->generic.bde.addrLow = bpl[0].addrLow; 595 wqe->generic.bde.addrHigh = bpl[0].addrHigh; 596 597 /* Word 3 */ 598 wqe->gen_req.request_payload_len = first_len; 599 600 /* Word 4 */ 601 602 /* Word 5 */ 603 bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0); 604 bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1); 605 bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1); 606 bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ); 607 bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME); 608 609 /* Word 6 */ 610 bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com, 611 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 612 bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag); 613 614 /* Word 7 */ 615 bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1)); 616 bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3); 617 bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE); 618 bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI); 619 620 /* Word 8 */ 621 wqe->gen_req.wqe_com.abort_tag = genwqe->iotag; 622 623 /* Word 9 */ 624 bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag); 625 626 /* Word 10 */ 627 bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1); 628 bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ); 629 bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1); 630 bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE); 631 bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0); 632 633 /* Word 11 */ 634 bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 635 bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND); 636 637 638 /* Issue GEN REQ WQE for NPORT <did> */ 639 genwqe->wqe_cmpl = cmpl; 640 genwqe->iocb_cmpl = NULL; 641 genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT; 642 genwqe->vport = vport; 643 genwqe->retry = retry; 644 645 lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n", 646 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID); 647 648 rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe); 649 if (rc) { 650 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC | LOG_ELS, 651 "6045 Issue GEN REQ WQE to NPORT x%x " 652 "Data: x%x x%x rc x%x\n", 653 ndlp->nlp_DID, genwqe->iotag, 654 vport->port_state, rc); 655 lpfc_sli_release_iocbq(phba, genwqe); 656 return 1; 657 } 658 659 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS, 660 "6050 Issue GEN REQ WQE to NPORT x%x " 661 "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px " 662 "bmp:x%px xmit:%d 1st:%d\n", 663 ndlp->nlp_DID, genwqe->sli4_xritag, 664 vport->port_state, 665 genwqe, pnvme_lsreq, bmp, xmit_len, first_len); 666 return 0; 667 } 668 669 670 /** 671 * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request 672 * @vport: The local port issuing the LS 673 * @ndlp: The remote port to send the LS to 674 * @pnvme_lsreq: Pointer to LS request structure from the transport 675 * 676 * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST 677 * WQE to perform the LS operation. 678 * 679 * Return value : 680 * 0 - Success 681 * non-zero: various error codes, in form of -Exxx 682 **/ 683 int 684 __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, 685 struct nvmefc_ls_req *pnvme_lsreq, 686 void (*gen_req_cmp)(struct lpfc_hba *phba, 687 struct lpfc_iocbq *cmdwqe, 688 struct lpfc_wcqe_complete *wcqe)) 689 { 690 struct lpfc_dmabuf *bmp; 691 struct ulp_bde64 *bpl; 692 int ret; 693 uint16_t ntype, nstate; 694 695 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 696 lpfc_printf_vlog(vport, KERN_ERR, 697 LOG_NVME_DISC | LOG_NODE | LOG_NVME_IOERR, 698 "6051 NVMEx LS REQ: Bad NDLP x%px, Failing " 699 "LS Req\n", 700 ndlp); 701 return -ENODEV; 702 } 703 704 ntype = ndlp->nlp_type; 705 nstate = ndlp->nlp_state; 706 if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) || 707 (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) { 708 lpfc_printf_vlog(vport, KERN_ERR, 709 LOG_NVME_DISC | LOG_NODE | LOG_NVME_IOERR, 710 "6088 NVMEx LS REQ: Fail DID x%06x not " 711 "ready for IO. Type x%x, State x%x\n", 712 ndlp->nlp_DID, ntype, nstate); 713 return -ENODEV; 714 } 715 716 /* 717 * there are two dma buf in the request, actually there is one and 718 * the second one is just the start address + cmd size. 719 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped 720 * in a lpfc_dmabuf struct. When freeing we just free the wrapper 721 * because the nvem layer owns the data bufs. 722 * We do not have to break these packets open, we don't care what is 723 * in them. And we do not have to look at the resonse data, we only 724 * care that we got a response. All of the caring is going to happen 725 * in the nvme-fc layer. 726 */ 727 728 bmp = kmalloc(sizeof(*bmp), GFP_KERNEL); 729 if (!bmp) { 730 731 lpfc_printf_vlog(vport, KERN_ERR, 732 LOG_NVME_DISC | LOG_NVME_IOERR, 733 "6044 NVMEx LS REQ: Could not alloc LS buf " 734 "for DID %x\n", 735 ndlp->nlp_DID); 736 return -ENOMEM; 737 } 738 739 bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys)); 740 if (!bmp->virt) { 741 lpfc_printf_vlog(vport, KERN_ERR, 742 LOG_NVME_DISC | LOG_NVME_IOERR, 743 "6042 NVMEx LS REQ: Could not alloc mbuf " 744 "for DID %x\n", 745 ndlp->nlp_DID); 746 kfree(bmp); 747 return -ENOMEM; 748 } 749 750 INIT_LIST_HEAD(&bmp->list); 751 752 bpl = (struct ulp_bde64 *)bmp->virt; 753 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma)); 754 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma)); 755 bpl->tus.f.bdeFlags = 0; 756 bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen; 757 bpl->tus.w = le32_to_cpu(bpl->tus.w); 758 bpl++; 759 760 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma)); 761 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma)); 762 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I; 763 bpl->tus.f.bdeSize = pnvme_lsreq->rsplen; 764 bpl->tus.w = le32_to_cpu(bpl->tus.w); 765 766 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 767 "6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, " 768 "rqstlen:%d rsplen:%d %pad %pad\n", 769 ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen, 770 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 771 &pnvme_lsreq->rspdma); 772 773 ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr, 774 pnvme_lsreq, gen_req_cmp, ndlp, 2, 775 LPFC_NVME_LS_TIMEOUT, 0); 776 if (ret != WQE_SUCCESS) { 777 lpfc_printf_vlog(vport, KERN_ERR, 778 LOG_NVME_DISC | LOG_NVME_IOERR, 779 "6052 NVMEx REQ: EXIT. issue ls wqe failed " 780 "lsreq x%px Status %x DID %x\n", 781 pnvme_lsreq, ret, ndlp->nlp_DID); 782 lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys); 783 kfree(bmp); 784 return -EIO; 785 } 786 787 return 0; 788 } 789 790 /** 791 * lpfc_nvme_ls_req - Issue an NVME Link Service request 792 * @lpfc_nvme_lport: Transport localport that LS is to be issued from. 793 * @lpfc_nvme_rport: Transport remoteport that LS is to be sent to. 794 * @pnvme_lsreq - the transport nvme_ls_req structure for the LS 795 * 796 * Driver registers this routine to handle any link service request 797 * from the nvme_fc transport to a remote nvme-aware port. 798 * 799 * Return value : 800 * 0 - Success 801 * non-zero: various error codes, in form of -Exxx 802 **/ 803 static int 804 lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport, 805 struct nvme_fc_remote_port *pnvme_rport, 806 struct nvmefc_ls_req *pnvme_lsreq) 807 { 808 struct lpfc_nvme_lport *lport; 809 struct lpfc_nvme_rport *rport; 810 struct lpfc_vport *vport; 811 int ret; 812 813 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 814 rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 815 if (unlikely(!lport) || unlikely(!rport)) 816 return -EINVAL; 817 818 vport = lport->vport; 819 if (vport->load_flag & FC_UNLOADING) 820 return -ENODEV; 821 822 atomic_inc(&lport->fc4NvmeLsRequests); 823 824 ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq, 825 lpfc_nvme_ls_req_cmp); 826 if (ret) 827 atomic_inc(&lport->xmt_ls_err); 828 829 return ret; 830 } 831 832 /** 833 * __lpfc_nvme_ls_abort - Generic service routine to abort a prior 834 * NVME LS request 835 * @vport: The local port that issued the LS 836 * @ndlp: The remote port the LS was sent to 837 * @pnvme_lsreq: Pointer to LS request structure from the transport 838 * 839 * The driver validates the ndlp, looks for the LS, and aborts the 840 * LS if found. 841 * 842 * Returns: 843 * 0 : if LS found and aborted 844 * non-zero: various error conditions in form -Exxx 845 **/ 846 int 847 __lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, 848 struct nvmefc_ls_req *pnvme_lsreq) 849 { 850 struct lpfc_hba *phba = vport->phba; 851 struct lpfc_sli_ring *pring; 852 struct lpfc_iocbq *wqe, *next_wqe; 853 bool foundit = false; 854 855 if (!ndlp) { 856 lpfc_printf_log(phba, KERN_ERR, 857 LOG_NVME_DISC | LOG_NODE | 858 LOG_NVME_IOERR | LOG_NVME_ABTS, 859 "6049 NVMEx LS REQ Abort: Bad NDLP x%px DID " 860 "x%06x, Failing LS Req\n", 861 ndlp, ndlp ? ndlp->nlp_DID : 0); 862 return -EINVAL; 863 } 864 865 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, 866 "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq " 867 "x%p rqstlen:%d rsplen:%d %pad %pad\n", 868 pnvme_lsreq, pnvme_lsreq->rqstlen, 869 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 870 &pnvme_lsreq->rspdma); 871 872 /* 873 * Lock the ELS ring txcmplq and look for the wqe that matches 874 * this ELS. If found, issue an abort on the wqe. 875 */ 876 pring = phba->sli4_hba.nvmels_wq->pring; 877 spin_lock_irq(&phba->hbalock); 878 spin_lock(&pring->ring_lock); 879 list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) { 880 if (wqe->context2 == pnvme_lsreq) { 881 wqe->iocb_flag |= LPFC_DRIVER_ABORTED; 882 foundit = true; 883 break; 884 } 885 } 886 spin_unlock(&pring->ring_lock); 887 888 if (foundit) 889 lpfc_sli_issue_abort_iotag(phba, pring, wqe); 890 spin_unlock_irq(&phba->hbalock); 891 892 if (foundit) 893 return 0; 894 895 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, 896 "6213 NVMEx LS REQ Abort: Unable to locate req x%p\n", 897 pnvme_lsreq); 898 return -EINVAL; 899 } 900 901 static int 902 lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport, 903 struct nvme_fc_remote_port *remoteport, 904 struct nvmefc_ls_rsp *ls_rsp) 905 { 906 struct lpfc_async_xchg_ctx *axchg = 907 container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp); 908 struct lpfc_nvme_lport *lport; 909 int rc; 910 911 if (axchg->phba->pport->load_flag & FC_UNLOADING) 912 return -ENODEV; 913 914 lport = (struct lpfc_nvme_lport *)localport->private; 915 916 rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp); 917 918 if (rc) { 919 /* 920 * unless the failure is due to having already sent 921 * the response, an abort will be generated for the 922 * exchange if the rsp can't be sent. 923 */ 924 if (rc != -EALREADY) 925 atomic_inc(&lport->xmt_ls_abort); 926 return rc; 927 } 928 929 return 0; 930 } 931 932 /** 933 * lpfc_nvme_ls_abort - Abort a prior NVME LS request 934 * @lpfc_nvme_lport: Transport localport that LS is to be issued from. 935 * @lpfc_nvme_rport: Transport remoteport that LS is to be sent to. 936 * @pnvme_lsreq - the transport nvme_ls_req structure for the LS 937 * 938 * Driver registers this routine to abort a NVME LS request that is 939 * in progress (from the transports perspective). 940 **/ 941 static void 942 lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, 943 struct nvme_fc_remote_port *pnvme_rport, 944 struct nvmefc_ls_req *pnvme_lsreq) 945 { 946 struct lpfc_nvme_lport *lport; 947 struct lpfc_vport *vport; 948 struct lpfc_hba *phba; 949 struct lpfc_nodelist *ndlp; 950 int ret; 951 952 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 953 if (unlikely(!lport)) 954 return; 955 vport = lport->vport; 956 phba = vport->phba; 957 958 if (vport->load_flag & FC_UNLOADING) 959 return; 960 961 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); 962 963 ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq); 964 if (!ret) 965 atomic_inc(&lport->xmt_ls_abort); 966 } 967 968 /* Fix up the existing sgls for NVME IO. */ 969 static inline void 970 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, 971 struct lpfc_io_buf *lpfc_ncmd, 972 struct nvmefc_fcp_req *nCmd) 973 { 974 struct lpfc_hba *phba = vport->phba; 975 struct sli4_sge *sgl; 976 union lpfc_wqe128 *wqe; 977 uint32_t *wptr, *dptr; 978 979 /* 980 * Get a local pointer to the built-in wqe and correct 981 * the cmd size to match NVME's 96 bytes and fix 982 * the dma address. 983 */ 984 985 wqe = &lpfc_ncmd->cur_iocbq.wqe; 986 987 /* 988 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to 989 * match NVME. NVME sends 96 bytes. Also, use the 990 * nvme commands command and response dma addresses 991 * rather than the virtual memory to ease the restore 992 * operation. 993 */ 994 sgl = lpfc_ncmd->dma_sgl; 995 sgl->sge_len = cpu_to_le32(nCmd->cmdlen); 996 if (phba->cfg_nvme_embed_cmd) { 997 sgl->addr_hi = 0; 998 sgl->addr_lo = 0; 999 1000 /* Word 0-2 - NVME CMND IU (embedded payload) */ 1001 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED; 1002 wqe->generic.bde.tus.f.bdeSize = 56; 1003 wqe->generic.bde.addrHigh = 0; 1004 wqe->generic.bde.addrLow = 64; /* Word 16 */ 1005 1006 /* Word 10 - dbde is 0, wqes is 1 in template */ 1007 1008 /* 1009 * Embed the payload in the last half of the WQE 1010 * WQE words 16-30 get the NVME CMD IU payload 1011 * 1012 * WQE words 16-19 get payload Words 1-4 1013 * WQE words 20-21 get payload Words 6-7 1014 * WQE words 22-29 get payload Words 16-23 1015 */ 1016 wptr = &wqe->words[16]; /* WQE ptr */ 1017 dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */ 1018 dptr++; /* Skip Word 0 in payload */ 1019 1020 *wptr++ = *dptr++; /* Word 1 */ 1021 *wptr++ = *dptr++; /* Word 2 */ 1022 *wptr++ = *dptr++; /* Word 3 */ 1023 *wptr++ = *dptr++; /* Word 4 */ 1024 dptr++; /* Skip Word 5 in payload */ 1025 *wptr++ = *dptr++; /* Word 6 */ 1026 *wptr++ = *dptr++; /* Word 7 */ 1027 dptr += 8; /* Skip Words 8-15 in payload */ 1028 *wptr++ = *dptr++; /* Word 16 */ 1029 *wptr++ = *dptr++; /* Word 17 */ 1030 *wptr++ = *dptr++; /* Word 18 */ 1031 *wptr++ = *dptr++; /* Word 19 */ 1032 *wptr++ = *dptr++; /* Word 20 */ 1033 *wptr++ = *dptr++; /* Word 21 */ 1034 *wptr++ = *dptr++; /* Word 22 */ 1035 *wptr = *dptr; /* Word 23 */ 1036 } else { 1037 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma)); 1038 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma)); 1039 1040 /* Word 0-2 - NVME CMND IU Inline BDE */ 1041 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 1042 wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen; 1043 wqe->generic.bde.addrHigh = sgl->addr_hi; 1044 wqe->generic.bde.addrLow = sgl->addr_lo; 1045 1046 /* Word 10 */ 1047 bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); 1048 bf_set(wqe_wqes, &wqe->generic.wqe_com, 0); 1049 } 1050 1051 sgl++; 1052 1053 /* Setup the physical region for the FCP RSP */ 1054 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma)); 1055 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma)); 1056 sgl->word2 = le32_to_cpu(sgl->word2); 1057 if (nCmd->sg_cnt) 1058 bf_set(lpfc_sli4_sge_last, sgl, 0); 1059 else 1060 bf_set(lpfc_sli4_sge_last, sgl, 1); 1061 sgl->word2 = cpu_to_le32(sgl->word2); 1062 sgl->sge_len = cpu_to_le32(nCmd->rsplen); 1063 } 1064 1065 1066 /** 1067 * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO 1068 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1069 * @lpfc_nvme_lport: Pointer to the driver's local port data 1070 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1071 * 1072 * Driver registers this routine as it io request handler. This 1073 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1074 * data structure to the rport indicated in @lpfc_nvme_rport. 1075 * 1076 * Return value : 1077 * 0 - Success 1078 * TODO: What are the failure codes. 1079 **/ 1080 static void 1081 lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, 1082 struct lpfc_wcqe_complete *wcqe) 1083 { 1084 struct lpfc_io_buf *lpfc_ncmd = 1085 (struct lpfc_io_buf *)pwqeIn->context1; 1086 struct lpfc_vport *vport = pwqeIn->vport; 1087 struct nvmefc_fcp_req *nCmd; 1088 struct nvme_fc_ersp_iu *ep; 1089 struct nvme_fc_cmd_iu *cp; 1090 struct lpfc_nodelist *ndlp; 1091 struct lpfc_nvme_fcpreq_priv *freqpriv; 1092 struct lpfc_nvme_lport *lport; 1093 uint32_t code, status, idx; 1094 uint16_t cid, sqhd, data; 1095 uint32_t *ptr; 1096 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1097 int cpu; 1098 #endif 1099 1100 /* Sanity check on return of outstanding command */ 1101 if (!lpfc_ncmd) { 1102 lpfc_printf_vlog(vport, KERN_ERR, 1103 LOG_NODE | LOG_NVME_IOERR, 1104 "6071 Null lpfc_ncmd pointer. No " 1105 "release, skip completion\n"); 1106 return; 1107 } 1108 1109 /* Guard against abort handler being called at same time */ 1110 spin_lock(&lpfc_ncmd->buf_lock); 1111 1112 if (!lpfc_ncmd->nvmeCmd) { 1113 spin_unlock(&lpfc_ncmd->buf_lock); 1114 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 1115 "6066 Missing cmpl ptrs: lpfc_ncmd x%px, " 1116 "nvmeCmd x%px\n", 1117 lpfc_ncmd, lpfc_ncmd->nvmeCmd); 1118 1119 /* Release the lpfc_ncmd regardless of the missing elements. */ 1120 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1121 return; 1122 } 1123 nCmd = lpfc_ncmd->nvmeCmd; 1124 status = bf_get(lpfc_wcqe_c_status, wcqe); 1125 1126 idx = lpfc_ncmd->cur_iocbq.hba_wqidx; 1127 phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++; 1128 1129 if (unlikely(status && vport->localport)) { 1130 lport = (struct lpfc_nvme_lport *)vport->localport->private; 1131 if (lport) { 1132 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 1133 atomic_inc(&lport->cmpl_fcp_xb); 1134 atomic_inc(&lport->cmpl_fcp_err); 1135 } 1136 } 1137 1138 lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n", 1139 lpfc_ncmd->cur_iocbq.sli4_xritag, 1140 status, wcqe->parameter); 1141 /* 1142 * Catch race where our node has transitioned, but the 1143 * transport is still transitioning. 1144 */ 1145 ndlp = lpfc_ncmd->ndlp; 1146 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 1147 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 1148 "6062 Ignoring NVME cmpl. No ndlp\n"); 1149 goto out_err; 1150 } 1151 1152 code = bf_get(lpfc_wcqe_c_code, wcqe); 1153 if (code == CQE_CODE_NVME_ERSP) { 1154 /* For this type of CQE, we need to rebuild the rsp */ 1155 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr; 1156 1157 /* 1158 * Get Command Id from cmd to plug into response. This 1159 * code is not needed in the next NVME Transport drop. 1160 */ 1161 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr; 1162 cid = cp->sqe.common.command_id; 1163 1164 /* 1165 * RSN is in CQE word 2 1166 * SQHD is in CQE Word 3 bits 15:0 1167 * Cmd Specific info is in CQE Word 1 1168 * and in CQE Word 0 bits 15:0 1169 */ 1170 sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe); 1171 1172 /* Now lets build the NVME ERSP IU */ 1173 ep->iu_len = cpu_to_be16(8); 1174 ep->rsn = wcqe->parameter; 1175 ep->xfrd_len = cpu_to_be32(nCmd->payload_length); 1176 ep->rsvd12 = 0; 1177 ptr = (uint32_t *)&ep->cqe.result.u64; 1178 *ptr++ = wcqe->total_data_placed; 1179 data = bf_get(lpfc_wcqe_c_ersp0, wcqe); 1180 *ptr = (uint32_t)data; 1181 ep->cqe.sq_head = sqhd; 1182 ep->cqe.sq_id = nCmd->sqid; 1183 ep->cqe.command_id = cid; 1184 ep->cqe.status = 0; 1185 1186 lpfc_ncmd->status = IOSTAT_SUCCESS; 1187 lpfc_ncmd->result = 0; 1188 nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN; 1189 nCmd->transferred_length = nCmd->payload_length; 1190 } else { 1191 lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK); 1192 lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK); 1193 1194 /* For NVME, the only failure path that results in an 1195 * IO error is when the adapter rejects it. All other 1196 * conditions are a success case and resolved by the 1197 * transport. 1198 * IOSTAT_FCP_RSP_ERROR means: 1199 * 1. Length of data received doesn't match total 1200 * transfer length in WQE 1201 * 2. If the RSP payload does NOT match these cases: 1202 * a. RSP length 12/24 bytes and all zeros 1203 * b. NVME ERSP 1204 */ 1205 switch (lpfc_ncmd->status) { 1206 case IOSTAT_SUCCESS: 1207 nCmd->transferred_length = wcqe->total_data_placed; 1208 nCmd->rcv_rsplen = 0; 1209 nCmd->status = 0; 1210 break; 1211 case IOSTAT_FCP_RSP_ERROR: 1212 nCmd->transferred_length = wcqe->total_data_placed; 1213 nCmd->rcv_rsplen = wcqe->parameter; 1214 nCmd->status = 0; 1215 /* Sanity check */ 1216 if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) 1217 break; 1218 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 1219 "6081 NVME Completion Protocol Error: " 1220 "xri %x status x%x result x%x " 1221 "placed x%x\n", 1222 lpfc_ncmd->cur_iocbq.sli4_xritag, 1223 lpfc_ncmd->status, lpfc_ncmd->result, 1224 wcqe->total_data_placed); 1225 break; 1226 case IOSTAT_LOCAL_REJECT: 1227 /* Let fall through to set command final state. */ 1228 if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED) 1229 lpfc_printf_vlog(vport, KERN_INFO, 1230 LOG_NVME_IOERR, 1231 "6032 Delay Aborted cmd x%px " 1232 "nvme cmd x%px, xri x%x, " 1233 "xb %d\n", 1234 lpfc_ncmd, nCmd, 1235 lpfc_ncmd->cur_iocbq.sli4_xritag, 1236 bf_get(lpfc_wcqe_c_xb, wcqe)); 1237 /* fall through */ 1238 default: 1239 out_err: 1240 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1241 "6072 NVME Completion Error: xri %x " 1242 "status x%x result x%x [x%x] " 1243 "placed x%x\n", 1244 lpfc_ncmd->cur_iocbq.sli4_xritag, 1245 lpfc_ncmd->status, lpfc_ncmd->result, 1246 wcqe->parameter, 1247 wcqe->total_data_placed); 1248 nCmd->transferred_length = 0; 1249 nCmd->rcv_rsplen = 0; 1250 nCmd->status = NVME_SC_INTERNAL; 1251 } 1252 } 1253 1254 /* pick up SLI4 exhange busy condition */ 1255 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 1256 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; 1257 else 1258 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 1259 1260 /* Update stats and complete the IO. There is 1261 * no need for dma unprep because the nvme_transport 1262 * owns the dma address. 1263 */ 1264 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1265 if (lpfc_ncmd->ts_cmd_start) { 1266 lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp; 1267 lpfc_ncmd->ts_data_io = ktime_get_ns(); 1268 phba->ktime_last_cmd = lpfc_ncmd->ts_data_io; 1269 lpfc_io_ktime(phba, lpfc_ncmd); 1270 } 1271 if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) { 1272 cpu = raw_smp_processor_id(); 1273 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io); 1274 if (lpfc_ncmd->cpu != cpu) 1275 lpfc_printf_vlog(vport, 1276 KERN_INFO, LOG_NVME_IOERR, 1277 "6701 CPU Check cmpl: " 1278 "cpu %d expect %d\n", 1279 cpu, lpfc_ncmd->cpu); 1280 } 1281 #endif 1282 1283 /* NVME targets need completion held off until the abort exchange 1284 * completes unless the NVME Rport is getting unregistered. 1285 */ 1286 1287 if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 1288 freqpriv = nCmd->private; 1289 freqpriv->nvme_buf = NULL; 1290 lpfc_ncmd->nvmeCmd = NULL; 1291 spin_unlock(&lpfc_ncmd->buf_lock); 1292 nCmd->done(nCmd); 1293 } else 1294 spin_unlock(&lpfc_ncmd->buf_lock); 1295 1296 /* Call release with XB=1 to queue the IO into the abort list. */ 1297 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1298 } 1299 1300 1301 /** 1302 * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO 1303 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1304 * @lpfc_nvme_lport: Pointer to the driver's local port data 1305 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1306 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1307 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1308 * 1309 * Driver registers this routine as it io request handler. This 1310 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1311 * data structure to the rport indicated in @lpfc_nvme_rport. 1312 * 1313 * Return value : 1314 * 0 - Success 1315 * TODO: What are the failure codes. 1316 **/ 1317 static int 1318 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, 1319 struct lpfc_io_buf *lpfc_ncmd, 1320 struct lpfc_nodelist *pnode, 1321 struct lpfc_fc4_ctrl_stat *cstat) 1322 { 1323 struct lpfc_hba *phba = vport->phba; 1324 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1325 struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq); 1326 union lpfc_wqe128 *wqe = &pwqeq->wqe; 1327 uint32_t req_len; 1328 1329 if (!NLP_CHK_NODE_ACT(pnode)) 1330 return -EINVAL; 1331 1332 /* 1333 * There are three possibilities here - use scatter-gather segment, use 1334 * the single mapping, or neither. 1335 */ 1336 if (nCmd->sg_cnt) { 1337 if (nCmd->io_dir == NVMEFC_FCP_WRITE) { 1338 /* From the iwrite template, initialize words 7 - 11 */ 1339 memcpy(&wqe->words[7], 1340 &lpfc_iwrite_cmd_template.words[7], 1341 sizeof(uint32_t) * 5); 1342 1343 /* Word 4 */ 1344 wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length; 1345 1346 /* Word 5 */ 1347 if ((phba->cfg_nvme_enable_fb) && 1348 (pnode->nlp_flag & NLP_FIRSTBURST)) { 1349 req_len = lpfc_ncmd->nvmeCmd->payload_length; 1350 if (req_len < pnode->nvme_fb_size) 1351 wqe->fcp_iwrite.initial_xfer_len = 1352 req_len; 1353 else 1354 wqe->fcp_iwrite.initial_xfer_len = 1355 pnode->nvme_fb_size; 1356 } else { 1357 wqe->fcp_iwrite.initial_xfer_len = 0; 1358 } 1359 cstat->output_requests++; 1360 } else { 1361 /* From the iread template, initialize words 7 - 11 */ 1362 memcpy(&wqe->words[7], 1363 &lpfc_iread_cmd_template.words[7], 1364 sizeof(uint32_t) * 5); 1365 1366 /* Word 4 */ 1367 wqe->fcp_iread.total_xfer_len = nCmd->payload_length; 1368 1369 /* Word 5 */ 1370 wqe->fcp_iread.rsrvd5 = 0; 1371 1372 cstat->input_requests++; 1373 } 1374 } else { 1375 /* From the icmnd template, initialize words 4 - 11 */ 1376 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4], 1377 sizeof(uint32_t) * 8); 1378 cstat->control_requests++; 1379 } 1380 1381 if (pnode->nlp_nvme_info & NLP_NVME_NSLER) 1382 bf_set(wqe_erp, &wqe->generic.wqe_com, 1); 1383 /* 1384 * Finish initializing those WQE fields that are independent 1385 * of the nvme_cmnd request_buffer 1386 */ 1387 1388 /* Word 3 */ 1389 bf_set(payload_offset_len, &wqe->fcp_icmd, 1390 (nCmd->rsplen + nCmd->cmdlen)); 1391 1392 /* Word 6 */ 1393 bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com, 1394 phba->sli4_hba.rpi_ids[pnode->nlp_rpi]); 1395 bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag); 1396 1397 /* Word 8 */ 1398 wqe->generic.wqe_com.abort_tag = pwqeq->iotag; 1399 1400 /* Word 9 */ 1401 bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag); 1402 1403 /* Words 13 14 15 are for PBDE support */ 1404 1405 pwqeq->vport = vport; 1406 return 0; 1407 } 1408 1409 1410 /** 1411 * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO 1412 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1413 * @lpfc_nvme_lport: Pointer to the driver's local port data 1414 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1415 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1416 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1417 * 1418 * Driver registers this routine as it io request handler. This 1419 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1420 * data structure to the rport indicated in @lpfc_nvme_rport. 1421 * 1422 * Return value : 1423 * 0 - Success 1424 * TODO: What are the failure codes. 1425 **/ 1426 static int 1427 lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, 1428 struct lpfc_io_buf *lpfc_ncmd) 1429 { 1430 struct lpfc_hba *phba = vport->phba; 1431 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1432 union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe; 1433 struct sli4_sge *sgl = lpfc_ncmd->dma_sgl; 1434 struct sli4_hybrid_sgl *sgl_xtra = NULL; 1435 struct scatterlist *data_sg; 1436 struct sli4_sge *first_data_sgl; 1437 struct ulp_bde64 *bde; 1438 dma_addr_t physaddr = 0; 1439 uint32_t num_bde = 0; 1440 uint32_t dma_len = 0; 1441 uint32_t dma_offset = 0; 1442 int nseg, i, j; 1443 bool lsp_just_set = false; 1444 1445 /* Fix up the command and response DMA stuff. */ 1446 lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd); 1447 1448 /* 1449 * There are three possibilities here - use scatter-gather segment, use 1450 * the single mapping, or neither. 1451 */ 1452 if (nCmd->sg_cnt) { 1453 /* 1454 * Jump over the cmd and rsp SGEs. The fix routine 1455 * has already adjusted for this. 1456 */ 1457 sgl += 2; 1458 1459 first_data_sgl = sgl; 1460 lpfc_ncmd->seg_cnt = nCmd->sg_cnt; 1461 if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) { 1462 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1463 "6058 Too many sg segments from " 1464 "NVME Transport. Max %d, " 1465 "nvmeIO sg_cnt %d\n", 1466 phba->cfg_nvme_seg_cnt + 1, 1467 lpfc_ncmd->seg_cnt); 1468 lpfc_ncmd->seg_cnt = 0; 1469 return 1; 1470 } 1471 1472 /* 1473 * The driver established a maximum scatter-gather segment count 1474 * during probe that limits the number of sg elements in any 1475 * single nvme command. Just run through the seg_cnt and format 1476 * the sge's. 1477 */ 1478 nseg = nCmd->sg_cnt; 1479 data_sg = nCmd->first_sgl; 1480 1481 /* for tracking the segment boundaries */ 1482 j = 2; 1483 for (i = 0; i < nseg; i++) { 1484 if (data_sg == NULL) { 1485 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1486 "6059 dptr err %d, nseg %d\n", 1487 i, nseg); 1488 lpfc_ncmd->seg_cnt = 0; 1489 return 1; 1490 } 1491 1492 sgl->word2 = 0; 1493 if ((num_bde + 1) == nseg) { 1494 bf_set(lpfc_sli4_sge_last, sgl, 1); 1495 bf_set(lpfc_sli4_sge_type, sgl, 1496 LPFC_SGE_TYPE_DATA); 1497 } else { 1498 bf_set(lpfc_sli4_sge_last, sgl, 0); 1499 1500 /* expand the segment */ 1501 if (!lsp_just_set && 1502 !((j + 1) % phba->border_sge_num) && 1503 ((nseg - 1) != i)) { 1504 /* set LSP type */ 1505 bf_set(lpfc_sli4_sge_type, sgl, 1506 LPFC_SGE_TYPE_LSP); 1507 1508 sgl_xtra = lpfc_get_sgl_per_hdwq( 1509 phba, lpfc_ncmd); 1510 1511 if (unlikely(!sgl_xtra)) { 1512 lpfc_ncmd->seg_cnt = 0; 1513 return 1; 1514 } 1515 sgl->addr_lo = cpu_to_le32(putPaddrLow( 1516 sgl_xtra->dma_phys_sgl)); 1517 sgl->addr_hi = cpu_to_le32(putPaddrHigh( 1518 sgl_xtra->dma_phys_sgl)); 1519 1520 } else { 1521 bf_set(lpfc_sli4_sge_type, sgl, 1522 LPFC_SGE_TYPE_DATA); 1523 } 1524 } 1525 1526 if (!(bf_get(lpfc_sli4_sge_type, sgl) & 1527 LPFC_SGE_TYPE_LSP)) { 1528 if ((nseg - 1) == i) 1529 bf_set(lpfc_sli4_sge_last, sgl, 1); 1530 1531 physaddr = data_sg->dma_address; 1532 dma_len = data_sg->length; 1533 sgl->addr_lo = cpu_to_le32( 1534 putPaddrLow(physaddr)); 1535 sgl->addr_hi = cpu_to_le32( 1536 putPaddrHigh(physaddr)); 1537 1538 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset); 1539 sgl->word2 = cpu_to_le32(sgl->word2); 1540 sgl->sge_len = cpu_to_le32(dma_len); 1541 1542 dma_offset += dma_len; 1543 data_sg = sg_next(data_sg); 1544 1545 sgl++; 1546 1547 lsp_just_set = false; 1548 } else { 1549 sgl->word2 = cpu_to_le32(sgl->word2); 1550 1551 sgl->sge_len = cpu_to_le32( 1552 phba->cfg_sg_dma_buf_size); 1553 1554 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl; 1555 i = i - 1; 1556 1557 lsp_just_set = true; 1558 } 1559 1560 j++; 1561 } 1562 if (phba->cfg_enable_pbde) { 1563 /* Use PBDE support for first SGL only, offset == 0 */ 1564 /* Words 13-15 */ 1565 bde = (struct ulp_bde64 *) 1566 &wqe->words[13]; 1567 bde->addrLow = first_data_sgl->addr_lo; 1568 bde->addrHigh = first_data_sgl->addr_hi; 1569 bde->tus.f.bdeSize = 1570 le32_to_cpu(first_data_sgl->sge_len); 1571 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64; 1572 bde->tus.w = cpu_to_le32(bde->tus.w); 1573 /* wqe_pbde is 1 in template */ 1574 } else { 1575 memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3)); 1576 bf_set(wqe_pbde, &wqe->generic.wqe_com, 0); 1577 } 1578 1579 } else { 1580 lpfc_ncmd->seg_cnt = 0; 1581 1582 /* For this clause to be valid, the payload_length 1583 * and sg_cnt must zero. 1584 */ 1585 if (nCmd->payload_length != 0) { 1586 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1587 "6063 NVME DMA Prep Err: sg_cnt %d " 1588 "payload_length x%x\n", 1589 nCmd->sg_cnt, nCmd->payload_length); 1590 return 1; 1591 } 1592 } 1593 return 0; 1594 } 1595 1596 /** 1597 * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO 1598 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1599 * @lpfc_nvme_lport: Pointer to the driver's local port data 1600 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1601 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1602 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1603 * 1604 * Driver registers this routine as it io request handler. This 1605 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1606 * data structure to the rport 1607 indicated in @lpfc_nvme_rport. 1608 * 1609 * Return value : 1610 * 0 - Success 1611 * TODO: What are the failure codes. 1612 **/ 1613 static int 1614 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, 1615 struct nvme_fc_remote_port *pnvme_rport, 1616 void *hw_queue_handle, 1617 struct nvmefc_fcp_req *pnvme_fcreq) 1618 { 1619 int ret = 0; 1620 int expedite = 0; 1621 int idx, cpu; 1622 struct lpfc_nvme_lport *lport; 1623 struct lpfc_fc4_ctrl_stat *cstat; 1624 struct lpfc_vport *vport; 1625 struct lpfc_hba *phba; 1626 struct lpfc_nodelist *ndlp; 1627 struct lpfc_io_buf *lpfc_ncmd; 1628 struct lpfc_nvme_rport *rport; 1629 struct lpfc_nvme_qhandle *lpfc_queue_info; 1630 struct lpfc_nvme_fcpreq_priv *freqpriv; 1631 struct nvme_common_command *sqe; 1632 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1633 uint64_t start = 0; 1634 #endif 1635 1636 /* Validate pointers. LLDD fault handling with transport does 1637 * have timing races. 1638 */ 1639 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1640 if (unlikely(!lport)) { 1641 ret = -EINVAL; 1642 goto out_fail; 1643 } 1644 1645 vport = lport->vport; 1646 1647 if (unlikely(!hw_queue_handle)) { 1648 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1649 "6117 Fail IO, NULL hw_queue_handle\n"); 1650 atomic_inc(&lport->xmt_fcp_err); 1651 ret = -EBUSY; 1652 goto out_fail; 1653 } 1654 1655 phba = vport->phba; 1656 1657 if (vport->load_flag & FC_UNLOADING) { 1658 ret = -ENODEV; 1659 goto out_fail; 1660 } 1661 1662 if (unlikely(vport->load_flag & FC_UNLOADING)) { 1663 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1664 "6124 Fail IO, Driver unload\n"); 1665 atomic_inc(&lport->xmt_fcp_err); 1666 ret = -ENODEV; 1667 goto out_fail; 1668 } 1669 1670 freqpriv = pnvme_fcreq->private; 1671 if (unlikely(!freqpriv)) { 1672 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1673 "6158 Fail IO, NULL request data\n"); 1674 atomic_inc(&lport->xmt_fcp_err); 1675 ret = -EINVAL; 1676 goto out_fail; 1677 } 1678 1679 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1680 if (phba->ktime_on) 1681 start = ktime_get_ns(); 1682 #endif 1683 rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 1684 lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle; 1685 1686 /* 1687 * Catch race where our node has transitioned, but the 1688 * transport is still transitioning. 1689 */ 1690 ndlp = rport->ndlp; 1691 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 1692 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR, 1693 "6053 Busy IO, ndlp not ready: rport x%px " 1694 "ndlp x%px, DID x%06x\n", 1695 rport, ndlp, pnvme_rport->port_id); 1696 atomic_inc(&lport->xmt_fcp_err); 1697 ret = -EBUSY; 1698 goto out_fail; 1699 } 1700 1701 /* The remote node has to be a mapped target or it's an error. */ 1702 if ((ndlp->nlp_type & NLP_NVME_TARGET) && 1703 (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) { 1704 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR, 1705 "6036 Fail IO, DID x%06x not ready for " 1706 "IO. State x%x, Type x%x Flg x%x\n", 1707 pnvme_rport->port_id, 1708 ndlp->nlp_state, ndlp->nlp_type, 1709 ndlp->upcall_flags); 1710 atomic_inc(&lport->xmt_fcp_bad_ndlp); 1711 ret = -EBUSY; 1712 goto out_fail; 1713 1714 } 1715 1716 /* Currently only NVME Keep alive commands should be expedited 1717 * if the driver runs out of a resource. These should only be 1718 * issued on the admin queue, qidx 0 1719 */ 1720 if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) { 1721 sqe = &((struct nvme_fc_cmd_iu *) 1722 pnvme_fcreq->cmdaddr)->sqe.common; 1723 if (sqe->opcode == nvme_admin_keep_alive) 1724 expedite = 1; 1725 } 1726 1727 /* The node is shared with FCP IO, make sure the IO pending count does 1728 * not exceed the programmed depth. 1729 */ 1730 if (lpfc_ndlp_check_qdepth(phba, ndlp)) { 1731 if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) && 1732 !expedite) { 1733 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1734 "6174 Fail IO, ndlp qdepth exceeded: " 1735 "idx %d DID %x pend %d qdepth %d\n", 1736 lpfc_queue_info->index, ndlp->nlp_DID, 1737 atomic_read(&ndlp->cmd_pending), 1738 ndlp->cmd_qdepth); 1739 atomic_inc(&lport->xmt_fcp_qdepth); 1740 ret = -EBUSY; 1741 goto out_fail; 1742 } 1743 } 1744 1745 /* Lookup Hardware Queue index based on fcp_io_sched module parameter */ 1746 if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) { 1747 idx = lpfc_queue_info->index; 1748 } else { 1749 cpu = raw_smp_processor_id(); 1750 idx = phba->sli4_hba.cpu_map[cpu].hdwq; 1751 } 1752 1753 lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite); 1754 if (lpfc_ncmd == NULL) { 1755 atomic_inc(&lport->xmt_fcp_noxri); 1756 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1757 "6065 Fail IO, driver buffer pool is empty: " 1758 "idx %d DID %x\n", 1759 lpfc_queue_info->index, ndlp->nlp_DID); 1760 ret = -EBUSY; 1761 goto out_fail; 1762 } 1763 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1764 if (start) { 1765 lpfc_ncmd->ts_cmd_start = start; 1766 lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd; 1767 } else { 1768 lpfc_ncmd->ts_cmd_start = 0; 1769 } 1770 #endif 1771 1772 /* 1773 * Store the data needed by the driver to issue, abort, and complete 1774 * an IO. 1775 * Do not let the IO hang out forever. There is no midlayer issuing 1776 * an abort so inform the FW of the maximum IO pending time. 1777 */ 1778 freqpriv->nvme_buf = lpfc_ncmd; 1779 lpfc_ncmd->nvmeCmd = pnvme_fcreq; 1780 lpfc_ncmd->ndlp = ndlp; 1781 lpfc_ncmd->qidx = lpfc_queue_info->qidx; 1782 1783 /* 1784 * Issue the IO on the WQ indicated by index in the hw_queue_handle. 1785 * This identfier was create in our hardware queue create callback 1786 * routine. The driver now is dependent on the IO queue steering from 1787 * the transport. We are trusting the upper NVME layers know which 1788 * index to use and that they have affinitized a CPU to this hardware 1789 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ. 1790 */ 1791 lpfc_ncmd->cur_iocbq.hba_wqidx = idx; 1792 cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat; 1793 1794 lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat); 1795 ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd); 1796 if (ret) { 1797 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1798 "6175 Fail IO, Prep DMA: " 1799 "idx %d DID %x\n", 1800 lpfc_queue_info->index, ndlp->nlp_DID); 1801 atomic_inc(&lport->xmt_fcp_err); 1802 ret = -ENOMEM; 1803 goto out_free_nvme_buf; 1804 } 1805 1806 lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n", 1807 lpfc_ncmd->cur_iocbq.sli4_xritag, 1808 lpfc_queue_info->index, ndlp->nlp_DID); 1809 1810 ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq); 1811 if (ret) { 1812 atomic_inc(&lport->xmt_fcp_wqerr); 1813 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1814 "6113 Fail IO, Could not issue WQE err %x " 1815 "sid: x%x did: x%x oxid: x%x\n", 1816 ret, vport->fc_myDID, ndlp->nlp_DID, 1817 lpfc_ncmd->cur_iocbq.sli4_xritag); 1818 goto out_free_nvme_buf; 1819 } 1820 1821 if (phba->cfg_xri_rebalancing) 1822 lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no); 1823 1824 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1825 if (lpfc_ncmd->ts_cmd_start) 1826 lpfc_ncmd->ts_cmd_wqput = ktime_get_ns(); 1827 1828 if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) { 1829 cpu = raw_smp_processor_id(); 1830 this_cpu_inc(phba->sli4_hba.c_stat->xmt_io); 1831 lpfc_ncmd->cpu = cpu; 1832 if (idx != cpu) 1833 lpfc_printf_vlog(vport, 1834 KERN_INFO, LOG_NVME_IOERR, 1835 "6702 CPU Check cmd: " 1836 "cpu %d wq %d\n", 1837 lpfc_ncmd->cpu, 1838 lpfc_queue_info->index); 1839 } 1840 #endif 1841 return 0; 1842 1843 out_free_nvme_buf: 1844 if (lpfc_ncmd->nvmeCmd->sg_cnt) { 1845 if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE) 1846 cstat->output_requests--; 1847 else 1848 cstat->input_requests--; 1849 } else 1850 cstat->control_requests--; 1851 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1852 out_fail: 1853 return ret; 1854 } 1855 1856 /** 1857 * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request. 1858 * @phba: Pointer to HBA context object 1859 * @cmdiocb: Pointer to command iocb object. 1860 * @rspiocb: Pointer to response iocb object. 1861 * 1862 * This is the callback function for any NVME FCP IO that was aborted. 1863 * 1864 * Return value: 1865 * None 1866 **/ 1867 void 1868 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, 1869 struct lpfc_wcqe_complete *abts_cmpl) 1870 { 1871 lpfc_printf_log(phba, KERN_INFO, LOG_NVME, 1872 "6145 ABORT_XRI_CN completing on rpi x%x " 1873 "original iotag x%x, abort cmd iotag x%x " 1874 "req_tag x%x, status x%x, hwstatus x%x\n", 1875 cmdiocb->iocb.un.acxri.abortContextTag, 1876 cmdiocb->iocb.un.acxri.abortIoTag, 1877 cmdiocb->iotag, 1878 bf_get(lpfc_wcqe_c_request_tag, abts_cmpl), 1879 bf_get(lpfc_wcqe_c_status, abts_cmpl), 1880 bf_get(lpfc_wcqe_c_hw_status, abts_cmpl)); 1881 lpfc_sli_release_iocbq(phba, cmdiocb); 1882 } 1883 1884 /** 1885 * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS 1886 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1887 * @lpfc_nvme_lport: Pointer to the driver's local port data 1888 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1889 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1890 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1891 * 1892 * Driver registers this routine as its nvme request io abort handler. This 1893 * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq 1894 * data structure to the rport indicated in @lpfc_nvme_rport. This routine 1895 * is executed asynchronously - one the target is validated as "MAPPED" and 1896 * ready for IO, the driver issues the abort request and returns. 1897 * 1898 * Return value: 1899 * None 1900 **/ 1901 static void 1902 lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, 1903 struct nvme_fc_remote_port *pnvme_rport, 1904 void *hw_queue_handle, 1905 struct nvmefc_fcp_req *pnvme_fcreq) 1906 { 1907 struct lpfc_nvme_lport *lport; 1908 struct lpfc_vport *vport; 1909 struct lpfc_hba *phba; 1910 struct lpfc_io_buf *lpfc_nbuf; 1911 struct lpfc_iocbq *abts_buf; 1912 struct lpfc_iocbq *nvmereq_wqe; 1913 struct lpfc_nvme_fcpreq_priv *freqpriv; 1914 unsigned long flags; 1915 int ret_val; 1916 1917 /* Validate pointers. LLDD fault handling with transport does 1918 * have timing races. 1919 */ 1920 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1921 if (unlikely(!lport)) 1922 return; 1923 1924 vport = lport->vport; 1925 1926 if (unlikely(!hw_queue_handle)) { 1927 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1928 "6129 Fail Abort, HW Queue Handle NULL.\n"); 1929 return; 1930 } 1931 1932 phba = vport->phba; 1933 freqpriv = pnvme_fcreq->private; 1934 1935 if (unlikely(!freqpriv)) 1936 return; 1937 if (vport->load_flag & FC_UNLOADING) 1938 return; 1939 1940 /* Announce entry to new IO submit field. */ 1941 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1942 "6002 Abort Request to rport DID x%06x " 1943 "for nvme_fc_req x%px\n", 1944 pnvme_rport->port_id, 1945 pnvme_fcreq); 1946 1947 /* If the hba is getting reset, this flag is set. It is 1948 * cleared when the reset is complete and rings reestablished. 1949 */ 1950 spin_lock_irqsave(&phba->hbalock, flags); 1951 /* driver queued commands are in process of being flushed */ 1952 if (phba->hba_flag & HBA_IOQ_FLUSH) { 1953 spin_unlock_irqrestore(&phba->hbalock, flags); 1954 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1955 "6139 Driver in reset cleanup - flushing " 1956 "NVME Req now. hba_flag x%x\n", 1957 phba->hba_flag); 1958 return; 1959 } 1960 1961 lpfc_nbuf = freqpriv->nvme_buf; 1962 if (!lpfc_nbuf) { 1963 spin_unlock_irqrestore(&phba->hbalock, flags); 1964 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1965 "6140 NVME IO req has no matching lpfc nvme " 1966 "io buffer. Skipping abort req.\n"); 1967 return; 1968 } else if (!lpfc_nbuf->nvmeCmd) { 1969 spin_unlock_irqrestore(&phba->hbalock, flags); 1970 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1971 "6141 lpfc NVME IO req has no nvme_fcreq " 1972 "io buffer. Skipping abort req.\n"); 1973 return; 1974 } 1975 nvmereq_wqe = &lpfc_nbuf->cur_iocbq; 1976 1977 /* Guard against IO completion being called at same time */ 1978 spin_lock(&lpfc_nbuf->buf_lock); 1979 1980 /* 1981 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's 1982 * state must match the nvme_fcreq passed by the nvme 1983 * transport. If they don't match, it is likely the driver 1984 * has already completed the NVME IO and the nvme transport 1985 * has not seen it yet. 1986 */ 1987 if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) { 1988 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1989 "6143 NVME req mismatch: " 1990 "lpfc_nbuf x%px nvmeCmd x%px, " 1991 "pnvme_fcreq x%px. Skipping Abort xri x%x\n", 1992 lpfc_nbuf, lpfc_nbuf->nvmeCmd, 1993 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1994 goto out_unlock; 1995 } 1996 1997 /* Don't abort IOs no longer on the pending queue. */ 1998 if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) { 1999 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 2000 "6142 NVME IO req x%px not queued - skipping " 2001 "abort req xri x%x\n", 2002 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 2003 goto out_unlock; 2004 } 2005 2006 atomic_inc(&lport->xmt_fcp_abort); 2007 lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n", 2008 nvmereq_wqe->sli4_xritag, 2009 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id); 2010 2011 /* Outstanding abort is in progress */ 2012 if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) { 2013 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 2014 "6144 Outstanding NVME I/O Abort Request " 2015 "still pending on nvme_fcreq x%px, " 2016 "lpfc_ncmd %px xri x%x\n", 2017 pnvme_fcreq, lpfc_nbuf, 2018 nvmereq_wqe->sli4_xritag); 2019 goto out_unlock; 2020 } 2021 2022 abts_buf = __lpfc_sli_get_iocbq(phba); 2023 if (!abts_buf) { 2024 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 2025 "6136 No available abort wqes. Skipping " 2026 "Abts req for nvme_fcreq x%px xri x%x\n", 2027 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 2028 goto out_unlock; 2029 } 2030 2031 /* Ready - mark outstanding as aborted by driver. */ 2032 nvmereq_wqe->iocb_flag |= LPFC_DRIVER_ABORTED; 2033 2034 lpfc_nvme_prep_abort_wqe(abts_buf, nvmereq_wqe->sli4_xritag, 0); 2035 2036 /* ABTS WQE must go to the same WQ as the WQE to be aborted */ 2037 abts_buf->iocb_flag |= LPFC_IO_NVME; 2038 abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx; 2039 abts_buf->vport = vport; 2040 abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl; 2041 ret_val = lpfc_sli4_issue_wqe(phba, lpfc_nbuf->hdwq, abts_buf); 2042 spin_unlock(&lpfc_nbuf->buf_lock); 2043 spin_unlock_irqrestore(&phba->hbalock, flags); 2044 if (ret_val) { 2045 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 2046 "6137 Failed abts issue_wqe with status x%x " 2047 "for nvme_fcreq x%px.\n", 2048 ret_val, pnvme_fcreq); 2049 lpfc_sli_release_iocbq(phba, abts_buf); 2050 return; 2051 } 2052 2053 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 2054 "6138 Transport Abort NVME Request Issued for " 2055 "ox_id x%x on reqtag x%x\n", 2056 nvmereq_wqe->sli4_xritag, 2057 abts_buf->iotag); 2058 return; 2059 2060 out_unlock: 2061 spin_unlock(&lpfc_nbuf->buf_lock); 2062 spin_unlock_irqrestore(&phba->hbalock, flags); 2063 return; 2064 } 2065 2066 /* Declare and initialization an instance of the FC NVME template. */ 2067 static struct nvme_fc_port_template lpfc_nvme_template = { 2068 /* initiator-based functions */ 2069 .localport_delete = lpfc_nvme_localport_delete, 2070 .remoteport_delete = lpfc_nvme_remoteport_delete, 2071 .create_queue = lpfc_nvme_create_queue, 2072 .delete_queue = lpfc_nvme_delete_queue, 2073 .ls_req = lpfc_nvme_ls_req, 2074 .fcp_io = lpfc_nvme_fcp_io_submit, 2075 .ls_abort = lpfc_nvme_ls_abort, 2076 .fcp_abort = lpfc_nvme_fcp_abort, 2077 .xmt_ls_rsp = lpfc_nvme_xmt_ls_rsp, 2078 2079 .max_hw_queues = 1, 2080 .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 2081 .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 2082 .dma_boundary = 0xFFFFFFFF, 2083 2084 /* Sizes of additional private data for data structures. 2085 * No use for the last two sizes at this time. 2086 */ 2087 .local_priv_sz = sizeof(struct lpfc_nvme_lport), 2088 .remote_priv_sz = sizeof(struct lpfc_nvme_rport), 2089 .lsrqst_priv_sz = 0, 2090 .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv), 2091 }; 2092 2093 /** 2094 * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA 2095 * @phba: The HBA for which this call is being executed. 2096 * 2097 * This routine removes a nvme buffer from head of @hdwq io_buf_list 2098 * and returns to caller. 2099 * 2100 * Return codes: 2101 * NULL - Error 2102 * Pointer to lpfc_nvme_buf - Success 2103 **/ 2104 static struct lpfc_io_buf * 2105 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 2106 int idx, int expedite) 2107 { 2108 struct lpfc_io_buf *lpfc_ncmd; 2109 struct lpfc_sli4_hdw_queue *qp; 2110 struct sli4_sge *sgl; 2111 struct lpfc_iocbq *pwqeq; 2112 union lpfc_wqe128 *wqe; 2113 2114 lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite); 2115 2116 if (lpfc_ncmd) { 2117 pwqeq = &(lpfc_ncmd->cur_iocbq); 2118 wqe = &pwqeq->wqe; 2119 2120 /* Setup key fields in buffer that may have been changed 2121 * if other protocols used this buffer. 2122 */ 2123 pwqeq->iocb_flag = LPFC_IO_NVME; 2124 pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl; 2125 lpfc_ncmd->start_time = jiffies; 2126 lpfc_ncmd->flags = 0; 2127 2128 /* Rsp SGE will be filled in when we rcv an IO 2129 * from the NVME Layer to be sent. 2130 * The cmd is going to be embedded so we need a SKIP SGE. 2131 */ 2132 sgl = lpfc_ncmd->dma_sgl; 2133 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2134 bf_set(lpfc_sli4_sge_last, sgl, 0); 2135 sgl->word2 = cpu_to_le32(sgl->word2); 2136 /* Fill in word 3 / sgl_len during cmd submission */ 2137 2138 /* Initialize 64 bytes only */ 2139 memset(wqe, 0, sizeof(union lpfc_wqe)); 2140 2141 if (lpfc_ndlp_check_qdepth(phba, ndlp)) { 2142 atomic_inc(&ndlp->cmd_pending); 2143 lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH; 2144 } 2145 2146 } else { 2147 qp = &phba->sli4_hba.hdwq[idx]; 2148 qp->empty_io_bufs++; 2149 } 2150 2151 return lpfc_ncmd; 2152 } 2153 2154 /** 2155 * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list. 2156 * @phba: The Hba for which this call is being executed. 2157 * @lpfc_ncmd: The nvme buffer which is being released. 2158 * 2159 * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba 2160 * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer 2161 * and cannot be reused for at least RA_TOV amount of time if it was 2162 * aborted. 2163 **/ 2164 static void 2165 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd) 2166 { 2167 struct lpfc_sli4_hdw_queue *qp; 2168 unsigned long iflag = 0; 2169 2170 if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp) 2171 atomic_dec(&lpfc_ncmd->ndlp->cmd_pending); 2172 2173 lpfc_ncmd->ndlp = NULL; 2174 lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH; 2175 2176 qp = lpfc_ncmd->hdwq; 2177 if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 2178 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2179 "6310 XB release deferred for " 2180 "ox_id x%x on reqtag x%x\n", 2181 lpfc_ncmd->cur_iocbq.sli4_xritag, 2182 lpfc_ncmd->cur_iocbq.iotag); 2183 2184 spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag); 2185 list_add_tail(&lpfc_ncmd->list, 2186 &qp->lpfc_abts_io_buf_list); 2187 qp->abts_nvme_io_bufs++; 2188 spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag); 2189 } else 2190 lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp); 2191 } 2192 2193 /** 2194 * lpfc_nvme_create_localport - Create/Bind an nvme localport instance. 2195 * @pvport - the lpfc_vport instance requesting a localport. 2196 * 2197 * This routine is invoked to create an nvme localport instance to bind 2198 * to the nvme_fc_transport. It is called once during driver load 2199 * like lpfc_create_shost after all other services are initialized. 2200 * It requires a vport, vpi, and wwns at call time. Other localport 2201 * parameters are modified as the driver's FCID and the Fabric WWN 2202 * are established. 2203 * 2204 * Return codes 2205 * 0 - successful 2206 * -ENOMEM - no heap memory available 2207 * other values - from nvme registration upcall 2208 **/ 2209 int 2210 lpfc_nvme_create_localport(struct lpfc_vport *vport) 2211 { 2212 int ret = 0; 2213 struct lpfc_hba *phba = vport->phba; 2214 struct nvme_fc_port_info nfcp_info; 2215 struct nvme_fc_local_port *localport; 2216 struct lpfc_nvme_lport *lport; 2217 2218 /* Initialize this localport instance. The vport wwn usage ensures 2219 * that NPIV is accounted for. 2220 */ 2221 memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info)); 2222 nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR; 2223 nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn); 2224 nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn); 2225 2226 /* We need to tell the transport layer + 1 because it takes page 2227 * alignment into account. When space for the SGL is allocated we 2228 * allocate + 3, one for cmd, one for rsp and one for this alignment 2229 */ 2230 lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; 2231 2232 /* Advertise how many hw queues we support based on cfg_hdw_queue, 2233 * which will not exceed cpu count. 2234 */ 2235 lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue; 2236 2237 if (!IS_ENABLED(CONFIG_NVME_FC)) 2238 return ret; 2239 2240 /* localport is allocated from the stack, but the registration 2241 * call allocates heap memory as well as the private area. 2242 */ 2243 2244 ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template, 2245 &vport->phba->pcidev->dev, &localport); 2246 if (!ret) { 2247 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC, 2248 "6005 Successfully registered local " 2249 "NVME port num %d, localP x%px, private " 2250 "x%px, sg_seg %d\n", 2251 localport->port_num, localport, 2252 localport->private, 2253 lpfc_nvme_template.max_sgl_segments); 2254 2255 /* Private is our lport size declared in the template. */ 2256 lport = (struct lpfc_nvme_lport *)localport->private; 2257 vport->localport = localport; 2258 lport->vport = vport; 2259 vport->nvmei_support = 1; 2260 2261 atomic_set(&lport->xmt_fcp_noxri, 0); 2262 atomic_set(&lport->xmt_fcp_bad_ndlp, 0); 2263 atomic_set(&lport->xmt_fcp_qdepth, 0); 2264 atomic_set(&lport->xmt_fcp_err, 0); 2265 atomic_set(&lport->xmt_fcp_wqerr, 0); 2266 atomic_set(&lport->xmt_fcp_abort, 0); 2267 atomic_set(&lport->xmt_ls_abort, 0); 2268 atomic_set(&lport->xmt_ls_err, 0); 2269 atomic_set(&lport->cmpl_fcp_xb, 0); 2270 atomic_set(&lport->cmpl_fcp_err, 0); 2271 atomic_set(&lport->cmpl_ls_xb, 0); 2272 atomic_set(&lport->cmpl_ls_err, 0); 2273 2274 atomic_set(&lport->fc4NvmeLsRequests, 0); 2275 atomic_set(&lport->fc4NvmeLsCmpls, 0); 2276 } 2277 2278 return ret; 2279 } 2280 2281 #if (IS_ENABLED(CONFIG_NVME_FC)) 2282 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg. 2283 * 2284 * The driver has to wait for the host nvme transport to callback 2285 * indicating the localport has successfully unregistered all 2286 * resources. Since this is an uninterruptible wait, loop every ten 2287 * seconds and print a message indicating no progress. 2288 * 2289 * An uninterruptible wait is used because of the risk of transport-to- 2290 * driver state mismatch. 2291 */ 2292 static void 2293 lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport, 2294 struct lpfc_nvme_lport *lport, 2295 struct completion *lport_unreg_cmp) 2296 { 2297 u32 wait_tmo; 2298 int ret, i, pending = 0; 2299 struct lpfc_sli_ring *pring; 2300 struct lpfc_hba *phba = vport->phba; 2301 2302 /* Host transport has to clean up and confirm requiring an indefinite 2303 * wait. Print a message if a 10 second wait expires and renew the 2304 * wait. This is unexpected. 2305 */ 2306 wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000); 2307 while (true) { 2308 ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo); 2309 if (unlikely(!ret)) { 2310 pending = 0; 2311 for (i = 0; i < phba->cfg_hdw_queue; i++) { 2312 pring = phba->sli4_hba.hdwq[i].io_wq->pring; 2313 if (!pring) 2314 continue; 2315 if (pring->txcmplq_cnt) 2316 pending += pring->txcmplq_cnt; 2317 } 2318 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 2319 "6176 Lport x%px Localport x%px wait " 2320 "timed out. Pending %d. Renewing.\n", 2321 lport, vport->localport, pending); 2322 continue; 2323 } 2324 break; 2325 } 2326 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 2327 "6177 Lport x%px Localport x%px Complete Success\n", 2328 lport, vport->localport); 2329 } 2330 #endif 2331 2332 /** 2333 * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport. 2334 * @pnvme: pointer to lpfc nvme data structure. 2335 * 2336 * This routine is invoked to destroy all lports bound to the phba. 2337 * The lport memory was allocated by the nvme fc transport and is 2338 * released there. This routine ensures all rports bound to the 2339 * lport have been disconnected. 2340 * 2341 **/ 2342 void 2343 lpfc_nvme_destroy_localport(struct lpfc_vport *vport) 2344 { 2345 #if (IS_ENABLED(CONFIG_NVME_FC)) 2346 struct nvme_fc_local_port *localport; 2347 struct lpfc_nvme_lport *lport; 2348 int ret; 2349 DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp); 2350 2351 if (vport->nvmei_support == 0) 2352 return; 2353 2354 localport = vport->localport; 2355 lport = (struct lpfc_nvme_lport *)localport->private; 2356 2357 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 2358 "6011 Destroying NVME localport x%px\n", 2359 localport); 2360 2361 /* lport's rport list is clear. Unregister 2362 * lport and release resources. 2363 */ 2364 lport->lport_unreg_cmp = &lport_unreg_cmp; 2365 ret = nvme_fc_unregister_localport(localport); 2366 2367 /* Wait for completion. This either blocks 2368 * indefinitely or succeeds 2369 */ 2370 lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp); 2371 vport->localport = NULL; 2372 2373 /* Regardless of the unregister upcall response, clear 2374 * nvmei_support. All rports are unregistered and the 2375 * driver will clean up. 2376 */ 2377 vport->nvmei_support = 0; 2378 if (ret == 0) { 2379 lpfc_printf_vlog(vport, 2380 KERN_INFO, LOG_NVME_DISC, 2381 "6009 Unregistered lport Success\n"); 2382 } else { 2383 lpfc_printf_vlog(vport, 2384 KERN_INFO, LOG_NVME_DISC, 2385 "6010 Unregistered lport " 2386 "Failed, status x%x\n", 2387 ret); 2388 } 2389 #endif 2390 } 2391 2392 void 2393 lpfc_nvme_update_localport(struct lpfc_vport *vport) 2394 { 2395 #if (IS_ENABLED(CONFIG_NVME_FC)) 2396 struct nvme_fc_local_port *localport; 2397 struct lpfc_nvme_lport *lport; 2398 2399 localport = vport->localport; 2400 if (!localport) { 2401 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 2402 "6710 Update NVME fail. No localport\n"); 2403 return; 2404 } 2405 lport = (struct lpfc_nvme_lport *)localport->private; 2406 if (!lport) { 2407 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 2408 "6171 Update NVME fail. localP x%px, No lport\n", 2409 localport); 2410 return; 2411 } 2412 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 2413 "6012 Update NVME lport x%px did x%x\n", 2414 localport, vport->fc_myDID); 2415 2416 localport->port_id = vport->fc_myDID; 2417 if (localport->port_id == 0) 2418 localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY; 2419 else 2420 localport->port_role = FC_PORT_ROLE_NVME_INITIATOR; 2421 2422 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2423 "6030 bound lport x%px to DID x%06x\n", 2424 lport, localport->port_id); 2425 #endif 2426 } 2427 2428 int 2429 lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2430 { 2431 #if (IS_ENABLED(CONFIG_NVME_FC)) 2432 int ret = 0; 2433 struct nvme_fc_local_port *localport; 2434 struct lpfc_nvme_lport *lport; 2435 struct lpfc_nvme_rport *rport; 2436 struct lpfc_nvme_rport *oldrport; 2437 struct nvme_fc_remote_port *remote_port; 2438 struct nvme_fc_port_info rpinfo; 2439 struct lpfc_nodelist *prev_ndlp = NULL; 2440 2441 lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC, 2442 "6006 Register NVME PORT. DID x%06x nlptype x%x\n", 2443 ndlp->nlp_DID, ndlp->nlp_type); 2444 2445 localport = vport->localport; 2446 if (!localport) 2447 return 0; 2448 2449 lport = (struct lpfc_nvme_lport *)localport->private; 2450 2451 /* NVME rports are not preserved across devloss. 2452 * Just register this instance. Note, rpinfo->dev_loss_tmo 2453 * is left 0 to indicate accept transport defaults. The 2454 * driver communicates port role capabilities consistent 2455 * with the PRLI response data. 2456 */ 2457 memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info)); 2458 rpinfo.port_id = ndlp->nlp_DID; 2459 if (ndlp->nlp_type & NLP_NVME_TARGET) 2460 rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET; 2461 if (ndlp->nlp_type & NLP_NVME_INITIATOR) 2462 rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR; 2463 2464 if (ndlp->nlp_type & NLP_NVME_DISCOVERY) 2465 rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY; 2466 2467 rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn); 2468 rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn); 2469 2470 spin_lock_irq(&vport->phba->hbalock); 2471 oldrport = lpfc_ndlp_get_nrport(ndlp); 2472 if (oldrport) { 2473 prev_ndlp = oldrport->ndlp; 2474 spin_unlock_irq(&vport->phba->hbalock); 2475 } else { 2476 spin_unlock_irq(&vport->phba->hbalock); 2477 lpfc_nlp_get(ndlp); 2478 } 2479 2480 ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port); 2481 if (!ret) { 2482 /* If the ndlp already has an nrport, this is just 2483 * a resume of the existing rport. Else this is a 2484 * new rport. 2485 */ 2486 /* Guard against an unregister/reregister 2487 * race that leaves the WAIT flag set. 2488 */ 2489 spin_lock_irq(&vport->phba->hbalock); 2490 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 2491 spin_unlock_irq(&vport->phba->hbalock); 2492 rport = remote_port->private; 2493 if (oldrport) { 2494 /* New remoteport record does not guarantee valid 2495 * host private memory area. 2496 */ 2497 if (oldrport == remote_port->private) { 2498 /* Same remoteport - ndlp should match. 2499 * Just reuse. 2500 */ 2501 lpfc_printf_vlog(ndlp->vport, KERN_INFO, 2502 LOG_NVME_DISC, 2503 "6014 Rebind lport to current " 2504 "remoteport x%px wwpn 0x%llx, " 2505 "Data: x%x x%x x%px x%px x%x " 2506 " x%06x\n", 2507 remote_port, 2508 remote_port->port_name, 2509 remote_port->port_id, 2510 remote_port->port_role, 2511 oldrport->ndlp, 2512 ndlp, 2513 ndlp->nlp_type, 2514 ndlp->nlp_DID); 2515 2516 /* It's a complete rebind only if the driver 2517 * is registering with the same ndlp. Otherwise 2518 * the driver likely executed a node swap 2519 * prior to this registration and the ndlp to 2520 * remoteport binding needs to be redone. 2521 */ 2522 if (prev_ndlp == ndlp) 2523 return 0; 2524 2525 } 2526 2527 /* Sever the ndlp<->rport association 2528 * before dropping the ndlp ref from 2529 * register. 2530 */ 2531 spin_lock_irq(&vport->phba->hbalock); 2532 ndlp->nrport = NULL; 2533 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 2534 spin_unlock_irq(&vport->phba->hbalock); 2535 rport->ndlp = NULL; 2536 rport->remoteport = NULL; 2537 2538 /* Reference only removed if previous NDLP is no longer 2539 * active. It might be just a swap and removing the 2540 * reference would cause a premature cleanup. 2541 */ 2542 if (prev_ndlp && prev_ndlp != ndlp) { 2543 if ((!NLP_CHK_NODE_ACT(prev_ndlp)) || 2544 (!prev_ndlp->nrport)) 2545 lpfc_nlp_put(prev_ndlp); 2546 } 2547 } 2548 2549 /* Clean bind the rport to the ndlp. */ 2550 rport->remoteport = remote_port; 2551 rport->lport = lport; 2552 rport->ndlp = ndlp; 2553 spin_lock_irq(&vport->phba->hbalock); 2554 ndlp->nrport = rport; 2555 spin_unlock_irq(&vport->phba->hbalock); 2556 lpfc_printf_vlog(vport, KERN_INFO, 2557 LOG_NVME_DISC | LOG_NODE, 2558 "6022 Bind lport x%px to remoteport x%px " 2559 "rport x%px WWNN 0x%llx, " 2560 "Rport WWPN 0x%llx DID " 2561 "x%06x Role x%x, ndlp %p prev_ndlp x%px\n", 2562 lport, remote_port, rport, 2563 rpinfo.node_name, rpinfo.port_name, 2564 rpinfo.port_id, rpinfo.port_role, 2565 ndlp, prev_ndlp); 2566 } else { 2567 lpfc_printf_vlog(vport, KERN_ERR, 2568 LOG_NVME_DISC | LOG_NODE, 2569 "6031 RemotePort Registration failed " 2570 "err: %d, DID x%06x\n", 2571 ret, ndlp->nlp_DID); 2572 } 2573 2574 return ret; 2575 #else 2576 return 0; 2577 #endif 2578 } 2579 2580 /** 2581 * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport 2582 * 2583 * If the ndlp represents an NVME Target, that we are logged into, 2584 * ping the NVME FC Transport layer to initiate a device rescan 2585 * on this remote NPort. 2586 */ 2587 void 2588 lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2589 { 2590 #if (IS_ENABLED(CONFIG_NVME_FC)) 2591 struct lpfc_nvme_rport *nrport; 2592 struct nvme_fc_remote_port *remoteport = NULL; 2593 2594 spin_lock_irq(&vport->phba->hbalock); 2595 nrport = lpfc_ndlp_get_nrport(ndlp); 2596 if (nrport) 2597 remoteport = nrport->remoteport; 2598 spin_unlock_irq(&vport->phba->hbalock); 2599 2600 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2601 "6170 Rescan NPort DID x%06x type x%x " 2602 "state x%x nrport x%px remoteport x%px\n", 2603 ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state, 2604 nrport, remoteport); 2605 2606 if (!nrport || !remoteport) 2607 goto rescan_exit; 2608 2609 /* Only rescan if we are an NVME target in the MAPPED state */ 2610 if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY && 2611 ndlp->nlp_state == NLP_STE_MAPPED_NODE) { 2612 nvme_fc_rescan_remoteport(remoteport); 2613 2614 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 2615 "6172 NVME rescanned DID x%06x " 2616 "port_state x%x\n", 2617 ndlp->nlp_DID, remoteport->port_state); 2618 } 2619 return; 2620 rescan_exit: 2621 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2622 "6169 Skip NVME Rport Rescan, NVME remoteport " 2623 "unregistered\n"); 2624 #endif 2625 } 2626 2627 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport. 2628 * 2629 * There is no notion of Devloss or rport recovery from the current 2630 * nvme_transport perspective. Loss of an rport just means IO cannot 2631 * be sent and recovery is completely up to the initator. 2632 * For now, the driver just unbinds the DID and port_role so that 2633 * no further IO can be issued. Changes are planned for later. 2634 * 2635 * Notes - the ndlp reference count is not decremented here since 2636 * since there is no nvme_transport api for devloss. Node ref count 2637 * is only adjusted in driver unload. 2638 */ 2639 void 2640 lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2641 { 2642 #if (IS_ENABLED(CONFIG_NVME_FC)) 2643 int ret; 2644 struct nvme_fc_local_port *localport; 2645 struct lpfc_nvme_lport *lport; 2646 struct lpfc_nvme_rport *rport; 2647 struct nvme_fc_remote_port *remoteport = NULL; 2648 2649 localport = vport->localport; 2650 2651 /* This is fundamental error. The localport is always 2652 * available until driver unload. Just exit. 2653 */ 2654 if (!localport) 2655 return; 2656 2657 lport = (struct lpfc_nvme_lport *)localport->private; 2658 if (!lport) 2659 goto input_err; 2660 2661 spin_lock_irq(&vport->phba->hbalock); 2662 rport = lpfc_ndlp_get_nrport(ndlp); 2663 if (rport) 2664 remoteport = rport->remoteport; 2665 spin_unlock_irq(&vport->phba->hbalock); 2666 if (!remoteport) 2667 goto input_err; 2668 2669 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2670 "6033 Unreg nvme remoteport x%px, portname x%llx, " 2671 "port_id x%06x, portstate x%x port type x%x\n", 2672 remoteport, remoteport->port_name, 2673 remoteport->port_id, remoteport->port_state, 2674 ndlp->nlp_type); 2675 2676 /* Sanity check ndlp type. Only call for NVME ports. Don't 2677 * clear any rport state until the transport calls back. 2678 */ 2679 2680 if (ndlp->nlp_type & NLP_NVME_TARGET) { 2681 /* No concern about the role change on the nvme remoteport. 2682 * The transport will update it. 2683 */ 2684 ndlp->upcall_flags |= NLP_WAIT_FOR_UNREG; 2685 2686 /* Don't let the host nvme transport keep sending keep-alives 2687 * on this remoteport. Vport is unloading, no recovery. The 2688 * return values is ignored. The upcall is a courtesy to the 2689 * transport. 2690 */ 2691 if (vport->load_flag & FC_UNLOADING) 2692 (void)nvme_fc_set_remoteport_devloss(remoteport, 0); 2693 2694 ret = nvme_fc_unregister_remoteport(remoteport); 2695 if (ret != 0) { 2696 lpfc_nlp_put(ndlp); 2697 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 2698 "6167 NVME unregister failed %d " 2699 "port_state x%x\n", 2700 ret, remoteport->port_state); 2701 } 2702 } 2703 return; 2704 2705 input_err: 2706 #endif 2707 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 2708 "6168 State error: lport x%px, rport x%px FCID x%06x\n", 2709 vport->localport, ndlp->rport, ndlp->nlp_DID); 2710 } 2711 2712 /** 2713 * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort 2714 * @phba: pointer to lpfc hba data structure. 2715 * @axri: pointer to the fcp xri abort wcqe structure. 2716 * @lpfc_ncmd: The nvme job structure for the request being aborted. 2717 * 2718 * This routine is invoked by the worker thread to process a SLI4 fast-path 2719 * NVME aborted xri. Aborted NVME IO commands are completed to the transport 2720 * here. 2721 **/ 2722 void 2723 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, 2724 struct sli4_wcqe_xri_aborted *axri, 2725 struct lpfc_io_buf *lpfc_ncmd) 2726 { 2727 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); 2728 struct nvmefc_fcp_req *nvme_cmd = NULL; 2729 struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp; 2730 2731 2732 if (ndlp) 2733 lpfc_sli4_abts_err_handler(phba, ndlp, axri); 2734 2735 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2736 "6311 nvme_cmd %p xri x%x tag x%x abort complete and " 2737 "xri released\n", 2738 lpfc_ncmd->nvmeCmd, xri, 2739 lpfc_ncmd->cur_iocbq.iotag); 2740 2741 /* Aborted NVME commands are required to not complete 2742 * before the abort exchange command fully completes. 2743 * Once completed, it is available via the put list. 2744 */ 2745 if (lpfc_ncmd->nvmeCmd) { 2746 nvme_cmd = lpfc_ncmd->nvmeCmd; 2747 nvme_cmd->done(nvme_cmd); 2748 lpfc_ncmd->nvmeCmd = NULL; 2749 } 2750 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2751 } 2752 2753 /** 2754 * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete 2755 * @phba: Pointer to HBA context object. 2756 * 2757 * This function flushes all wqes in the nvme rings and frees all resources 2758 * in the txcmplq. This function does not issue abort wqes for the IO 2759 * commands in txcmplq, they will just be returned with 2760 * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI 2761 * slot has been permanently disabled. 2762 **/ 2763 void 2764 lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba) 2765 { 2766 struct lpfc_sli_ring *pring; 2767 u32 i, wait_cnt = 0; 2768 2769 if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq) 2770 return; 2771 2772 /* Cycle through all IO rings and make sure all outstanding 2773 * WQEs have been removed from the txcmplqs. 2774 */ 2775 for (i = 0; i < phba->cfg_hdw_queue; i++) { 2776 if (!phba->sli4_hba.hdwq[i].io_wq) 2777 continue; 2778 pring = phba->sli4_hba.hdwq[i].io_wq->pring; 2779 2780 if (!pring) 2781 continue; 2782 2783 /* Retrieve everything on the txcmplq */ 2784 while (!list_empty(&pring->txcmplq)) { 2785 msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1); 2786 wait_cnt++; 2787 2788 /* The sleep is 10mS. Every ten seconds, 2789 * dump a message. Something is wrong. 2790 */ 2791 if ((wait_cnt % 1000) == 0) { 2792 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2793 "6178 NVME IO not empty, " 2794 "cnt %d\n", wait_cnt); 2795 } 2796 } 2797 } 2798 } 2799 2800 void 2801 lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn) 2802 { 2803 #if (IS_ENABLED(CONFIG_NVME_FC)) 2804 struct lpfc_io_buf *lpfc_ncmd; 2805 struct nvmefc_fcp_req *nCmd; 2806 struct lpfc_nvme_fcpreq_priv *freqpriv; 2807 2808 if (!pwqeIn->context1) { 2809 lpfc_sli_release_iocbq(phba, pwqeIn); 2810 return; 2811 } 2812 /* For abort iocb just return, IO iocb will do a done call */ 2813 if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) == 2814 CMD_ABORT_XRI_CX) { 2815 lpfc_sli_release_iocbq(phba, pwqeIn); 2816 return; 2817 } 2818 lpfc_ncmd = (struct lpfc_io_buf *)pwqeIn->context1; 2819 2820 spin_lock(&lpfc_ncmd->buf_lock); 2821 if (!lpfc_ncmd->nvmeCmd) { 2822 spin_unlock(&lpfc_ncmd->buf_lock); 2823 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2824 return; 2825 } 2826 2827 nCmd = lpfc_ncmd->nvmeCmd; 2828 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 2829 "6194 NVME Cancel xri %x\n", 2830 lpfc_ncmd->cur_iocbq.sli4_xritag); 2831 2832 nCmd->transferred_length = 0; 2833 nCmd->rcv_rsplen = 0; 2834 nCmd->status = NVME_SC_INTERNAL; 2835 freqpriv = nCmd->private; 2836 freqpriv->nvme_buf = NULL; 2837 lpfc_ncmd->nvmeCmd = NULL; 2838 2839 spin_unlock(&lpfc_ncmd->buf_lock); 2840 nCmd->done(nCmd); 2841 2842 /* Call release with XB=1 to queue the IO into the abort list. */ 2843 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2844 #endif 2845 } 2846