1 /******************************************************************* 2 * This file is part of the Emulex Linux Device Driver for * 3 * Fibre Channel Host Bus Adapters. * 4 * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term * 5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * 6 * Copyright (C) 2004-2016 Emulex. All rights reserved. * 7 * EMULEX and SLI are trademarks of Emulex. * 8 * www.broadcom.com * 9 * Portions Copyright (C) 2004-2005 Christoph Hellwig * 10 * * 11 * This program is free software; you can redistribute it and/or * 12 * modify it under the terms of version 2 of the GNU General * 13 * Public License as published by the Free Software Foundation. * 14 * This program is distributed in the hope that it will be useful. * 15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 19 * TO BE LEGALLY INVALID. See the GNU General Public License for * 20 * more details, a copy of which can be found in the file COPYING * 21 * included with this package. * 22 ********************************************************************/ 23 #include <linux/pci.h> 24 #include <linux/slab.h> 25 #include <linux/interrupt.h> 26 #include <linux/delay.h> 27 #include <asm/unaligned.h> 28 #include <linux/crc-t10dif.h> 29 #include <net/checksum.h> 30 31 #include <scsi/scsi.h> 32 #include <scsi/scsi_device.h> 33 #include <scsi/scsi_eh.h> 34 #include <scsi/scsi_host.h> 35 #include <scsi/scsi_tcq.h> 36 #include <scsi/scsi_transport_fc.h> 37 #include <scsi/fc/fc_fs.h> 38 39 #include "lpfc_version.h" 40 #include "lpfc_hw4.h" 41 #include "lpfc_hw.h" 42 #include "lpfc_sli.h" 43 #include "lpfc_sli4.h" 44 #include "lpfc_nl.h" 45 #include "lpfc_disc.h" 46 #include "lpfc.h" 47 #include "lpfc_nvme.h" 48 #include "lpfc_scsi.h" 49 #include "lpfc_logmsg.h" 50 #include "lpfc_crtn.h" 51 #include "lpfc_vport.h" 52 #include "lpfc_debugfs.h" 53 54 /* NVME initiator-based functions */ 55 56 static struct lpfc_io_buf * 57 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 58 int idx, int expedite); 59 60 static void 61 lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *); 62 63 static struct nvme_fc_port_template lpfc_nvme_template; 64 65 /** 66 * lpfc_nvme_create_queue - 67 * @pnvme_lport: Transport localport that LS is to be issued from 68 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 69 * @qsize: Size of the queue in bytes 70 * @handle: An opaque driver handle used in follow-up calls. 71 * 72 * Driver registers this routine to preallocate and initialize any 73 * internal data structures to bind the @qidx to its internal IO queues. 74 * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ. 75 * 76 * Return value : 77 * 0 - Success 78 * -EINVAL - Unsupported input value. 79 * -ENOMEM - Could not alloc necessary memory 80 **/ 81 static int 82 lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, 83 unsigned int qidx, u16 qsize, 84 void **handle) 85 { 86 struct lpfc_nvme_lport *lport; 87 struct lpfc_vport *vport; 88 struct lpfc_nvme_qhandle *qhandle; 89 char *str; 90 91 if (!pnvme_lport->private) 92 return -ENOMEM; 93 94 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 95 vport = lport->vport; 96 qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL); 97 if (qhandle == NULL) 98 return -ENOMEM; 99 100 qhandle->cpu_id = raw_smp_processor_id(); 101 qhandle->qidx = qidx; 102 /* 103 * NVME qidx == 0 is the admin queue, so both admin queue 104 * and first IO queue will use MSI-X vector and associated 105 * EQ/CQ/WQ at index 0. After that they are sequentially assigned. 106 */ 107 if (qidx) { 108 str = "IO "; /* IO queue */ 109 qhandle->index = ((qidx - 1) % 110 lpfc_nvme_template.max_hw_queues); 111 } else { 112 str = "ADM"; /* Admin queue */ 113 qhandle->index = qidx; 114 } 115 116 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 117 "6073 Binding %s HdwQueue %d (cpu %d) to " 118 "hdw_queue %d qhandle x%px\n", str, 119 qidx, qhandle->cpu_id, qhandle->index, qhandle); 120 *handle = (void *)qhandle; 121 return 0; 122 } 123 124 /** 125 * lpfc_nvme_delete_queue - 126 * @pnvme_lport: Transport localport that LS is to be issued from 127 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 128 * @handle: An opaque driver handle from lpfc_nvme_create_queue 129 * 130 * Driver registers this routine to free 131 * any internal data structures to bind the @qidx to its internal 132 * IO queues. 133 * 134 * Return value : 135 * 0 - Success 136 * TODO: What are the failure codes. 137 **/ 138 static void 139 lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport, 140 unsigned int qidx, 141 void *handle) 142 { 143 struct lpfc_nvme_lport *lport; 144 struct lpfc_vport *vport; 145 146 if (!pnvme_lport->private) 147 return; 148 149 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 150 vport = lport->vport; 151 152 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 153 "6001 ENTER. lpfc_pnvme x%px, qidx x%x qhandle x%px\n", 154 lport, qidx, handle); 155 kfree(handle); 156 } 157 158 static void 159 lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport) 160 { 161 struct lpfc_nvme_lport *lport = localport->private; 162 163 lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME, 164 "6173 localport x%px delete complete\n", 165 lport); 166 167 /* release any threads waiting for the unreg to complete */ 168 if (lport->vport->localport) 169 complete(lport->lport_unreg_cmp); 170 } 171 172 /* lpfc_nvme_remoteport_delete 173 * 174 * @remoteport: Pointer to an nvme transport remoteport instance. 175 * 176 * This is a template downcall. NVME transport calls this function 177 * when it has completed the unregistration of a previously 178 * registered remoteport. 179 * 180 * Return value : 181 * None 182 */ 183 static void 184 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport) 185 { 186 struct lpfc_nvme_rport *rport = remoteport->private; 187 struct lpfc_vport *vport; 188 struct lpfc_nodelist *ndlp; 189 u32 fc4_xpt_flags; 190 191 ndlp = rport->ndlp; 192 if (!ndlp) { 193 pr_err("**** %s: NULL ndlp on rport x%px remoteport x%px\n", 194 __func__, rport, remoteport); 195 goto rport_err; 196 } 197 198 vport = ndlp->vport; 199 if (!vport) { 200 pr_err("**** %s: Null vport on ndlp x%px, ste x%x rport x%px\n", 201 __func__, ndlp, ndlp->nlp_state, rport); 202 goto rport_err; 203 } 204 205 fc4_xpt_flags = NVME_XPT_REGD | SCSI_XPT_REGD; 206 207 /* Remove this rport from the lport's list - memory is owned by the 208 * transport. Remove the ndlp reference for the NVME transport before 209 * calling state machine to remove the node. 210 */ 211 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 212 "6146 remoteport delete of remoteport x%px, ndlp x%px " 213 "DID x%x xflags x%x\n", 214 remoteport, ndlp, ndlp->nlp_DID, ndlp->fc4_xpt_flags); 215 spin_lock_irq(&ndlp->lock); 216 217 /* The register rebind might have occurred before the delete 218 * downcall. Guard against this race. 219 */ 220 if (ndlp->fc4_xpt_flags & NVME_XPT_UNREG_WAIT) 221 ndlp->fc4_xpt_flags &= ~(NVME_XPT_UNREG_WAIT | NVME_XPT_REGD); 222 223 spin_unlock_irq(&ndlp->lock); 224 225 /* On a devloss timeout event, one more put is executed provided the 226 * NVME and SCSI rport unregister requests are complete. If the vport 227 * is unloading, this extra put is executed by lpfc_drop_node. 228 */ 229 if (!(ndlp->fc4_xpt_flags & fc4_xpt_flags)) 230 lpfc_disc_state_machine(vport, ndlp, NULL, NLP_EVT_DEVICE_RM); 231 232 rport_err: 233 return; 234 } 235 236 /** 237 * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request 238 * @phba: pointer to lpfc hba data structure. 239 * @axchg: pointer to exchange context for the NVME LS request 240 * 241 * This routine is used for processing an asychronously received NVME LS 242 * request. Any remaining validation is done and the LS is then forwarded 243 * to the nvme-fc transport via nvme_fc_rcv_ls_req(). 244 * 245 * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing) 246 * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done. 247 * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg. 248 * 249 * Returns 0 if LS was handled and delivered to the transport 250 * Returns 1 if LS failed to be handled and should be dropped 251 */ 252 int 253 lpfc_nvme_handle_lsreq(struct lpfc_hba *phba, 254 struct lpfc_async_xchg_ctx *axchg) 255 { 256 #if (IS_ENABLED(CONFIG_NVME_FC)) 257 struct lpfc_vport *vport; 258 struct lpfc_nvme_rport *lpfc_rport; 259 struct nvme_fc_remote_port *remoteport; 260 struct lpfc_nvme_lport *lport; 261 uint32_t *payload = axchg->payload; 262 int rc; 263 264 vport = axchg->ndlp->vport; 265 lpfc_rport = axchg->ndlp->nrport; 266 if (!lpfc_rport) 267 return -EINVAL; 268 269 remoteport = lpfc_rport->remoteport; 270 if (!vport->localport) 271 return -EINVAL; 272 273 lport = vport->localport->private; 274 if (!lport) 275 return -EINVAL; 276 277 rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload, 278 axchg->size); 279 280 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 281 "6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x " 282 "%08x %08x %08x\n", 283 axchg->size, rc, 284 *payload, *(payload+1), *(payload+2), 285 *(payload+3), *(payload+4), *(payload+5)); 286 287 if (!rc) 288 return 0; 289 #endif 290 return 1; 291 } 292 293 /** 294 * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME 295 * LS request. 296 * @phba: Pointer to HBA context object 297 * @vport: The local port that issued the LS 298 * @cmdwqe: Pointer to driver command WQE object. 299 * @wcqe: Pointer to driver response CQE object. 300 * 301 * This function is the generic completion handler for NVME LS requests. 302 * The function updates any states and statistics, calls the transport 303 * ls_req done() routine, then tears down the command and buffers used 304 * for the LS request. 305 **/ 306 void 307 __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport, 308 struct lpfc_iocbq *cmdwqe, 309 struct lpfc_wcqe_complete *wcqe) 310 { 311 struct nvmefc_ls_req *pnvme_lsreq; 312 struct lpfc_dmabuf *buf_ptr; 313 struct lpfc_nodelist *ndlp; 314 uint32_t status; 315 316 pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2; 317 ndlp = (struct lpfc_nodelist *)cmdwqe->context1; 318 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 319 320 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 321 "6047 NVMEx LS REQ x%px cmpl DID %x Xri: %x " 322 "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px " 323 "ndlp:x%px\n", 324 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 325 cmdwqe->sli4_xritag, status, 326 (wcqe->parameter & 0xffff), 327 cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp); 328 329 lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n", 330 cmdwqe->sli4_xritag, status, wcqe->parameter); 331 332 if (cmdwqe->context3) { 333 buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3; 334 lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys); 335 kfree(buf_ptr); 336 cmdwqe->context3 = NULL; 337 } 338 if (pnvme_lsreq->done) 339 pnvme_lsreq->done(pnvme_lsreq, status); 340 else 341 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 342 "6046 NVMEx cmpl without done call back? " 343 "Data x%px DID %x Xri: %x status %x\n", 344 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 345 cmdwqe->sli4_xritag, status); 346 if (ndlp) { 347 lpfc_nlp_put(ndlp); 348 cmdwqe->context1 = NULL; 349 } 350 lpfc_sli_release_iocbq(phba, cmdwqe); 351 } 352 353 static void 354 lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 355 struct lpfc_iocbq *rspwqe) 356 { 357 struct lpfc_vport *vport = cmdwqe->vport; 358 struct lpfc_nvme_lport *lport; 359 uint32_t status; 360 struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl; 361 362 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 363 364 if (vport->localport) { 365 lport = (struct lpfc_nvme_lport *)vport->localport->private; 366 if (lport) { 367 atomic_inc(&lport->fc4NvmeLsCmpls); 368 if (status) { 369 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 370 atomic_inc(&lport->cmpl_ls_xb); 371 atomic_inc(&lport->cmpl_ls_err); 372 } 373 } 374 } 375 376 __lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe); 377 } 378 379 static int 380 lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp, 381 struct lpfc_dmabuf *inp, 382 struct nvmefc_ls_req *pnvme_lsreq, 383 void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *, 384 struct lpfc_iocbq *), 385 struct lpfc_nodelist *ndlp, uint32_t num_entry, 386 uint32_t tmo, uint8_t retry) 387 { 388 struct lpfc_hba *phba = vport->phba; 389 union lpfc_wqe128 *wqe; 390 struct lpfc_iocbq *genwqe; 391 struct ulp_bde64 *bpl; 392 struct ulp_bde64 bde; 393 int i, rc, xmit_len, first_len; 394 395 /* Allocate buffer for command WQE */ 396 genwqe = lpfc_sli_get_iocbq(phba); 397 if (genwqe == NULL) 398 return 1; 399 400 wqe = &genwqe->wqe; 401 /* Initialize only 64 bytes */ 402 memset(wqe, 0, sizeof(union lpfc_wqe)); 403 404 genwqe->context3 = (uint8_t *)bmp; 405 genwqe->cmd_flag |= LPFC_IO_NVME_LS; 406 407 /* Save for completion so we can release these resources */ 408 genwqe->context1 = lpfc_nlp_get(ndlp); 409 if (!genwqe->context1) { 410 dev_warn(&phba->pcidev->dev, 411 "Warning: Failed node ref, not sending LS_REQ\n"); 412 lpfc_sli_release_iocbq(phba, genwqe); 413 return 1; 414 } 415 416 genwqe->context2 = (uint8_t *)pnvme_lsreq; 417 /* Fill in payload, bp points to frame payload */ 418 419 if (!tmo) 420 /* FC spec states we need 3 * ratov for CT requests */ 421 tmo = (3 * phba->fc_ratov); 422 423 /* For this command calculate the xmit length of the request bde. */ 424 xmit_len = 0; 425 first_len = 0; 426 bpl = (struct ulp_bde64 *)bmp->virt; 427 for (i = 0; i < num_entry; i++) { 428 bde.tus.w = bpl[i].tus.w; 429 if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64) 430 break; 431 xmit_len += bde.tus.f.bdeSize; 432 if (i == 0) 433 first_len = xmit_len; 434 } 435 436 genwqe->num_bdes = num_entry; 437 genwqe->hba_wqidx = 0; 438 439 /* Words 0 - 2 */ 440 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 441 wqe->generic.bde.tus.f.bdeSize = first_len; 442 wqe->generic.bde.addrLow = bpl[0].addrLow; 443 wqe->generic.bde.addrHigh = bpl[0].addrHigh; 444 445 /* Word 3 */ 446 wqe->gen_req.request_payload_len = first_len; 447 448 /* Word 4 */ 449 450 /* Word 5 */ 451 bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0); 452 bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1); 453 bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1); 454 bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ); 455 bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME); 456 457 /* Word 6 */ 458 bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com, 459 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 460 bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag); 461 462 /* Word 7 */ 463 bf_set(wqe_tmo, &wqe->gen_req.wqe_com, tmo); 464 bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3); 465 bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE); 466 bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI); 467 468 /* Word 8 */ 469 wqe->gen_req.wqe_com.abort_tag = genwqe->iotag; 470 471 /* Word 9 */ 472 bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag); 473 474 /* Word 10 */ 475 bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1); 476 bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ); 477 bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1); 478 bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE); 479 bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0); 480 481 /* Word 11 */ 482 bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 483 bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND); 484 485 486 /* Issue GEN REQ WQE for NPORT <did> */ 487 genwqe->cmd_cmpl = cmpl; 488 genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT; 489 genwqe->vport = vport; 490 genwqe->retry = retry; 491 492 lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n", 493 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID); 494 495 rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe); 496 if (rc) { 497 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 498 "6045 Issue GEN REQ WQE to NPORT x%x " 499 "Data: x%x x%x rc x%x\n", 500 ndlp->nlp_DID, genwqe->iotag, 501 vport->port_state, rc); 502 lpfc_nlp_put(ndlp); 503 lpfc_sli_release_iocbq(phba, genwqe); 504 return 1; 505 } 506 507 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS, 508 "6050 Issue GEN REQ WQE to NPORT x%x " 509 "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px " 510 "bmp:x%px xmit:%d 1st:%d\n", 511 ndlp->nlp_DID, genwqe->sli4_xritag, 512 vport->port_state, 513 genwqe, pnvme_lsreq, bmp, xmit_len, first_len); 514 return 0; 515 } 516 517 518 /** 519 * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request 520 * @vport: The local port issuing the LS 521 * @ndlp: The remote port to send the LS to 522 * @pnvme_lsreq: Pointer to LS request structure from the transport 523 * @gen_req_cmp: Completion call-back 524 * 525 * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST 526 * WQE to perform the LS operation. 527 * 528 * Return value : 529 * 0 - Success 530 * non-zero: various error codes, in form of -Exxx 531 **/ 532 int 533 __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, 534 struct nvmefc_ls_req *pnvme_lsreq, 535 void (*gen_req_cmp)(struct lpfc_hba *phba, 536 struct lpfc_iocbq *cmdwqe, 537 struct lpfc_iocbq *rspwqe)) 538 { 539 struct lpfc_dmabuf *bmp; 540 struct ulp_bde64 *bpl; 541 int ret; 542 uint16_t ntype, nstate; 543 544 if (!ndlp) { 545 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 546 "6051 NVMEx LS REQ: Bad NDLP x%px, Failing " 547 "LS Req\n", 548 ndlp); 549 return -ENODEV; 550 } 551 552 ntype = ndlp->nlp_type; 553 nstate = ndlp->nlp_state; 554 if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) || 555 (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) { 556 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 557 "6088 NVMEx LS REQ: Fail DID x%06x not " 558 "ready for IO. Type x%x, State x%x\n", 559 ndlp->nlp_DID, ntype, nstate); 560 return -ENODEV; 561 } 562 563 if (!vport->phba->sli4_hba.nvmels_wq) 564 return -ENOMEM; 565 566 /* 567 * there are two dma buf in the request, actually there is one and 568 * the second one is just the start address + cmd size. 569 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped 570 * in a lpfc_dmabuf struct. When freeing we just free the wrapper 571 * because the nvem layer owns the data bufs. 572 * We do not have to break these packets open, we don't care what is 573 * in them. And we do not have to look at the resonse data, we only 574 * care that we got a response. All of the caring is going to happen 575 * in the nvme-fc layer. 576 */ 577 578 bmp = kmalloc(sizeof(*bmp), GFP_KERNEL); 579 if (!bmp) { 580 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 581 "6044 NVMEx LS REQ: Could not alloc LS buf " 582 "for DID %x\n", 583 ndlp->nlp_DID); 584 return -ENOMEM; 585 } 586 587 bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys)); 588 if (!bmp->virt) { 589 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 590 "6042 NVMEx LS REQ: Could not alloc mbuf " 591 "for DID %x\n", 592 ndlp->nlp_DID); 593 kfree(bmp); 594 return -ENOMEM; 595 } 596 597 INIT_LIST_HEAD(&bmp->list); 598 599 bpl = (struct ulp_bde64 *)bmp->virt; 600 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma)); 601 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma)); 602 bpl->tus.f.bdeFlags = 0; 603 bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen; 604 bpl->tus.w = le32_to_cpu(bpl->tus.w); 605 bpl++; 606 607 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma)); 608 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma)); 609 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I; 610 bpl->tus.f.bdeSize = pnvme_lsreq->rsplen; 611 bpl->tus.w = le32_to_cpu(bpl->tus.w); 612 613 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 614 "6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, " 615 "rqstlen:%d rsplen:%d %pad %pad\n", 616 ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen, 617 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 618 &pnvme_lsreq->rspdma); 619 620 ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr, 621 pnvme_lsreq, gen_req_cmp, ndlp, 2, 622 pnvme_lsreq->timeout, 0); 623 if (ret != WQE_SUCCESS) { 624 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 625 "6052 NVMEx REQ: EXIT. issue ls wqe failed " 626 "lsreq x%px Status %x DID %x\n", 627 pnvme_lsreq, ret, ndlp->nlp_DID); 628 lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys); 629 kfree(bmp); 630 return -EIO; 631 } 632 633 return 0; 634 } 635 636 /** 637 * lpfc_nvme_ls_req - Issue an NVME Link Service request 638 * @pnvme_lport: Transport localport that LS is to be issued from. 639 * @pnvme_rport: Transport remoteport that LS is to be sent to. 640 * @pnvme_lsreq: the transport nvme_ls_req structure for the LS 641 * 642 * Driver registers this routine to handle any link service request 643 * from the nvme_fc transport to a remote nvme-aware port. 644 * 645 * Return value : 646 * 0 - Success 647 * non-zero: various error codes, in form of -Exxx 648 **/ 649 static int 650 lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport, 651 struct nvme_fc_remote_port *pnvme_rport, 652 struct nvmefc_ls_req *pnvme_lsreq) 653 { 654 struct lpfc_nvme_lport *lport; 655 struct lpfc_nvme_rport *rport; 656 struct lpfc_vport *vport; 657 int ret; 658 659 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 660 rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 661 if (unlikely(!lport) || unlikely(!rport)) 662 return -EINVAL; 663 664 vport = lport->vport; 665 if (vport->load_flag & FC_UNLOADING) 666 return -ENODEV; 667 668 atomic_inc(&lport->fc4NvmeLsRequests); 669 670 ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq, 671 lpfc_nvme_ls_req_cmp); 672 if (ret) 673 atomic_inc(&lport->xmt_ls_err); 674 675 return ret; 676 } 677 678 /** 679 * __lpfc_nvme_ls_abort - Generic service routine to abort a prior 680 * NVME LS request 681 * @vport: The local port that issued the LS 682 * @ndlp: The remote port the LS was sent to 683 * @pnvme_lsreq: Pointer to LS request structure from the transport 684 * 685 * The driver validates the ndlp, looks for the LS, and aborts the 686 * LS if found. 687 * 688 * Returns: 689 * 0 : if LS found and aborted 690 * non-zero: various error conditions in form -Exxx 691 **/ 692 int 693 __lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, 694 struct nvmefc_ls_req *pnvme_lsreq) 695 { 696 struct lpfc_hba *phba = vport->phba; 697 struct lpfc_sli_ring *pring; 698 struct lpfc_iocbq *wqe, *next_wqe; 699 bool foundit = false; 700 701 if (!ndlp) { 702 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 703 "6049 NVMEx LS REQ Abort: Bad NDLP x%px DID " 704 "x%06x, Failing LS Req\n", 705 ndlp, ndlp ? ndlp->nlp_DID : 0); 706 return -EINVAL; 707 } 708 709 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, 710 "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq " 711 "x%px rqstlen:%d rsplen:%d %pad %pad\n", 712 pnvme_lsreq, pnvme_lsreq->rqstlen, 713 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 714 &pnvme_lsreq->rspdma); 715 716 /* 717 * Lock the ELS ring txcmplq and look for the wqe that matches 718 * this ELS. If found, issue an abort on the wqe. 719 */ 720 pring = phba->sli4_hba.nvmels_wq->pring; 721 spin_lock_irq(&phba->hbalock); 722 spin_lock(&pring->ring_lock); 723 list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) { 724 if (wqe->context2 == pnvme_lsreq) { 725 wqe->cmd_flag |= LPFC_DRIVER_ABORTED; 726 foundit = true; 727 break; 728 } 729 } 730 spin_unlock(&pring->ring_lock); 731 732 if (foundit) 733 lpfc_sli_issue_abort_iotag(phba, pring, wqe, NULL); 734 spin_unlock_irq(&phba->hbalock); 735 736 if (foundit) 737 return 0; 738 739 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, 740 "6213 NVMEx LS REQ Abort: Unable to locate req x%px\n", 741 pnvme_lsreq); 742 return -EINVAL; 743 } 744 745 static int 746 lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport, 747 struct nvme_fc_remote_port *remoteport, 748 struct nvmefc_ls_rsp *ls_rsp) 749 { 750 struct lpfc_async_xchg_ctx *axchg = 751 container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp); 752 struct lpfc_nvme_lport *lport; 753 int rc; 754 755 if (axchg->phba->pport->load_flag & FC_UNLOADING) 756 return -ENODEV; 757 758 lport = (struct lpfc_nvme_lport *)localport->private; 759 760 rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp); 761 762 if (rc) { 763 /* 764 * unless the failure is due to having already sent 765 * the response, an abort will be generated for the 766 * exchange if the rsp can't be sent. 767 */ 768 if (rc != -EALREADY) 769 atomic_inc(&lport->xmt_ls_abort); 770 return rc; 771 } 772 773 return 0; 774 } 775 776 /** 777 * lpfc_nvme_ls_abort - Abort a prior NVME LS request 778 * @pnvme_lport: Transport localport that LS is to be issued from. 779 * @pnvme_rport: Transport remoteport that LS is to be sent to. 780 * @pnvme_lsreq: the transport nvme_ls_req structure for the LS 781 * 782 * Driver registers this routine to abort a NVME LS request that is 783 * in progress (from the transports perspective). 784 **/ 785 static void 786 lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, 787 struct nvme_fc_remote_port *pnvme_rport, 788 struct nvmefc_ls_req *pnvme_lsreq) 789 { 790 struct lpfc_nvme_lport *lport; 791 struct lpfc_vport *vport; 792 struct lpfc_nodelist *ndlp; 793 int ret; 794 795 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 796 if (unlikely(!lport)) 797 return; 798 vport = lport->vport; 799 800 if (vport->load_flag & FC_UNLOADING) 801 return; 802 803 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); 804 805 ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq); 806 if (!ret) 807 atomic_inc(&lport->xmt_ls_abort); 808 } 809 810 /* Fix up the existing sgls for NVME IO. */ 811 static inline void 812 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, 813 struct lpfc_io_buf *lpfc_ncmd, 814 struct nvmefc_fcp_req *nCmd) 815 { 816 struct lpfc_hba *phba = vport->phba; 817 struct sli4_sge *sgl; 818 union lpfc_wqe128 *wqe; 819 uint32_t *wptr, *dptr; 820 821 /* 822 * Get a local pointer to the built-in wqe and correct 823 * the cmd size to match NVME's 96 bytes and fix 824 * the dma address. 825 */ 826 827 wqe = &lpfc_ncmd->cur_iocbq.wqe; 828 829 /* 830 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to 831 * match NVME. NVME sends 96 bytes. Also, use the 832 * nvme commands command and response dma addresses 833 * rather than the virtual memory to ease the restore 834 * operation. 835 */ 836 sgl = lpfc_ncmd->dma_sgl; 837 sgl->sge_len = cpu_to_le32(nCmd->cmdlen); 838 if (phba->cfg_nvme_embed_cmd) { 839 sgl->addr_hi = 0; 840 sgl->addr_lo = 0; 841 842 /* Word 0-2 - NVME CMND IU (embedded payload) */ 843 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED; 844 wqe->generic.bde.tus.f.bdeSize = 56; 845 wqe->generic.bde.addrHigh = 0; 846 wqe->generic.bde.addrLow = 64; /* Word 16 */ 847 848 /* Word 10 - dbde is 0, wqes is 1 in template */ 849 850 /* 851 * Embed the payload in the last half of the WQE 852 * WQE words 16-30 get the NVME CMD IU payload 853 * 854 * WQE words 16-19 get payload Words 1-4 855 * WQE words 20-21 get payload Words 6-7 856 * WQE words 22-29 get payload Words 16-23 857 */ 858 wptr = &wqe->words[16]; /* WQE ptr */ 859 dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */ 860 dptr++; /* Skip Word 0 in payload */ 861 862 *wptr++ = *dptr++; /* Word 1 */ 863 *wptr++ = *dptr++; /* Word 2 */ 864 *wptr++ = *dptr++; /* Word 3 */ 865 *wptr++ = *dptr++; /* Word 4 */ 866 dptr++; /* Skip Word 5 in payload */ 867 *wptr++ = *dptr++; /* Word 6 */ 868 *wptr++ = *dptr++; /* Word 7 */ 869 dptr += 8; /* Skip Words 8-15 in payload */ 870 *wptr++ = *dptr++; /* Word 16 */ 871 *wptr++ = *dptr++; /* Word 17 */ 872 *wptr++ = *dptr++; /* Word 18 */ 873 *wptr++ = *dptr++; /* Word 19 */ 874 *wptr++ = *dptr++; /* Word 20 */ 875 *wptr++ = *dptr++; /* Word 21 */ 876 *wptr++ = *dptr++; /* Word 22 */ 877 *wptr = *dptr; /* Word 23 */ 878 } else { 879 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma)); 880 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma)); 881 882 /* Word 0-2 - NVME CMND IU Inline BDE */ 883 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 884 wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen; 885 wqe->generic.bde.addrHigh = sgl->addr_hi; 886 wqe->generic.bde.addrLow = sgl->addr_lo; 887 888 /* Word 10 */ 889 bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); 890 bf_set(wqe_wqes, &wqe->generic.wqe_com, 0); 891 } 892 893 sgl++; 894 895 /* Setup the physical region for the FCP RSP */ 896 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma)); 897 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma)); 898 sgl->word2 = le32_to_cpu(sgl->word2); 899 if (nCmd->sg_cnt) 900 bf_set(lpfc_sli4_sge_last, sgl, 0); 901 else 902 bf_set(lpfc_sli4_sge_last, sgl, 1); 903 sgl->word2 = cpu_to_le32(sgl->word2); 904 sgl->sge_len = cpu_to_le32(nCmd->rsplen); 905 } 906 907 908 /* 909 * lpfc_nvme_io_cmd_cmpl - Complete an NVME-over-FCP IO 910 * 911 * Driver registers this routine as it io request handler. This 912 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 913 * data structure to the rport indicated in @lpfc_nvme_rport. 914 * 915 * Return value : 916 * 0 - Success 917 * TODO: What are the failure codes. 918 **/ 919 static void 920 lpfc_nvme_io_cmd_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, 921 struct lpfc_iocbq *pwqeOut) 922 { 923 struct lpfc_io_buf *lpfc_ncmd = 924 (struct lpfc_io_buf *)pwqeIn->context1; 925 struct lpfc_wcqe_complete *wcqe = &pwqeOut->wcqe_cmpl; 926 struct lpfc_vport *vport = pwqeIn->vport; 927 struct nvmefc_fcp_req *nCmd; 928 struct nvme_fc_ersp_iu *ep; 929 struct nvme_fc_cmd_iu *cp; 930 struct lpfc_nodelist *ndlp; 931 struct lpfc_nvme_fcpreq_priv *freqpriv; 932 struct lpfc_nvme_lport *lport; 933 uint32_t code, status, idx; 934 uint16_t cid, sqhd, data; 935 uint32_t *ptr; 936 uint32_t lat; 937 bool call_done = false; 938 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 939 int cpu; 940 #endif 941 int offline = 0; 942 943 /* Sanity check on return of outstanding command */ 944 if (!lpfc_ncmd) { 945 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 946 "6071 Null lpfc_ncmd pointer. No " 947 "release, skip completion\n"); 948 return; 949 } 950 951 /* Guard against abort handler being called at same time */ 952 spin_lock(&lpfc_ncmd->buf_lock); 953 954 if (!lpfc_ncmd->nvmeCmd) { 955 spin_unlock(&lpfc_ncmd->buf_lock); 956 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 957 "6066 Missing cmpl ptrs: lpfc_ncmd x%px, " 958 "nvmeCmd x%px\n", 959 lpfc_ncmd, lpfc_ncmd->nvmeCmd); 960 961 /* Release the lpfc_ncmd regardless of the missing elements. */ 962 lpfc_release_nvme_buf(phba, lpfc_ncmd); 963 return; 964 } 965 nCmd = lpfc_ncmd->nvmeCmd; 966 status = bf_get(lpfc_wcqe_c_status, wcqe); 967 968 idx = lpfc_ncmd->cur_iocbq.hba_wqidx; 969 phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++; 970 971 if (unlikely(status && vport->localport)) { 972 lport = (struct lpfc_nvme_lport *)vport->localport->private; 973 if (lport) { 974 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 975 atomic_inc(&lport->cmpl_fcp_xb); 976 atomic_inc(&lport->cmpl_fcp_err); 977 } 978 } 979 980 lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n", 981 lpfc_ncmd->cur_iocbq.sli4_xritag, 982 status, wcqe->parameter); 983 /* 984 * Catch race where our node has transitioned, but the 985 * transport is still transitioning. 986 */ 987 ndlp = lpfc_ncmd->ndlp; 988 if (!ndlp) { 989 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 990 "6062 Ignoring NVME cmpl. No ndlp\n"); 991 goto out_err; 992 } 993 994 code = bf_get(lpfc_wcqe_c_code, wcqe); 995 if (code == CQE_CODE_NVME_ERSP) { 996 /* For this type of CQE, we need to rebuild the rsp */ 997 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr; 998 999 /* 1000 * Get Command Id from cmd to plug into response. This 1001 * code is not needed in the next NVME Transport drop. 1002 */ 1003 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr; 1004 cid = cp->sqe.common.command_id; 1005 1006 /* 1007 * RSN is in CQE word 2 1008 * SQHD is in CQE Word 3 bits 15:0 1009 * Cmd Specific info is in CQE Word 1 1010 * and in CQE Word 0 bits 15:0 1011 */ 1012 sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe); 1013 1014 /* Now lets build the NVME ERSP IU */ 1015 ep->iu_len = cpu_to_be16(8); 1016 ep->rsn = wcqe->parameter; 1017 ep->xfrd_len = cpu_to_be32(nCmd->payload_length); 1018 ep->rsvd12 = 0; 1019 ptr = (uint32_t *)&ep->cqe.result.u64; 1020 *ptr++ = wcqe->total_data_placed; 1021 data = bf_get(lpfc_wcqe_c_ersp0, wcqe); 1022 *ptr = (uint32_t)data; 1023 ep->cqe.sq_head = sqhd; 1024 ep->cqe.sq_id = nCmd->sqid; 1025 ep->cqe.command_id = cid; 1026 ep->cqe.status = 0; 1027 1028 lpfc_ncmd->status = IOSTAT_SUCCESS; 1029 lpfc_ncmd->result = 0; 1030 nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN; 1031 nCmd->transferred_length = nCmd->payload_length; 1032 } else { 1033 lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK); 1034 lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK); 1035 1036 /* For NVME, the only failure path that results in an 1037 * IO error is when the adapter rejects it. All other 1038 * conditions are a success case and resolved by the 1039 * transport. 1040 * IOSTAT_FCP_RSP_ERROR means: 1041 * 1. Length of data received doesn't match total 1042 * transfer length in WQE 1043 * 2. If the RSP payload does NOT match these cases: 1044 * a. RSP length 12/24 bytes and all zeros 1045 * b. NVME ERSP 1046 */ 1047 switch (lpfc_ncmd->status) { 1048 case IOSTAT_SUCCESS: 1049 nCmd->transferred_length = wcqe->total_data_placed; 1050 nCmd->rcv_rsplen = 0; 1051 nCmd->status = 0; 1052 break; 1053 case IOSTAT_FCP_RSP_ERROR: 1054 nCmd->transferred_length = wcqe->total_data_placed; 1055 nCmd->rcv_rsplen = wcqe->parameter; 1056 nCmd->status = 0; 1057 1058 /* Check if this is really an ERSP */ 1059 if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) { 1060 lpfc_ncmd->status = IOSTAT_SUCCESS; 1061 lpfc_ncmd->result = 0; 1062 1063 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 1064 "6084 NVME Completion ERSP: " 1065 "xri %x placed x%x\n", 1066 lpfc_ncmd->cur_iocbq.sli4_xritag, 1067 wcqe->total_data_placed); 1068 break; 1069 } 1070 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1071 "6081 NVME Completion Protocol Error: " 1072 "xri %x status x%x result x%x " 1073 "placed x%x\n", 1074 lpfc_ncmd->cur_iocbq.sli4_xritag, 1075 lpfc_ncmd->status, lpfc_ncmd->result, 1076 wcqe->total_data_placed); 1077 break; 1078 case IOSTAT_LOCAL_REJECT: 1079 /* Let fall through to set command final state. */ 1080 if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED) 1081 lpfc_printf_vlog(vport, KERN_INFO, 1082 LOG_NVME_IOERR, 1083 "6032 Delay Aborted cmd x%px " 1084 "nvme cmd x%px, xri x%x, " 1085 "xb %d\n", 1086 lpfc_ncmd, nCmd, 1087 lpfc_ncmd->cur_iocbq.sli4_xritag, 1088 bf_get(lpfc_wcqe_c_xb, wcqe)); 1089 fallthrough; 1090 default: 1091 out_err: 1092 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1093 "6072 NVME Completion Error: xri %x " 1094 "status x%x result x%x [x%x] " 1095 "placed x%x\n", 1096 lpfc_ncmd->cur_iocbq.sli4_xritag, 1097 lpfc_ncmd->status, lpfc_ncmd->result, 1098 wcqe->parameter, 1099 wcqe->total_data_placed); 1100 nCmd->transferred_length = 0; 1101 nCmd->rcv_rsplen = 0; 1102 nCmd->status = NVME_SC_INTERNAL; 1103 offline = pci_channel_offline(vport->phba->pcidev); 1104 } 1105 } 1106 1107 /* pick up SLI4 exhange busy condition */ 1108 if (bf_get(lpfc_wcqe_c_xb, wcqe) && !offline) 1109 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; 1110 else 1111 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 1112 1113 /* Update stats and complete the IO. There is 1114 * no need for dma unprep because the nvme_transport 1115 * owns the dma address. 1116 */ 1117 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1118 if (lpfc_ncmd->ts_cmd_start) { 1119 lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp; 1120 lpfc_ncmd->ts_data_io = ktime_get_ns(); 1121 phba->ktime_last_cmd = lpfc_ncmd->ts_data_io; 1122 lpfc_io_ktime(phba, lpfc_ncmd); 1123 } 1124 if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) { 1125 cpu = raw_smp_processor_id(); 1126 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io); 1127 if (lpfc_ncmd->cpu != cpu) 1128 lpfc_printf_vlog(vport, 1129 KERN_INFO, LOG_NVME_IOERR, 1130 "6701 CPU Check cmpl: " 1131 "cpu %d expect %d\n", 1132 cpu, lpfc_ncmd->cpu); 1133 } 1134 #endif 1135 1136 /* NVME targets need completion held off until the abort exchange 1137 * completes unless the NVME Rport is getting unregistered. 1138 */ 1139 1140 if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 1141 freqpriv = nCmd->private; 1142 freqpriv->nvme_buf = NULL; 1143 lpfc_ncmd->nvmeCmd = NULL; 1144 call_done = true; 1145 } 1146 spin_unlock(&lpfc_ncmd->buf_lock); 1147 1148 /* Check if IO qualified for CMF */ 1149 if (phba->cmf_active_mode != LPFC_CFG_OFF && 1150 nCmd->io_dir == NVMEFC_FCP_READ && 1151 nCmd->payload_length) { 1152 /* Used when calculating average latency */ 1153 lat = ktime_get_ns() - lpfc_ncmd->rx_cmd_start; 1154 lpfc_update_cmf_cmpl(phba, lat, nCmd->payload_length, NULL); 1155 } 1156 1157 if (call_done) 1158 nCmd->done(nCmd); 1159 1160 /* Call release with XB=1 to queue the IO into the abort list. */ 1161 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1162 } 1163 1164 1165 /** 1166 * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO 1167 * @vport: pointer to a host virtual N_Port data structure 1168 * @lpfc_ncmd: Pointer to lpfc scsi command 1169 * @pnode: pointer to a node-list data structure 1170 * @cstat: pointer to the control status structure 1171 * 1172 * Driver registers this routine as it io request handler. This 1173 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1174 * data structure to the rport indicated in @lpfc_nvme_rport. 1175 * 1176 * Return value : 1177 * 0 - Success 1178 * TODO: What are the failure codes. 1179 **/ 1180 static int 1181 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, 1182 struct lpfc_io_buf *lpfc_ncmd, 1183 struct lpfc_nodelist *pnode, 1184 struct lpfc_fc4_ctrl_stat *cstat) 1185 { 1186 struct lpfc_hba *phba = vport->phba; 1187 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1188 struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq); 1189 union lpfc_wqe128 *wqe = &pwqeq->wqe; 1190 uint32_t req_len; 1191 1192 /* 1193 * There are three possibilities here - use scatter-gather segment, use 1194 * the single mapping, or neither. 1195 */ 1196 if (nCmd->sg_cnt) { 1197 if (nCmd->io_dir == NVMEFC_FCP_WRITE) { 1198 /* From the iwrite template, initialize words 7 - 11 */ 1199 memcpy(&wqe->words[7], 1200 &lpfc_iwrite_cmd_template.words[7], 1201 sizeof(uint32_t) * 5); 1202 1203 /* Word 4 */ 1204 wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length; 1205 1206 /* Word 5 */ 1207 if ((phba->cfg_nvme_enable_fb) && 1208 (pnode->nlp_flag & NLP_FIRSTBURST)) { 1209 req_len = lpfc_ncmd->nvmeCmd->payload_length; 1210 if (req_len < pnode->nvme_fb_size) 1211 wqe->fcp_iwrite.initial_xfer_len = 1212 req_len; 1213 else 1214 wqe->fcp_iwrite.initial_xfer_len = 1215 pnode->nvme_fb_size; 1216 } else { 1217 wqe->fcp_iwrite.initial_xfer_len = 0; 1218 } 1219 cstat->output_requests++; 1220 } else { 1221 /* From the iread template, initialize words 7 - 11 */ 1222 memcpy(&wqe->words[7], 1223 &lpfc_iread_cmd_template.words[7], 1224 sizeof(uint32_t) * 5); 1225 1226 /* Word 4 */ 1227 wqe->fcp_iread.total_xfer_len = nCmd->payload_length; 1228 1229 /* Word 5 */ 1230 wqe->fcp_iread.rsrvd5 = 0; 1231 1232 /* For a CMF Managed port, iod must be zero'ed */ 1233 if (phba->cmf_active_mode == LPFC_CFG_MANAGED) 1234 bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, 1235 LPFC_WQE_IOD_NONE); 1236 cstat->input_requests++; 1237 } 1238 } else { 1239 /* From the icmnd template, initialize words 4 - 11 */ 1240 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4], 1241 sizeof(uint32_t) * 8); 1242 cstat->control_requests++; 1243 } 1244 1245 if (pnode->nlp_nvme_info & NLP_NVME_NSLER) 1246 bf_set(wqe_erp, &wqe->generic.wqe_com, 1); 1247 /* 1248 * Finish initializing those WQE fields that are independent 1249 * of the nvme_cmnd request_buffer 1250 */ 1251 1252 /* Word 3 */ 1253 bf_set(payload_offset_len, &wqe->fcp_icmd, 1254 (nCmd->rsplen + nCmd->cmdlen)); 1255 1256 /* Word 6 */ 1257 bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com, 1258 phba->sli4_hba.rpi_ids[pnode->nlp_rpi]); 1259 bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag); 1260 1261 /* Word 8 */ 1262 wqe->generic.wqe_com.abort_tag = pwqeq->iotag; 1263 1264 /* Word 9 */ 1265 bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag); 1266 1267 /* Word 10 */ 1268 bf_set(wqe_xchg, &wqe->fcp_iwrite.wqe_com, LPFC_NVME_XCHG); 1269 1270 /* Words 13 14 15 are for PBDE support */ 1271 1272 pwqeq->vport = vport; 1273 return 0; 1274 } 1275 1276 1277 /** 1278 * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO 1279 * @vport: pointer to a host virtual N_Port data structure 1280 * @lpfc_ncmd: Pointer to lpfc scsi command 1281 * 1282 * Driver registers this routine as it io request handler. This 1283 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1284 * data structure to the rport indicated in @lpfc_nvme_rport. 1285 * 1286 * Return value : 1287 * 0 - Success 1288 * TODO: What are the failure codes. 1289 **/ 1290 static int 1291 lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, 1292 struct lpfc_io_buf *lpfc_ncmd) 1293 { 1294 struct lpfc_hba *phba = vport->phba; 1295 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1296 union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe; 1297 struct sli4_sge *sgl = lpfc_ncmd->dma_sgl; 1298 struct sli4_hybrid_sgl *sgl_xtra = NULL; 1299 struct scatterlist *data_sg; 1300 struct sli4_sge *first_data_sgl; 1301 struct ulp_bde64 *bde; 1302 dma_addr_t physaddr = 0; 1303 uint32_t dma_len = 0; 1304 uint32_t dma_offset = 0; 1305 int nseg, i, j; 1306 bool lsp_just_set = false; 1307 1308 /* Fix up the command and response DMA stuff. */ 1309 lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd); 1310 1311 /* 1312 * There are three possibilities here - use scatter-gather segment, use 1313 * the single mapping, or neither. 1314 */ 1315 if (nCmd->sg_cnt) { 1316 /* 1317 * Jump over the cmd and rsp SGEs. The fix routine 1318 * has already adjusted for this. 1319 */ 1320 sgl += 2; 1321 1322 first_data_sgl = sgl; 1323 lpfc_ncmd->seg_cnt = nCmd->sg_cnt; 1324 if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) { 1325 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, 1326 "6058 Too many sg segments from " 1327 "NVME Transport. Max %d, " 1328 "nvmeIO sg_cnt %d\n", 1329 phba->cfg_nvme_seg_cnt + 1, 1330 lpfc_ncmd->seg_cnt); 1331 lpfc_ncmd->seg_cnt = 0; 1332 return 1; 1333 } 1334 1335 /* 1336 * The driver established a maximum scatter-gather segment count 1337 * during probe that limits the number of sg elements in any 1338 * single nvme command. Just run through the seg_cnt and format 1339 * the sge's. 1340 */ 1341 nseg = nCmd->sg_cnt; 1342 data_sg = nCmd->first_sgl; 1343 1344 /* for tracking the segment boundaries */ 1345 j = 2; 1346 for (i = 0; i < nseg; i++) { 1347 if (data_sg == NULL) { 1348 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, 1349 "6059 dptr err %d, nseg %d\n", 1350 i, nseg); 1351 lpfc_ncmd->seg_cnt = 0; 1352 return 1; 1353 } 1354 1355 sgl->word2 = 0; 1356 if (nseg == 1) { 1357 bf_set(lpfc_sli4_sge_last, sgl, 1); 1358 bf_set(lpfc_sli4_sge_type, sgl, 1359 LPFC_SGE_TYPE_DATA); 1360 } else { 1361 bf_set(lpfc_sli4_sge_last, sgl, 0); 1362 1363 /* expand the segment */ 1364 if (!lsp_just_set && 1365 !((j + 1) % phba->border_sge_num) && 1366 ((nseg - 1) != i)) { 1367 /* set LSP type */ 1368 bf_set(lpfc_sli4_sge_type, sgl, 1369 LPFC_SGE_TYPE_LSP); 1370 1371 sgl_xtra = lpfc_get_sgl_per_hdwq( 1372 phba, lpfc_ncmd); 1373 1374 if (unlikely(!sgl_xtra)) { 1375 lpfc_ncmd->seg_cnt = 0; 1376 return 1; 1377 } 1378 sgl->addr_lo = cpu_to_le32(putPaddrLow( 1379 sgl_xtra->dma_phys_sgl)); 1380 sgl->addr_hi = cpu_to_le32(putPaddrHigh( 1381 sgl_xtra->dma_phys_sgl)); 1382 1383 } else { 1384 bf_set(lpfc_sli4_sge_type, sgl, 1385 LPFC_SGE_TYPE_DATA); 1386 } 1387 } 1388 1389 if (!(bf_get(lpfc_sli4_sge_type, sgl) & 1390 LPFC_SGE_TYPE_LSP)) { 1391 if ((nseg - 1) == i) 1392 bf_set(lpfc_sli4_sge_last, sgl, 1); 1393 1394 physaddr = data_sg->dma_address; 1395 dma_len = data_sg->length; 1396 sgl->addr_lo = cpu_to_le32( 1397 putPaddrLow(physaddr)); 1398 sgl->addr_hi = cpu_to_le32( 1399 putPaddrHigh(physaddr)); 1400 1401 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset); 1402 sgl->word2 = cpu_to_le32(sgl->word2); 1403 sgl->sge_len = cpu_to_le32(dma_len); 1404 1405 dma_offset += dma_len; 1406 data_sg = sg_next(data_sg); 1407 1408 sgl++; 1409 1410 lsp_just_set = false; 1411 } else { 1412 sgl->word2 = cpu_to_le32(sgl->word2); 1413 1414 sgl->sge_len = cpu_to_le32( 1415 phba->cfg_sg_dma_buf_size); 1416 1417 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl; 1418 i = i - 1; 1419 1420 lsp_just_set = true; 1421 } 1422 1423 j++; 1424 } 1425 1426 /* PBDE support for first data SGE only */ 1427 if (nseg == 1 && phba->cfg_enable_pbde) { 1428 /* Words 13-15 */ 1429 bde = (struct ulp_bde64 *) 1430 &wqe->words[13]; 1431 bde->addrLow = first_data_sgl->addr_lo; 1432 bde->addrHigh = first_data_sgl->addr_hi; 1433 bde->tus.f.bdeSize = 1434 le32_to_cpu(first_data_sgl->sge_len); 1435 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64; 1436 bde->tus.w = cpu_to_le32(bde->tus.w); 1437 1438 /* Word 11 - set PBDE bit */ 1439 bf_set(wqe_pbde, &wqe->generic.wqe_com, 1); 1440 } else { 1441 memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3)); 1442 /* Word 11 - PBDE bit disabled by default template */ 1443 } 1444 1445 } else { 1446 lpfc_ncmd->seg_cnt = 0; 1447 1448 /* For this clause to be valid, the payload_length 1449 * and sg_cnt must zero. 1450 */ 1451 if (nCmd->payload_length != 0) { 1452 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, 1453 "6063 NVME DMA Prep Err: sg_cnt %d " 1454 "payload_length x%x\n", 1455 nCmd->sg_cnt, nCmd->payload_length); 1456 return 1; 1457 } 1458 } 1459 return 0; 1460 } 1461 1462 /** 1463 * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO 1464 * @pnvme_lport: Pointer to the driver's local port data 1465 * @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1466 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1467 * @pnvme_fcreq: IO request from nvme fc to driver. 1468 * 1469 * Driver registers this routine as it io request handler. This 1470 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1471 * data structure to the rport indicated in @lpfc_nvme_rport. 1472 * 1473 * Return value : 1474 * 0 - Success 1475 * TODO: What are the failure codes. 1476 **/ 1477 static int 1478 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, 1479 struct nvme_fc_remote_port *pnvme_rport, 1480 void *hw_queue_handle, 1481 struct nvmefc_fcp_req *pnvme_fcreq) 1482 { 1483 int ret = 0; 1484 int expedite = 0; 1485 int idx, cpu; 1486 struct lpfc_nvme_lport *lport; 1487 struct lpfc_fc4_ctrl_stat *cstat; 1488 struct lpfc_vport *vport; 1489 struct lpfc_hba *phba; 1490 struct lpfc_nodelist *ndlp; 1491 struct lpfc_io_buf *lpfc_ncmd; 1492 struct lpfc_nvme_rport *rport; 1493 struct lpfc_nvme_qhandle *lpfc_queue_info; 1494 struct lpfc_nvme_fcpreq_priv *freqpriv; 1495 struct nvme_common_command *sqe; 1496 uint64_t start = 0; 1497 1498 /* Validate pointers. LLDD fault handling with transport does 1499 * have timing races. 1500 */ 1501 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1502 if (unlikely(!lport)) { 1503 ret = -EINVAL; 1504 goto out_fail; 1505 } 1506 1507 vport = lport->vport; 1508 1509 if (unlikely(!hw_queue_handle)) { 1510 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1511 "6117 Fail IO, NULL hw_queue_handle\n"); 1512 atomic_inc(&lport->xmt_fcp_err); 1513 ret = -EBUSY; 1514 goto out_fail; 1515 } 1516 1517 phba = vport->phba; 1518 1519 if (unlikely(vport->load_flag & FC_UNLOADING)) { 1520 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1521 "6124 Fail IO, Driver unload\n"); 1522 atomic_inc(&lport->xmt_fcp_err); 1523 ret = -ENODEV; 1524 goto out_fail; 1525 } 1526 1527 freqpriv = pnvme_fcreq->private; 1528 if (unlikely(!freqpriv)) { 1529 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1530 "6158 Fail IO, NULL request data\n"); 1531 atomic_inc(&lport->xmt_fcp_err); 1532 ret = -EINVAL; 1533 goto out_fail; 1534 } 1535 1536 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1537 if (phba->ktime_on) 1538 start = ktime_get_ns(); 1539 #endif 1540 rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 1541 lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle; 1542 1543 /* 1544 * Catch race where our node has transitioned, but the 1545 * transport is still transitioning. 1546 */ 1547 ndlp = rport->ndlp; 1548 if (!ndlp) { 1549 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR, 1550 "6053 Busy IO, ndlp not ready: rport x%px " 1551 "ndlp x%px, DID x%06x\n", 1552 rport, ndlp, pnvme_rport->port_id); 1553 atomic_inc(&lport->xmt_fcp_err); 1554 ret = -EBUSY; 1555 goto out_fail; 1556 } 1557 1558 /* The remote node has to be a mapped target or it's an error. */ 1559 if ((ndlp->nlp_type & NLP_NVME_TARGET) && 1560 (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) { 1561 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR, 1562 "6036 Fail IO, DID x%06x not ready for " 1563 "IO. State x%x, Type x%x Flg x%x\n", 1564 pnvme_rport->port_id, 1565 ndlp->nlp_state, ndlp->nlp_type, 1566 ndlp->fc4_xpt_flags); 1567 atomic_inc(&lport->xmt_fcp_bad_ndlp); 1568 ret = -EBUSY; 1569 goto out_fail; 1570 1571 } 1572 1573 /* Currently only NVME Keep alive commands should be expedited 1574 * if the driver runs out of a resource. These should only be 1575 * issued on the admin queue, qidx 0 1576 */ 1577 if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) { 1578 sqe = &((struct nvme_fc_cmd_iu *) 1579 pnvme_fcreq->cmdaddr)->sqe.common; 1580 if (sqe->opcode == nvme_admin_keep_alive) 1581 expedite = 1; 1582 } 1583 1584 /* Check if IO qualifies for CMF */ 1585 if (phba->cmf_active_mode != LPFC_CFG_OFF && 1586 pnvme_fcreq->io_dir == NVMEFC_FCP_READ && 1587 pnvme_fcreq->payload_length) { 1588 ret = lpfc_update_cmf_cmd(phba, pnvme_fcreq->payload_length); 1589 if (ret) { 1590 ret = -EBUSY; 1591 goto out_fail; 1592 } 1593 /* Get start time for IO latency */ 1594 start = ktime_get_ns(); 1595 } 1596 1597 /* The node is shared with FCP IO, make sure the IO pending count does 1598 * not exceed the programmed depth. 1599 */ 1600 if (lpfc_ndlp_check_qdepth(phba, ndlp)) { 1601 if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) && 1602 !expedite) { 1603 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1604 "6174 Fail IO, ndlp qdepth exceeded: " 1605 "idx %d DID %x pend %d qdepth %d\n", 1606 lpfc_queue_info->index, ndlp->nlp_DID, 1607 atomic_read(&ndlp->cmd_pending), 1608 ndlp->cmd_qdepth); 1609 atomic_inc(&lport->xmt_fcp_qdepth); 1610 ret = -EBUSY; 1611 goto out_fail1; 1612 } 1613 } 1614 1615 /* Lookup Hardware Queue index based on fcp_io_sched module parameter */ 1616 if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) { 1617 idx = lpfc_queue_info->index; 1618 } else { 1619 cpu = raw_smp_processor_id(); 1620 idx = phba->sli4_hba.cpu_map[cpu].hdwq; 1621 } 1622 1623 lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite); 1624 if (lpfc_ncmd == NULL) { 1625 atomic_inc(&lport->xmt_fcp_noxri); 1626 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1627 "6065 Fail IO, driver buffer pool is empty: " 1628 "idx %d DID %x\n", 1629 lpfc_queue_info->index, ndlp->nlp_DID); 1630 ret = -EBUSY; 1631 goto out_fail1; 1632 } 1633 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1634 if (start) { 1635 lpfc_ncmd->ts_cmd_start = start; 1636 lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd; 1637 } else { 1638 lpfc_ncmd->ts_cmd_start = 0; 1639 } 1640 #endif 1641 lpfc_ncmd->rx_cmd_start = start; 1642 1643 /* 1644 * Store the data needed by the driver to issue, abort, and complete 1645 * an IO. 1646 * Do not let the IO hang out forever. There is no midlayer issuing 1647 * an abort so inform the FW of the maximum IO pending time. 1648 */ 1649 freqpriv->nvme_buf = lpfc_ncmd; 1650 lpfc_ncmd->nvmeCmd = pnvme_fcreq; 1651 lpfc_ncmd->ndlp = ndlp; 1652 lpfc_ncmd->qidx = lpfc_queue_info->qidx; 1653 1654 /* 1655 * Issue the IO on the WQ indicated by index in the hw_queue_handle. 1656 * This identfier was create in our hardware queue create callback 1657 * routine. The driver now is dependent on the IO queue steering from 1658 * the transport. We are trusting the upper NVME layers know which 1659 * index to use and that they have affinitized a CPU to this hardware 1660 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ. 1661 */ 1662 lpfc_ncmd->cur_iocbq.hba_wqidx = idx; 1663 cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat; 1664 1665 lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat); 1666 ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd); 1667 if (ret) { 1668 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1669 "6175 Fail IO, Prep DMA: " 1670 "idx %d DID %x\n", 1671 lpfc_queue_info->index, ndlp->nlp_DID); 1672 atomic_inc(&lport->xmt_fcp_err); 1673 ret = -ENOMEM; 1674 goto out_free_nvme_buf; 1675 } 1676 1677 lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n", 1678 lpfc_ncmd->cur_iocbq.sli4_xritag, 1679 lpfc_queue_info->index, ndlp->nlp_DID); 1680 1681 ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq); 1682 if (ret) { 1683 atomic_inc(&lport->xmt_fcp_wqerr); 1684 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1685 "6113 Fail IO, Could not issue WQE err %x " 1686 "sid: x%x did: x%x oxid: x%x\n", 1687 ret, vport->fc_myDID, ndlp->nlp_DID, 1688 lpfc_ncmd->cur_iocbq.sli4_xritag); 1689 goto out_free_nvme_buf; 1690 } 1691 1692 if (phba->cfg_xri_rebalancing) 1693 lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no); 1694 1695 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1696 if (lpfc_ncmd->ts_cmd_start) 1697 lpfc_ncmd->ts_cmd_wqput = ktime_get_ns(); 1698 1699 if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) { 1700 cpu = raw_smp_processor_id(); 1701 this_cpu_inc(phba->sli4_hba.c_stat->xmt_io); 1702 lpfc_ncmd->cpu = cpu; 1703 if (idx != cpu) 1704 lpfc_printf_vlog(vport, 1705 KERN_INFO, LOG_NVME_IOERR, 1706 "6702 CPU Check cmd: " 1707 "cpu %d wq %d\n", 1708 lpfc_ncmd->cpu, 1709 lpfc_queue_info->index); 1710 } 1711 #endif 1712 return 0; 1713 1714 out_free_nvme_buf: 1715 if (lpfc_ncmd->nvmeCmd->sg_cnt) { 1716 if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE) 1717 cstat->output_requests--; 1718 else 1719 cstat->input_requests--; 1720 } else 1721 cstat->control_requests--; 1722 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1723 out_fail1: 1724 lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT, 1725 pnvme_fcreq->payload_length, NULL); 1726 out_fail: 1727 return ret; 1728 } 1729 1730 /** 1731 * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request. 1732 * @phba: Pointer to HBA context object 1733 * @cmdiocb: Pointer to command iocb object. 1734 * @abts_cmpl: Pointer to wcqe complete object. 1735 * 1736 * This is the callback function for any NVME FCP IO that was aborted. 1737 * 1738 * Return value: 1739 * None 1740 **/ 1741 void 1742 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, 1743 struct lpfc_wcqe_complete *abts_cmpl) 1744 { 1745 lpfc_printf_log(phba, KERN_INFO, LOG_NVME, 1746 "6145 ABORT_XRI_CN completing on rpi x%x " 1747 "original iotag x%x, abort cmd iotag x%x " 1748 "req_tag x%x, status x%x, hwstatus x%x\n", 1749 bf_get(wqe_ctxt_tag, &cmdiocb->wqe.generic.wqe_com), 1750 get_job_abtsiotag(phba, cmdiocb), cmdiocb->iotag, 1751 bf_get(lpfc_wcqe_c_request_tag, abts_cmpl), 1752 bf_get(lpfc_wcqe_c_status, abts_cmpl), 1753 bf_get(lpfc_wcqe_c_hw_status, abts_cmpl)); 1754 lpfc_sli_release_iocbq(phba, cmdiocb); 1755 } 1756 1757 /** 1758 * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS 1759 * @pnvme_lport: Pointer to the driver's local port data 1760 * @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1761 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1762 * @pnvme_fcreq: IO request from nvme fc to driver. 1763 * 1764 * Driver registers this routine as its nvme request io abort handler. This 1765 * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq 1766 * data structure to the rport indicated in @lpfc_nvme_rport. This routine 1767 * is executed asynchronously - one the target is validated as "MAPPED" and 1768 * ready for IO, the driver issues the abort request and returns. 1769 * 1770 * Return value: 1771 * None 1772 **/ 1773 static void 1774 lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, 1775 struct nvme_fc_remote_port *pnvme_rport, 1776 void *hw_queue_handle, 1777 struct nvmefc_fcp_req *pnvme_fcreq) 1778 { 1779 struct lpfc_nvme_lport *lport; 1780 struct lpfc_vport *vport; 1781 struct lpfc_hba *phba; 1782 struct lpfc_io_buf *lpfc_nbuf; 1783 struct lpfc_iocbq *nvmereq_wqe; 1784 struct lpfc_nvme_fcpreq_priv *freqpriv; 1785 unsigned long flags; 1786 int ret_val; 1787 1788 /* Validate pointers. LLDD fault handling with transport does 1789 * have timing races. 1790 */ 1791 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1792 if (unlikely(!lport)) 1793 return; 1794 1795 vport = lport->vport; 1796 1797 if (unlikely(!hw_queue_handle)) { 1798 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1799 "6129 Fail Abort, HW Queue Handle NULL.\n"); 1800 return; 1801 } 1802 1803 phba = vport->phba; 1804 freqpriv = pnvme_fcreq->private; 1805 1806 if (unlikely(!freqpriv)) 1807 return; 1808 if (vport->load_flag & FC_UNLOADING) 1809 return; 1810 1811 /* Announce entry to new IO submit field. */ 1812 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1813 "6002 Abort Request to rport DID x%06x " 1814 "for nvme_fc_req x%px\n", 1815 pnvme_rport->port_id, 1816 pnvme_fcreq); 1817 1818 /* If the hba is getting reset, this flag is set. It is 1819 * cleared when the reset is complete and rings reestablished. 1820 */ 1821 spin_lock_irqsave(&phba->hbalock, flags); 1822 /* driver queued commands are in process of being flushed */ 1823 if (phba->hba_flag & HBA_IOQ_FLUSH) { 1824 spin_unlock_irqrestore(&phba->hbalock, flags); 1825 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1826 "6139 Driver in reset cleanup - flushing " 1827 "NVME Req now. hba_flag x%x\n", 1828 phba->hba_flag); 1829 return; 1830 } 1831 1832 lpfc_nbuf = freqpriv->nvme_buf; 1833 if (!lpfc_nbuf) { 1834 spin_unlock_irqrestore(&phba->hbalock, flags); 1835 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1836 "6140 NVME IO req has no matching lpfc nvme " 1837 "io buffer. Skipping abort req.\n"); 1838 return; 1839 } else if (!lpfc_nbuf->nvmeCmd) { 1840 spin_unlock_irqrestore(&phba->hbalock, flags); 1841 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1842 "6141 lpfc NVME IO req has no nvme_fcreq " 1843 "io buffer. Skipping abort req.\n"); 1844 return; 1845 } 1846 nvmereq_wqe = &lpfc_nbuf->cur_iocbq; 1847 1848 /* Guard against IO completion being called at same time */ 1849 spin_lock(&lpfc_nbuf->buf_lock); 1850 1851 /* 1852 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's 1853 * state must match the nvme_fcreq passed by the nvme 1854 * transport. If they don't match, it is likely the driver 1855 * has already completed the NVME IO and the nvme transport 1856 * has not seen it yet. 1857 */ 1858 if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) { 1859 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1860 "6143 NVME req mismatch: " 1861 "lpfc_nbuf x%px nvmeCmd x%px, " 1862 "pnvme_fcreq x%px. Skipping Abort xri x%x\n", 1863 lpfc_nbuf, lpfc_nbuf->nvmeCmd, 1864 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1865 goto out_unlock; 1866 } 1867 1868 /* Don't abort IOs no longer on the pending queue. */ 1869 if (!(nvmereq_wqe->cmd_flag & LPFC_IO_ON_TXCMPLQ)) { 1870 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1871 "6142 NVME IO req x%px not queued - skipping " 1872 "abort req xri x%x\n", 1873 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1874 goto out_unlock; 1875 } 1876 1877 atomic_inc(&lport->xmt_fcp_abort); 1878 lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n", 1879 nvmereq_wqe->sli4_xritag, 1880 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id); 1881 1882 /* Outstanding abort is in progress */ 1883 if (nvmereq_wqe->cmd_flag & LPFC_DRIVER_ABORTED) { 1884 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1885 "6144 Outstanding NVME I/O Abort Request " 1886 "still pending on nvme_fcreq x%px, " 1887 "lpfc_ncmd x%px xri x%x\n", 1888 pnvme_fcreq, lpfc_nbuf, 1889 nvmereq_wqe->sli4_xritag); 1890 goto out_unlock; 1891 } 1892 1893 ret_val = lpfc_sli4_issue_abort_iotag(phba, nvmereq_wqe, 1894 lpfc_nvme_abort_fcreq_cmpl); 1895 1896 spin_unlock(&lpfc_nbuf->buf_lock); 1897 spin_unlock_irqrestore(&phba->hbalock, flags); 1898 1899 /* Make sure HBA is alive */ 1900 lpfc_issue_hb_tmo(phba); 1901 1902 if (ret_val != WQE_SUCCESS) { 1903 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 1904 "6137 Failed abts issue_wqe with status x%x " 1905 "for nvme_fcreq x%px.\n", 1906 ret_val, pnvme_fcreq); 1907 return; 1908 } 1909 1910 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1911 "6138 Transport Abort NVME Request Issued for " 1912 "ox_id x%x\n", 1913 nvmereq_wqe->sli4_xritag); 1914 return; 1915 1916 out_unlock: 1917 spin_unlock(&lpfc_nbuf->buf_lock); 1918 spin_unlock_irqrestore(&phba->hbalock, flags); 1919 return; 1920 } 1921 1922 /* Declare and initialization an instance of the FC NVME template. */ 1923 static struct nvme_fc_port_template lpfc_nvme_template = { 1924 /* initiator-based functions */ 1925 .localport_delete = lpfc_nvme_localport_delete, 1926 .remoteport_delete = lpfc_nvme_remoteport_delete, 1927 .create_queue = lpfc_nvme_create_queue, 1928 .delete_queue = lpfc_nvme_delete_queue, 1929 .ls_req = lpfc_nvme_ls_req, 1930 .fcp_io = lpfc_nvme_fcp_io_submit, 1931 .ls_abort = lpfc_nvme_ls_abort, 1932 .fcp_abort = lpfc_nvme_fcp_abort, 1933 .xmt_ls_rsp = lpfc_nvme_xmt_ls_rsp, 1934 1935 .max_hw_queues = 1, 1936 .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 1937 .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 1938 .dma_boundary = 0xFFFFFFFF, 1939 1940 /* Sizes of additional private data for data structures. 1941 * No use for the last two sizes at this time. 1942 */ 1943 .local_priv_sz = sizeof(struct lpfc_nvme_lport), 1944 .remote_priv_sz = sizeof(struct lpfc_nvme_rport), 1945 .lsrqst_priv_sz = 0, 1946 .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv), 1947 }; 1948 1949 /* 1950 * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA 1951 * 1952 * This routine removes a nvme buffer from head of @hdwq io_buf_list 1953 * and returns to caller. 1954 * 1955 * Return codes: 1956 * NULL - Error 1957 * Pointer to lpfc_nvme_buf - Success 1958 **/ 1959 static struct lpfc_io_buf * 1960 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 1961 int idx, int expedite) 1962 { 1963 struct lpfc_io_buf *lpfc_ncmd; 1964 struct lpfc_sli4_hdw_queue *qp; 1965 struct sli4_sge *sgl; 1966 struct lpfc_iocbq *pwqeq; 1967 union lpfc_wqe128 *wqe; 1968 1969 lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite); 1970 1971 if (lpfc_ncmd) { 1972 pwqeq = &(lpfc_ncmd->cur_iocbq); 1973 wqe = &pwqeq->wqe; 1974 1975 /* Setup key fields in buffer that may have been changed 1976 * if other protocols used this buffer. 1977 */ 1978 pwqeq->cmd_flag = LPFC_IO_NVME; 1979 pwqeq->cmd_cmpl = lpfc_nvme_io_cmd_cmpl; 1980 lpfc_ncmd->start_time = jiffies; 1981 lpfc_ncmd->flags = 0; 1982 1983 /* Rsp SGE will be filled in when we rcv an IO 1984 * from the NVME Layer to be sent. 1985 * The cmd is going to be embedded so we need a SKIP SGE. 1986 */ 1987 sgl = lpfc_ncmd->dma_sgl; 1988 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 1989 bf_set(lpfc_sli4_sge_last, sgl, 0); 1990 sgl->word2 = cpu_to_le32(sgl->word2); 1991 /* Fill in word 3 / sgl_len during cmd submission */ 1992 1993 /* Initialize 64 bytes only */ 1994 memset(wqe, 0, sizeof(union lpfc_wqe)); 1995 1996 if (lpfc_ndlp_check_qdepth(phba, ndlp)) { 1997 atomic_inc(&ndlp->cmd_pending); 1998 lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH; 1999 } 2000 2001 } else { 2002 qp = &phba->sli4_hba.hdwq[idx]; 2003 qp->empty_io_bufs++; 2004 } 2005 2006 return lpfc_ncmd; 2007 } 2008 2009 /** 2010 * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list. 2011 * @phba: The Hba for which this call is being executed. 2012 * @lpfc_ncmd: The nvme buffer which is being released. 2013 * 2014 * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba 2015 * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer 2016 * and cannot be reused for at least RA_TOV amount of time if it was 2017 * aborted. 2018 **/ 2019 static void 2020 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd) 2021 { 2022 struct lpfc_sli4_hdw_queue *qp; 2023 unsigned long iflag = 0; 2024 2025 if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp) 2026 atomic_dec(&lpfc_ncmd->ndlp->cmd_pending); 2027 2028 lpfc_ncmd->ndlp = NULL; 2029 lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH; 2030 2031 qp = lpfc_ncmd->hdwq; 2032 if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 2033 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2034 "6310 XB release deferred for " 2035 "ox_id x%x on reqtag x%x\n", 2036 lpfc_ncmd->cur_iocbq.sli4_xritag, 2037 lpfc_ncmd->cur_iocbq.iotag); 2038 2039 spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag); 2040 list_add_tail(&lpfc_ncmd->list, 2041 &qp->lpfc_abts_io_buf_list); 2042 qp->abts_nvme_io_bufs++; 2043 spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag); 2044 } else 2045 lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp); 2046 } 2047 2048 /** 2049 * lpfc_nvme_create_localport - Create/Bind an nvme localport instance. 2050 * @vport: the lpfc_vport instance requesting a localport. 2051 * 2052 * This routine is invoked to create an nvme localport instance to bind 2053 * to the nvme_fc_transport. It is called once during driver load 2054 * like lpfc_create_shost after all other services are initialized. 2055 * It requires a vport, vpi, and wwns at call time. Other localport 2056 * parameters are modified as the driver's FCID and the Fabric WWN 2057 * are established. 2058 * 2059 * Return codes 2060 * 0 - successful 2061 * -ENOMEM - no heap memory available 2062 * other values - from nvme registration upcall 2063 **/ 2064 int 2065 lpfc_nvme_create_localport(struct lpfc_vport *vport) 2066 { 2067 int ret = 0; 2068 struct lpfc_hba *phba = vport->phba; 2069 struct nvme_fc_port_info nfcp_info; 2070 struct nvme_fc_local_port *localport; 2071 struct lpfc_nvme_lport *lport; 2072 2073 /* Initialize this localport instance. The vport wwn usage ensures 2074 * that NPIV is accounted for. 2075 */ 2076 memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info)); 2077 nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR; 2078 nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn); 2079 nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn); 2080 2081 /* We need to tell the transport layer + 1 because it takes page 2082 * alignment into account. When space for the SGL is allocated we 2083 * allocate + 3, one for cmd, one for rsp and one for this alignment 2084 */ 2085 lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; 2086 2087 /* Advertise how many hw queues we support based on cfg_hdw_queue, 2088 * which will not exceed cpu count. 2089 */ 2090 lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue; 2091 2092 if (!IS_ENABLED(CONFIG_NVME_FC)) 2093 return ret; 2094 2095 /* localport is allocated from the stack, but the registration 2096 * call allocates heap memory as well as the private area. 2097 */ 2098 2099 ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template, 2100 &vport->phba->pcidev->dev, &localport); 2101 if (!ret) { 2102 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC, 2103 "6005 Successfully registered local " 2104 "NVME port num %d, localP x%px, private " 2105 "x%px, sg_seg %d\n", 2106 localport->port_num, localport, 2107 localport->private, 2108 lpfc_nvme_template.max_sgl_segments); 2109 2110 /* Private is our lport size declared in the template. */ 2111 lport = (struct lpfc_nvme_lport *)localport->private; 2112 vport->localport = localport; 2113 lport->vport = vport; 2114 vport->nvmei_support = 1; 2115 2116 atomic_set(&lport->xmt_fcp_noxri, 0); 2117 atomic_set(&lport->xmt_fcp_bad_ndlp, 0); 2118 atomic_set(&lport->xmt_fcp_qdepth, 0); 2119 atomic_set(&lport->xmt_fcp_err, 0); 2120 atomic_set(&lport->xmt_fcp_wqerr, 0); 2121 atomic_set(&lport->xmt_fcp_abort, 0); 2122 atomic_set(&lport->xmt_ls_abort, 0); 2123 atomic_set(&lport->xmt_ls_err, 0); 2124 atomic_set(&lport->cmpl_fcp_xb, 0); 2125 atomic_set(&lport->cmpl_fcp_err, 0); 2126 atomic_set(&lport->cmpl_ls_xb, 0); 2127 atomic_set(&lport->cmpl_ls_err, 0); 2128 2129 atomic_set(&lport->fc4NvmeLsRequests, 0); 2130 atomic_set(&lport->fc4NvmeLsCmpls, 0); 2131 } 2132 2133 return ret; 2134 } 2135 2136 #if (IS_ENABLED(CONFIG_NVME_FC)) 2137 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg. 2138 * 2139 * The driver has to wait for the host nvme transport to callback 2140 * indicating the localport has successfully unregistered all 2141 * resources. Since this is an uninterruptible wait, loop every ten 2142 * seconds and print a message indicating no progress. 2143 * 2144 * An uninterruptible wait is used because of the risk of transport-to- 2145 * driver state mismatch. 2146 */ 2147 static void 2148 lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport, 2149 struct lpfc_nvme_lport *lport, 2150 struct completion *lport_unreg_cmp) 2151 { 2152 u32 wait_tmo; 2153 int ret, i, pending = 0; 2154 struct lpfc_sli_ring *pring; 2155 struct lpfc_hba *phba = vport->phba; 2156 struct lpfc_sli4_hdw_queue *qp; 2157 int abts_scsi, abts_nvme; 2158 2159 /* Host transport has to clean up and confirm requiring an indefinite 2160 * wait. Print a message if a 10 second wait expires and renew the 2161 * wait. This is unexpected. 2162 */ 2163 wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000); 2164 while (true) { 2165 ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo); 2166 if (unlikely(!ret)) { 2167 pending = 0; 2168 abts_scsi = 0; 2169 abts_nvme = 0; 2170 for (i = 0; i < phba->cfg_hdw_queue; i++) { 2171 qp = &phba->sli4_hba.hdwq[i]; 2172 if (!vport || !vport->localport || 2173 !qp || !qp->io_wq) 2174 return; 2175 2176 pring = qp->io_wq->pring; 2177 if (!pring) 2178 continue; 2179 pending += pring->txcmplq_cnt; 2180 abts_scsi += qp->abts_scsi_io_bufs; 2181 abts_nvme += qp->abts_nvme_io_bufs; 2182 } 2183 if (!vport || !vport->localport || 2184 vport->phba->hba_flag & HBA_PCI_ERR) 2185 return; 2186 2187 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 2188 "6176 Lport x%px Localport x%px wait " 2189 "timed out. Pending %d [%d:%d]. " 2190 "Renewing.\n", 2191 lport, vport->localport, pending, 2192 abts_scsi, abts_nvme); 2193 continue; 2194 } 2195 break; 2196 } 2197 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 2198 "6177 Lport x%px Localport x%px Complete Success\n", 2199 lport, vport->localport); 2200 } 2201 #endif 2202 2203 /** 2204 * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport. 2205 * @vport: pointer to a host virtual N_Port data structure 2206 * 2207 * This routine is invoked to destroy all lports bound to the phba. 2208 * The lport memory was allocated by the nvme fc transport and is 2209 * released there. This routine ensures all rports bound to the 2210 * lport have been disconnected. 2211 * 2212 **/ 2213 void 2214 lpfc_nvme_destroy_localport(struct lpfc_vport *vport) 2215 { 2216 #if (IS_ENABLED(CONFIG_NVME_FC)) 2217 struct nvme_fc_local_port *localport; 2218 struct lpfc_nvme_lport *lport; 2219 int ret; 2220 DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp); 2221 2222 if (vport->nvmei_support == 0) 2223 return; 2224 2225 localport = vport->localport; 2226 if (!localport) 2227 return; 2228 lport = (struct lpfc_nvme_lport *)localport->private; 2229 2230 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 2231 "6011 Destroying NVME localport x%px\n", 2232 localport); 2233 2234 /* lport's rport list is clear. Unregister 2235 * lport and release resources. 2236 */ 2237 lport->lport_unreg_cmp = &lport_unreg_cmp; 2238 ret = nvme_fc_unregister_localport(localport); 2239 2240 /* Wait for completion. This either blocks 2241 * indefinitely or succeeds 2242 */ 2243 lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp); 2244 vport->localport = NULL; 2245 2246 /* Regardless of the unregister upcall response, clear 2247 * nvmei_support. All rports are unregistered and the 2248 * driver will clean up. 2249 */ 2250 vport->nvmei_support = 0; 2251 if (ret == 0) { 2252 lpfc_printf_vlog(vport, 2253 KERN_INFO, LOG_NVME_DISC, 2254 "6009 Unregistered lport Success\n"); 2255 } else { 2256 lpfc_printf_vlog(vport, 2257 KERN_INFO, LOG_NVME_DISC, 2258 "6010 Unregistered lport " 2259 "Failed, status x%x\n", 2260 ret); 2261 } 2262 #endif 2263 } 2264 2265 void 2266 lpfc_nvme_update_localport(struct lpfc_vport *vport) 2267 { 2268 #if (IS_ENABLED(CONFIG_NVME_FC)) 2269 struct nvme_fc_local_port *localport; 2270 struct lpfc_nvme_lport *lport; 2271 2272 localport = vport->localport; 2273 if (!localport) { 2274 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 2275 "6710 Update NVME fail. No localport\n"); 2276 return; 2277 } 2278 lport = (struct lpfc_nvme_lport *)localport->private; 2279 if (!lport) { 2280 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 2281 "6171 Update NVME fail. localP x%px, No lport\n", 2282 localport); 2283 return; 2284 } 2285 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 2286 "6012 Update NVME lport x%px did x%x\n", 2287 localport, vport->fc_myDID); 2288 2289 localport->port_id = vport->fc_myDID; 2290 if (localport->port_id == 0) 2291 localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY; 2292 else 2293 localport->port_role = FC_PORT_ROLE_NVME_INITIATOR; 2294 2295 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2296 "6030 bound lport x%px to DID x%06x\n", 2297 lport, localport->port_id); 2298 #endif 2299 } 2300 2301 int 2302 lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2303 { 2304 #if (IS_ENABLED(CONFIG_NVME_FC)) 2305 int ret = 0; 2306 struct nvme_fc_local_port *localport; 2307 struct lpfc_nvme_lport *lport; 2308 struct lpfc_nvme_rport *rport; 2309 struct lpfc_nvme_rport *oldrport; 2310 struct nvme_fc_remote_port *remote_port; 2311 struct nvme_fc_port_info rpinfo; 2312 struct lpfc_nodelist *prev_ndlp = NULL; 2313 struct fc_rport *srport = ndlp->rport; 2314 2315 lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC, 2316 "6006 Register NVME PORT. DID x%06x nlptype x%x\n", 2317 ndlp->nlp_DID, ndlp->nlp_type); 2318 2319 localport = vport->localport; 2320 if (!localport) 2321 return 0; 2322 2323 lport = (struct lpfc_nvme_lport *)localport->private; 2324 2325 /* NVME rports are not preserved across devloss. 2326 * Just register this instance. Note, rpinfo->dev_loss_tmo 2327 * is left 0 to indicate accept transport defaults. The 2328 * driver communicates port role capabilities consistent 2329 * with the PRLI response data. 2330 */ 2331 memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info)); 2332 rpinfo.port_id = ndlp->nlp_DID; 2333 if (ndlp->nlp_type & NLP_NVME_TARGET) 2334 rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET; 2335 if (ndlp->nlp_type & NLP_NVME_INITIATOR) 2336 rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR; 2337 2338 if (ndlp->nlp_type & NLP_NVME_DISCOVERY) 2339 rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY; 2340 2341 rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn); 2342 rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn); 2343 if (srport) 2344 rpinfo.dev_loss_tmo = srport->dev_loss_tmo; 2345 else 2346 rpinfo.dev_loss_tmo = vport->cfg_devloss_tmo; 2347 2348 spin_lock_irq(&ndlp->lock); 2349 oldrport = lpfc_ndlp_get_nrport(ndlp); 2350 if (oldrport) { 2351 prev_ndlp = oldrport->ndlp; 2352 spin_unlock_irq(&ndlp->lock); 2353 } else { 2354 spin_unlock_irq(&ndlp->lock); 2355 if (!lpfc_nlp_get(ndlp)) { 2356 dev_warn(&vport->phba->pcidev->dev, 2357 "Warning - No node ref - exit register\n"); 2358 return 0; 2359 } 2360 } 2361 2362 ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port); 2363 if (!ret) { 2364 /* If the ndlp already has an nrport, this is just 2365 * a resume of the existing rport. Else this is a 2366 * new rport. 2367 */ 2368 /* Guard against an unregister/reregister 2369 * race that leaves the WAIT flag set. 2370 */ 2371 spin_lock_irq(&ndlp->lock); 2372 ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT; 2373 ndlp->fc4_xpt_flags |= NVME_XPT_REGD; 2374 spin_unlock_irq(&ndlp->lock); 2375 rport = remote_port->private; 2376 if (oldrport) { 2377 2378 /* Sever the ndlp<->rport association 2379 * before dropping the ndlp ref from 2380 * register. 2381 */ 2382 spin_lock_irq(&ndlp->lock); 2383 ndlp->nrport = NULL; 2384 ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT; 2385 spin_unlock_irq(&ndlp->lock); 2386 rport->ndlp = NULL; 2387 rport->remoteport = NULL; 2388 2389 /* Reference only removed if previous NDLP is no longer 2390 * active. It might be just a swap and removing the 2391 * reference would cause a premature cleanup. 2392 */ 2393 if (prev_ndlp && prev_ndlp != ndlp) { 2394 if (!prev_ndlp->nrport) 2395 lpfc_nlp_put(prev_ndlp); 2396 } 2397 } 2398 2399 /* Clean bind the rport to the ndlp. */ 2400 rport->remoteport = remote_port; 2401 rport->lport = lport; 2402 rport->ndlp = ndlp; 2403 spin_lock_irq(&ndlp->lock); 2404 ndlp->nrport = rport; 2405 spin_unlock_irq(&ndlp->lock); 2406 lpfc_printf_vlog(vport, KERN_INFO, 2407 LOG_NVME_DISC | LOG_NODE, 2408 "6022 Bind lport x%px to remoteport x%px " 2409 "rport x%px WWNN 0x%llx, " 2410 "Rport WWPN 0x%llx DID " 2411 "x%06x Role x%x, ndlp %p prev_ndlp x%px\n", 2412 lport, remote_port, rport, 2413 rpinfo.node_name, rpinfo.port_name, 2414 rpinfo.port_id, rpinfo.port_role, 2415 ndlp, prev_ndlp); 2416 } else { 2417 lpfc_printf_vlog(vport, KERN_ERR, 2418 LOG_TRACE_EVENT, 2419 "6031 RemotePort Registration failed " 2420 "err: %d, DID x%06x\n", 2421 ret, ndlp->nlp_DID); 2422 } 2423 2424 return ret; 2425 #else 2426 return 0; 2427 #endif 2428 } 2429 2430 /* 2431 * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport 2432 * 2433 * If the ndlp represents an NVME Target, that we are logged into, 2434 * ping the NVME FC Transport layer to initiate a device rescan 2435 * on this remote NPort. 2436 */ 2437 void 2438 lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2439 { 2440 #if (IS_ENABLED(CONFIG_NVME_FC)) 2441 struct lpfc_nvme_rport *nrport; 2442 struct nvme_fc_remote_port *remoteport = NULL; 2443 2444 spin_lock_irq(&ndlp->lock); 2445 nrport = lpfc_ndlp_get_nrport(ndlp); 2446 if (nrport) 2447 remoteport = nrport->remoteport; 2448 spin_unlock_irq(&ndlp->lock); 2449 2450 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2451 "6170 Rescan NPort DID x%06x type x%x " 2452 "state x%x nrport x%px remoteport x%px\n", 2453 ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state, 2454 nrport, remoteport); 2455 2456 if (!nrport || !remoteport) 2457 goto rescan_exit; 2458 2459 /* Only rescan if we are an NVME target in the MAPPED state */ 2460 if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY && 2461 ndlp->nlp_state == NLP_STE_MAPPED_NODE) { 2462 nvme_fc_rescan_remoteport(remoteport); 2463 2464 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 2465 "6172 NVME rescanned DID x%06x " 2466 "port_state x%x\n", 2467 ndlp->nlp_DID, remoteport->port_state); 2468 } 2469 return; 2470 rescan_exit: 2471 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2472 "6169 Skip NVME Rport Rescan, NVME remoteport " 2473 "unregistered\n"); 2474 #endif 2475 } 2476 2477 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport. 2478 * 2479 * There is no notion of Devloss or rport recovery from the current 2480 * nvme_transport perspective. Loss of an rport just means IO cannot 2481 * be sent and recovery is completely up to the initator. 2482 * For now, the driver just unbinds the DID and port_role so that 2483 * no further IO can be issued. Changes are planned for later. 2484 * 2485 * Notes - the ndlp reference count is not decremented here since 2486 * since there is no nvme_transport api for devloss. Node ref count 2487 * is only adjusted in driver unload. 2488 */ 2489 void 2490 lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2491 { 2492 #if (IS_ENABLED(CONFIG_NVME_FC)) 2493 int ret; 2494 struct nvme_fc_local_port *localport; 2495 struct lpfc_nvme_lport *lport; 2496 struct lpfc_nvme_rport *rport; 2497 struct nvme_fc_remote_port *remoteport = NULL; 2498 2499 localport = vport->localport; 2500 2501 /* This is fundamental error. The localport is always 2502 * available until driver unload. Just exit. 2503 */ 2504 if (!localport) 2505 return; 2506 2507 lport = (struct lpfc_nvme_lport *)localport->private; 2508 if (!lport) 2509 goto input_err; 2510 2511 spin_lock_irq(&ndlp->lock); 2512 rport = lpfc_ndlp_get_nrport(ndlp); 2513 if (rport) 2514 remoteport = rport->remoteport; 2515 spin_unlock_irq(&ndlp->lock); 2516 if (!remoteport) 2517 goto input_err; 2518 2519 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2520 "6033 Unreg nvme remoteport x%px, portname x%llx, " 2521 "port_id x%06x, portstate x%x port type x%x " 2522 "refcnt %d\n", 2523 remoteport, remoteport->port_name, 2524 remoteport->port_id, remoteport->port_state, 2525 ndlp->nlp_type, kref_read(&ndlp->kref)); 2526 2527 /* Sanity check ndlp type. Only call for NVME ports. Don't 2528 * clear any rport state until the transport calls back. 2529 */ 2530 2531 if (ndlp->nlp_type & NLP_NVME_TARGET) { 2532 /* No concern about the role change on the nvme remoteport. 2533 * The transport will update it. 2534 */ 2535 spin_lock_irq(&vport->phba->hbalock); 2536 ndlp->fc4_xpt_flags |= NVME_XPT_UNREG_WAIT; 2537 spin_unlock_irq(&vport->phba->hbalock); 2538 2539 /* Don't let the host nvme transport keep sending keep-alives 2540 * on this remoteport. Vport is unloading, no recovery. The 2541 * return values is ignored. The upcall is a courtesy to the 2542 * transport. 2543 */ 2544 if (vport->load_flag & FC_UNLOADING || 2545 unlikely(vport->phba->hba_flag & HBA_PCI_ERR)) 2546 (void)nvme_fc_set_remoteport_devloss(remoteport, 0); 2547 2548 ret = nvme_fc_unregister_remoteport(remoteport); 2549 2550 /* The driver no longer knows if the nrport memory is valid. 2551 * because the controller teardown process has begun and 2552 * is asynchronous. Break the binding in the ndlp. Also 2553 * remove the register ndlp reference to setup node release. 2554 */ 2555 ndlp->nrport = NULL; 2556 lpfc_nlp_put(ndlp); 2557 if (ret != 0) { 2558 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 2559 "6167 NVME unregister failed %d " 2560 "port_state x%x\n", 2561 ret, remoteport->port_state); 2562 } 2563 } 2564 return; 2565 2566 input_err: 2567 #endif 2568 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 2569 "6168 State error: lport x%px, rport x%px FCID x%06x\n", 2570 vport->localport, ndlp->rport, ndlp->nlp_DID); 2571 } 2572 2573 /** 2574 * lpfc_sli4_nvme_pci_offline_aborted - Fast-path process of NVME xri abort 2575 * @phba: pointer to lpfc hba data structure. 2576 * @lpfc_ncmd: The nvme job structure for the request being aborted. 2577 * 2578 * This routine is invoked by the worker thread to process a SLI4 fast-path 2579 * NVME aborted xri. Aborted NVME IO commands are completed to the transport 2580 * here. 2581 **/ 2582 void 2583 lpfc_sli4_nvme_pci_offline_aborted(struct lpfc_hba *phba, 2584 struct lpfc_io_buf *lpfc_ncmd) 2585 { 2586 struct nvmefc_fcp_req *nvme_cmd = NULL; 2587 2588 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2589 "6533 %s nvme_cmd %p tag x%x abort complete and " 2590 "xri released\n", __func__, 2591 lpfc_ncmd->nvmeCmd, 2592 lpfc_ncmd->cur_iocbq.iotag); 2593 2594 /* Aborted NVME commands are required to not complete 2595 * before the abort exchange command fully completes. 2596 * Once completed, it is available via the put list. 2597 */ 2598 if (lpfc_ncmd->nvmeCmd) { 2599 nvme_cmd = lpfc_ncmd->nvmeCmd; 2600 nvme_cmd->transferred_length = 0; 2601 nvme_cmd->rcv_rsplen = 0; 2602 nvme_cmd->status = NVME_SC_INTERNAL; 2603 nvme_cmd->done(nvme_cmd); 2604 lpfc_ncmd->nvmeCmd = NULL; 2605 } 2606 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2607 } 2608 2609 /** 2610 * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort 2611 * @phba: pointer to lpfc hba data structure. 2612 * @axri: pointer to the fcp xri abort wcqe structure. 2613 * @lpfc_ncmd: The nvme job structure for the request being aborted. 2614 * 2615 * This routine is invoked by the worker thread to process a SLI4 fast-path 2616 * NVME aborted xri. Aborted NVME IO commands are completed to the transport 2617 * here. 2618 **/ 2619 void 2620 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, 2621 struct sli4_wcqe_xri_aborted *axri, 2622 struct lpfc_io_buf *lpfc_ncmd) 2623 { 2624 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); 2625 struct nvmefc_fcp_req *nvme_cmd = NULL; 2626 struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp; 2627 2628 2629 if (ndlp) 2630 lpfc_sli4_abts_err_handler(phba, ndlp, axri); 2631 2632 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2633 "6311 nvme_cmd %p xri x%x tag x%x abort complete and " 2634 "xri released\n", 2635 lpfc_ncmd->nvmeCmd, xri, 2636 lpfc_ncmd->cur_iocbq.iotag); 2637 2638 /* Aborted NVME commands are required to not complete 2639 * before the abort exchange command fully completes. 2640 * Once completed, it is available via the put list. 2641 */ 2642 if (lpfc_ncmd->nvmeCmd) { 2643 nvme_cmd = lpfc_ncmd->nvmeCmd; 2644 nvme_cmd->done(nvme_cmd); 2645 lpfc_ncmd->nvmeCmd = NULL; 2646 } 2647 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2648 } 2649 2650 /** 2651 * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete 2652 * @phba: Pointer to HBA context object. 2653 * 2654 * This function flushes all wqes in the nvme rings and frees all resources 2655 * in the txcmplq. This function does not issue abort wqes for the IO 2656 * commands in txcmplq, they will just be returned with 2657 * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI 2658 * slot has been permanently disabled. 2659 **/ 2660 void 2661 lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba) 2662 { 2663 struct lpfc_sli_ring *pring; 2664 u32 i, wait_cnt = 0; 2665 2666 if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq) 2667 return; 2668 2669 /* Cycle through all IO rings and make sure all outstanding 2670 * WQEs have been removed from the txcmplqs. 2671 */ 2672 for (i = 0; i < phba->cfg_hdw_queue; i++) { 2673 if (!phba->sli4_hba.hdwq[i].io_wq) 2674 continue; 2675 pring = phba->sli4_hba.hdwq[i].io_wq->pring; 2676 2677 if (!pring) 2678 continue; 2679 2680 /* Retrieve everything on the txcmplq */ 2681 while (!list_empty(&pring->txcmplq)) { 2682 msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1); 2683 wait_cnt++; 2684 2685 /* The sleep is 10mS. Every ten seconds, 2686 * dump a message. Something is wrong. 2687 */ 2688 if ((wait_cnt % 1000) == 0) { 2689 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, 2690 "6178 NVME IO not empty, " 2691 "cnt %d\n", wait_cnt); 2692 } 2693 } 2694 } 2695 2696 /* Make sure HBA is alive */ 2697 lpfc_issue_hb_tmo(phba); 2698 2699 } 2700 2701 void 2702 lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, 2703 uint32_t stat, uint32_t param) 2704 { 2705 #if (IS_ENABLED(CONFIG_NVME_FC)) 2706 struct lpfc_io_buf *lpfc_ncmd; 2707 struct nvmefc_fcp_req *nCmd; 2708 struct lpfc_wcqe_complete wcqe; 2709 struct lpfc_wcqe_complete *wcqep = &wcqe; 2710 2711 lpfc_ncmd = (struct lpfc_io_buf *)pwqeIn->context1; 2712 if (!lpfc_ncmd) { 2713 lpfc_sli_release_iocbq(phba, pwqeIn); 2714 return; 2715 } 2716 /* For abort iocb just return, IO iocb will do a done call */ 2717 if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) == 2718 CMD_ABORT_XRI_CX) { 2719 lpfc_sli_release_iocbq(phba, pwqeIn); 2720 return; 2721 } 2722 2723 spin_lock(&lpfc_ncmd->buf_lock); 2724 nCmd = lpfc_ncmd->nvmeCmd; 2725 if (!nCmd) { 2726 spin_unlock(&lpfc_ncmd->buf_lock); 2727 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2728 return; 2729 } 2730 spin_unlock(&lpfc_ncmd->buf_lock); 2731 2732 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 2733 "6194 NVME Cancel xri %x\n", 2734 lpfc_ncmd->cur_iocbq.sli4_xritag); 2735 2736 wcqep->word0 = 0; 2737 bf_set(lpfc_wcqe_c_status, wcqep, stat); 2738 wcqep->parameter = param; 2739 wcqep->word3 = 0; /* xb is 0 */ 2740 2741 /* Call release with XB=1 to queue the IO into the abort list. */ 2742 if (phba->sli.sli_flag & LPFC_SLI_ACTIVE) 2743 bf_set(lpfc_wcqe_c_xb, wcqep, 1); 2744 2745 memcpy(&pwqeIn->wcqe_cmpl, wcqep, sizeof(*wcqep)); 2746 (pwqeIn->cmd_cmpl)(phba, pwqeIn, pwqeIn); 2747 #endif 2748 } 2749