1 /******************************************************************* 2 * This file is part of the Emulex Linux Device Driver for * 3 * Fibre Channel Host Bus Adapters. * 4 * Copyright (C) 2017 Broadcom. All Rights Reserved. The term * 5 * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. * 6 * Copyright (C) 2004-2016 Emulex. All rights reserved. * 7 * EMULEX and SLI are trademarks of Emulex. * 8 * www.broadcom.com * 9 * Portions Copyright (C) 2004-2005 Christoph Hellwig * 10 * * 11 * This program is free software; you can redistribute it and/or * 12 * modify it under the terms of version 2 of the GNU General * 13 * Public License as published by the Free Software Foundation. * 14 * This program is distributed in the hope that it will be useful. * 15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 19 * TO BE LEGALLY INVALID. See the GNU General Public License for * 20 * more details, a copy of which can be found in the file COPYING * 21 * included with this package. * 22 ********************************************************************/ 23 #include <linux/pci.h> 24 #include <linux/slab.h> 25 #include <linux/interrupt.h> 26 #include <linux/delay.h> 27 #include <asm/unaligned.h> 28 #include <linux/crc-t10dif.h> 29 #include <net/checksum.h> 30 31 #include <scsi/scsi.h> 32 #include <scsi/scsi_device.h> 33 #include <scsi/scsi_eh.h> 34 #include <scsi/scsi_host.h> 35 #include <scsi/scsi_tcq.h> 36 #include <scsi/scsi_transport_fc.h> 37 #include <scsi/fc/fc_fs.h> 38 39 #include <linux/nvme.h> 40 #include <linux/nvme-fc-driver.h> 41 #include <linux/nvme-fc.h> 42 #include "lpfc_version.h" 43 #include "lpfc_hw4.h" 44 #include "lpfc_hw.h" 45 #include "lpfc_sli.h" 46 #include "lpfc_sli4.h" 47 #include "lpfc_nl.h" 48 #include "lpfc_disc.h" 49 #include "lpfc.h" 50 #include "lpfc_nvme.h" 51 #include "lpfc_scsi.h" 52 #include "lpfc_logmsg.h" 53 #include "lpfc_crtn.h" 54 #include "lpfc_vport.h" 55 #include "lpfc_debugfs.h" 56 57 /* NVME initiator-based functions */ 58 59 static struct lpfc_nvme_buf * 60 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp); 61 62 static void 63 lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_nvme_buf *); 64 65 66 /** 67 * lpfc_nvme_create_queue - 68 * @lpfc_pnvme: Pointer to the driver's nvme instance data 69 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 70 * @handle: An opaque driver handle used in follow-up calls. 71 * 72 * Driver registers this routine to preallocate and initialize any 73 * internal data structures to bind the @qidx to its internal IO queues. 74 * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ. 75 * 76 * Return value : 77 * 0 - Success 78 * -EINVAL - Unsupported input value. 79 * -ENOMEM - Could not alloc necessary memory 80 **/ 81 static int 82 lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, 83 unsigned int qidx, u16 qsize, 84 void **handle) 85 { 86 struct lpfc_nvme_lport *lport; 87 struct lpfc_vport *vport; 88 struct lpfc_nvme_qhandle *qhandle; 89 char *str; 90 91 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 92 vport = lport->vport; 93 qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL); 94 if (qhandle == NULL) 95 return -ENOMEM; 96 97 qhandle->cpu_id = smp_processor_id(); 98 qhandle->qidx = qidx; 99 /* 100 * NVME qidx == 0 is the admin queue, so both admin queue 101 * and first IO queue will use MSI-X vector and associated 102 * EQ/CQ/WQ at index 0. After that they are sequentially assigned. 103 */ 104 if (qidx) { 105 str = "IO "; /* IO queue */ 106 qhandle->index = ((qidx - 1) % 107 vport->phba->cfg_nvme_io_channel); 108 } else { 109 str = "ADM"; /* Admin queue */ 110 qhandle->index = qidx; 111 } 112 113 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 114 "6073 Binding %s HdwQueue %d (cpu %d) to " 115 "io_channel %d qhandle %p\n", str, 116 qidx, qhandle->cpu_id, qhandle->index, qhandle); 117 *handle = (void *)qhandle; 118 return 0; 119 } 120 121 /** 122 * lpfc_nvme_delete_queue - 123 * @lpfc_pnvme: Pointer to the driver's nvme instance data 124 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 125 * @handle: An opaque driver handle from lpfc_nvme_create_queue 126 * 127 * Driver registers this routine to free 128 * any internal data structures to bind the @qidx to its internal 129 * IO queues. 130 * 131 * Return value : 132 * 0 - Success 133 * TODO: What are the failure codes. 134 **/ 135 static void 136 lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport, 137 unsigned int qidx, 138 void *handle) 139 { 140 struct lpfc_nvme_lport *lport; 141 struct lpfc_vport *vport; 142 143 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 144 vport = lport->vport; 145 146 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 147 "6001 ENTER. lpfc_pnvme %p, qidx x%xi qhandle %p\n", 148 lport, qidx, handle); 149 kfree(handle); 150 } 151 152 static void 153 lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport) 154 { 155 struct lpfc_nvme_lport *lport = localport->private; 156 157 /* release any threads waiting for the unreg to complete */ 158 complete(&lport->lport_unreg_done); 159 } 160 161 /* lpfc_nvme_remoteport_delete 162 * 163 * @remoteport: Pointer to an nvme transport remoteport instance. 164 * 165 * This is a template downcall. NVME transport calls this function 166 * when it has completed the unregistration of a previously 167 * registered remoteport. 168 * 169 * Return value : 170 * None 171 */ 172 void 173 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport) 174 { 175 struct lpfc_nvme_rport *rport = remoteport->private; 176 struct lpfc_vport *vport; 177 struct lpfc_nodelist *ndlp; 178 179 ndlp = rport->ndlp; 180 if (!ndlp) 181 goto rport_err; 182 183 vport = ndlp->vport; 184 if (!vport) 185 goto rport_err; 186 187 /* Remove this rport from the lport's list - memory is owned by the 188 * transport. Remove the ndlp reference for the NVME transport before 189 * calling state machine to remove the node. 190 */ 191 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 192 "6146 remoteport delete complete %p\n", 193 remoteport); 194 ndlp->nrport = NULL; 195 lpfc_nlp_put(ndlp); 196 197 rport_err: 198 /* This call has to execute as long as the rport is valid. 199 * Release any threads waiting for the unreg to complete. 200 */ 201 complete(&rport->rport_unreg_done); 202 } 203 204 static void 205 lpfc_nvme_cmpl_gen_req(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 206 struct lpfc_wcqe_complete *wcqe) 207 { 208 struct lpfc_vport *vport = cmdwqe->vport; 209 uint32_t status; 210 struct nvmefc_ls_req *pnvme_lsreq; 211 struct lpfc_dmabuf *buf_ptr; 212 struct lpfc_nodelist *ndlp; 213 214 atomic_inc(&vport->phba->fc4NvmeLsCmpls); 215 216 pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2; 217 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 218 ndlp = (struct lpfc_nodelist *)cmdwqe->context1; 219 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 220 "6047 nvme cmpl Enter " 221 "Data %p DID %x Xri: %x status %x cmd:%p lsreg:%p " 222 "bmp:%p ndlp:%p\n", 223 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 224 cmdwqe->sli4_xritag, status, 225 cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp); 226 227 lpfc_nvmeio_data(phba, "NVME LS CMPL: xri x%x stat x%x parm x%x\n", 228 cmdwqe->sli4_xritag, status, wcqe->parameter); 229 230 if (cmdwqe->context3) { 231 buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3; 232 lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys); 233 kfree(buf_ptr); 234 cmdwqe->context3 = NULL; 235 } 236 if (pnvme_lsreq->done) 237 pnvme_lsreq->done(pnvme_lsreq, status); 238 else 239 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 240 "6046 nvme cmpl without done call back? " 241 "Data %p DID %x Xri: %x status %x\n", 242 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 243 cmdwqe->sli4_xritag, status); 244 if (ndlp) { 245 lpfc_nlp_put(ndlp); 246 cmdwqe->context1 = NULL; 247 } 248 lpfc_sli_release_iocbq(phba, cmdwqe); 249 } 250 251 static int 252 lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp, 253 struct lpfc_dmabuf *inp, 254 struct nvmefc_ls_req *pnvme_lsreq, 255 void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *, 256 struct lpfc_wcqe_complete *), 257 struct lpfc_nodelist *ndlp, uint32_t num_entry, 258 uint32_t tmo, uint8_t retry) 259 { 260 struct lpfc_hba *phba = vport->phba; 261 union lpfc_wqe *wqe; 262 struct lpfc_iocbq *genwqe; 263 struct ulp_bde64 *bpl; 264 struct ulp_bde64 bde; 265 int i, rc, xmit_len, first_len; 266 267 /* Allocate buffer for command WQE */ 268 genwqe = lpfc_sli_get_iocbq(phba); 269 if (genwqe == NULL) 270 return 1; 271 272 wqe = &genwqe->wqe; 273 memset(wqe, 0, sizeof(union lpfc_wqe)); 274 275 genwqe->context3 = (uint8_t *)bmp; 276 genwqe->iocb_flag |= LPFC_IO_NVME_LS; 277 278 /* Save for completion so we can release these resources */ 279 genwqe->context1 = lpfc_nlp_get(ndlp); 280 genwqe->context2 = (uint8_t *)pnvme_lsreq; 281 /* Fill in payload, bp points to frame payload */ 282 283 if (!tmo) 284 /* FC spec states we need 3 * ratov for CT requests */ 285 tmo = (3 * phba->fc_ratov); 286 287 /* For this command calculate the xmit length of the request bde. */ 288 xmit_len = 0; 289 first_len = 0; 290 bpl = (struct ulp_bde64 *)bmp->virt; 291 for (i = 0; i < num_entry; i++) { 292 bde.tus.w = bpl[i].tus.w; 293 if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64) 294 break; 295 xmit_len += bde.tus.f.bdeSize; 296 if (i == 0) 297 first_len = xmit_len; 298 } 299 300 genwqe->rsvd2 = num_entry; 301 genwqe->hba_wqidx = 0; 302 303 /* Words 0 - 2 */ 304 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 305 wqe->generic.bde.tus.f.bdeSize = first_len; 306 wqe->generic.bde.addrLow = bpl[0].addrLow; 307 wqe->generic.bde.addrHigh = bpl[0].addrHigh; 308 309 /* Word 3 */ 310 wqe->gen_req.request_payload_len = first_len; 311 312 /* Word 4 */ 313 314 /* Word 5 */ 315 bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0); 316 bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1); 317 bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1); 318 bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ); 319 bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME); 320 321 /* Word 6 */ 322 bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com, 323 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 324 bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag); 325 326 /* Word 7 */ 327 bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1)); 328 bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3); 329 bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE); 330 bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI); 331 332 /* Word 8 */ 333 wqe->gen_req.wqe_com.abort_tag = genwqe->iotag; 334 335 /* Word 9 */ 336 bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag); 337 338 /* Word 10 */ 339 bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1); 340 bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ); 341 bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1); 342 bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE); 343 bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0); 344 345 /* Word 11 */ 346 bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 347 bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND); 348 349 350 /* Issue GEN REQ WQE for NPORT <did> */ 351 lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS, 352 "6050 Issue GEN REQ WQE to NPORT x%x " 353 "Data: x%x x%x wq:%p lsreq:%p bmp:%p xmit:%d 1st:%d\n", 354 ndlp->nlp_DID, genwqe->iotag, 355 vport->port_state, 356 genwqe, pnvme_lsreq, bmp, xmit_len, first_len); 357 genwqe->wqe_cmpl = cmpl; 358 genwqe->iocb_cmpl = NULL; 359 genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT; 360 genwqe->vport = vport; 361 genwqe->retry = retry; 362 363 lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n", 364 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID); 365 366 rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, genwqe); 367 if (rc) { 368 lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS, 369 "6045 Issue GEN REQ WQE to NPORT x%x " 370 "Data: x%x x%x\n", 371 ndlp->nlp_DID, genwqe->iotag, 372 vport->port_state); 373 lpfc_sli_release_iocbq(phba, genwqe); 374 return 1; 375 } 376 return 0; 377 } 378 379 /** 380 * lpfc_nvme_ls_req - Issue an Link Service request 381 * @lpfc_pnvme: Pointer to the driver's nvme instance data 382 * @lpfc_nvme_lport: Pointer to the driver's local port data 383 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 384 * 385 * Driver registers this routine to handle any link service request 386 * from the nvme_fc transport to a remote nvme-aware port. 387 * 388 * Return value : 389 * 0 - Success 390 * TODO: What are the failure codes. 391 **/ 392 static int 393 lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport, 394 struct nvme_fc_remote_port *pnvme_rport, 395 struct nvmefc_ls_req *pnvme_lsreq) 396 { 397 int ret = 0; 398 struct lpfc_nvme_lport *lport; 399 struct lpfc_vport *vport; 400 struct lpfc_nodelist *ndlp; 401 struct ulp_bde64 *bpl; 402 struct lpfc_dmabuf *bmp; 403 uint16_t ntype, nstate; 404 405 /* there are two dma buf in the request, actually there is one and 406 * the second one is just the start address + cmd size. 407 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped 408 * in a lpfc_dmabuf struct. When freeing we just free the wrapper 409 * because the nvem layer owns the data bufs. 410 * We do not have to break these packets open, we don't care what is in 411 * them. And we do not have to look at the resonse data, we only care 412 * that we got a response. All of the caring is going to happen in the 413 * nvme-fc layer. 414 */ 415 416 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 417 vport = lport->vport; 418 419 if (vport->load_flag & FC_UNLOADING) 420 return -ENODEV; 421 422 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); 423 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 424 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 425 "6051 DID x%06x not an active rport.\n", 426 pnvme_rport->port_id); 427 return -ENODEV; 428 } 429 430 /* The remote node has to be a mapped nvme target or an 431 * unmapped nvme initiator or it's an error. 432 */ 433 ntype = ndlp->nlp_type; 434 nstate = ndlp->nlp_state; 435 if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) || 436 (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) { 437 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 438 "6088 DID x%06x not ready for " 439 "IO. State x%x, Type x%x\n", 440 pnvme_rport->port_id, 441 ndlp->nlp_state, ndlp->nlp_type); 442 return -ENODEV; 443 } 444 bmp = kmalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL); 445 if (!bmp) { 446 447 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 448 "6044 Could not find node for DID %x\n", 449 pnvme_rport->port_id); 450 return 2; 451 } 452 INIT_LIST_HEAD(&bmp->list); 453 bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys)); 454 if (!bmp->virt) { 455 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 456 "6042 Could not find node for DID %x\n", 457 pnvme_rport->port_id); 458 kfree(bmp); 459 return 3; 460 } 461 bpl = (struct ulp_bde64 *)bmp->virt; 462 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma)); 463 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma)); 464 bpl->tus.f.bdeFlags = 0; 465 bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen; 466 bpl->tus.w = le32_to_cpu(bpl->tus.w); 467 bpl++; 468 469 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma)); 470 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma)); 471 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I; 472 bpl->tus.f.bdeSize = pnvme_lsreq->rsplen; 473 bpl->tus.w = le32_to_cpu(bpl->tus.w); 474 475 /* Expand print to include key fields. */ 476 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 477 "6149 ENTER. lport %p, rport %p lsreq%p rqstlen:%d " 478 "rsplen:%d %pad %pad\n", 479 pnvme_lport, pnvme_rport, 480 pnvme_lsreq, pnvme_lsreq->rqstlen, 481 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 482 &pnvme_lsreq->rspdma); 483 484 atomic_inc(&vport->phba->fc4NvmeLsRequests); 485 486 /* Hardcode the wait to 30 seconds. Connections are failing otherwise. 487 * This code allows it all to work. 488 */ 489 ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr, 490 pnvme_lsreq, lpfc_nvme_cmpl_gen_req, 491 ndlp, 2, 30, 0); 492 if (ret != WQE_SUCCESS) { 493 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 494 "6052 EXIT. issue ls wqe failed lport %p, " 495 "rport %p lsreq%p Status %x DID %x\n", 496 pnvme_lport, pnvme_rport, pnvme_lsreq, 497 ret, ndlp->nlp_DID); 498 lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys); 499 kfree(bmp); 500 return ret; 501 } 502 503 /* Stub in routine and return 0 for now. */ 504 return ret; 505 } 506 507 /** 508 * lpfc_nvme_ls_abort - Issue an Link Service request 509 * @lpfc_pnvme: Pointer to the driver's nvme instance data 510 * @lpfc_nvme_lport: Pointer to the driver's local port data 511 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 512 * 513 * Driver registers this routine to handle any link service request 514 * from the nvme_fc transport to a remote nvme-aware port. 515 * 516 * Return value : 517 * 0 - Success 518 * TODO: What are the failure codes. 519 **/ 520 static void 521 lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, 522 struct nvme_fc_remote_port *pnvme_rport, 523 struct nvmefc_ls_req *pnvme_lsreq) 524 { 525 struct lpfc_nvme_lport *lport; 526 struct lpfc_vport *vport; 527 struct lpfc_hba *phba; 528 struct lpfc_nodelist *ndlp; 529 LIST_HEAD(abort_list); 530 struct lpfc_sli_ring *pring; 531 struct lpfc_iocbq *wqe, *next_wqe; 532 533 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 534 vport = lport->vport; 535 phba = vport->phba; 536 537 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); 538 if (!ndlp) { 539 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 540 "6049 Could not find node for DID %x\n", 541 pnvme_rport->port_id); 542 return; 543 } 544 545 /* Expand print to include key fields. */ 546 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 547 "6040 ENTER. lport %p, rport %p lsreq %p rqstlen:%d " 548 "rsplen:%d %pad %pad\n", 549 pnvme_lport, pnvme_rport, 550 pnvme_lsreq, pnvme_lsreq->rqstlen, 551 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 552 &pnvme_lsreq->rspdma); 553 554 /* 555 * Lock the ELS ring txcmplq and build a local list of all ELS IOs 556 * that need an ABTS. The IOs need to stay on the txcmplq so that 557 * the abort operation completes them successfully. 558 */ 559 pring = phba->sli4_hba.nvmels_wq->pring; 560 spin_lock_irq(&phba->hbalock); 561 spin_lock(&pring->ring_lock); 562 list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) { 563 /* Add to abort_list on on NDLP match. */ 564 if (lpfc_check_sli_ndlp(phba, pring, wqe, ndlp)) { 565 wqe->iocb_flag |= LPFC_DRIVER_ABORTED; 566 list_add_tail(&wqe->dlist, &abort_list); 567 } 568 } 569 spin_unlock(&pring->ring_lock); 570 spin_unlock_irq(&phba->hbalock); 571 572 /* Abort the targeted IOs and remove them from the abort list. */ 573 list_for_each_entry_safe(wqe, next_wqe, &abort_list, dlist) { 574 spin_lock_irq(&phba->hbalock); 575 list_del_init(&wqe->dlist); 576 lpfc_sli_issue_abort_iotag(phba, pring, wqe); 577 spin_unlock_irq(&phba->hbalock); 578 } 579 } 580 581 /* Fix up the existing sgls for NVME IO. */ 582 static void 583 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, 584 struct lpfc_nvme_buf *lpfc_ncmd, 585 struct nvmefc_fcp_req *nCmd) 586 { 587 struct sli4_sge *sgl; 588 union lpfc_wqe128 *wqe; 589 uint32_t *wptr, *dptr; 590 591 /* 592 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to 593 * match NVME. NVME sends 96 bytes. Also, use the 594 * nvme commands command and response dma addresses 595 * rather than the virtual memory to ease the restore 596 * operation. 597 */ 598 sgl = lpfc_ncmd->nvme_sgl; 599 sgl->sge_len = cpu_to_le32(nCmd->cmdlen); 600 601 sgl++; 602 603 /* Setup the physical region for the FCP RSP */ 604 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma)); 605 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma)); 606 sgl->word2 = le32_to_cpu(sgl->word2); 607 if (nCmd->sg_cnt) 608 bf_set(lpfc_sli4_sge_last, sgl, 0); 609 else 610 bf_set(lpfc_sli4_sge_last, sgl, 1); 611 sgl->word2 = cpu_to_le32(sgl->word2); 612 sgl->sge_len = cpu_to_le32(nCmd->rsplen); 613 614 /* 615 * Get a local pointer to the built-in wqe and correct 616 * the cmd size to match NVME's 96 bytes and fix 617 * the dma address. 618 */ 619 620 /* 128 byte wqe support here */ 621 wqe = (union lpfc_wqe128 *)&lpfc_ncmd->cur_iocbq.wqe; 622 623 /* Word 0-2 - NVME CMND IU (embedded payload) */ 624 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED; 625 wqe->generic.bde.tus.f.bdeSize = 60; 626 wqe->generic.bde.addrHigh = 0; 627 wqe->generic.bde.addrLow = 64; /* Word 16 */ 628 629 /* Word 3 */ 630 bf_set(payload_offset_len, &wqe->fcp_icmd, 631 (nCmd->rsplen + nCmd->cmdlen)); 632 633 /* Word 10 */ 634 bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1); 635 bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1); 636 637 /* 638 * Embed the payload in the last half of the WQE 639 * WQE words 16-30 get the NVME CMD IU payload 640 * 641 * WQE words 16-19 get payload Words 1-4 642 * WQE words 20-21 get payload Words 6-7 643 * WQE words 22-29 get payload Words 16-23 644 */ 645 wptr = &wqe->words[16]; /* WQE ptr */ 646 dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */ 647 dptr++; /* Skip Word 0 in payload */ 648 649 *wptr++ = *dptr++; /* Word 1 */ 650 *wptr++ = *dptr++; /* Word 2 */ 651 *wptr++ = *dptr++; /* Word 3 */ 652 *wptr++ = *dptr++; /* Word 4 */ 653 dptr++; /* Skip Word 5 in payload */ 654 *wptr++ = *dptr++; /* Word 6 */ 655 *wptr++ = *dptr++; /* Word 7 */ 656 dptr += 8; /* Skip Words 8-15 in payload */ 657 *wptr++ = *dptr++; /* Word 16 */ 658 *wptr++ = *dptr++; /* Word 17 */ 659 *wptr++ = *dptr++; /* Word 18 */ 660 *wptr++ = *dptr++; /* Word 19 */ 661 *wptr++ = *dptr++; /* Word 20 */ 662 *wptr++ = *dptr++; /* Word 21 */ 663 *wptr++ = *dptr++; /* Word 22 */ 664 *wptr = *dptr; /* Word 23 */ 665 } 666 667 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 668 static void 669 lpfc_nvme_ktime(struct lpfc_hba *phba, 670 struct lpfc_nvme_buf *lpfc_ncmd) 671 { 672 uint64_t seg1, seg2, seg3, seg4; 673 uint64_t segsum; 674 675 if (!lpfc_ncmd->ts_last_cmd || 676 !lpfc_ncmd->ts_cmd_start || 677 !lpfc_ncmd->ts_cmd_wqput || 678 !lpfc_ncmd->ts_isr_cmpl || 679 !lpfc_ncmd->ts_data_nvme) 680 return; 681 682 if (lpfc_ncmd->ts_data_nvme < lpfc_ncmd->ts_cmd_start) 683 return; 684 if (lpfc_ncmd->ts_cmd_start < lpfc_ncmd->ts_last_cmd) 685 return; 686 if (lpfc_ncmd->ts_cmd_wqput < lpfc_ncmd->ts_cmd_start) 687 return; 688 if (lpfc_ncmd->ts_isr_cmpl < lpfc_ncmd->ts_cmd_wqput) 689 return; 690 if (lpfc_ncmd->ts_data_nvme < lpfc_ncmd->ts_isr_cmpl) 691 return; 692 /* 693 * Segment 1 - Time from Last FCP command cmpl is handed 694 * off to NVME Layer to start of next command. 695 * Segment 2 - Time from Driver receives a IO cmd start 696 * from NVME Layer to WQ put is done on IO cmd. 697 * Segment 3 - Time from Driver WQ put is done on IO cmd 698 * to MSI-X ISR for IO cmpl. 699 * Segment 4 - Time from MSI-X ISR for IO cmpl to when 700 * cmpl is handled off to the NVME Layer. 701 */ 702 seg1 = lpfc_ncmd->ts_cmd_start - lpfc_ncmd->ts_last_cmd; 703 if (seg1 > 5000000) /* 5 ms - for sequential IOs only */ 704 seg1 = 0; 705 706 /* Calculate times relative to start of IO */ 707 seg2 = (lpfc_ncmd->ts_cmd_wqput - lpfc_ncmd->ts_cmd_start); 708 segsum = seg2; 709 seg3 = lpfc_ncmd->ts_isr_cmpl - lpfc_ncmd->ts_cmd_start; 710 if (segsum > seg3) 711 return; 712 seg3 -= segsum; 713 segsum += seg3; 714 715 seg4 = lpfc_ncmd->ts_data_nvme - lpfc_ncmd->ts_cmd_start; 716 if (segsum > seg4) 717 return; 718 seg4 -= segsum; 719 720 phba->ktime_data_samples++; 721 phba->ktime_seg1_total += seg1; 722 if (seg1 < phba->ktime_seg1_min) 723 phba->ktime_seg1_min = seg1; 724 else if (seg1 > phba->ktime_seg1_max) 725 phba->ktime_seg1_max = seg1; 726 phba->ktime_seg2_total += seg2; 727 if (seg2 < phba->ktime_seg2_min) 728 phba->ktime_seg2_min = seg2; 729 else if (seg2 > phba->ktime_seg2_max) 730 phba->ktime_seg2_max = seg2; 731 phba->ktime_seg3_total += seg3; 732 if (seg3 < phba->ktime_seg3_min) 733 phba->ktime_seg3_min = seg3; 734 else if (seg3 > phba->ktime_seg3_max) 735 phba->ktime_seg3_max = seg3; 736 phba->ktime_seg4_total += seg4; 737 if (seg4 < phba->ktime_seg4_min) 738 phba->ktime_seg4_min = seg4; 739 else if (seg4 > phba->ktime_seg4_max) 740 phba->ktime_seg4_max = seg4; 741 742 lpfc_ncmd->ts_last_cmd = 0; 743 lpfc_ncmd->ts_cmd_start = 0; 744 lpfc_ncmd->ts_cmd_wqput = 0; 745 lpfc_ncmd->ts_isr_cmpl = 0; 746 lpfc_ncmd->ts_data_nvme = 0; 747 } 748 #endif 749 750 /** 751 * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO 752 * @lpfc_pnvme: Pointer to the driver's nvme instance data 753 * @lpfc_nvme_lport: Pointer to the driver's local port data 754 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 755 * 756 * Driver registers this routine as it io request handler. This 757 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 758 * data structure to the rport indicated in @lpfc_nvme_rport. 759 * 760 * Return value : 761 * 0 - Success 762 * TODO: What are the failure codes. 763 **/ 764 static void 765 lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, 766 struct lpfc_wcqe_complete *wcqe) 767 { 768 struct lpfc_nvme_buf *lpfc_ncmd = 769 (struct lpfc_nvme_buf *)pwqeIn->context1; 770 struct lpfc_vport *vport = pwqeIn->vport; 771 struct nvmefc_fcp_req *nCmd; 772 struct nvme_fc_ersp_iu *ep; 773 struct nvme_fc_cmd_iu *cp; 774 struct lpfc_nvme_rport *rport; 775 struct lpfc_nodelist *ndlp; 776 struct lpfc_nvme_fcpreq_priv *freqpriv; 777 unsigned long flags; 778 uint32_t code; 779 uint16_t cid, sqhd, data; 780 uint32_t *ptr; 781 782 /* Sanity check on return of outstanding command */ 783 if (!lpfc_ncmd || !lpfc_ncmd->nvmeCmd || !lpfc_ncmd->nrport) { 784 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 785 "6071 Completion pointers bad on wqe %p.\n", 786 wcqe); 787 return; 788 } 789 atomic_inc(&phba->fc4NvmeIoCmpls); 790 791 nCmd = lpfc_ncmd->nvmeCmd; 792 rport = lpfc_ncmd->nrport; 793 794 lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n", 795 lpfc_ncmd->cur_iocbq.sli4_xritag, 796 bf_get(lpfc_wcqe_c_status, wcqe), wcqe->parameter); 797 /* 798 * Catch race where our node has transitioned, but the 799 * transport is still transitioning. 800 */ 801 ndlp = rport->ndlp; 802 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 803 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 804 "6061 rport %p, DID x%06x node not ready.\n", 805 rport, rport->remoteport->port_id); 806 807 ndlp = lpfc_findnode_did(vport, rport->remoteport->port_id); 808 if (!ndlp) { 809 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 810 "6062 Ignoring NVME cmpl. No ndlp\n"); 811 goto out_err; 812 } 813 } 814 815 code = bf_get(lpfc_wcqe_c_code, wcqe); 816 if (code == CQE_CODE_NVME_ERSP) { 817 /* For this type of CQE, we need to rebuild the rsp */ 818 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr; 819 820 /* 821 * Get Command Id from cmd to plug into response. This 822 * code is not needed in the next NVME Transport drop. 823 */ 824 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr; 825 cid = cp->sqe.common.command_id; 826 827 /* 828 * RSN is in CQE word 2 829 * SQHD is in CQE Word 3 bits 15:0 830 * Cmd Specific info is in CQE Word 1 831 * and in CQE Word 0 bits 15:0 832 */ 833 sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe); 834 835 /* Now lets build the NVME ERSP IU */ 836 ep->iu_len = cpu_to_be16(8); 837 ep->rsn = wcqe->parameter; 838 ep->xfrd_len = cpu_to_be32(nCmd->payload_length); 839 ep->rsvd12 = 0; 840 ptr = (uint32_t *)&ep->cqe.result.u64; 841 *ptr++ = wcqe->total_data_placed; 842 data = bf_get(lpfc_wcqe_c_ersp0, wcqe); 843 *ptr = (uint32_t)data; 844 ep->cqe.sq_head = sqhd; 845 ep->cqe.sq_id = nCmd->sqid; 846 ep->cqe.command_id = cid; 847 ep->cqe.status = 0; 848 849 lpfc_ncmd->status = IOSTAT_SUCCESS; 850 lpfc_ncmd->result = 0; 851 nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN; 852 nCmd->transferred_length = nCmd->payload_length; 853 } else { 854 lpfc_ncmd->status = (bf_get(lpfc_wcqe_c_status, wcqe) & 855 LPFC_IOCB_STATUS_MASK); 856 lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK); 857 858 /* For NVME, the only failure path that results in an 859 * IO error is when the adapter rejects it. All other 860 * conditions are a success case and resolved by the 861 * transport. 862 * IOSTAT_FCP_RSP_ERROR means: 863 * 1. Length of data received doesn't match total 864 * transfer length in WQE 865 * 2. If the RSP payload does NOT match these cases: 866 * a. RSP length 12/24 bytes and all zeros 867 * b. NVME ERSP 868 */ 869 switch (lpfc_ncmd->status) { 870 case IOSTAT_SUCCESS: 871 nCmd->transferred_length = wcqe->total_data_placed; 872 nCmd->rcv_rsplen = 0; 873 nCmd->status = 0; 874 break; 875 case IOSTAT_FCP_RSP_ERROR: 876 nCmd->transferred_length = wcqe->total_data_placed; 877 nCmd->rcv_rsplen = wcqe->parameter; 878 nCmd->status = 0; 879 /* Sanity check */ 880 if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) 881 break; 882 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 883 "6081 NVME Completion Protocol Error: " 884 "xri %x status x%x result x%x " 885 "placed x%x\n", 886 lpfc_ncmd->cur_iocbq.sli4_xritag, 887 lpfc_ncmd->status, lpfc_ncmd->result, 888 wcqe->total_data_placed); 889 break; 890 case IOSTAT_LOCAL_REJECT: 891 /* Let fall through to set command final state. */ 892 if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED) 893 lpfc_printf_vlog(vport, KERN_INFO, 894 LOG_NVME_IOERR, 895 "6032 Delay Aborted cmd %p " 896 "nvme cmd %p, xri x%x, " 897 "xb %d\n", 898 lpfc_ncmd, nCmd, 899 lpfc_ncmd->cur_iocbq.sli4_xritag, 900 bf_get(lpfc_wcqe_c_xb, wcqe)); 901 default: 902 out_err: 903 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 904 "6072 NVME Completion Error: xri %x " 905 "status x%x result x%x placed x%x\n", 906 lpfc_ncmd->cur_iocbq.sli4_xritag, 907 lpfc_ncmd->status, lpfc_ncmd->result, 908 wcqe->total_data_placed); 909 nCmd->transferred_length = 0; 910 nCmd->rcv_rsplen = 0; 911 nCmd->status = NVME_SC_INTERNAL; 912 } 913 } 914 915 /* pick up SLI4 exhange busy condition */ 916 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 917 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; 918 else 919 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 920 921 if (ndlp && NLP_CHK_NODE_ACT(ndlp)) 922 atomic_dec(&ndlp->cmd_pending); 923 924 /* Update stats and complete the IO. There is 925 * no need for dma unprep because the nvme_transport 926 * owns the dma address. 927 */ 928 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 929 if (lpfc_ncmd->ts_cmd_start) { 930 lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp; 931 lpfc_ncmd->ts_data_nvme = ktime_get_ns(); 932 phba->ktime_last_cmd = lpfc_ncmd->ts_data_nvme; 933 lpfc_nvme_ktime(phba, lpfc_ncmd); 934 } 935 if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) { 936 if (lpfc_ncmd->cpu != smp_processor_id()) 937 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 938 "6701 CPU Check cmpl: " 939 "cpu %d expect %d\n", 940 smp_processor_id(), lpfc_ncmd->cpu); 941 if (lpfc_ncmd->cpu < LPFC_CHECK_CPU_CNT) 942 phba->cpucheck_cmpl_io[lpfc_ncmd->cpu]++; 943 } 944 #endif 945 freqpriv = nCmd->private; 946 freqpriv->nvme_buf = NULL; 947 948 /* NVME targets need completion held off until the abort exchange 949 * completes. 950 */ 951 if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) 952 nCmd->done(nCmd); 953 954 spin_lock_irqsave(&phba->hbalock, flags); 955 lpfc_ncmd->nrport = NULL; 956 spin_unlock_irqrestore(&phba->hbalock, flags); 957 958 /* Call release with XB=1 to queue the IO into the abort list. */ 959 lpfc_release_nvme_buf(phba, lpfc_ncmd); 960 } 961 962 963 /** 964 * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO 965 * @lpfc_pnvme: Pointer to the driver's nvme instance data 966 * @lpfc_nvme_lport: Pointer to the driver's local port data 967 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 968 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 969 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 970 * 971 * Driver registers this routine as it io request handler. This 972 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 973 * data structure to the rport indicated in @lpfc_nvme_rport. 974 * 975 * Return value : 976 * 0 - Success 977 * TODO: What are the failure codes. 978 **/ 979 static int 980 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, 981 struct lpfc_nvme_buf *lpfc_ncmd, 982 struct lpfc_nodelist *pnode) 983 { 984 struct lpfc_hba *phba = vport->phba; 985 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 986 struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq); 987 union lpfc_wqe128 *wqe = (union lpfc_wqe128 *)&pwqeq->wqe; 988 uint32_t req_len; 989 990 if (!pnode || !NLP_CHK_NODE_ACT(pnode)) 991 return -EINVAL; 992 993 /* 994 * There are three possibilities here - use scatter-gather segment, use 995 * the single mapping, or neither. 996 */ 997 wqe->fcp_iwrite.initial_xfer_len = 0; 998 if (nCmd->sg_cnt) { 999 if (nCmd->io_dir == NVMEFC_FCP_WRITE) { 1000 /* Word 5 */ 1001 if ((phba->cfg_nvme_enable_fb) && 1002 (pnode->nlp_flag & NLP_FIRSTBURST)) { 1003 req_len = lpfc_ncmd->nvmeCmd->payload_length; 1004 if (req_len < pnode->nvme_fb_size) 1005 wqe->fcp_iwrite.initial_xfer_len = 1006 req_len; 1007 else 1008 wqe->fcp_iwrite.initial_xfer_len = 1009 pnode->nvme_fb_size; 1010 } 1011 1012 /* Word 7 */ 1013 bf_set(wqe_cmnd, &wqe->generic.wqe_com, 1014 CMD_FCP_IWRITE64_WQE); 1015 bf_set(wqe_pu, &wqe->generic.wqe_com, 1016 PARM_READ_CHECK); 1017 1018 /* Word 10 */ 1019 bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0); 1020 bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, 1021 LPFC_WQE_IOD_WRITE); 1022 bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, 1023 LPFC_WQE_LENLOC_WORD4); 1024 if (phba->cfg_nvme_oas) 1025 bf_set(wqe_oas, &wqe->fcp_iwrite.wqe_com, 1); 1026 1027 /* Word 11 */ 1028 bf_set(wqe_cmd_type, &wqe->generic.wqe_com, 1029 NVME_WRITE_CMD); 1030 1031 atomic_inc(&phba->fc4NvmeOutputRequests); 1032 } else { 1033 /* Word 7 */ 1034 bf_set(wqe_cmnd, &wqe->generic.wqe_com, 1035 CMD_FCP_IREAD64_WQE); 1036 bf_set(wqe_pu, &wqe->generic.wqe_com, 1037 PARM_READ_CHECK); 1038 1039 /* Word 10 */ 1040 bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0); 1041 bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, 1042 LPFC_WQE_IOD_READ); 1043 bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, 1044 LPFC_WQE_LENLOC_WORD4); 1045 if (phba->cfg_nvme_oas) 1046 bf_set(wqe_oas, &wqe->fcp_iread.wqe_com, 1); 1047 1048 /* Word 11 */ 1049 bf_set(wqe_cmd_type, &wqe->generic.wqe_com, 1050 NVME_READ_CMD); 1051 1052 atomic_inc(&phba->fc4NvmeInputRequests); 1053 } 1054 } else { 1055 /* Word 4 */ 1056 wqe->fcp_icmd.rsrvd4 = 0; 1057 1058 /* Word 7 */ 1059 bf_set(wqe_cmnd, &wqe->generic.wqe_com, CMD_FCP_ICMND64_WQE); 1060 bf_set(wqe_pu, &wqe->generic.wqe_com, 0); 1061 1062 /* Word 10 */ 1063 bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1); 1064 bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_WRITE); 1065 bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, 1066 LPFC_WQE_LENLOC_NONE); 1067 if (phba->cfg_nvme_oas) 1068 bf_set(wqe_oas, &wqe->fcp_icmd.wqe_com, 1); 1069 1070 /* Word 11 */ 1071 bf_set(wqe_cmd_type, &wqe->generic.wqe_com, NVME_READ_CMD); 1072 1073 atomic_inc(&phba->fc4NvmeControlRequests); 1074 } 1075 /* 1076 * Finish initializing those WQE fields that are independent 1077 * of the nvme_cmnd request_buffer 1078 */ 1079 1080 /* Word 6 */ 1081 bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com, 1082 phba->sli4_hba.rpi_ids[pnode->nlp_rpi]); 1083 bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag); 1084 1085 /* Word 7 */ 1086 /* Preserve Class data in the ndlp. */ 1087 bf_set(wqe_class, &wqe->generic.wqe_com, 1088 (pnode->nlp_fcp_info & 0x0f)); 1089 1090 /* Word 8 */ 1091 wqe->generic.wqe_com.abort_tag = pwqeq->iotag; 1092 1093 /* Word 9 */ 1094 bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag); 1095 1096 /* Word 11 */ 1097 bf_set(wqe_cqid, &wqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 1098 1099 pwqeq->vport = vport; 1100 return 0; 1101 } 1102 1103 1104 /** 1105 * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO 1106 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1107 * @lpfc_nvme_lport: Pointer to the driver's local port data 1108 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1109 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1110 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1111 * 1112 * Driver registers this routine as it io request handler. This 1113 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1114 * data structure to the rport indicated in @lpfc_nvme_rport. 1115 * 1116 * Return value : 1117 * 0 - Success 1118 * TODO: What are the failure codes. 1119 **/ 1120 static int 1121 lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, 1122 struct lpfc_nvme_buf *lpfc_ncmd) 1123 { 1124 struct lpfc_hba *phba = vport->phba; 1125 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1126 union lpfc_wqe128 *wqe = (union lpfc_wqe128 *)&lpfc_ncmd->cur_iocbq.wqe; 1127 struct sli4_sge *sgl = lpfc_ncmd->nvme_sgl; 1128 struct scatterlist *data_sg; 1129 struct sli4_sge *first_data_sgl; 1130 dma_addr_t physaddr; 1131 uint32_t num_bde = 0; 1132 uint32_t dma_len; 1133 uint32_t dma_offset = 0; 1134 int nseg, i; 1135 1136 /* Fix up the command and response DMA stuff. */ 1137 lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd); 1138 1139 /* 1140 * There are three possibilities here - use scatter-gather segment, use 1141 * the single mapping, or neither. 1142 */ 1143 if (nCmd->sg_cnt) { 1144 /* 1145 * Jump over the cmd and rsp SGEs. The fix routine 1146 * has already adjusted for this. 1147 */ 1148 sgl += 2; 1149 1150 first_data_sgl = sgl; 1151 lpfc_ncmd->seg_cnt = nCmd->sg_cnt; 1152 if (lpfc_ncmd->seg_cnt > phba->cfg_nvme_seg_cnt + 1) { 1153 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1154 "6058 Too many sg segments from " 1155 "NVME Transport. Max %d, " 1156 "nvmeIO sg_cnt %d\n", 1157 phba->cfg_nvme_seg_cnt + 1, 1158 lpfc_ncmd->seg_cnt); 1159 lpfc_ncmd->seg_cnt = 0; 1160 return 1; 1161 } 1162 1163 /* 1164 * The driver established a maximum scatter-gather segment count 1165 * during probe that limits the number of sg elements in any 1166 * single nvme command. Just run through the seg_cnt and format 1167 * the sge's. 1168 */ 1169 nseg = nCmd->sg_cnt; 1170 data_sg = nCmd->first_sgl; 1171 for (i = 0; i < nseg; i++) { 1172 if (data_sg == NULL) { 1173 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1174 "6059 dptr err %d, nseg %d\n", 1175 i, nseg); 1176 lpfc_ncmd->seg_cnt = 0; 1177 return 1; 1178 } 1179 physaddr = data_sg->dma_address; 1180 dma_len = data_sg->length; 1181 sgl->addr_lo = cpu_to_le32(putPaddrLow(physaddr)); 1182 sgl->addr_hi = cpu_to_le32(putPaddrHigh(physaddr)); 1183 sgl->word2 = le32_to_cpu(sgl->word2); 1184 if ((num_bde + 1) == nseg) 1185 bf_set(lpfc_sli4_sge_last, sgl, 1); 1186 else 1187 bf_set(lpfc_sli4_sge_last, sgl, 0); 1188 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset); 1189 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA); 1190 sgl->word2 = cpu_to_le32(sgl->word2); 1191 sgl->sge_len = cpu_to_le32(dma_len); 1192 1193 dma_offset += dma_len; 1194 data_sg = sg_next(data_sg); 1195 sgl++; 1196 } 1197 } else { 1198 /* For this clause to be valid, the payload_length 1199 * and sg_cnt must zero. 1200 */ 1201 if (nCmd->payload_length != 0) { 1202 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1203 "6063 NVME DMA Prep Err: sg_cnt %d " 1204 "payload_length x%x\n", 1205 nCmd->sg_cnt, nCmd->payload_length); 1206 return 1; 1207 } 1208 } 1209 1210 /* 1211 * Due to difference in data length between DIF/non-DIF paths, 1212 * we need to set word 4 of WQE here 1213 */ 1214 wqe->fcp_iread.total_xfer_len = nCmd->payload_length; 1215 return 0; 1216 } 1217 1218 /** 1219 * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO 1220 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1221 * @lpfc_nvme_lport: Pointer to the driver's local port data 1222 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1223 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1224 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1225 * 1226 * Driver registers this routine as it io request handler. This 1227 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1228 * data structure to the rport 1229 indicated in @lpfc_nvme_rport. 1230 * 1231 * Return value : 1232 * 0 - Success 1233 * TODO: What are the failure codes. 1234 **/ 1235 static int 1236 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, 1237 struct nvme_fc_remote_port *pnvme_rport, 1238 void *hw_queue_handle, 1239 struct nvmefc_fcp_req *pnvme_fcreq) 1240 { 1241 int ret = 0; 1242 struct lpfc_nvme_lport *lport; 1243 struct lpfc_vport *vport; 1244 struct lpfc_hba *phba; 1245 struct lpfc_nodelist *ndlp; 1246 struct lpfc_nvme_buf *lpfc_ncmd; 1247 struct lpfc_nvme_rport *rport; 1248 struct lpfc_nvme_qhandle *lpfc_queue_info; 1249 struct lpfc_nvme_fcpreq_priv *freqpriv = pnvme_fcreq->private; 1250 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1251 uint64_t start = 0; 1252 #endif 1253 1254 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1255 vport = lport->vport; 1256 phba = vport->phba; 1257 1258 if (vport->load_flag & FC_UNLOADING) { 1259 ret = -ENODEV; 1260 goto out_fail; 1261 } 1262 1263 /* Validate pointers. */ 1264 if (!pnvme_lport || !pnvme_rport || !freqpriv) { 1265 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR | LOG_NODE, 1266 "6117 No Send:IO submit ptrs NULL, lport %p, " 1267 "rport %p fcreq_priv %p\n", 1268 pnvme_lport, pnvme_rport, freqpriv); 1269 ret = -ENODEV; 1270 goto out_fail; 1271 } 1272 1273 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1274 if (phba->ktime_on) 1275 start = ktime_get_ns(); 1276 #endif 1277 rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 1278 lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle; 1279 1280 /* 1281 * Catch race where our node has transitioned, but the 1282 * transport is still transitioning. 1283 */ 1284 ndlp = rport->ndlp; 1285 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 1286 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 1287 "6053 rport %p, ndlp %p, DID x%06x " 1288 "ndlp not ready.\n", 1289 rport, ndlp, pnvme_rport->port_id); 1290 1291 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); 1292 if (!ndlp) { 1293 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 1294 "6066 Missing node for DID %x\n", 1295 pnvme_rport->port_id); 1296 ret = -ENODEV; 1297 goto out_fail; 1298 } 1299 } 1300 1301 /* The remote node has to be a mapped target or it's an error. */ 1302 if ((ndlp->nlp_type & NLP_NVME_TARGET) && 1303 (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) { 1304 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 1305 "6036 rport %p, DID x%06x not ready for " 1306 "IO. State x%x, Type x%x\n", 1307 rport, pnvme_rport->port_id, 1308 ndlp->nlp_state, ndlp->nlp_type); 1309 ret = -ENODEV; 1310 goto out_fail; 1311 1312 } 1313 1314 /* The node is shared with FCP IO, make sure the IO pending count does 1315 * not exceed the programmed depth. 1316 */ 1317 if (atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) { 1318 ret = -EBUSY; 1319 goto out_fail; 1320 } 1321 1322 lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp); 1323 if (lpfc_ncmd == NULL) { 1324 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1325 "6065 driver's buffer pool is empty, " 1326 "IO failed\n"); 1327 ret = -EBUSY; 1328 goto out_fail; 1329 } 1330 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1331 if (start) { 1332 lpfc_ncmd->ts_cmd_start = start; 1333 lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd; 1334 } else { 1335 lpfc_ncmd->ts_cmd_start = 0; 1336 } 1337 #endif 1338 1339 /* 1340 * Store the data needed by the driver to issue, abort, and complete 1341 * an IO. 1342 * Do not let the IO hang out forever. There is no midlayer issuing 1343 * an abort so inform the FW of the maximum IO pending time. 1344 */ 1345 freqpriv->nvme_buf = lpfc_ncmd; 1346 lpfc_ncmd->nvmeCmd = pnvme_fcreq; 1347 lpfc_ncmd->nrport = rport; 1348 lpfc_ncmd->ndlp = ndlp; 1349 lpfc_ncmd->start_time = jiffies; 1350 1351 lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp); 1352 ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd); 1353 if (ret) { 1354 ret = -ENOMEM; 1355 goto out_free_nvme_buf; 1356 } 1357 1358 atomic_inc(&ndlp->cmd_pending); 1359 1360 /* 1361 * Issue the IO on the WQ indicated by index in the hw_queue_handle. 1362 * This identfier was create in our hardware queue create callback 1363 * routine. The driver now is dependent on the IO queue steering from 1364 * the transport. We are trusting the upper NVME layers know which 1365 * index to use and that they have affinitized a CPU to this hardware 1366 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ. 1367 */ 1368 lpfc_ncmd->cur_iocbq.hba_wqidx = lpfc_queue_info->index; 1369 1370 lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n", 1371 lpfc_ncmd->cur_iocbq.sli4_xritag, 1372 lpfc_queue_info->index, ndlp->nlp_DID); 1373 1374 ret = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, &lpfc_ncmd->cur_iocbq); 1375 if (ret) { 1376 atomic_dec(&ndlp->cmd_pending); 1377 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1378 "6113 FCP could not issue WQE err %x " 1379 "sid: x%x did: x%x oxid: x%x\n", 1380 ret, vport->fc_myDID, ndlp->nlp_DID, 1381 lpfc_ncmd->cur_iocbq.sli4_xritag); 1382 goto out_free_nvme_buf; 1383 } 1384 1385 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1386 if (lpfc_ncmd->ts_cmd_start) 1387 lpfc_ncmd->ts_cmd_wqput = ktime_get_ns(); 1388 1389 if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) { 1390 lpfc_ncmd->cpu = smp_processor_id(); 1391 if (lpfc_ncmd->cpu != lpfc_queue_info->index) { 1392 /* Check for admin queue */ 1393 if (lpfc_queue_info->qidx) { 1394 lpfc_printf_vlog(vport, 1395 KERN_ERR, LOG_NVME_IOERR, 1396 "6702 CPU Check cmd: " 1397 "cpu %d wq %d\n", 1398 lpfc_ncmd->cpu, 1399 lpfc_queue_info->index); 1400 } 1401 lpfc_ncmd->cpu = lpfc_queue_info->index; 1402 } 1403 if (lpfc_ncmd->cpu < LPFC_CHECK_CPU_CNT) 1404 phba->cpucheck_xmt_io[lpfc_ncmd->cpu]++; 1405 } 1406 #endif 1407 return 0; 1408 1409 out_free_nvme_buf: 1410 if (lpfc_ncmd->nvmeCmd->sg_cnt) { 1411 if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE) 1412 atomic_dec(&phba->fc4NvmeOutputRequests); 1413 else 1414 atomic_dec(&phba->fc4NvmeInputRequests); 1415 } else 1416 atomic_dec(&phba->fc4NvmeControlRequests); 1417 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1418 out_fail: 1419 return ret; 1420 } 1421 1422 /** 1423 * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request. 1424 * @phba: Pointer to HBA context object 1425 * @cmdiocb: Pointer to command iocb object. 1426 * @rspiocb: Pointer to response iocb object. 1427 * 1428 * This is the callback function for any NVME FCP IO that was aborted. 1429 * 1430 * Return value: 1431 * None 1432 **/ 1433 void 1434 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, 1435 struct lpfc_wcqe_complete *abts_cmpl) 1436 { 1437 lpfc_printf_log(phba, KERN_INFO, LOG_NVME, 1438 "6145 ABORT_XRI_CN completing on rpi x%x " 1439 "original iotag x%x, abort cmd iotag x%x " 1440 "req_tag x%x, status x%x, hwstatus x%x\n", 1441 cmdiocb->iocb.un.acxri.abortContextTag, 1442 cmdiocb->iocb.un.acxri.abortIoTag, 1443 cmdiocb->iotag, 1444 bf_get(lpfc_wcqe_c_request_tag, abts_cmpl), 1445 bf_get(lpfc_wcqe_c_status, abts_cmpl), 1446 bf_get(lpfc_wcqe_c_hw_status, abts_cmpl)); 1447 lpfc_sli_release_iocbq(phba, cmdiocb); 1448 } 1449 1450 /** 1451 * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS 1452 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1453 * @lpfc_nvme_lport: Pointer to the driver's local port data 1454 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1455 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1456 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1457 * 1458 * Driver registers this routine as its nvme request io abort handler. This 1459 * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq 1460 * data structure to the rport indicated in @lpfc_nvme_rport. This routine 1461 * is executed asynchronously - one the target is validated as "MAPPED" and 1462 * ready for IO, the driver issues the abort request and returns. 1463 * 1464 * Return value: 1465 * None 1466 **/ 1467 static void 1468 lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, 1469 struct nvme_fc_remote_port *pnvme_rport, 1470 void *hw_queue_handle, 1471 struct nvmefc_fcp_req *pnvme_fcreq) 1472 { 1473 struct lpfc_nvme_lport *lport; 1474 struct lpfc_vport *vport; 1475 struct lpfc_hba *phba; 1476 struct lpfc_nvme_rport *rport; 1477 struct lpfc_nvme_buf *lpfc_nbuf; 1478 struct lpfc_iocbq *abts_buf; 1479 struct lpfc_iocbq *nvmereq_wqe; 1480 struct lpfc_nvme_fcpreq_priv *freqpriv = pnvme_fcreq->private; 1481 union lpfc_wqe *abts_wqe; 1482 unsigned long flags; 1483 int ret_val; 1484 1485 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1486 rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 1487 vport = lport->vport; 1488 phba = vport->phba; 1489 1490 /* Announce entry to new IO submit field. */ 1491 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1492 "6002 Abort Request to rport DID x%06x " 1493 "for nvme_fc_req %p\n", 1494 pnvme_rport->port_id, 1495 pnvme_fcreq); 1496 1497 /* If the hba is getting reset, this flag is set. It is 1498 * cleared when the reset is complete and rings reestablished. 1499 */ 1500 spin_lock_irqsave(&phba->hbalock, flags); 1501 /* driver queued commands are in process of being flushed */ 1502 if (phba->hba_flag & HBA_NVME_IOQ_FLUSH) { 1503 spin_unlock_irqrestore(&phba->hbalock, flags); 1504 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1505 "6139 Driver in reset cleanup - flushing " 1506 "NVME Req now. hba_flag x%x\n", 1507 phba->hba_flag); 1508 return; 1509 } 1510 1511 lpfc_nbuf = freqpriv->nvme_buf; 1512 if (!lpfc_nbuf) { 1513 spin_unlock_irqrestore(&phba->hbalock, flags); 1514 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1515 "6140 NVME IO req has no matching lpfc nvme " 1516 "io buffer. Skipping abort req.\n"); 1517 return; 1518 } else if (!lpfc_nbuf->nvmeCmd) { 1519 spin_unlock_irqrestore(&phba->hbalock, flags); 1520 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1521 "6141 lpfc NVME IO req has no nvme_fcreq " 1522 "io buffer. Skipping abort req.\n"); 1523 return; 1524 } 1525 nvmereq_wqe = &lpfc_nbuf->cur_iocbq; 1526 1527 /* 1528 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's 1529 * state must match the nvme_fcreq passed by the nvme 1530 * transport. If they don't match, it is likely the driver 1531 * has already completed the NVME IO and the nvme transport 1532 * has not seen it yet. 1533 */ 1534 if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) { 1535 spin_unlock_irqrestore(&phba->hbalock, flags); 1536 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1537 "6143 NVME req mismatch: " 1538 "lpfc_nbuf %p nvmeCmd %p, " 1539 "pnvme_fcreq %p. Skipping Abort xri x%x\n", 1540 lpfc_nbuf, lpfc_nbuf->nvmeCmd, 1541 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1542 return; 1543 } 1544 1545 /* Don't abort IOs no longer on the pending queue. */ 1546 if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) { 1547 spin_unlock_irqrestore(&phba->hbalock, flags); 1548 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1549 "6142 NVME IO req %p not queued - skipping " 1550 "abort req xri x%x\n", 1551 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1552 return; 1553 } 1554 1555 lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n", 1556 nvmereq_wqe->sli4_xritag, 1557 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id); 1558 1559 /* Outstanding abort is in progress */ 1560 if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) { 1561 spin_unlock_irqrestore(&phba->hbalock, flags); 1562 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1563 "6144 Outstanding NVME I/O Abort Request " 1564 "still pending on nvme_fcreq %p, " 1565 "lpfc_ncmd %p xri x%x\n", 1566 pnvme_fcreq, lpfc_nbuf, 1567 nvmereq_wqe->sli4_xritag); 1568 return; 1569 } 1570 1571 abts_buf = __lpfc_sli_get_iocbq(phba); 1572 if (!abts_buf) { 1573 spin_unlock_irqrestore(&phba->hbalock, flags); 1574 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1575 "6136 No available abort wqes. Skipping " 1576 "Abts req for nvme_fcreq %p xri x%x\n", 1577 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1578 return; 1579 } 1580 1581 /* Ready - mark outstanding as aborted by driver. */ 1582 nvmereq_wqe->iocb_flag |= LPFC_DRIVER_ABORTED; 1583 1584 /* Complete prepping the abort wqe and issue to the FW. */ 1585 abts_wqe = &abts_buf->wqe; 1586 1587 /* WQEs are reused. Clear stale data and set key fields to 1588 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag. 1589 */ 1590 memset(abts_wqe, 0, sizeof(union lpfc_wqe)); 1591 bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG); 1592 1593 /* word 7 */ 1594 bf_set(wqe_ct, &abts_wqe->abort_cmd.wqe_com, 0); 1595 bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX); 1596 bf_set(wqe_class, &abts_wqe->abort_cmd.wqe_com, 1597 nvmereq_wqe->iocb.ulpClass); 1598 1599 /* word 8 - tell the FW to abort the IO associated with this 1600 * outstanding exchange ID. 1601 */ 1602 abts_wqe->abort_cmd.wqe_com.abort_tag = nvmereq_wqe->sli4_xritag; 1603 1604 /* word 9 - this is the iotag for the abts_wqe completion. */ 1605 bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com, 1606 abts_buf->iotag); 1607 1608 /* word 10 */ 1609 bf_set(wqe_wqid, &abts_wqe->abort_cmd.wqe_com, nvmereq_wqe->hba_wqidx); 1610 bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1); 1611 bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE); 1612 1613 /* word 11 */ 1614 bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND); 1615 bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1); 1616 bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 1617 1618 /* ABTS WQE must go to the same WQ as the WQE to be aborted */ 1619 abts_buf->iocb_flag |= LPFC_IO_NVME; 1620 abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx; 1621 abts_buf->vport = vport; 1622 abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl; 1623 ret_val = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_buf); 1624 spin_unlock_irqrestore(&phba->hbalock, flags); 1625 if (ret_val) { 1626 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1627 "6137 Failed abts issue_wqe with status x%x " 1628 "for nvme_fcreq %p.\n", 1629 ret_val, pnvme_fcreq); 1630 lpfc_sli_release_iocbq(phba, abts_buf); 1631 return; 1632 } 1633 1634 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1635 "6138 Transport Abort NVME Request Issued for " 1636 "ox_id x%x on reqtag x%x\n", 1637 nvmereq_wqe->sli4_xritag, 1638 abts_buf->iotag); 1639 } 1640 1641 /* Declare and initialization an instance of the FC NVME template. */ 1642 static struct nvme_fc_port_template lpfc_nvme_template = { 1643 /* initiator-based functions */ 1644 .localport_delete = lpfc_nvme_localport_delete, 1645 .remoteport_delete = lpfc_nvme_remoteport_delete, 1646 .create_queue = lpfc_nvme_create_queue, 1647 .delete_queue = lpfc_nvme_delete_queue, 1648 .ls_req = lpfc_nvme_ls_req, 1649 .fcp_io = lpfc_nvme_fcp_io_submit, 1650 .ls_abort = lpfc_nvme_ls_abort, 1651 .fcp_abort = lpfc_nvme_fcp_abort, 1652 1653 .max_hw_queues = 1, 1654 .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 1655 .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 1656 .dma_boundary = 0xFFFFFFFF, 1657 1658 /* Sizes of additional private data for data structures. 1659 * No use for the last two sizes at this time. 1660 */ 1661 .local_priv_sz = sizeof(struct lpfc_nvme_lport), 1662 .remote_priv_sz = sizeof(struct lpfc_nvme_rport), 1663 .lsrqst_priv_sz = 0, 1664 .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv), 1665 }; 1666 1667 /** 1668 * lpfc_sli4_post_nvme_sgl_block - post a block of nvme sgl list to firmware 1669 * @phba: pointer to lpfc hba data structure. 1670 * @nblist: pointer to nvme buffer list. 1671 * @count: number of scsi buffers on the list. 1672 * 1673 * This routine is invoked to post a block of @count scsi sgl pages from a 1674 * SCSI buffer list @nblist to the HBA using non-embedded mailbox command. 1675 * No Lock is held. 1676 * 1677 **/ 1678 static int 1679 lpfc_sli4_post_nvme_sgl_block(struct lpfc_hba *phba, 1680 struct list_head *nblist, 1681 int count) 1682 { 1683 struct lpfc_nvme_buf *lpfc_ncmd; 1684 struct lpfc_mbx_post_uembed_sgl_page1 *sgl; 1685 struct sgl_page_pairs *sgl_pg_pairs; 1686 void *viraddr; 1687 LPFC_MBOXQ_t *mbox; 1688 uint32_t reqlen, alloclen, pg_pairs; 1689 uint32_t mbox_tmo; 1690 uint16_t xritag_start = 0; 1691 int rc = 0; 1692 uint32_t shdr_status, shdr_add_status; 1693 dma_addr_t pdma_phys_bpl1; 1694 union lpfc_sli4_cfg_shdr *shdr; 1695 1696 /* Calculate the requested length of the dma memory */ 1697 reqlen = count * sizeof(struct sgl_page_pairs) + 1698 sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t); 1699 if (reqlen > SLI4_PAGE_SIZE) { 1700 lpfc_printf_log(phba, KERN_WARNING, LOG_INIT, 1701 "6118 Block sgl registration required DMA " 1702 "size (%d) great than a page\n", reqlen); 1703 return -ENOMEM; 1704 } 1705 mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); 1706 if (!mbox) { 1707 lpfc_printf_log(phba, KERN_ERR, LOG_INIT, 1708 "6119 Failed to allocate mbox cmd memory\n"); 1709 return -ENOMEM; 1710 } 1711 1712 /* Allocate DMA memory and set up the non-embedded mailbox command */ 1713 alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE, 1714 LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES, reqlen, 1715 LPFC_SLI4_MBX_NEMBED); 1716 1717 if (alloclen < reqlen) { 1718 lpfc_printf_log(phba, KERN_ERR, LOG_INIT, 1719 "6120 Allocated DMA memory size (%d) is " 1720 "less than the requested DMA memory " 1721 "size (%d)\n", alloclen, reqlen); 1722 lpfc_sli4_mbox_cmd_free(phba, mbox); 1723 return -ENOMEM; 1724 } 1725 1726 /* Get the first SGE entry from the non-embedded DMA memory */ 1727 viraddr = mbox->sge_array->addr[0]; 1728 1729 /* Set up the SGL pages in the non-embedded DMA pages */ 1730 sgl = (struct lpfc_mbx_post_uembed_sgl_page1 *)viraddr; 1731 sgl_pg_pairs = &sgl->sgl_pg_pairs; 1732 1733 pg_pairs = 0; 1734 list_for_each_entry(lpfc_ncmd, nblist, list) { 1735 /* Set up the sge entry */ 1736 sgl_pg_pairs->sgl_pg0_addr_lo = 1737 cpu_to_le32(putPaddrLow(lpfc_ncmd->dma_phys_sgl)); 1738 sgl_pg_pairs->sgl_pg0_addr_hi = 1739 cpu_to_le32(putPaddrHigh(lpfc_ncmd->dma_phys_sgl)); 1740 if (phba->cfg_sg_dma_buf_size > SGL_PAGE_SIZE) 1741 pdma_phys_bpl1 = lpfc_ncmd->dma_phys_sgl + 1742 SGL_PAGE_SIZE; 1743 else 1744 pdma_phys_bpl1 = 0; 1745 sgl_pg_pairs->sgl_pg1_addr_lo = 1746 cpu_to_le32(putPaddrLow(pdma_phys_bpl1)); 1747 sgl_pg_pairs->sgl_pg1_addr_hi = 1748 cpu_to_le32(putPaddrHigh(pdma_phys_bpl1)); 1749 /* Keep the first xritag on the list */ 1750 if (pg_pairs == 0) 1751 xritag_start = lpfc_ncmd->cur_iocbq.sli4_xritag; 1752 sgl_pg_pairs++; 1753 pg_pairs++; 1754 } 1755 bf_set(lpfc_post_sgl_pages_xri, sgl, xritag_start); 1756 bf_set(lpfc_post_sgl_pages_xricnt, sgl, pg_pairs); 1757 /* Perform endian conversion if necessary */ 1758 sgl->word0 = cpu_to_le32(sgl->word0); 1759 1760 if (!phba->sli4_hba.intr_enable) 1761 rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL); 1762 else { 1763 mbox_tmo = lpfc_mbox_tmo_val(phba, mbox); 1764 rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo); 1765 } 1766 shdr = (union lpfc_sli4_cfg_shdr *)&sgl->cfg_shdr; 1767 shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); 1768 shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); 1769 if (rc != MBX_TIMEOUT) 1770 lpfc_sli4_mbox_cmd_free(phba, mbox); 1771 if (shdr_status || shdr_add_status || rc) { 1772 lpfc_printf_log(phba, KERN_ERR, LOG_SLI, 1773 "6125 POST_SGL_BLOCK mailbox command failed " 1774 "status x%x add_status x%x mbx status x%x\n", 1775 shdr_status, shdr_add_status, rc); 1776 rc = -ENXIO; 1777 } 1778 return rc; 1779 } 1780 1781 /** 1782 * lpfc_post_nvme_sgl_list - Post blocks of nvme buffer sgls from a list 1783 * @phba: pointer to lpfc hba data structure. 1784 * @post_nblist: pointer to the nvme buffer list. 1785 * 1786 * This routine walks a list of nvme buffers that was passed in. It attempts 1787 * to construct blocks of nvme buffer sgls which contains contiguous xris and 1788 * uses the non-embedded SGL block post mailbox commands to post to the port. 1789 * For single NVME buffer sgl with non-contiguous xri, if any, it shall use 1790 * embedded SGL post mailbox command for posting. The @post_nblist passed in 1791 * must be local list, thus no lock is needed when manipulate the list. 1792 * 1793 * Returns: 0 = failure, non-zero number of successfully posted buffers. 1794 **/ 1795 static int 1796 lpfc_post_nvme_sgl_list(struct lpfc_hba *phba, 1797 struct list_head *post_nblist, int sb_count) 1798 { 1799 struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; 1800 int status, sgl_size; 1801 int post_cnt = 0, block_cnt = 0, num_posting = 0, num_posted = 0; 1802 dma_addr_t pdma_phys_sgl1; 1803 int last_xritag = NO_XRI; 1804 int cur_xritag; 1805 LIST_HEAD(prep_nblist); 1806 LIST_HEAD(blck_nblist); 1807 LIST_HEAD(nvme_nblist); 1808 1809 /* sanity check */ 1810 if (sb_count <= 0) 1811 return -EINVAL; 1812 1813 sgl_size = phba->cfg_sg_dma_buf_size; 1814 1815 list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, post_nblist, list) { 1816 list_del_init(&lpfc_ncmd->list); 1817 block_cnt++; 1818 if ((last_xritag != NO_XRI) && 1819 (lpfc_ncmd->cur_iocbq.sli4_xritag != last_xritag + 1)) { 1820 /* a hole in xri block, form a sgl posting block */ 1821 list_splice_init(&prep_nblist, &blck_nblist); 1822 post_cnt = block_cnt - 1; 1823 /* prepare list for next posting block */ 1824 list_add_tail(&lpfc_ncmd->list, &prep_nblist); 1825 block_cnt = 1; 1826 } else { 1827 /* prepare list for next posting block */ 1828 list_add_tail(&lpfc_ncmd->list, &prep_nblist); 1829 /* enough sgls for non-embed sgl mbox command */ 1830 if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) { 1831 list_splice_init(&prep_nblist, &blck_nblist); 1832 post_cnt = block_cnt; 1833 block_cnt = 0; 1834 } 1835 } 1836 num_posting++; 1837 last_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag; 1838 1839 /* end of repost sgl list condition for NVME buffers */ 1840 if (num_posting == sb_count) { 1841 if (post_cnt == 0) { 1842 /* last sgl posting block */ 1843 list_splice_init(&prep_nblist, &blck_nblist); 1844 post_cnt = block_cnt; 1845 } else if (block_cnt == 1) { 1846 /* last single sgl with non-contiguous xri */ 1847 if (sgl_size > SGL_PAGE_SIZE) 1848 pdma_phys_sgl1 = 1849 lpfc_ncmd->dma_phys_sgl + 1850 SGL_PAGE_SIZE; 1851 else 1852 pdma_phys_sgl1 = 0; 1853 cur_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag; 1854 status = lpfc_sli4_post_sgl(phba, 1855 lpfc_ncmd->dma_phys_sgl, 1856 pdma_phys_sgl1, cur_xritag); 1857 if (status) { 1858 /* failure, put on abort nvme list */ 1859 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; 1860 } else { 1861 /* success, put on NVME buffer list */ 1862 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 1863 lpfc_ncmd->status = IOSTAT_SUCCESS; 1864 num_posted++; 1865 } 1866 /* success, put on NVME buffer sgl list */ 1867 list_add_tail(&lpfc_ncmd->list, &nvme_nblist); 1868 } 1869 } 1870 1871 /* continue until a nembed page worth of sgls */ 1872 if (post_cnt == 0) 1873 continue; 1874 1875 /* post block of NVME buffer list sgls */ 1876 status = lpfc_sli4_post_nvme_sgl_block(phba, &blck_nblist, 1877 post_cnt); 1878 1879 /* don't reset xirtag due to hole in xri block */ 1880 if (block_cnt == 0) 1881 last_xritag = NO_XRI; 1882 1883 /* reset NVME buffer post count for next round of posting */ 1884 post_cnt = 0; 1885 1886 /* put posted NVME buffer-sgl posted on NVME buffer sgl list */ 1887 while (!list_empty(&blck_nblist)) { 1888 list_remove_head(&blck_nblist, lpfc_ncmd, 1889 struct lpfc_nvme_buf, list); 1890 if (status) { 1891 /* failure, put on abort nvme list */ 1892 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; 1893 } else { 1894 /* success, put on NVME buffer list */ 1895 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 1896 lpfc_ncmd->status = IOSTAT_SUCCESS; 1897 num_posted++; 1898 } 1899 list_add_tail(&lpfc_ncmd->list, &nvme_nblist); 1900 } 1901 } 1902 /* Push NVME buffers with sgl posted to the available list */ 1903 while (!list_empty(&nvme_nblist)) { 1904 list_remove_head(&nvme_nblist, lpfc_ncmd, 1905 struct lpfc_nvme_buf, list); 1906 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1907 } 1908 return num_posted; 1909 } 1910 1911 /** 1912 * lpfc_repost_nvme_sgl_list - Repost all the allocated nvme buffer sgls 1913 * @phba: pointer to lpfc hba data structure. 1914 * 1915 * This routine walks the list of nvme buffers that have been allocated and 1916 * repost them to the port by using SGL block post. This is needed after a 1917 * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine 1918 * is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list 1919 * to the lpfc_nvme_buf_list. If the repost fails, reject all nvme buffers. 1920 * 1921 * Returns: 0 = success, non-zero failure. 1922 **/ 1923 int 1924 lpfc_repost_nvme_sgl_list(struct lpfc_hba *phba) 1925 { 1926 LIST_HEAD(post_nblist); 1927 int num_posted, rc = 0; 1928 1929 /* get all NVME buffers need to repost to a local list */ 1930 spin_lock_irq(&phba->nvme_buf_list_get_lock); 1931 spin_lock(&phba->nvme_buf_list_put_lock); 1932 list_splice_init(&phba->lpfc_nvme_buf_list_get, &post_nblist); 1933 list_splice(&phba->lpfc_nvme_buf_list_put, &post_nblist); 1934 spin_unlock(&phba->nvme_buf_list_put_lock); 1935 spin_unlock_irq(&phba->nvme_buf_list_get_lock); 1936 1937 /* post the list of nvme buffer sgls to port if available */ 1938 if (!list_empty(&post_nblist)) { 1939 num_posted = lpfc_post_nvme_sgl_list(phba, &post_nblist, 1940 phba->sli4_hba.nvme_xri_cnt); 1941 /* failed to post any nvme buffer, return error */ 1942 if (num_posted == 0) 1943 rc = -EIO; 1944 } 1945 return rc; 1946 } 1947 1948 /** 1949 * lpfc_new_nvme_buf - Scsi buffer allocator for HBA with SLI4 IF spec 1950 * @vport: The virtual port for which this call being executed. 1951 * @num_to_allocate: The requested number of buffers to allocate. 1952 * 1953 * This routine allocates nvme buffers for device with SLI-4 interface spec, 1954 * the nvme buffer contains all the necessary information needed to initiate 1955 * a NVME I/O. After allocating up to @num_to_allocate NVME buffers and put 1956 * them on a list, it post them to the port by using SGL block post. 1957 * 1958 * Return codes: 1959 * int - number of nvme buffers that were allocated and posted. 1960 * 0 = failure, less than num_to_alloc is a partial failure. 1961 **/ 1962 static int 1963 lpfc_new_nvme_buf(struct lpfc_vport *vport, int num_to_alloc) 1964 { 1965 struct lpfc_hba *phba = vport->phba; 1966 struct lpfc_nvme_buf *lpfc_ncmd; 1967 struct lpfc_iocbq *pwqeq; 1968 union lpfc_wqe128 *wqe; 1969 struct sli4_sge *sgl; 1970 dma_addr_t pdma_phys_sgl; 1971 uint16_t iotag, lxri = 0; 1972 int bcnt, num_posted, sgl_size; 1973 LIST_HEAD(prep_nblist); 1974 LIST_HEAD(post_nblist); 1975 LIST_HEAD(nvme_nblist); 1976 1977 sgl_size = phba->cfg_sg_dma_buf_size; 1978 1979 for (bcnt = 0; bcnt < num_to_alloc; bcnt++) { 1980 lpfc_ncmd = kzalloc(sizeof(struct lpfc_nvme_buf), GFP_KERNEL); 1981 if (!lpfc_ncmd) 1982 break; 1983 /* 1984 * Get memory from the pci pool to map the virt space to 1985 * pci bus space for an I/O. The DMA buffer includes the 1986 * number of SGE's necessary to support the sg_tablesize. 1987 */ 1988 lpfc_ncmd->data = dma_pool_zalloc(phba->lpfc_sg_dma_buf_pool, 1989 GFP_KERNEL, 1990 &lpfc_ncmd->dma_handle); 1991 if (!lpfc_ncmd->data) { 1992 kfree(lpfc_ncmd); 1993 break; 1994 } 1995 1996 lxri = lpfc_sli4_next_xritag(phba); 1997 if (lxri == NO_XRI) { 1998 dma_pool_free(phba->lpfc_sg_dma_buf_pool, 1999 lpfc_ncmd->data, lpfc_ncmd->dma_handle); 2000 kfree(lpfc_ncmd); 2001 break; 2002 } 2003 pwqeq = &(lpfc_ncmd->cur_iocbq); 2004 wqe = (union lpfc_wqe128 *)&pwqeq->wqe; 2005 2006 /* Allocate iotag for lpfc_ncmd->cur_iocbq. */ 2007 iotag = lpfc_sli_next_iotag(phba, pwqeq); 2008 if (iotag == 0) { 2009 dma_pool_free(phba->lpfc_sg_dma_buf_pool, 2010 lpfc_ncmd->data, lpfc_ncmd->dma_handle); 2011 kfree(lpfc_ncmd); 2012 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2013 "6121 Failed to allocated IOTAG for" 2014 " XRI:0x%x\n", lxri); 2015 lpfc_sli4_free_xri(phba, lxri); 2016 break; 2017 } 2018 pwqeq->sli4_lxritag = lxri; 2019 pwqeq->sli4_xritag = phba->sli4_hba.xri_ids[lxri]; 2020 pwqeq->iocb_flag |= LPFC_IO_NVME; 2021 pwqeq->context1 = lpfc_ncmd; 2022 pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl; 2023 2024 /* Initialize local short-hand pointers. */ 2025 lpfc_ncmd->nvme_sgl = lpfc_ncmd->data; 2026 sgl = lpfc_ncmd->nvme_sgl; 2027 pdma_phys_sgl = lpfc_ncmd->dma_handle; 2028 lpfc_ncmd->dma_phys_sgl = pdma_phys_sgl; 2029 2030 /* Rsp SGE will be filled in when we rcv an IO 2031 * from the NVME Layer to be sent. 2032 * The cmd is going to be embedded so we need a SKIP SGE. 2033 */ 2034 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2035 bf_set(lpfc_sli4_sge_last, sgl, 0); 2036 sgl->word2 = cpu_to_le32(sgl->word2); 2037 /* Fill in word 3 / sgl_len during cmd submission */ 2038 2039 lpfc_ncmd->cur_iocbq.context1 = lpfc_ncmd; 2040 2041 /* Word 7 */ 2042 bf_set(wqe_erp, &wqe->generic.wqe_com, 0); 2043 /* NVME upper layers will time things out, if needed */ 2044 bf_set(wqe_tmo, &wqe->generic.wqe_com, 0); 2045 2046 /* Word 10 */ 2047 bf_set(wqe_ebde_cnt, &wqe->generic.wqe_com, 0); 2048 bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); 2049 2050 /* add the nvme buffer to a post list */ 2051 list_add_tail(&lpfc_ncmd->list, &post_nblist); 2052 spin_lock_irq(&phba->nvme_buf_list_get_lock); 2053 phba->sli4_hba.nvme_xri_cnt++; 2054 spin_unlock_irq(&phba->nvme_buf_list_get_lock); 2055 } 2056 lpfc_printf_log(phba, KERN_INFO, LOG_NVME, 2057 "6114 Allocate %d out of %d requested new NVME " 2058 "buffers\n", bcnt, num_to_alloc); 2059 2060 /* post the list of nvme buffer sgls to port if available */ 2061 if (!list_empty(&post_nblist)) 2062 num_posted = lpfc_post_nvme_sgl_list(phba, 2063 &post_nblist, bcnt); 2064 else 2065 num_posted = 0; 2066 2067 return num_posted; 2068 } 2069 2070 /** 2071 * lpfc_get_nvme_buf - Get a nvme buffer from lpfc_nvme_buf_list of the HBA 2072 * @phba: The HBA for which this call is being executed. 2073 * 2074 * This routine removes a nvme buffer from head of @phba lpfc_nvme_buf_list list 2075 * and returns to caller. 2076 * 2077 * Return codes: 2078 * NULL - Error 2079 * Pointer to lpfc_nvme_buf - Success 2080 **/ 2081 static struct lpfc_nvme_buf * 2082 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) 2083 { 2084 struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; 2085 unsigned long iflag = 0; 2086 int found = 0; 2087 2088 spin_lock_irqsave(&phba->nvme_buf_list_get_lock, iflag); 2089 list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, 2090 &phba->lpfc_nvme_buf_list_get, list) { 2091 list_del_init(&lpfc_ncmd->list); 2092 found = 1; 2093 break; 2094 } 2095 if (!found) { 2096 spin_lock(&phba->nvme_buf_list_put_lock); 2097 list_splice(&phba->lpfc_nvme_buf_list_put, 2098 &phba->lpfc_nvme_buf_list_get); 2099 INIT_LIST_HEAD(&phba->lpfc_nvme_buf_list_put); 2100 spin_unlock(&phba->nvme_buf_list_put_lock); 2101 list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, 2102 &phba->lpfc_nvme_buf_list_get, list) { 2103 list_del_init(&lpfc_ncmd->list); 2104 found = 1; 2105 break; 2106 } 2107 } 2108 spin_unlock_irqrestore(&phba->nvme_buf_list_get_lock, iflag); 2109 if (!found) 2110 return NULL; 2111 return lpfc_ncmd; 2112 } 2113 2114 /** 2115 * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list. 2116 * @phba: The Hba for which this call is being executed. 2117 * @lpfc_ncmd: The nvme buffer which is being released. 2118 * 2119 * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba 2120 * lpfc_nvme_buf_list list. For SLI4 XRI's are tied to the nvme buffer 2121 * and cannot be reused for at least RA_TOV amount of time if it was 2122 * aborted. 2123 **/ 2124 static void 2125 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_nvme_buf *lpfc_ncmd) 2126 { 2127 unsigned long iflag = 0; 2128 2129 lpfc_ncmd->nonsg_phys = 0; 2130 if (lpfc_ncmd->flags & LPFC_SBUF_XBUSY) { 2131 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2132 "6310 XB release deferred for " 2133 "ox_id x%x on reqtag x%x\n", 2134 lpfc_ncmd->cur_iocbq.sli4_xritag, 2135 lpfc_ncmd->cur_iocbq.iotag); 2136 2137 spin_lock_irqsave(&phba->sli4_hba.abts_nvme_buf_list_lock, 2138 iflag); 2139 list_add_tail(&lpfc_ncmd->list, 2140 &phba->sli4_hba.lpfc_abts_nvme_buf_list); 2141 spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, 2142 iflag); 2143 } else { 2144 lpfc_ncmd->nvmeCmd = NULL; 2145 lpfc_ncmd->cur_iocbq.iocb_flag = LPFC_IO_NVME; 2146 spin_lock_irqsave(&phba->nvme_buf_list_put_lock, iflag); 2147 list_add_tail(&lpfc_ncmd->list, &phba->lpfc_nvme_buf_list_put); 2148 spin_unlock_irqrestore(&phba->nvme_buf_list_put_lock, iflag); 2149 } 2150 } 2151 2152 /** 2153 * lpfc_nvme_create_localport - Create/Bind an nvme localport instance. 2154 * @pvport - the lpfc_vport instance requesting a localport. 2155 * 2156 * This routine is invoked to create an nvme localport instance to bind 2157 * to the nvme_fc_transport. It is called once during driver load 2158 * like lpfc_create_shost after all other services are initialized. 2159 * It requires a vport, vpi, and wwns at call time. Other localport 2160 * parameters are modified as the driver's FCID and the Fabric WWN 2161 * are established. 2162 * 2163 * Return codes 2164 * 0 - successful 2165 * -ENOMEM - no heap memory available 2166 * other values - from nvme registration upcall 2167 **/ 2168 int 2169 lpfc_nvme_create_localport(struct lpfc_vport *vport) 2170 { 2171 int ret = 0; 2172 struct lpfc_hba *phba = vport->phba; 2173 struct nvme_fc_port_info nfcp_info; 2174 struct nvme_fc_local_port *localport; 2175 struct lpfc_nvme_lport *lport; 2176 int len; 2177 2178 /* Initialize this localport instance. The vport wwn usage ensures 2179 * that NPIV is accounted for. 2180 */ 2181 memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info)); 2182 nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR; 2183 nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn); 2184 nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn); 2185 2186 /* Limit to LPFC_MAX_NVME_SEG_CNT. 2187 * For now need + 1 to get around NVME transport logic. 2188 */ 2189 if (phba->cfg_sg_seg_cnt > LPFC_MAX_NVME_SEG_CNT) { 2190 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_INIT, 2191 "6300 Reducing sg segment cnt to %d\n", 2192 LPFC_MAX_NVME_SEG_CNT); 2193 phba->cfg_nvme_seg_cnt = LPFC_MAX_NVME_SEG_CNT; 2194 } else { 2195 phba->cfg_nvme_seg_cnt = phba->cfg_sg_seg_cnt; 2196 } 2197 lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; 2198 lpfc_nvme_template.max_hw_queues = phba->cfg_nvme_io_channel; 2199 2200 /* localport is allocated from the stack, but the registration 2201 * call allocates heap memory as well as the private area. 2202 */ 2203 #if (IS_ENABLED(CONFIG_NVME_FC)) 2204 ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template, 2205 &vport->phba->pcidev->dev, &localport); 2206 #else 2207 ret = -ENOMEM; 2208 #endif 2209 if (!ret) { 2210 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC, 2211 "6005 Successfully registered local " 2212 "NVME port num %d, localP %p, private %p, " 2213 "sg_seg %d\n", 2214 localport->port_num, localport, 2215 localport->private, 2216 lpfc_nvme_template.max_sgl_segments); 2217 2218 /* Private is our lport size declared in the template. */ 2219 lport = (struct lpfc_nvme_lport *)localport->private; 2220 vport->localport = localport; 2221 lport->vport = vport; 2222 vport->nvmei_support = 1; 2223 2224 /* Don't post more new bufs if repost already recovered 2225 * the nvme sgls. 2226 */ 2227 if (phba->sli4_hba.nvme_xri_cnt == 0) { 2228 len = lpfc_new_nvme_buf(vport, 2229 phba->sli4_hba.nvme_xri_max); 2230 vport->phba->total_nvme_bufs += len; 2231 } 2232 } 2233 2234 return ret; 2235 } 2236 2237 /** 2238 * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport. 2239 * @pnvme: pointer to lpfc nvme data structure. 2240 * 2241 * This routine is invoked to destroy all lports bound to the phba. 2242 * The lport memory was allocated by the nvme fc transport and is 2243 * released there. This routine ensures all rports bound to the 2244 * lport have been disconnected. 2245 * 2246 **/ 2247 void 2248 lpfc_nvme_destroy_localport(struct lpfc_vport *vport) 2249 { 2250 #if (IS_ENABLED(CONFIG_NVME_FC)) 2251 struct nvme_fc_local_port *localport; 2252 struct lpfc_nvme_lport *lport; 2253 int ret; 2254 2255 if (vport->nvmei_support == 0) 2256 return; 2257 2258 localport = vport->localport; 2259 vport->localport = NULL; 2260 lport = (struct lpfc_nvme_lport *)localport->private; 2261 2262 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 2263 "6011 Destroying NVME localport %p\n", 2264 localport); 2265 2266 /* lport's rport list is clear. Unregister 2267 * lport and release resources. 2268 */ 2269 init_completion(&lport->lport_unreg_done); 2270 ret = nvme_fc_unregister_localport(localport); 2271 wait_for_completion_timeout(&lport->lport_unreg_done, 5); 2272 2273 /* Regardless of the unregister upcall response, clear 2274 * nvmei_support. All rports are unregistered and the 2275 * driver will clean up. 2276 */ 2277 vport->nvmei_support = 0; 2278 if (ret == 0) { 2279 lpfc_printf_vlog(vport, 2280 KERN_INFO, LOG_NVME_DISC, 2281 "6009 Unregistered lport Success\n"); 2282 } else { 2283 lpfc_printf_vlog(vport, 2284 KERN_INFO, LOG_NVME_DISC, 2285 "6010 Unregistered lport " 2286 "Failed, status x%x\n", 2287 ret); 2288 } 2289 #endif 2290 } 2291 2292 void 2293 lpfc_nvme_update_localport(struct lpfc_vport *vport) 2294 { 2295 #if (IS_ENABLED(CONFIG_NVME_FC)) 2296 struct nvme_fc_local_port *localport; 2297 struct lpfc_nvme_lport *lport; 2298 2299 localport = vport->localport; 2300 if (!localport) { 2301 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 2302 "6710 Update NVME fail. No localport\n"); 2303 return; 2304 } 2305 lport = (struct lpfc_nvme_lport *)localport->private; 2306 if (!lport) { 2307 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 2308 "6171 Update NVME fail. localP %p, No lport\n", 2309 localport); 2310 return; 2311 } 2312 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 2313 "6012 Update NVME lport %p did x%x\n", 2314 localport, vport->fc_myDID); 2315 2316 localport->port_id = vport->fc_myDID; 2317 if (localport->port_id == 0) 2318 localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY; 2319 else 2320 localport->port_role = FC_PORT_ROLE_NVME_INITIATOR; 2321 2322 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2323 "6030 bound lport %p to DID x%06x\n", 2324 lport, localport->port_id); 2325 #endif 2326 } 2327 2328 int 2329 lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2330 { 2331 #if (IS_ENABLED(CONFIG_NVME_FC)) 2332 int ret = 0; 2333 struct nvme_fc_local_port *localport; 2334 struct lpfc_nvme_lport *lport; 2335 struct lpfc_nvme_rport *rport; 2336 struct nvme_fc_remote_port *remote_port; 2337 struct nvme_fc_port_info rpinfo; 2338 struct lpfc_nodelist *prev_ndlp; 2339 2340 lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC, 2341 "6006 Register NVME PORT. DID x%06x nlptype x%x\n", 2342 ndlp->nlp_DID, ndlp->nlp_type); 2343 2344 localport = vport->localport; 2345 if (!localport) 2346 return 0; 2347 2348 lport = (struct lpfc_nvme_lport *)localport->private; 2349 2350 /* NVME rports are not preserved across devloss. 2351 * Just register this instance. Note, rpinfo->dev_loss_tmo 2352 * is left 0 to indicate accept transport defaults. The 2353 * driver communicates port role capabilities consistent 2354 * with the PRLI response data. 2355 */ 2356 memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info)); 2357 rpinfo.port_id = ndlp->nlp_DID; 2358 if (ndlp->nlp_type & NLP_NVME_TARGET) 2359 rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET; 2360 if (ndlp->nlp_type & NLP_NVME_INITIATOR) 2361 rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR; 2362 2363 if (ndlp->nlp_type & NLP_NVME_DISCOVERY) 2364 rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY; 2365 2366 rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn); 2367 rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn); 2368 ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port); 2369 if (!ret) { 2370 /* If the ndlp already has an nrport, this is just 2371 * a resume of the existing rport. Else this is a 2372 * new rport. 2373 */ 2374 rport = remote_port->private; 2375 if (ndlp->nrport) { 2376 lpfc_printf_vlog(ndlp->vport, KERN_INFO, 2377 LOG_NVME_DISC, 2378 "6014 Rebinding lport to " 2379 "rport wwpn 0x%llx, " 2380 "Data: x%x x%x x%x x%06x\n", 2381 remote_port->port_name, 2382 remote_port->port_id, 2383 remote_port->port_role, 2384 ndlp->nlp_type, 2385 ndlp->nlp_DID); 2386 prev_ndlp = rport->ndlp; 2387 2388 /* Sever the ndlp<->rport connection before dropping 2389 * the ndlp ref from register. 2390 */ 2391 ndlp->nrport = NULL; 2392 rport->ndlp = NULL; 2393 if (prev_ndlp) 2394 lpfc_nlp_put(ndlp); 2395 } 2396 2397 /* Clean bind the rport to the ndlp. */ 2398 rport->remoteport = remote_port; 2399 rport->lport = lport; 2400 rport->ndlp = lpfc_nlp_get(ndlp); 2401 if (!rport->ndlp) 2402 return -1; 2403 ndlp->nrport = rport; 2404 lpfc_printf_vlog(vport, KERN_INFO, 2405 LOG_NVME_DISC | LOG_NODE, 2406 "6022 Binding new rport to " 2407 "lport %p Rport WWNN 0x%llx, " 2408 "Rport WWPN 0x%llx DID " 2409 "x%06x Role x%x\n", 2410 lport, 2411 rpinfo.node_name, rpinfo.port_name, 2412 rpinfo.port_id, rpinfo.port_role); 2413 } else { 2414 lpfc_printf_vlog(vport, KERN_ERR, 2415 LOG_NVME_DISC | LOG_NODE, 2416 "6031 RemotePort Registration failed " 2417 "err: %d, DID x%06x\n", 2418 ret, ndlp->nlp_DID); 2419 } 2420 2421 return ret; 2422 #else 2423 return 0; 2424 #endif 2425 } 2426 2427 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport. 2428 * 2429 * There is no notion of Devloss or rport recovery from the current 2430 * nvme_transport perspective. Loss of an rport just means IO cannot 2431 * be sent and recovery is completely up to the initator. 2432 * For now, the driver just unbinds the DID and port_role so that 2433 * no further IO can be issued. Changes are planned for later. 2434 * 2435 * Notes - the ndlp reference count is not decremented here since 2436 * since there is no nvme_transport api for devloss. Node ref count 2437 * is only adjusted in driver unload. 2438 */ 2439 void 2440 lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2441 { 2442 #if (IS_ENABLED(CONFIG_NVME_FC)) 2443 int ret; 2444 struct nvme_fc_local_port *localport; 2445 struct lpfc_nvme_lport *lport; 2446 struct lpfc_nvme_rport *rport; 2447 struct nvme_fc_remote_port *remoteport; 2448 2449 localport = vport->localport; 2450 2451 /* This is fundamental error. The localport is always 2452 * available until driver unload. Just exit. 2453 */ 2454 if (!localport) 2455 return; 2456 2457 lport = (struct lpfc_nvme_lport *)localport->private; 2458 if (!lport) 2459 goto input_err; 2460 2461 rport = ndlp->nrport; 2462 if (!rport) 2463 goto input_err; 2464 2465 remoteport = rport->remoteport; 2466 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2467 "6033 Unreg nvme remoteport %p, portname x%llx, " 2468 "port_id x%06x, portstate x%x port type x%x\n", 2469 remoteport, remoteport->port_name, 2470 remoteport->port_id, remoteport->port_state, 2471 ndlp->nlp_type); 2472 2473 /* Sanity check ndlp type. Only call for NVME ports. Don't 2474 * clear any rport state until the transport calls back. 2475 */ 2476 if (ndlp->nlp_type & (NLP_NVME_TARGET | NLP_NVME_INITIATOR)) { 2477 init_completion(&rport->rport_unreg_done); 2478 2479 /* No concern about the role change on the nvme remoteport. 2480 * The transport will update it. 2481 */ 2482 ret = nvme_fc_unregister_remoteport(remoteport); 2483 if (ret != 0) { 2484 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 2485 "6167 NVME unregister failed %d " 2486 "port_state x%x\n", 2487 ret, remoteport->port_state); 2488 } 2489 2490 } 2491 return; 2492 2493 input_err: 2494 #endif 2495 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 2496 "6168 State error: lport %p, rport%p FCID x%06x\n", 2497 vport->localport, ndlp->rport, ndlp->nlp_DID); 2498 } 2499 2500 /** 2501 * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort 2502 * @phba: pointer to lpfc hba data structure. 2503 * @axri: pointer to the fcp xri abort wcqe structure. 2504 * 2505 * This routine is invoked by the worker thread to process a SLI4 fast-path 2506 * NVME aborted xri. Aborted NVME IO commands are completed to the transport 2507 * here. 2508 **/ 2509 void 2510 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, 2511 struct sli4_wcqe_xri_aborted *axri) 2512 { 2513 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); 2514 struct lpfc_nvme_buf *lpfc_ncmd, *next_lpfc_ncmd; 2515 struct nvmefc_fcp_req *nvme_cmd = NULL; 2516 struct lpfc_nodelist *ndlp; 2517 unsigned long iflag = 0; 2518 2519 if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) 2520 return; 2521 spin_lock_irqsave(&phba->hbalock, iflag); 2522 spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); 2523 list_for_each_entry_safe(lpfc_ncmd, next_lpfc_ncmd, 2524 &phba->sli4_hba.lpfc_abts_nvme_buf_list, 2525 list) { 2526 if (lpfc_ncmd->cur_iocbq.sli4_xritag == xri) { 2527 list_del_init(&lpfc_ncmd->list); 2528 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 2529 lpfc_ncmd->status = IOSTAT_SUCCESS; 2530 spin_unlock( 2531 &phba->sli4_hba.abts_nvme_buf_list_lock); 2532 2533 spin_unlock_irqrestore(&phba->hbalock, iflag); 2534 ndlp = lpfc_ncmd->ndlp; 2535 if (ndlp) 2536 lpfc_sli4_abts_err_handler(phba, ndlp, axri); 2537 2538 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2539 "6311 nvme_cmd %p xri x%x tag x%x " 2540 "abort complete and xri released\n", 2541 lpfc_ncmd->nvmeCmd, xri, 2542 lpfc_ncmd->cur_iocbq.iotag); 2543 2544 /* Aborted NVME commands are required to not complete 2545 * before the abort exchange command fully completes. 2546 * Once completed, it is available via the put list. 2547 */ 2548 nvme_cmd = lpfc_ncmd->nvmeCmd; 2549 nvme_cmd->done(nvme_cmd); 2550 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2551 return; 2552 } 2553 } 2554 spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); 2555 spin_unlock_irqrestore(&phba->hbalock, iflag); 2556 2557 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2558 "6312 XRI Aborted xri x%x not found\n", xri); 2559 2560 } 2561