1 /******************************************************************* 2 * This file is part of the Emulex Linux Device Driver for * 3 * Fibre Channel Host Bus Adapters. * 4 * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * 5 * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. * 6 * Copyright (C) 2004-2016 Emulex. All rights reserved. * 7 * EMULEX and SLI are trademarks of Emulex. * 8 * www.broadcom.com * 9 * Portions Copyright (C) 2004-2005 Christoph Hellwig * 10 * * 11 * This program is free software; you can redistribute it and/or * 12 * modify it under the terms of version 2 of the GNU General * 13 * Public License as published by the Free Software Foundation. * 14 * This program is distributed in the hope that it will be useful. * 15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 19 * TO BE LEGALLY INVALID. See the GNU General Public License for * 20 * more details, a copy of which can be found in the file COPYING * 21 * included with this package. * 22 ********************************************************************/ 23 #include <linux/pci.h> 24 #include <linux/slab.h> 25 #include <linux/interrupt.h> 26 #include <linux/delay.h> 27 #include <asm/unaligned.h> 28 #include <linux/crc-t10dif.h> 29 #include <net/checksum.h> 30 31 #include <scsi/scsi.h> 32 #include <scsi/scsi_device.h> 33 #include <scsi/scsi_eh.h> 34 #include <scsi/scsi_host.h> 35 #include <scsi/scsi_tcq.h> 36 #include <scsi/scsi_transport_fc.h> 37 #include <scsi/fc/fc_fs.h> 38 39 #include <linux/nvme.h> 40 #include <linux/nvme-fc-driver.h> 41 #include <linux/nvme-fc.h> 42 #include "lpfc_version.h" 43 #include "lpfc_hw4.h" 44 #include "lpfc_hw.h" 45 #include "lpfc_sli.h" 46 #include "lpfc_sli4.h" 47 #include "lpfc_nl.h" 48 #include "lpfc_disc.h" 49 #include "lpfc.h" 50 #include "lpfc_nvme.h" 51 #include "lpfc_scsi.h" 52 #include "lpfc_logmsg.h" 53 #include "lpfc_crtn.h" 54 #include "lpfc_vport.h" 55 #include "lpfc_debugfs.h" 56 57 /* NVME initiator-based functions */ 58 59 static struct lpfc_nvme_buf * 60 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 61 int expedite); 62 63 static void 64 lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_nvme_buf *); 65 66 static struct nvme_fc_port_template lpfc_nvme_template; 67 68 static union lpfc_wqe128 lpfc_iread_cmd_template; 69 static union lpfc_wqe128 lpfc_iwrite_cmd_template; 70 static union lpfc_wqe128 lpfc_icmnd_cmd_template; 71 72 /* Setup WQE templates for NVME IOs */ 73 void 74 lpfc_nvme_cmd_template(void) 75 { 76 union lpfc_wqe128 *wqe; 77 78 /* IREAD template */ 79 wqe = &lpfc_iread_cmd_template; 80 memset(wqe, 0, sizeof(union lpfc_wqe128)); 81 82 /* Word 0, 1, 2 - BDE is variable */ 83 84 /* Word 3 - cmd_buff_len, payload_offset_len is zero */ 85 86 /* Word 4 - total_xfer_len is variable */ 87 88 /* Word 5 - is zero */ 89 90 /* Word 6 - ctxt_tag, xri_tag is variable */ 91 92 /* Word 7 */ 93 bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE); 94 bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK); 95 bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3); 96 bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI); 97 98 /* Word 8 - abort_tag is variable */ 99 100 /* Word 9 - reqtag is variable */ 101 102 /* Word 10 - dbde, wqes is variable */ 103 bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0); 104 bf_set(wqe_nvme, &wqe->fcp_iread.wqe_com, 1); 105 bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ); 106 bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4); 107 bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0); 108 bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1); 109 110 /* Word 11 - pbde is variable */ 111 bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, NVME_READ_CMD); 112 bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 113 bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1); 114 115 /* Word 12 - is zero */ 116 117 /* Word 13, 14, 15 - PBDE is variable */ 118 119 /* IWRITE template */ 120 wqe = &lpfc_iwrite_cmd_template; 121 memset(wqe, 0, sizeof(union lpfc_wqe128)); 122 123 /* Word 0, 1, 2 - BDE is variable */ 124 125 /* Word 3 - cmd_buff_len, payload_offset_len is zero */ 126 127 /* Word 4 - total_xfer_len is variable */ 128 129 /* Word 5 - initial_xfer_len is variable */ 130 131 /* Word 6 - ctxt_tag, xri_tag is variable */ 132 133 /* Word 7 */ 134 bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE); 135 bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK); 136 bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3); 137 bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI); 138 139 /* Word 8 - abort_tag is variable */ 140 141 /* Word 9 - reqtag is variable */ 142 143 /* Word 10 - dbde, wqes is variable */ 144 bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0); 145 bf_set(wqe_nvme, &wqe->fcp_iwrite.wqe_com, 1); 146 bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE); 147 bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4); 148 bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0); 149 bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1); 150 151 /* Word 11 - pbde is variable */ 152 bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, NVME_WRITE_CMD); 153 bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 154 bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1); 155 156 /* Word 12 - is zero */ 157 158 /* Word 13, 14, 15 - PBDE is variable */ 159 160 /* ICMND template */ 161 wqe = &lpfc_icmnd_cmd_template; 162 memset(wqe, 0, sizeof(union lpfc_wqe128)); 163 164 /* Word 0, 1, 2 - BDE is variable */ 165 166 /* Word 3 - payload_offset_len is variable */ 167 168 /* Word 4, 5 - is zero */ 169 170 /* Word 6 - ctxt_tag, xri_tag is variable */ 171 172 /* Word 7 */ 173 bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE); 174 bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0); 175 bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3); 176 bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI); 177 178 /* Word 8 - abort_tag is variable */ 179 180 /* Word 9 - reqtag is variable */ 181 182 /* Word 10 - dbde, wqes is variable */ 183 bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1); 184 bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1); 185 bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE); 186 bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE); 187 bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0); 188 bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1); 189 190 /* Word 11 */ 191 bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, FCP_COMMAND); 192 bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 193 bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0); 194 195 /* Word 12, 13, 14, 15 - is zero */ 196 } 197 198 /** 199 * lpfc_nvme_create_queue - 200 * @lpfc_pnvme: Pointer to the driver's nvme instance data 201 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 202 * @handle: An opaque driver handle used in follow-up calls. 203 * 204 * Driver registers this routine to preallocate and initialize any 205 * internal data structures to bind the @qidx to its internal IO queues. 206 * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ. 207 * 208 * Return value : 209 * 0 - Success 210 * -EINVAL - Unsupported input value. 211 * -ENOMEM - Could not alloc necessary memory 212 **/ 213 static int 214 lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, 215 unsigned int qidx, u16 qsize, 216 void **handle) 217 { 218 struct lpfc_nvme_lport *lport; 219 struct lpfc_vport *vport; 220 struct lpfc_nvme_qhandle *qhandle; 221 char *str; 222 223 if (!pnvme_lport->private) 224 return -ENOMEM; 225 226 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 227 vport = lport->vport; 228 qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL); 229 if (qhandle == NULL) 230 return -ENOMEM; 231 232 qhandle->cpu_id = smp_processor_id(); 233 qhandle->qidx = qidx; 234 /* 235 * NVME qidx == 0 is the admin queue, so both admin queue 236 * and first IO queue will use MSI-X vector and associated 237 * EQ/CQ/WQ at index 0. After that they are sequentially assigned. 238 */ 239 if (qidx) { 240 str = "IO "; /* IO queue */ 241 qhandle->index = ((qidx - 1) % 242 vport->phba->cfg_nvme_io_channel); 243 } else { 244 str = "ADM"; /* Admin queue */ 245 qhandle->index = qidx; 246 } 247 248 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 249 "6073 Binding %s HdwQueue %d (cpu %d) to " 250 "io_channel %d qhandle %p\n", str, 251 qidx, qhandle->cpu_id, qhandle->index, qhandle); 252 *handle = (void *)qhandle; 253 return 0; 254 } 255 256 /** 257 * lpfc_nvme_delete_queue - 258 * @lpfc_pnvme: Pointer to the driver's nvme instance data 259 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 260 * @handle: An opaque driver handle from lpfc_nvme_create_queue 261 * 262 * Driver registers this routine to free 263 * any internal data structures to bind the @qidx to its internal 264 * IO queues. 265 * 266 * Return value : 267 * 0 - Success 268 * TODO: What are the failure codes. 269 **/ 270 static void 271 lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport, 272 unsigned int qidx, 273 void *handle) 274 { 275 struct lpfc_nvme_lport *lport; 276 struct lpfc_vport *vport; 277 278 if (!pnvme_lport->private) 279 return; 280 281 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 282 vport = lport->vport; 283 284 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 285 "6001 ENTER. lpfc_pnvme %p, qidx x%xi qhandle %p\n", 286 lport, qidx, handle); 287 kfree(handle); 288 } 289 290 static void 291 lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport) 292 { 293 struct lpfc_nvme_lport *lport = localport->private; 294 295 lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME, 296 "6173 localport %p delete complete\n", 297 lport); 298 299 /* release any threads waiting for the unreg to complete */ 300 complete(&lport->lport_unreg_done); 301 } 302 303 /* lpfc_nvme_remoteport_delete 304 * 305 * @remoteport: Pointer to an nvme transport remoteport instance. 306 * 307 * This is a template downcall. NVME transport calls this function 308 * when it has completed the unregistration of a previously 309 * registered remoteport. 310 * 311 * Return value : 312 * None 313 */ 314 void 315 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport) 316 { 317 struct lpfc_nvme_rport *rport = remoteport->private; 318 struct lpfc_vport *vport; 319 struct lpfc_nodelist *ndlp; 320 321 ndlp = rport->ndlp; 322 if (!ndlp) 323 goto rport_err; 324 325 vport = ndlp->vport; 326 if (!vport) 327 goto rport_err; 328 329 /* Remove this rport from the lport's list - memory is owned by the 330 * transport. Remove the ndlp reference for the NVME transport before 331 * calling state machine to remove the node. 332 */ 333 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 334 "6146 remoteport delete of remoteport %p\n", 335 remoteport); 336 spin_lock_irq(&vport->phba->hbalock); 337 ndlp->nrport = NULL; 338 spin_unlock_irq(&vport->phba->hbalock); 339 340 /* Remove original register reference. The host transport 341 * won't reference this rport/remoteport any further. 342 */ 343 lpfc_nlp_put(ndlp); 344 345 rport_err: 346 return; 347 } 348 349 static void 350 lpfc_nvme_cmpl_gen_req(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 351 struct lpfc_wcqe_complete *wcqe) 352 { 353 struct lpfc_vport *vport = cmdwqe->vport; 354 struct lpfc_nvme_lport *lport; 355 uint32_t status; 356 struct nvmefc_ls_req *pnvme_lsreq; 357 struct lpfc_dmabuf *buf_ptr; 358 struct lpfc_nodelist *ndlp; 359 360 atomic_inc(&vport->phba->fc4NvmeLsCmpls); 361 362 pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2; 363 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 364 if (status) { 365 lport = (struct lpfc_nvme_lport *)vport->localport->private; 366 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 367 atomic_inc(&lport->cmpl_ls_xb); 368 atomic_inc(&lport->cmpl_ls_err); 369 } 370 371 ndlp = (struct lpfc_nodelist *)cmdwqe->context1; 372 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 373 "6047 nvme cmpl Enter " 374 "Data %p DID %x Xri: %x status %x reason x%x cmd:%p " 375 "lsreg:%p bmp:%p ndlp:%p\n", 376 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 377 cmdwqe->sli4_xritag, status, 378 (wcqe->parameter & 0xffff), 379 cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp); 380 381 lpfc_nvmeio_data(phba, "NVME LS CMPL: xri x%x stat x%x parm x%x\n", 382 cmdwqe->sli4_xritag, status, wcqe->parameter); 383 384 if (cmdwqe->context3) { 385 buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3; 386 lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys); 387 kfree(buf_ptr); 388 cmdwqe->context3 = NULL; 389 } 390 if (pnvme_lsreq->done) 391 pnvme_lsreq->done(pnvme_lsreq, status); 392 else 393 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 394 "6046 nvme cmpl without done call back? " 395 "Data %p DID %x Xri: %x status %x\n", 396 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 397 cmdwqe->sli4_xritag, status); 398 if (ndlp) { 399 lpfc_nlp_put(ndlp); 400 cmdwqe->context1 = NULL; 401 } 402 lpfc_sli_release_iocbq(phba, cmdwqe); 403 } 404 405 static int 406 lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp, 407 struct lpfc_dmabuf *inp, 408 struct nvmefc_ls_req *pnvme_lsreq, 409 void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *, 410 struct lpfc_wcqe_complete *), 411 struct lpfc_nodelist *ndlp, uint32_t num_entry, 412 uint32_t tmo, uint8_t retry) 413 { 414 struct lpfc_hba *phba = vport->phba; 415 union lpfc_wqe128 *wqe; 416 struct lpfc_iocbq *genwqe; 417 struct ulp_bde64 *bpl; 418 struct ulp_bde64 bde; 419 int i, rc, xmit_len, first_len; 420 421 /* Allocate buffer for command WQE */ 422 genwqe = lpfc_sli_get_iocbq(phba); 423 if (genwqe == NULL) 424 return 1; 425 426 wqe = &genwqe->wqe; 427 memset(wqe, 0, sizeof(union lpfc_wqe)); 428 429 genwqe->context3 = (uint8_t *)bmp; 430 genwqe->iocb_flag |= LPFC_IO_NVME_LS; 431 432 /* Save for completion so we can release these resources */ 433 genwqe->context1 = lpfc_nlp_get(ndlp); 434 genwqe->context2 = (uint8_t *)pnvme_lsreq; 435 /* Fill in payload, bp points to frame payload */ 436 437 if (!tmo) 438 /* FC spec states we need 3 * ratov for CT requests */ 439 tmo = (3 * phba->fc_ratov); 440 441 /* For this command calculate the xmit length of the request bde. */ 442 xmit_len = 0; 443 first_len = 0; 444 bpl = (struct ulp_bde64 *)bmp->virt; 445 for (i = 0; i < num_entry; i++) { 446 bde.tus.w = bpl[i].tus.w; 447 if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64) 448 break; 449 xmit_len += bde.tus.f.bdeSize; 450 if (i == 0) 451 first_len = xmit_len; 452 } 453 454 genwqe->rsvd2 = num_entry; 455 genwqe->hba_wqidx = 0; 456 457 /* Words 0 - 2 */ 458 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 459 wqe->generic.bde.tus.f.bdeSize = first_len; 460 wqe->generic.bde.addrLow = bpl[0].addrLow; 461 wqe->generic.bde.addrHigh = bpl[0].addrHigh; 462 463 /* Word 3 */ 464 wqe->gen_req.request_payload_len = first_len; 465 466 /* Word 4 */ 467 468 /* Word 5 */ 469 bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0); 470 bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1); 471 bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1); 472 bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ); 473 bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME); 474 475 /* Word 6 */ 476 bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com, 477 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 478 bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag); 479 480 /* Word 7 */ 481 bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1)); 482 bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3); 483 bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE); 484 bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI); 485 486 /* Word 8 */ 487 wqe->gen_req.wqe_com.abort_tag = genwqe->iotag; 488 489 /* Word 9 */ 490 bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag); 491 492 /* Word 10 */ 493 bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1); 494 bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ); 495 bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1); 496 bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE); 497 bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0); 498 499 /* Word 11 */ 500 bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 501 bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND); 502 503 504 /* Issue GEN REQ WQE for NPORT <did> */ 505 lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS, 506 "6050 Issue GEN REQ WQE to NPORT x%x " 507 "Data: x%x x%x wq:%p lsreq:%p bmp:%p xmit:%d 1st:%d\n", 508 ndlp->nlp_DID, genwqe->iotag, 509 vport->port_state, 510 genwqe, pnvme_lsreq, bmp, xmit_len, first_len); 511 genwqe->wqe_cmpl = cmpl; 512 genwqe->iocb_cmpl = NULL; 513 genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT; 514 genwqe->vport = vport; 515 genwqe->retry = retry; 516 517 lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n", 518 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID); 519 520 rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, genwqe); 521 if (rc) { 522 lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS, 523 "6045 Issue GEN REQ WQE to NPORT x%x " 524 "Data: x%x x%x\n", 525 ndlp->nlp_DID, genwqe->iotag, 526 vport->port_state); 527 lpfc_sli_release_iocbq(phba, genwqe); 528 return 1; 529 } 530 return 0; 531 } 532 533 /** 534 * lpfc_nvme_ls_req - Issue an Link Service request 535 * @lpfc_pnvme: Pointer to the driver's nvme instance data 536 * @lpfc_nvme_lport: Pointer to the driver's local port data 537 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 538 * 539 * Driver registers this routine to handle any link service request 540 * from the nvme_fc transport to a remote nvme-aware port. 541 * 542 * Return value : 543 * 0 - Success 544 * TODO: What are the failure codes. 545 **/ 546 static int 547 lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport, 548 struct nvme_fc_remote_port *pnvme_rport, 549 struct nvmefc_ls_req *pnvme_lsreq) 550 { 551 int ret = 0; 552 struct lpfc_nvme_lport *lport; 553 struct lpfc_nvme_rport *rport; 554 struct lpfc_vport *vport; 555 struct lpfc_nodelist *ndlp; 556 struct ulp_bde64 *bpl; 557 struct lpfc_dmabuf *bmp; 558 uint16_t ntype, nstate; 559 560 /* there are two dma buf in the request, actually there is one and 561 * the second one is just the start address + cmd size. 562 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped 563 * in a lpfc_dmabuf struct. When freeing we just free the wrapper 564 * because the nvem layer owns the data bufs. 565 * We do not have to break these packets open, we don't care what is in 566 * them. And we do not have to look at the resonse data, we only care 567 * that we got a response. All of the caring is going to happen in the 568 * nvme-fc layer. 569 */ 570 571 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 572 rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 573 vport = lport->vport; 574 575 if (vport->load_flag & FC_UNLOADING) 576 return -ENODEV; 577 578 /* Need the ndlp. It is stored in the driver's rport. */ 579 ndlp = rport->ndlp; 580 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 581 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 582 "6051 Remoteport %p, rport has invalid ndlp. " 583 "Failing LS Req\n", pnvme_rport); 584 return -ENODEV; 585 } 586 587 /* The remote node has to be a mapped nvme target or an 588 * unmapped nvme initiator or it's an error. 589 */ 590 ntype = ndlp->nlp_type; 591 nstate = ndlp->nlp_state; 592 if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) || 593 (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) { 594 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 595 "6088 DID x%06x not ready for " 596 "IO. State x%x, Type x%x\n", 597 pnvme_rport->port_id, 598 ndlp->nlp_state, ndlp->nlp_type); 599 return -ENODEV; 600 } 601 bmp = kmalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL); 602 if (!bmp) { 603 604 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 605 "6044 Could not find node for DID %x\n", 606 pnvme_rport->port_id); 607 return 2; 608 } 609 INIT_LIST_HEAD(&bmp->list); 610 bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys)); 611 if (!bmp->virt) { 612 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 613 "6042 Could not find node for DID %x\n", 614 pnvme_rport->port_id); 615 kfree(bmp); 616 return 3; 617 } 618 bpl = (struct ulp_bde64 *)bmp->virt; 619 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma)); 620 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma)); 621 bpl->tus.f.bdeFlags = 0; 622 bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen; 623 bpl->tus.w = le32_to_cpu(bpl->tus.w); 624 bpl++; 625 626 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma)); 627 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma)); 628 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I; 629 bpl->tus.f.bdeSize = pnvme_lsreq->rsplen; 630 bpl->tus.w = le32_to_cpu(bpl->tus.w); 631 632 /* Expand print to include key fields. */ 633 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 634 "6149 Issue LS Req to DID 0x%06x lport %p, rport %p " 635 "lsreq%p rqstlen:%d rsplen:%d %pad %pad\n", 636 ndlp->nlp_DID, 637 pnvme_lport, pnvme_rport, 638 pnvme_lsreq, pnvme_lsreq->rqstlen, 639 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 640 &pnvme_lsreq->rspdma); 641 642 atomic_inc(&vport->phba->fc4NvmeLsRequests); 643 644 /* Hardcode the wait to 30 seconds. Connections are failing otherwise. 645 * This code allows it all to work. 646 */ 647 ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr, 648 pnvme_lsreq, lpfc_nvme_cmpl_gen_req, 649 ndlp, 2, 30, 0); 650 if (ret != WQE_SUCCESS) { 651 atomic_inc(&lport->xmt_ls_err); 652 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 653 "6052 EXIT. issue ls wqe failed lport %p, " 654 "rport %p lsreq%p Status %x DID %x\n", 655 pnvme_lport, pnvme_rport, pnvme_lsreq, 656 ret, ndlp->nlp_DID); 657 lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys); 658 kfree(bmp); 659 return ret; 660 } 661 662 /* Stub in routine and return 0 for now. */ 663 return ret; 664 } 665 666 /** 667 * lpfc_nvme_ls_abort - Issue an Link Service request 668 * @lpfc_pnvme: Pointer to the driver's nvme instance data 669 * @lpfc_nvme_lport: Pointer to the driver's local port data 670 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 671 * 672 * Driver registers this routine to handle any link service request 673 * from the nvme_fc transport to a remote nvme-aware port. 674 * 675 * Return value : 676 * 0 - Success 677 * TODO: What are the failure codes. 678 **/ 679 static void 680 lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, 681 struct nvme_fc_remote_port *pnvme_rport, 682 struct nvmefc_ls_req *pnvme_lsreq) 683 { 684 struct lpfc_nvme_lport *lport; 685 struct lpfc_vport *vport; 686 struct lpfc_hba *phba; 687 struct lpfc_nodelist *ndlp; 688 LIST_HEAD(abort_list); 689 struct lpfc_sli_ring *pring; 690 struct lpfc_iocbq *wqe, *next_wqe; 691 692 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 693 vport = lport->vport; 694 phba = vport->phba; 695 696 if (vport->load_flag & FC_UNLOADING) 697 return; 698 699 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); 700 if (!ndlp) { 701 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 702 "6049 Could not find node for DID %x\n", 703 pnvme_rport->port_id); 704 return; 705 } 706 707 /* Expand print to include key fields. */ 708 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 709 "6040 ENTER. lport %p, rport %p lsreq %p rqstlen:%d " 710 "rsplen:%d %pad %pad\n", 711 pnvme_lport, pnvme_rport, 712 pnvme_lsreq, pnvme_lsreq->rqstlen, 713 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 714 &pnvme_lsreq->rspdma); 715 716 /* 717 * Lock the ELS ring txcmplq and build a local list of all ELS IOs 718 * that need an ABTS. The IOs need to stay on the txcmplq so that 719 * the abort operation completes them successfully. 720 */ 721 pring = phba->sli4_hba.nvmels_wq->pring; 722 spin_lock_irq(&phba->hbalock); 723 spin_lock(&pring->ring_lock); 724 list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) { 725 /* Add to abort_list on on NDLP match. */ 726 if (lpfc_check_sli_ndlp(phba, pring, wqe, ndlp)) { 727 wqe->iocb_flag |= LPFC_DRIVER_ABORTED; 728 list_add_tail(&wqe->dlist, &abort_list); 729 } 730 } 731 spin_unlock(&pring->ring_lock); 732 spin_unlock_irq(&phba->hbalock); 733 734 /* Abort the targeted IOs and remove them from the abort list. */ 735 list_for_each_entry_safe(wqe, next_wqe, &abort_list, dlist) { 736 atomic_inc(&lport->xmt_ls_abort); 737 spin_lock_irq(&phba->hbalock); 738 list_del_init(&wqe->dlist); 739 lpfc_sli_issue_abort_iotag(phba, pring, wqe); 740 spin_unlock_irq(&phba->hbalock); 741 } 742 } 743 744 /* Fix up the existing sgls for NVME IO. */ 745 static inline void 746 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, 747 struct lpfc_nvme_buf *lpfc_ncmd, 748 struct nvmefc_fcp_req *nCmd) 749 { 750 struct lpfc_hba *phba = vport->phba; 751 struct sli4_sge *sgl; 752 union lpfc_wqe128 *wqe; 753 uint32_t *wptr, *dptr; 754 755 /* 756 * Get a local pointer to the built-in wqe and correct 757 * the cmd size to match NVME's 96 bytes and fix 758 * the dma address. 759 */ 760 761 wqe = &lpfc_ncmd->cur_iocbq.wqe; 762 763 /* 764 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to 765 * match NVME. NVME sends 96 bytes. Also, use the 766 * nvme commands command and response dma addresses 767 * rather than the virtual memory to ease the restore 768 * operation. 769 */ 770 sgl = lpfc_ncmd->nvme_sgl; 771 sgl->sge_len = cpu_to_le32(nCmd->cmdlen); 772 if (phba->cfg_nvme_embed_cmd) { 773 sgl->addr_hi = 0; 774 sgl->addr_lo = 0; 775 776 /* Word 0-2 - NVME CMND IU (embedded payload) */ 777 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED; 778 wqe->generic.bde.tus.f.bdeSize = 56; 779 wqe->generic.bde.addrHigh = 0; 780 wqe->generic.bde.addrLow = 64; /* Word 16 */ 781 782 /* Word 10 - dbde is 0, wqes is 1 in template */ 783 784 /* 785 * Embed the payload in the last half of the WQE 786 * WQE words 16-30 get the NVME CMD IU payload 787 * 788 * WQE words 16-19 get payload Words 1-4 789 * WQE words 20-21 get payload Words 6-7 790 * WQE words 22-29 get payload Words 16-23 791 */ 792 wptr = &wqe->words[16]; /* WQE ptr */ 793 dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */ 794 dptr++; /* Skip Word 0 in payload */ 795 796 *wptr++ = *dptr++; /* Word 1 */ 797 *wptr++ = *dptr++; /* Word 2 */ 798 *wptr++ = *dptr++; /* Word 3 */ 799 *wptr++ = *dptr++; /* Word 4 */ 800 dptr++; /* Skip Word 5 in payload */ 801 *wptr++ = *dptr++; /* Word 6 */ 802 *wptr++ = *dptr++; /* Word 7 */ 803 dptr += 8; /* Skip Words 8-15 in payload */ 804 *wptr++ = *dptr++; /* Word 16 */ 805 *wptr++ = *dptr++; /* Word 17 */ 806 *wptr++ = *dptr++; /* Word 18 */ 807 *wptr++ = *dptr++; /* Word 19 */ 808 *wptr++ = *dptr++; /* Word 20 */ 809 *wptr++ = *dptr++; /* Word 21 */ 810 *wptr++ = *dptr++; /* Word 22 */ 811 *wptr = *dptr; /* Word 23 */ 812 } else { 813 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma)); 814 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma)); 815 816 /* Word 0-2 - NVME CMND IU Inline BDE */ 817 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 818 wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen; 819 wqe->generic.bde.addrHigh = sgl->addr_hi; 820 wqe->generic.bde.addrLow = sgl->addr_lo; 821 822 /* Word 10 */ 823 bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); 824 bf_set(wqe_wqes, &wqe->generic.wqe_com, 0); 825 } 826 827 sgl++; 828 829 /* Setup the physical region for the FCP RSP */ 830 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma)); 831 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma)); 832 sgl->word2 = le32_to_cpu(sgl->word2); 833 if (nCmd->sg_cnt) 834 bf_set(lpfc_sli4_sge_last, sgl, 0); 835 else 836 bf_set(lpfc_sli4_sge_last, sgl, 1); 837 sgl->word2 = cpu_to_le32(sgl->word2); 838 sgl->sge_len = cpu_to_le32(nCmd->rsplen); 839 } 840 841 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 842 static void 843 lpfc_nvme_ktime(struct lpfc_hba *phba, 844 struct lpfc_nvme_buf *lpfc_ncmd) 845 { 846 uint64_t seg1, seg2, seg3, seg4; 847 uint64_t segsum; 848 849 if (!lpfc_ncmd->ts_last_cmd || 850 !lpfc_ncmd->ts_cmd_start || 851 !lpfc_ncmd->ts_cmd_wqput || 852 !lpfc_ncmd->ts_isr_cmpl || 853 !lpfc_ncmd->ts_data_nvme) 854 return; 855 856 if (lpfc_ncmd->ts_data_nvme < lpfc_ncmd->ts_cmd_start) 857 return; 858 if (lpfc_ncmd->ts_cmd_start < lpfc_ncmd->ts_last_cmd) 859 return; 860 if (lpfc_ncmd->ts_cmd_wqput < lpfc_ncmd->ts_cmd_start) 861 return; 862 if (lpfc_ncmd->ts_isr_cmpl < lpfc_ncmd->ts_cmd_wqput) 863 return; 864 if (lpfc_ncmd->ts_data_nvme < lpfc_ncmd->ts_isr_cmpl) 865 return; 866 /* 867 * Segment 1 - Time from Last FCP command cmpl is handed 868 * off to NVME Layer to start of next command. 869 * Segment 2 - Time from Driver receives a IO cmd start 870 * from NVME Layer to WQ put is done on IO cmd. 871 * Segment 3 - Time from Driver WQ put is done on IO cmd 872 * to MSI-X ISR for IO cmpl. 873 * Segment 4 - Time from MSI-X ISR for IO cmpl to when 874 * cmpl is handled off to the NVME Layer. 875 */ 876 seg1 = lpfc_ncmd->ts_cmd_start - lpfc_ncmd->ts_last_cmd; 877 if (seg1 > 5000000) /* 5 ms - for sequential IOs only */ 878 seg1 = 0; 879 880 /* Calculate times relative to start of IO */ 881 seg2 = (lpfc_ncmd->ts_cmd_wqput - lpfc_ncmd->ts_cmd_start); 882 segsum = seg2; 883 seg3 = lpfc_ncmd->ts_isr_cmpl - lpfc_ncmd->ts_cmd_start; 884 if (segsum > seg3) 885 return; 886 seg3 -= segsum; 887 segsum += seg3; 888 889 seg4 = lpfc_ncmd->ts_data_nvme - lpfc_ncmd->ts_cmd_start; 890 if (segsum > seg4) 891 return; 892 seg4 -= segsum; 893 894 phba->ktime_data_samples++; 895 phba->ktime_seg1_total += seg1; 896 if (seg1 < phba->ktime_seg1_min) 897 phba->ktime_seg1_min = seg1; 898 else if (seg1 > phba->ktime_seg1_max) 899 phba->ktime_seg1_max = seg1; 900 phba->ktime_seg2_total += seg2; 901 if (seg2 < phba->ktime_seg2_min) 902 phba->ktime_seg2_min = seg2; 903 else if (seg2 > phba->ktime_seg2_max) 904 phba->ktime_seg2_max = seg2; 905 phba->ktime_seg3_total += seg3; 906 if (seg3 < phba->ktime_seg3_min) 907 phba->ktime_seg3_min = seg3; 908 else if (seg3 > phba->ktime_seg3_max) 909 phba->ktime_seg3_max = seg3; 910 phba->ktime_seg4_total += seg4; 911 if (seg4 < phba->ktime_seg4_min) 912 phba->ktime_seg4_min = seg4; 913 else if (seg4 > phba->ktime_seg4_max) 914 phba->ktime_seg4_max = seg4; 915 916 lpfc_ncmd->ts_last_cmd = 0; 917 lpfc_ncmd->ts_cmd_start = 0; 918 lpfc_ncmd->ts_cmd_wqput = 0; 919 lpfc_ncmd->ts_isr_cmpl = 0; 920 lpfc_ncmd->ts_data_nvme = 0; 921 } 922 #endif 923 924 /** 925 * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO 926 * @lpfc_pnvme: Pointer to the driver's nvme instance data 927 * @lpfc_nvme_lport: Pointer to the driver's local port data 928 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 929 * 930 * Driver registers this routine as it io request handler. This 931 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 932 * data structure to the rport indicated in @lpfc_nvme_rport. 933 * 934 * Return value : 935 * 0 - Success 936 * TODO: What are the failure codes. 937 **/ 938 static void 939 lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, 940 struct lpfc_wcqe_complete *wcqe) 941 { 942 struct lpfc_nvme_buf *lpfc_ncmd = 943 (struct lpfc_nvme_buf *)pwqeIn->context1; 944 struct lpfc_vport *vport = pwqeIn->vport; 945 struct nvmefc_fcp_req *nCmd; 946 struct nvme_fc_ersp_iu *ep; 947 struct nvme_fc_cmd_iu *cp; 948 struct lpfc_nvme_rport *rport; 949 struct lpfc_nodelist *ndlp; 950 struct lpfc_nvme_fcpreq_priv *freqpriv; 951 struct lpfc_nvme_lport *lport; 952 unsigned long flags; 953 uint32_t code, status; 954 uint16_t cid, sqhd, data; 955 uint32_t *ptr; 956 957 /* Sanity check on return of outstanding command */ 958 if (!lpfc_ncmd || !lpfc_ncmd->nvmeCmd || !lpfc_ncmd->nrport) { 959 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 960 "6071 Completion pointers bad on wqe %p.\n", 961 wcqe); 962 return; 963 } 964 atomic_inc(&phba->fc4NvmeIoCmpls); 965 966 nCmd = lpfc_ncmd->nvmeCmd; 967 rport = lpfc_ncmd->nrport; 968 status = bf_get(lpfc_wcqe_c_status, wcqe); 969 if (status) { 970 lport = (struct lpfc_nvme_lport *)vport->localport->private; 971 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 972 atomic_inc(&lport->cmpl_fcp_xb); 973 atomic_inc(&lport->cmpl_fcp_err); 974 } 975 976 lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n", 977 lpfc_ncmd->cur_iocbq.sli4_xritag, 978 status, wcqe->parameter); 979 /* 980 * Catch race where our node has transitioned, but the 981 * transport is still transitioning. 982 */ 983 ndlp = rport->ndlp; 984 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 985 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 986 "6061 rport %p, DID x%06x node not ready.\n", 987 rport, rport->remoteport->port_id); 988 989 ndlp = lpfc_findnode_did(vport, rport->remoteport->port_id); 990 if (!ndlp) { 991 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 992 "6062 Ignoring NVME cmpl. No ndlp\n"); 993 goto out_err; 994 } 995 } 996 997 code = bf_get(lpfc_wcqe_c_code, wcqe); 998 if (code == CQE_CODE_NVME_ERSP) { 999 /* For this type of CQE, we need to rebuild the rsp */ 1000 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr; 1001 1002 /* 1003 * Get Command Id from cmd to plug into response. This 1004 * code is not needed in the next NVME Transport drop. 1005 */ 1006 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr; 1007 cid = cp->sqe.common.command_id; 1008 1009 /* 1010 * RSN is in CQE word 2 1011 * SQHD is in CQE Word 3 bits 15:0 1012 * Cmd Specific info is in CQE Word 1 1013 * and in CQE Word 0 bits 15:0 1014 */ 1015 sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe); 1016 1017 /* Now lets build the NVME ERSP IU */ 1018 ep->iu_len = cpu_to_be16(8); 1019 ep->rsn = wcqe->parameter; 1020 ep->xfrd_len = cpu_to_be32(nCmd->payload_length); 1021 ep->rsvd12 = 0; 1022 ptr = (uint32_t *)&ep->cqe.result.u64; 1023 *ptr++ = wcqe->total_data_placed; 1024 data = bf_get(lpfc_wcqe_c_ersp0, wcqe); 1025 *ptr = (uint32_t)data; 1026 ep->cqe.sq_head = sqhd; 1027 ep->cqe.sq_id = nCmd->sqid; 1028 ep->cqe.command_id = cid; 1029 ep->cqe.status = 0; 1030 1031 lpfc_ncmd->status = IOSTAT_SUCCESS; 1032 lpfc_ncmd->result = 0; 1033 nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN; 1034 nCmd->transferred_length = nCmd->payload_length; 1035 } else { 1036 lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK); 1037 lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK); 1038 1039 /* For NVME, the only failure path that results in an 1040 * IO error is when the adapter rejects it. All other 1041 * conditions are a success case and resolved by the 1042 * transport. 1043 * IOSTAT_FCP_RSP_ERROR means: 1044 * 1. Length of data received doesn't match total 1045 * transfer length in WQE 1046 * 2. If the RSP payload does NOT match these cases: 1047 * a. RSP length 12/24 bytes and all zeros 1048 * b. NVME ERSP 1049 */ 1050 switch (lpfc_ncmd->status) { 1051 case IOSTAT_SUCCESS: 1052 nCmd->transferred_length = wcqe->total_data_placed; 1053 nCmd->rcv_rsplen = 0; 1054 nCmd->status = 0; 1055 break; 1056 case IOSTAT_FCP_RSP_ERROR: 1057 nCmd->transferred_length = wcqe->total_data_placed; 1058 nCmd->rcv_rsplen = wcqe->parameter; 1059 nCmd->status = 0; 1060 /* Sanity check */ 1061 if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) 1062 break; 1063 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 1064 "6081 NVME Completion Protocol Error: " 1065 "xri %x status x%x result x%x " 1066 "placed x%x\n", 1067 lpfc_ncmd->cur_iocbq.sli4_xritag, 1068 lpfc_ncmd->status, lpfc_ncmd->result, 1069 wcqe->total_data_placed); 1070 break; 1071 case IOSTAT_LOCAL_REJECT: 1072 /* Let fall through to set command final state. */ 1073 if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED) 1074 lpfc_printf_vlog(vport, KERN_INFO, 1075 LOG_NVME_IOERR, 1076 "6032 Delay Aborted cmd %p " 1077 "nvme cmd %p, xri x%x, " 1078 "xb %d\n", 1079 lpfc_ncmd, nCmd, 1080 lpfc_ncmd->cur_iocbq.sli4_xritag, 1081 bf_get(lpfc_wcqe_c_xb, wcqe)); 1082 default: 1083 out_err: 1084 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1085 "6072 NVME Completion Error: xri %x " 1086 "status x%x result x%x placed x%x\n", 1087 lpfc_ncmd->cur_iocbq.sli4_xritag, 1088 lpfc_ncmd->status, lpfc_ncmd->result, 1089 wcqe->total_data_placed); 1090 nCmd->transferred_length = 0; 1091 nCmd->rcv_rsplen = 0; 1092 nCmd->status = NVME_SC_INTERNAL; 1093 } 1094 } 1095 1096 /* pick up SLI4 exhange busy condition */ 1097 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 1098 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; 1099 else 1100 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 1101 1102 if (ndlp && NLP_CHK_NODE_ACT(ndlp)) 1103 atomic_dec(&ndlp->cmd_pending); 1104 1105 /* Update stats and complete the IO. There is 1106 * no need for dma unprep because the nvme_transport 1107 * owns the dma address. 1108 */ 1109 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1110 if (lpfc_ncmd->ts_cmd_start) { 1111 lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp; 1112 lpfc_ncmd->ts_data_nvme = ktime_get_ns(); 1113 phba->ktime_last_cmd = lpfc_ncmd->ts_data_nvme; 1114 lpfc_nvme_ktime(phba, lpfc_ncmd); 1115 } 1116 if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) { 1117 if (lpfc_ncmd->cpu != smp_processor_id()) 1118 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 1119 "6701 CPU Check cmpl: " 1120 "cpu %d expect %d\n", 1121 smp_processor_id(), lpfc_ncmd->cpu); 1122 if (lpfc_ncmd->cpu < LPFC_CHECK_CPU_CNT) 1123 phba->cpucheck_cmpl_io[lpfc_ncmd->cpu]++; 1124 } 1125 #endif 1126 1127 /* NVME targets need completion held off until the abort exchange 1128 * completes unless the NVME Rport is getting unregistered. 1129 */ 1130 1131 if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 1132 freqpriv = nCmd->private; 1133 freqpriv->nvme_buf = NULL; 1134 nCmd->done(nCmd); 1135 lpfc_ncmd->nvmeCmd = NULL; 1136 } 1137 1138 spin_lock_irqsave(&phba->hbalock, flags); 1139 lpfc_ncmd->nrport = NULL; 1140 spin_unlock_irqrestore(&phba->hbalock, flags); 1141 1142 /* Call release with XB=1 to queue the IO into the abort list. */ 1143 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1144 } 1145 1146 1147 /** 1148 * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO 1149 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1150 * @lpfc_nvme_lport: Pointer to the driver's local port data 1151 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1152 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1153 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1154 * 1155 * Driver registers this routine as it io request handler. This 1156 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1157 * data structure to the rport indicated in @lpfc_nvme_rport. 1158 * 1159 * Return value : 1160 * 0 - Success 1161 * TODO: What are the failure codes. 1162 **/ 1163 static int 1164 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, 1165 struct lpfc_nvme_buf *lpfc_ncmd, 1166 struct lpfc_nodelist *pnode) 1167 { 1168 struct lpfc_hba *phba = vport->phba; 1169 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1170 struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq); 1171 union lpfc_wqe128 *wqe = &pwqeq->wqe; 1172 uint32_t req_len; 1173 1174 if (!pnode || !NLP_CHK_NODE_ACT(pnode)) 1175 return -EINVAL; 1176 1177 /* 1178 * There are three possibilities here - use scatter-gather segment, use 1179 * the single mapping, or neither. 1180 */ 1181 if (nCmd->sg_cnt) { 1182 if (nCmd->io_dir == NVMEFC_FCP_WRITE) { 1183 /* From the iwrite template, initialize words 7 - 11 */ 1184 memcpy(&wqe->words[7], 1185 &lpfc_iwrite_cmd_template.words[7], 1186 sizeof(uint32_t) * 5); 1187 1188 /* Word 4 */ 1189 wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length; 1190 1191 /* Word 5 */ 1192 if ((phba->cfg_nvme_enable_fb) && 1193 (pnode->nlp_flag & NLP_FIRSTBURST)) { 1194 req_len = lpfc_ncmd->nvmeCmd->payload_length; 1195 if (req_len < pnode->nvme_fb_size) 1196 wqe->fcp_iwrite.initial_xfer_len = 1197 req_len; 1198 else 1199 wqe->fcp_iwrite.initial_xfer_len = 1200 pnode->nvme_fb_size; 1201 } else { 1202 wqe->fcp_iwrite.initial_xfer_len = 0; 1203 } 1204 atomic_inc(&phba->fc4NvmeOutputRequests); 1205 } else { 1206 /* From the iread template, initialize words 7 - 11 */ 1207 memcpy(&wqe->words[7], 1208 &lpfc_iread_cmd_template.words[7], 1209 sizeof(uint32_t) * 5); 1210 1211 /* Word 4 */ 1212 wqe->fcp_iread.total_xfer_len = nCmd->payload_length; 1213 1214 /* Word 5 */ 1215 wqe->fcp_iread.rsrvd5 = 0; 1216 1217 atomic_inc(&phba->fc4NvmeInputRequests); 1218 } 1219 } else { 1220 /* From the icmnd template, initialize words 4 - 11 */ 1221 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4], 1222 sizeof(uint32_t) * 8); 1223 atomic_inc(&phba->fc4NvmeControlRequests); 1224 } 1225 /* 1226 * Finish initializing those WQE fields that are independent 1227 * of the nvme_cmnd request_buffer 1228 */ 1229 1230 /* Word 3 */ 1231 bf_set(payload_offset_len, &wqe->fcp_icmd, 1232 (nCmd->rsplen + nCmd->cmdlen)); 1233 1234 /* Word 6 */ 1235 bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com, 1236 phba->sli4_hba.rpi_ids[pnode->nlp_rpi]); 1237 bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag); 1238 1239 /* Word 8 */ 1240 wqe->generic.wqe_com.abort_tag = pwqeq->iotag; 1241 1242 /* Word 9 */ 1243 bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag); 1244 1245 pwqeq->vport = vport; 1246 return 0; 1247 } 1248 1249 1250 /** 1251 * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO 1252 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1253 * @lpfc_nvme_lport: Pointer to the driver's local port data 1254 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1255 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1256 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1257 * 1258 * Driver registers this routine as it io request handler. This 1259 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1260 * data structure to the rport indicated in @lpfc_nvme_rport. 1261 * 1262 * Return value : 1263 * 0 - Success 1264 * TODO: What are the failure codes. 1265 **/ 1266 static int 1267 lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, 1268 struct lpfc_nvme_buf *lpfc_ncmd) 1269 { 1270 struct lpfc_hba *phba = vport->phba; 1271 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1272 union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe; 1273 struct sli4_sge *sgl = lpfc_ncmd->nvme_sgl; 1274 struct scatterlist *data_sg; 1275 struct sli4_sge *first_data_sgl; 1276 struct ulp_bde64 *bde; 1277 dma_addr_t physaddr; 1278 uint32_t num_bde = 0; 1279 uint32_t dma_len; 1280 uint32_t dma_offset = 0; 1281 int nseg, i; 1282 1283 /* Fix up the command and response DMA stuff. */ 1284 lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd); 1285 1286 /* 1287 * There are three possibilities here - use scatter-gather segment, use 1288 * the single mapping, or neither. 1289 */ 1290 if (nCmd->sg_cnt) { 1291 /* 1292 * Jump over the cmd and rsp SGEs. The fix routine 1293 * has already adjusted for this. 1294 */ 1295 sgl += 2; 1296 1297 first_data_sgl = sgl; 1298 lpfc_ncmd->seg_cnt = nCmd->sg_cnt; 1299 if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) { 1300 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1301 "6058 Too many sg segments from " 1302 "NVME Transport. Max %d, " 1303 "nvmeIO sg_cnt %d\n", 1304 phba->cfg_nvme_seg_cnt + 1, 1305 lpfc_ncmd->seg_cnt); 1306 lpfc_ncmd->seg_cnt = 0; 1307 return 1; 1308 } 1309 1310 /* 1311 * The driver established a maximum scatter-gather segment count 1312 * during probe that limits the number of sg elements in any 1313 * single nvme command. Just run through the seg_cnt and format 1314 * the sge's. 1315 */ 1316 nseg = nCmd->sg_cnt; 1317 data_sg = nCmd->first_sgl; 1318 for (i = 0; i < nseg; i++) { 1319 if (data_sg == NULL) { 1320 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1321 "6059 dptr err %d, nseg %d\n", 1322 i, nseg); 1323 lpfc_ncmd->seg_cnt = 0; 1324 return 1; 1325 } 1326 physaddr = data_sg->dma_address; 1327 dma_len = data_sg->length; 1328 sgl->addr_lo = cpu_to_le32(putPaddrLow(physaddr)); 1329 sgl->addr_hi = cpu_to_le32(putPaddrHigh(physaddr)); 1330 sgl->word2 = le32_to_cpu(sgl->word2); 1331 if ((num_bde + 1) == nseg) 1332 bf_set(lpfc_sli4_sge_last, sgl, 1); 1333 else 1334 bf_set(lpfc_sli4_sge_last, sgl, 0); 1335 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset); 1336 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA); 1337 sgl->word2 = cpu_to_le32(sgl->word2); 1338 sgl->sge_len = cpu_to_le32(dma_len); 1339 1340 dma_offset += dma_len; 1341 data_sg = sg_next(data_sg); 1342 sgl++; 1343 } 1344 if (phba->nvme_embed_pbde) { 1345 /* Use PBDE support for first SGL only, offset == 0 */ 1346 /* Words 13-15 */ 1347 bde = (struct ulp_bde64 *) 1348 &wqe->words[13]; 1349 bde->addrLow = first_data_sgl->addr_lo; 1350 bde->addrHigh = first_data_sgl->addr_hi; 1351 bde->tus.f.bdeSize = 1352 le32_to_cpu(first_data_sgl->sge_len); 1353 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64; 1354 bde->tus.w = cpu_to_le32(bde->tus.w); 1355 /* wqe_pbde is 1 in template */ 1356 } else { 1357 memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3)); 1358 bf_set(wqe_pbde, &wqe->generic.wqe_com, 0); 1359 } 1360 } else { 1361 bf_set(wqe_pbde, &wqe->generic.wqe_com, 0); 1362 memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3)); 1363 1364 /* For this clause to be valid, the payload_length 1365 * and sg_cnt must zero. 1366 */ 1367 if (nCmd->payload_length != 0) { 1368 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1369 "6063 NVME DMA Prep Err: sg_cnt %d " 1370 "payload_length x%x\n", 1371 nCmd->sg_cnt, nCmd->payload_length); 1372 return 1; 1373 } 1374 } 1375 return 0; 1376 } 1377 1378 /** 1379 * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO 1380 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1381 * @lpfc_nvme_lport: Pointer to the driver's local port data 1382 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1383 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1384 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1385 * 1386 * Driver registers this routine as it io request handler. This 1387 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1388 * data structure to the rport 1389 indicated in @lpfc_nvme_rport. 1390 * 1391 * Return value : 1392 * 0 - Success 1393 * TODO: What are the failure codes. 1394 **/ 1395 static int 1396 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, 1397 struct nvme_fc_remote_port *pnvme_rport, 1398 void *hw_queue_handle, 1399 struct nvmefc_fcp_req *pnvme_fcreq) 1400 { 1401 int ret = 0; 1402 int expedite = 0; 1403 struct lpfc_nvme_lport *lport; 1404 struct lpfc_vport *vport; 1405 struct lpfc_hba *phba; 1406 struct lpfc_nodelist *ndlp; 1407 struct lpfc_nvme_buf *lpfc_ncmd; 1408 struct lpfc_nvme_rport *rport; 1409 struct lpfc_nvme_qhandle *lpfc_queue_info; 1410 struct lpfc_nvme_fcpreq_priv *freqpriv; 1411 struct nvme_common_command *sqe; 1412 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1413 uint64_t start = 0; 1414 #endif 1415 1416 /* Validate pointers. LLDD fault handling with transport does 1417 * have timing races. 1418 */ 1419 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1420 if (unlikely(!lport)) { 1421 ret = -EINVAL; 1422 goto out_fail; 1423 } 1424 1425 vport = lport->vport; 1426 1427 if (unlikely(!hw_queue_handle)) { 1428 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1429 "6129 Fail Abort, NULL hw_queue_handle\n"); 1430 ret = -EINVAL; 1431 goto out_fail; 1432 } 1433 1434 phba = vport->phba; 1435 1436 if (vport->load_flag & FC_UNLOADING) { 1437 ret = -ENODEV; 1438 goto out_fail; 1439 } 1440 1441 if (vport->load_flag & FC_UNLOADING) { 1442 ret = -ENODEV; 1443 goto out_fail; 1444 } 1445 1446 freqpriv = pnvme_fcreq->private; 1447 if (unlikely(!freqpriv)) { 1448 ret = -EINVAL; 1449 goto out_fail; 1450 } 1451 1452 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1453 if (phba->ktime_on) 1454 start = ktime_get_ns(); 1455 #endif 1456 rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 1457 lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle; 1458 1459 /* 1460 * Catch race where our node has transitioned, but the 1461 * transport is still transitioning. 1462 */ 1463 ndlp = rport->ndlp; 1464 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 1465 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 1466 "6053 rport %p, ndlp %p, DID x%06x " 1467 "ndlp not ready.\n", 1468 rport, ndlp, pnvme_rport->port_id); 1469 1470 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); 1471 if (!ndlp) { 1472 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 1473 "6066 Missing node for DID %x\n", 1474 pnvme_rport->port_id); 1475 atomic_inc(&lport->xmt_fcp_bad_ndlp); 1476 ret = -ENODEV; 1477 goto out_fail; 1478 } 1479 } 1480 1481 /* The remote node has to be a mapped target or it's an error. */ 1482 if ((ndlp->nlp_type & NLP_NVME_TARGET) && 1483 (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) { 1484 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 1485 "6036 rport %p, DID x%06x not ready for " 1486 "IO. State x%x, Type x%x\n", 1487 rport, pnvme_rport->port_id, 1488 ndlp->nlp_state, ndlp->nlp_type); 1489 atomic_inc(&lport->xmt_fcp_bad_ndlp); 1490 ret = -ENODEV; 1491 goto out_fail; 1492 1493 } 1494 1495 /* Currently only NVME Keep alive commands should be expedited 1496 * if the driver runs out of a resource. These should only be 1497 * issued on the admin queue, qidx 0 1498 */ 1499 if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) { 1500 sqe = &((struct nvme_fc_cmd_iu *) 1501 pnvme_fcreq->cmdaddr)->sqe.common; 1502 if (sqe->opcode == nvme_admin_keep_alive) 1503 expedite = 1; 1504 } 1505 1506 /* The node is shared with FCP IO, make sure the IO pending count does 1507 * not exceed the programmed depth. 1508 */ 1509 if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) && 1510 !expedite) { 1511 atomic_inc(&lport->xmt_fcp_qdepth); 1512 ret = -EBUSY; 1513 goto out_fail; 1514 } 1515 1516 lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, expedite); 1517 if (lpfc_ncmd == NULL) { 1518 atomic_inc(&lport->xmt_fcp_noxri); 1519 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1520 "6065 driver's buffer pool is empty, " 1521 "IO failed\n"); 1522 ret = -EBUSY; 1523 goto out_fail; 1524 } 1525 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1526 if (start) { 1527 lpfc_ncmd->ts_cmd_start = start; 1528 lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd; 1529 } else { 1530 lpfc_ncmd->ts_cmd_start = 0; 1531 } 1532 #endif 1533 1534 /* 1535 * Store the data needed by the driver to issue, abort, and complete 1536 * an IO. 1537 * Do not let the IO hang out forever. There is no midlayer issuing 1538 * an abort so inform the FW of the maximum IO pending time. 1539 */ 1540 freqpriv->nvme_buf = lpfc_ncmd; 1541 lpfc_ncmd->nvmeCmd = pnvme_fcreq; 1542 lpfc_ncmd->nrport = rport; 1543 lpfc_ncmd->ndlp = ndlp; 1544 lpfc_ncmd->start_time = jiffies; 1545 1546 lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp); 1547 ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd); 1548 if (ret) { 1549 ret = -ENOMEM; 1550 goto out_free_nvme_buf; 1551 } 1552 1553 atomic_inc(&ndlp->cmd_pending); 1554 1555 /* 1556 * Issue the IO on the WQ indicated by index in the hw_queue_handle. 1557 * This identfier was create in our hardware queue create callback 1558 * routine. The driver now is dependent on the IO queue steering from 1559 * the transport. We are trusting the upper NVME layers know which 1560 * index to use and that they have affinitized a CPU to this hardware 1561 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ. 1562 */ 1563 lpfc_ncmd->cur_iocbq.hba_wqidx = lpfc_queue_info->index; 1564 1565 lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n", 1566 lpfc_ncmd->cur_iocbq.sli4_xritag, 1567 lpfc_queue_info->index, ndlp->nlp_DID); 1568 1569 ret = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, &lpfc_ncmd->cur_iocbq); 1570 if (ret) { 1571 atomic_inc(&lport->xmt_fcp_wqerr); 1572 atomic_dec(&ndlp->cmd_pending); 1573 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1574 "6113 FCP could not issue WQE err %x " 1575 "sid: x%x did: x%x oxid: x%x\n", 1576 ret, vport->fc_myDID, ndlp->nlp_DID, 1577 lpfc_ncmd->cur_iocbq.sli4_xritag); 1578 goto out_free_nvme_buf; 1579 } 1580 1581 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1582 if (lpfc_ncmd->ts_cmd_start) 1583 lpfc_ncmd->ts_cmd_wqput = ktime_get_ns(); 1584 1585 if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) { 1586 lpfc_ncmd->cpu = smp_processor_id(); 1587 if (lpfc_ncmd->cpu != lpfc_queue_info->index) { 1588 /* Check for admin queue */ 1589 if (lpfc_queue_info->qidx) { 1590 lpfc_printf_vlog(vport, 1591 KERN_ERR, LOG_NVME_IOERR, 1592 "6702 CPU Check cmd: " 1593 "cpu %d wq %d\n", 1594 lpfc_ncmd->cpu, 1595 lpfc_queue_info->index); 1596 } 1597 lpfc_ncmd->cpu = lpfc_queue_info->index; 1598 } 1599 if (lpfc_ncmd->cpu < LPFC_CHECK_CPU_CNT) 1600 phba->cpucheck_xmt_io[lpfc_ncmd->cpu]++; 1601 } 1602 #endif 1603 return 0; 1604 1605 out_free_nvme_buf: 1606 if (lpfc_ncmd->nvmeCmd->sg_cnt) { 1607 if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE) 1608 atomic_dec(&phba->fc4NvmeOutputRequests); 1609 else 1610 atomic_dec(&phba->fc4NvmeInputRequests); 1611 } else 1612 atomic_dec(&phba->fc4NvmeControlRequests); 1613 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1614 out_fail: 1615 return ret; 1616 } 1617 1618 /** 1619 * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request. 1620 * @phba: Pointer to HBA context object 1621 * @cmdiocb: Pointer to command iocb object. 1622 * @rspiocb: Pointer to response iocb object. 1623 * 1624 * This is the callback function for any NVME FCP IO that was aborted. 1625 * 1626 * Return value: 1627 * None 1628 **/ 1629 void 1630 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, 1631 struct lpfc_wcqe_complete *abts_cmpl) 1632 { 1633 lpfc_printf_log(phba, KERN_INFO, LOG_NVME, 1634 "6145 ABORT_XRI_CN completing on rpi x%x " 1635 "original iotag x%x, abort cmd iotag x%x " 1636 "req_tag x%x, status x%x, hwstatus x%x\n", 1637 cmdiocb->iocb.un.acxri.abortContextTag, 1638 cmdiocb->iocb.un.acxri.abortIoTag, 1639 cmdiocb->iotag, 1640 bf_get(lpfc_wcqe_c_request_tag, abts_cmpl), 1641 bf_get(lpfc_wcqe_c_status, abts_cmpl), 1642 bf_get(lpfc_wcqe_c_hw_status, abts_cmpl)); 1643 lpfc_sli_release_iocbq(phba, cmdiocb); 1644 } 1645 1646 /** 1647 * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS 1648 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1649 * @lpfc_nvme_lport: Pointer to the driver's local port data 1650 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1651 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1652 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1653 * 1654 * Driver registers this routine as its nvme request io abort handler. This 1655 * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq 1656 * data structure to the rport indicated in @lpfc_nvme_rport. This routine 1657 * is executed asynchronously - one the target is validated as "MAPPED" and 1658 * ready for IO, the driver issues the abort request and returns. 1659 * 1660 * Return value: 1661 * None 1662 **/ 1663 static void 1664 lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, 1665 struct nvme_fc_remote_port *pnvme_rport, 1666 void *hw_queue_handle, 1667 struct nvmefc_fcp_req *pnvme_fcreq) 1668 { 1669 struct lpfc_nvme_lport *lport; 1670 struct lpfc_vport *vport; 1671 struct lpfc_hba *phba; 1672 struct lpfc_nvme_buf *lpfc_nbuf; 1673 struct lpfc_iocbq *abts_buf; 1674 struct lpfc_iocbq *nvmereq_wqe; 1675 struct lpfc_nvme_fcpreq_priv *freqpriv; 1676 union lpfc_wqe128 *abts_wqe; 1677 unsigned long flags; 1678 int ret_val; 1679 1680 /* Validate pointers. LLDD fault handling with transport does 1681 * have timing races. 1682 */ 1683 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1684 if (unlikely(!lport)) 1685 return; 1686 1687 vport = lport->vport; 1688 1689 if (unlikely(!hw_queue_handle)) { 1690 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1691 "6129 Fail Abort, HW Queue Handle NULL.\n"); 1692 return; 1693 } 1694 1695 phba = vport->phba; 1696 freqpriv = pnvme_fcreq->private; 1697 1698 if (unlikely(!freqpriv)) 1699 return; 1700 if (vport->load_flag & FC_UNLOADING) 1701 return; 1702 1703 /* Announce entry to new IO submit field. */ 1704 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1705 "6002 Abort Request to rport DID x%06x " 1706 "for nvme_fc_req %p\n", 1707 pnvme_rport->port_id, 1708 pnvme_fcreq); 1709 1710 /* If the hba is getting reset, this flag is set. It is 1711 * cleared when the reset is complete and rings reestablished. 1712 */ 1713 spin_lock_irqsave(&phba->hbalock, flags); 1714 /* driver queued commands are in process of being flushed */ 1715 if (phba->hba_flag & HBA_NVME_IOQ_FLUSH) { 1716 spin_unlock_irqrestore(&phba->hbalock, flags); 1717 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1718 "6139 Driver in reset cleanup - flushing " 1719 "NVME Req now. hba_flag x%x\n", 1720 phba->hba_flag); 1721 return; 1722 } 1723 1724 lpfc_nbuf = freqpriv->nvme_buf; 1725 if (!lpfc_nbuf) { 1726 spin_unlock_irqrestore(&phba->hbalock, flags); 1727 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1728 "6140 NVME IO req has no matching lpfc nvme " 1729 "io buffer. Skipping abort req.\n"); 1730 return; 1731 } else if (!lpfc_nbuf->nvmeCmd) { 1732 spin_unlock_irqrestore(&phba->hbalock, flags); 1733 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1734 "6141 lpfc NVME IO req has no nvme_fcreq " 1735 "io buffer. Skipping abort req.\n"); 1736 return; 1737 } 1738 nvmereq_wqe = &lpfc_nbuf->cur_iocbq; 1739 1740 /* 1741 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's 1742 * state must match the nvme_fcreq passed by the nvme 1743 * transport. If they don't match, it is likely the driver 1744 * has already completed the NVME IO and the nvme transport 1745 * has not seen it yet. 1746 */ 1747 if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) { 1748 spin_unlock_irqrestore(&phba->hbalock, flags); 1749 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1750 "6143 NVME req mismatch: " 1751 "lpfc_nbuf %p nvmeCmd %p, " 1752 "pnvme_fcreq %p. Skipping Abort xri x%x\n", 1753 lpfc_nbuf, lpfc_nbuf->nvmeCmd, 1754 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1755 return; 1756 } 1757 1758 /* Don't abort IOs no longer on the pending queue. */ 1759 if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) { 1760 spin_unlock_irqrestore(&phba->hbalock, flags); 1761 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1762 "6142 NVME IO req %p not queued - skipping " 1763 "abort req xri x%x\n", 1764 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1765 return; 1766 } 1767 1768 atomic_inc(&lport->xmt_fcp_abort); 1769 lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n", 1770 nvmereq_wqe->sli4_xritag, 1771 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id); 1772 1773 /* Outstanding abort is in progress */ 1774 if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) { 1775 spin_unlock_irqrestore(&phba->hbalock, flags); 1776 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1777 "6144 Outstanding NVME I/O Abort Request " 1778 "still pending on nvme_fcreq %p, " 1779 "lpfc_ncmd %p xri x%x\n", 1780 pnvme_fcreq, lpfc_nbuf, 1781 nvmereq_wqe->sli4_xritag); 1782 return; 1783 } 1784 1785 abts_buf = __lpfc_sli_get_iocbq(phba); 1786 if (!abts_buf) { 1787 spin_unlock_irqrestore(&phba->hbalock, flags); 1788 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1789 "6136 No available abort wqes. Skipping " 1790 "Abts req for nvme_fcreq %p xri x%x\n", 1791 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1792 return; 1793 } 1794 1795 /* Ready - mark outstanding as aborted by driver. */ 1796 nvmereq_wqe->iocb_flag |= LPFC_DRIVER_ABORTED; 1797 1798 /* Complete prepping the abort wqe and issue to the FW. */ 1799 abts_wqe = &abts_buf->wqe; 1800 1801 /* WQEs are reused. Clear stale data and set key fields to 1802 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag. 1803 */ 1804 memset(abts_wqe, 0, sizeof(union lpfc_wqe)); 1805 bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG); 1806 1807 /* word 7 */ 1808 bf_set(wqe_ct, &abts_wqe->abort_cmd.wqe_com, 0); 1809 bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX); 1810 bf_set(wqe_class, &abts_wqe->abort_cmd.wqe_com, 1811 nvmereq_wqe->iocb.ulpClass); 1812 1813 /* word 8 - tell the FW to abort the IO associated with this 1814 * outstanding exchange ID. 1815 */ 1816 abts_wqe->abort_cmd.wqe_com.abort_tag = nvmereq_wqe->sli4_xritag; 1817 1818 /* word 9 - this is the iotag for the abts_wqe completion. */ 1819 bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com, 1820 abts_buf->iotag); 1821 1822 /* word 10 */ 1823 bf_set(wqe_wqid, &abts_wqe->abort_cmd.wqe_com, nvmereq_wqe->hba_wqidx); 1824 bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1); 1825 bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE); 1826 1827 /* word 11 */ 1828 bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND); 1829 bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1); 1830 bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 1831 1832 /* ABTS WQE must go to the same WQ as the WQE to be aborted */ 1833 abts_buf->iocb_flag |= LPFC_IO_NVME; 1834 abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx; 1835 abts_buf->vport = vport; 1836 abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl; 1837 ret_val = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_buf); 1838 spin_unlock_irqrestore(&phba->hbalock, flags); 1839 if (ret_val) { 1840 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1841 "6137 Failed abts issue_wqe with status x%x " 1842 "for nvme_fcreq %p.\n", 1843 ret_val, pnvme_fcreq); 1844 lpfc_sli_release_iocbq(phba, abts_buf); 1845 return; 1846 } 1847 1848 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1849 "6138 Transport Abort NVME Request Issued for " 1850 "ox_id x%x on reqtag x%x\n", 1851 nvmereq_wqe->sli4_xritag, 1852 abts_buf->iotag); 1853 } 1854 1855 /* Declare and initialization an instance of the FC NVME template. */ 1856 static struct nvme_fc_port_template lpfc_nvme_template = { 1857 /* initiator-based functions */ 1858 .localport_delete = lpfc_nvme_localport_delete, 1859 .remoteport_delete = lpfc_nvme_remoteport_delete, 1860 .create_queue = lpfc_nvme_create_queue, 1861 .delete_queue = lpfc_nvme_delete_queue, 1862 .ls_req = lpfc_nvme_ls_req, 1863 .fcp_io = lpfc_nvme_fcp_io_submit, 1864 .ls_abort = lpfc_nvme_ls_abort, 1865 .fcp_abort = lpfc_nvme_fcp_abort, 1866 1867 .max_hw_queues = 1, 1868 .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 1869 .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 1870 .dma_boundary = 0xFFFFFFFF, 1871 1872 /* Sizes of additional private data for data structures. 1873 * No use for the last two sizes at this time. 1874 */ 1875 .local_priv_sz = sizeof(struct lpfc_nvme_lport), 1876 .remote_priv_sz = sizeof(struct lpfc_nvme_rport), 1877 .lsrqst_priv_sz = 0, 1878 .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv), 1879 }; 1880 1881 /** 1882 * lpfc_sli4_post_nvme_sgl_block - post a block of nvme sgl list to firmware 1883 * @phba: pointer to lpfc hba data structure. 1884 * @nblist: pointer to nvme buffer list. 1885 * @count: number of scsi buffers on the list. 1886 * 1887 * This routine is invoked to post a block of @count scsi sgl pages from a 1888 * SCSI buffer list @nblist to the HBA using non-embedded mailbox command. 1889 * No Lock is held. 1890 * 1891 **/ 1892 static int 1893 lpfc_sli4_post_nvme_sgl_block(struct lpfc_hba *phba, 1894 struct list_head *nblist, 1895 int count) 1896 { 1897 struct lpfc_nvme_buf *lpfc_ncmd; 1898 struct lpfc_mbx_post_uembed_sgl_page1 *sgl; 1899 struct sgl_page_pairs *sgl_pg_pairs; 1900 void *viraddr; 1901 LPFC_MBOXQ_t *mbox; 1902 uint32_t reqlen, alloclen, pg_pairs; 1903 uint32_t mbox_tmo; 1904 uint16_t xritag_start = 0; 1905 int rc = 0; 1906 uint32_t shdr_status, shdr_add_status; 1907 dma_addr_t pdma_phys_bpl1; 1908 union lpfc_sli4_cfg_shdr *shdr; 1909 1910 /* Calculate the requested length of the dma memory */ 1911 reqlen = count * sizeof(struct sgl_page_pairs) + 1912 sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t); 1913 if (reqlen > SLI4_PAGE_SIZE) { 1914 lpfc_printf_log(phba, KERN_WARNING, LOG_INIT, 1915 "6118 Block sgl registration required DMA " 1916 "size (%d) great than a page\n", reqlen); 1917 return -ENOMEM; 1918 } 1919 mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); 1920 if (!mbox) { 1921 lpfc_printf_log(phba, KERN_ERR, LOG_INIT, 1922 "6119 Failed to allocate mbox cmd memory\n"); 1923 return -ENOMEM; 1924 } 1925 1926 /* Allocate DMA memory and set up the non-embedded mailbox command */ 1927 alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE, 1928 LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES, reqlen, 1929 LPFC_SLI4_MBX_NEMBED); 1930 1931 if (alloclen < reqlen) { 1932 lpfc_printf_log(phba, KERN_ERR, LOG_INIT, 1933 "6120 Allocated DMA memory size (%d) is " 1934 "less than the requested DMA memory " 1935 "size (%d)\n", alloclen, reqlen); 1936 lpfc_sli4_mbox_cmd_free(phba, mbox); 1937 return -ENOMEM; 1938 } 1939 1940 /* Get the first SGE entry from the non-embedded DMA memory */ 1941 viraddr = mbox->sge_array->addr[0]; 1942 1943 /* Set up the SGL pages in the non-embedded DMA pages */ 1944 sgl = (struct lpfc_mbx_post_uembed_sgl_page1 *)viraddr; 1945 sgl_pg_pairs = &sgl->sgl_pg_pairs; 1946 1947 pg_pairs = 0; 1948 list_for_each_entry(lpfc_ncmd, nblist, list) { 1949 /* Set up the sge entry */ 1950 sgl_pg_pairs->sgl_pg0_addr_lo = 1951 cpu_to_le32(putPaddrLow(lpfc_ncmd->dma_phys_sgl)); 1952 sgl_pg_pairs->sgl_pg0_addr_hi = 1953 cpu_to_le32(putPaddrHigh(lpfc_ncmd->dma_phys_sgl)); 1954 if (phba->cfg_sg_dma_buf_size > SGL_PAGE_SIZE) 1955 pdma_phys_bpl1 = lpfc_ncmd->dma_phys_sgl + 1956 SGL_PAGE_SIZE; 1957 else 1958 pdma_phys_bpl1 = 0; 1959 sgl_pg_pairs->sgl_pg1_addr_lo = 1960 cpu_to_le32(putPaddrLow(pdma_phys_bpl1)); 1961 sgl_pg_pairs->sgl_pg1_addr_hi = 1962 cpu_to_le32(putPaddrHigh(pdma_phys_bpl1)); 1963 /* Keep the first xritag on the list */ 1964 if (pg_pairs == 0) 1965 xritag_start = lpfc_ncmd->cur_iocbq.sli4_xritag; 1966 sgl_pg_pairs++; 1967 pg_pairs++; 1968 } 1969 bf_set(lpfc_post_sgl_pages_xri, sgl, xritag_start); 1970 bf_set(lpfc_post_sgl_pages_xricnt, sgl, pg_pairs); 1971 /* Perform endian conversion if necessary */ 1972 sgl->word0 = cpu_to_le32(sgl->word0); 1973 1974 if (!phba->sli4_hba.intr_enable) 1975 rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL); 1976 else { 1977 mbox_tmo = lpfc_mbox_tmo_val(phba, mbox); 1978 rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo); 1979 } 1980 shdr = (union lpfc_sli4_cfg_shdr *)&sgl->cfg_shdr; 1981 shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); 1982 shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); 1983 if (rc != MBX_TIMEOUT) 1984 lpfc_sli4_mbox_cmd_free(phba, mbox); 1985 if (shdr_status || shdr_add_status || rc) { 1986 lpfc_printf_log(phba, KERN_ERR, LOG_SLI, 1987 "6125 POST_SGL_BLOCK mailbox command failed " 1988 "status x%x add_status x%x mbx status x%x\n", 1989 shdr_status, shdr_add_status, rc); 1990 rc = -ENXIO; 1991 } 1992 return rc; 1993 } 1994 1995 /** 1996 * lpfc_post_nvme_sgl_list - Post blocks of nvme buffer sgls from a list 1997 * @phba: pointer to lpfc hba data structure. 1998 * @post_nblist: pointer to the nvme buffer list. 1999 * 2000 * This routine walks a list of nvme buffers that was passed in. It attempts 2001 * to construct blocks of nvme buffer sgls which contains contiguous xris and 2002 * uses the non-embedded SGL block post mailbox commands to post to the port. 2003 * For single NVME buffer sgl with non-contiguous xri, if any, it shall use 2004 * embedded SGL post mailbox command for posting. The @post_nblist passed in 2005 * must be local list, thus no lock is needed when manipulate the list. 2006 * 2007 * Returns: 0 = failure, non-zero number of successfully posted buffers. 2008 **/ 2009 static int 2010 lpfc_post_nvme_sgl_list(struct lpfc_hba *phba, 2011 struct list_head *post_nblist, int sb_count) 2012 { 2013 struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; 2014 int status, sgl_size; 2015 int post_cnt = 0, block_cnt = 0, num_posting = 0, num_posted = 0; 2016 dma_addr_t pdma_phys_sgl1; 2017 int last_xritag = NO_XRI; 2018 int cur_xritag; 2019 LIST_HEAD(prep_nblist); 2020 LIST_HEAD(blck_nblist); 2021 LIST_HEAD(nvme_nblist); 2022 2023 /* sanity check */ 2024 if (sb_count <= 0) 2025 return -EINVAL; 2026 2027 sgl_size = phba->cfg_sg_dma_buf_size; 2028 2029 list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, post_nblist, list) { 2030 list_del_init(&lpfc_ncmd->list); 2031 block_cnt++; 2032 if ((last_xritag != NO_XRI) && 2033 (lpfc_ncmd->cur_iocbq.sli4_xritag != last_xritag + 1)) { 2034 /* a hole in xri block, form a sgl posting block */ 2035 list_splice_init(&prep_nblist, &blck_nblist); 2036 post_cnt = block_cnt - 1; 2037 /* prepare list for next posting block */ 2038 list_add_tail(&lpfc_ncmd->list, &prep_nblist); 2039 block_cnt = 1; 2040 } else { 2041 /* prepare list for next posting block */ 2042 list_add_tail(&lpfc_ncmd->list, &prep_nblist); 2043 /* enough sgls for non-embed sgl mbox command */ 2044 if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) { 2045 list_splice_init(&prep_nblist, &blck_nblist); 2046 post_cnt = block_cnt; 2047 block_cnt = 0; 2048 } 2049 } 2050 num_posting++; 2051 last_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag; 2052 2053 /* end of repost sgl list condition for NVME buffers */ 2054 if (num_posting == sb_count) { 2055 if (post_cnt == 0) { 2056 /* last sgl posting block */ 2057 list_splice_init(&prep_nblist, &blck_nblist); 2058 post_cnt = block_cnt; 2059 } else if (block_cnt == 1) { 2060 /* last single sgl with non-contiguous xri */ 2061 if (sgl_size > SGL_PAGE_SIZE) 2062 pdma_phys_sgl1 = 2063 lpfc_ncmd->dma_phys_sgl + 2064 SGL_PAGE_SIZE; 2065 else 2066 pdma_phys_sgl1 = 0; 2067 cur_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag; 2068 status = lpfc_sli4_post_sgl(phba, 2069 lpfc_ncmd->dma_phys_sgl, 2070 pdma_phys_sgl1, cur_xritag); 2071 if (status) { 2072 /* failure, put on abort nvme list */ 2073 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; 2074 } else { 2075 /* success, put on NVME buffer list */ 2076 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 2077 lpfc_ncmd->status = IOSTAT_SUCCESS; 2078 num_posted++; 2079 } 2080 /* success, put on NVME buffer sgl list */ 2081 list_add_tail(&lpfc_ncmd->list, &nvme_nblist); 2082 } 2083 } 2084 2085 /* continue until a nembed page worth of sgls */ 2086 if (post_cnt == 0) 2087 continue; 2088 2089 /* post block of NVME buffer list sgls */ 2090 status = lpfc_sli4_post_nvme_sgl_block(phba, &blck_nblist, 2091 post_cnt); 2092 2093 /* don't reset xirtag due to hole in xri block */ 2094 if (block_cnt == 0) 2095 last_xritag = NO_XRI; 2096 2097 /* reset NVME buffer post count for next round of posting */ 2098 post_cnt = 0; 2099 2100 /* put posted NVME buffer-sgl posted on NVME buffer sgl list */ 2101 while (!list_empty(&blck_nblist)) { 2102 list_remove_head(&blck_nblist, lpfc_ncmd, 2103 struct lpfc_nvme_buf, list); 2104 if (status) { 2105 /* failure, put on abort nvme list */ 2106 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; 2107 } else { 2108 /* success, put on NVME buffer list */ 2109 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 2110 lpfc_ncmd->status = IOSTAT_SUCCESS; 2111 num_posted++; 2112 } 2113 list_add_tail(&lpfc_ncmd->list, &nvme_nblist); 2114 } 2115 } 2116 /* Push NVME buffers with sgl posted to the available list */ 2117 while (!list_empty(&nvme_nblist)) { 2118 list_remove_head(&nvme_nblist, lpfc_ncmd, 2119 struct lpfc_nvme_buf, list); 2120 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2121 } 2122 return num_posted; 2123 } 2124 2125 /** 2126 * lpfc_repost_nvme_sgl_list - Repost all the allocated nvme buffer sgls 2127 * @phba: pointer to lpfc hba data structure. 2128 * 2129 * This routine walks the list of nvme buffers that have been allocated and 2130 * repost them to the port by using SGL block post. This is needed after a 2131 * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine 2132 * is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list 2133 * to the lpfc_nvme_buf_list. If the repost fails, reject all nvme buffers. 2134 * 2135 * Returns: 0 = success, non-zero failure. 2136 **/ 2137 int 2138 lpfc_repost_nvme_sgl_list(struct lpfc_hba *phba) 2139 { 2140 LIST_HEAD(post_nblist); 2141 int num_posted, rc = 0; 2142 2143 /* get all NVME buffers need to repost to a local list */ 2144 spin_lock_irq(&phba->nvme_buf_list_get_lock); 2145 spin_lock(&phba->nvme_buf_list_put_lock); 2146 list_splice_init(&phba->lpfc_nvme_buf_list_get, &post_nblist); 2147 list_splice(&phba->lpfc_nvme_buf_list_put, &post_nblist); 2148 phba->get_nvme_bufs = 0; 2149 phba->put_nvme_bufs = 0; 2150 spin_unlock(&phba->nvme_buf_list_put_lock); 2151 spin_unlock_irq(&phba->nvme_buf_list_get_lock); 2152 2153 /* post the list of nvme buffer sgls to port if available */ 2154 if (!list_empty(&post_nblist)) { 2155 num_posted = lpfc_post_nvme_sgl_list(phba, &post_nblist, 2156 phba->sli4_hba.nvme_xri_cnt); 2157 /* failed to post any nvme buffer, return error */ 2158 if (num_posted == 0) 2159 rc = -EIO; 2160 } 2161 return rc; 2162 } 2163 2164 /** 2165 * lpfc_new_nvme_buf - Scsi buffer allocator for HBA with SLI4 IF spec 2166 * @vport: The virtual port for which this call being executed. 2167 * @num_to_allocate: The requested number of buffers to allocate. 2168 * 2169 * This routine allocates nvme buffers for device with SLI-4 interface spec, 2170 * the nvme buffer contains all the necessary information needed to initiate 2171 * a NVME I/O. After allocating up to @num_to_allocate NVME buffers and put 2172 * them on a list, it post them to the port by using SGL block post. 2173 * 2174 * Return codes: 2175 * int - number of nvme buffers that were allocated and posted. 2176 * 0 = failure, less than num_to_alloc is a partial failure. 2177 **/ 2178 static int 2179 lpfc_new_nvme_buf(struct lpfc_vport *vport, int num_to_alloc) 2180 { 2181 struct lpfc_hba *phba = vport->phba; 2182 struct lpfc_nvme_buf *lpfc_ncmd; 2183 struct lpfc_iocbq *pwqeq; 2184 union lpfc_wqe128 *wqe; 2185 struct sli4_sge *sgl; 2186 dma_addr_t pdma_phys_sgl; 2187 uint16_t iotag, lxri = 0; 2188 int bcnt, num_posted, sgl_size; 2189 LIST_HEAD(prep_nblist); 2190 LIST_HEAD(post_nblist); 2191 LIST_HEAD(nvme_nblist); 2192 2193 sgl_size = phba->cfg_sg_dma_buf_size; 2194 2195 for (bcnt = 0; bcnt < num_to_alloc; bcnt++) { 2196 lpfc_ncmd = kzalloc(sizeof(struct lpfc_nvme_buf), GFP_KERNEL); 2197 if (!lpfc_ncmd) 2198 break; 2199 /* 2200 * Get memory from the pci pool to map the virt space to 2201 * pci bus space for an I/O. The DMA buffer includes the 2202 * number of SGE's necessary to support the sg_tablesize. 2203 */ 2204 lpfc_ncmd->data = dma_pool_zalloc(phba->lpfc_sg_dma_buf_pool, 2205 GFP_KERNEL, 2206 &lpfc_ncmd->dma_handle); 2207 if (!lpfc_ncmd->data) { 2208 kfree(lpfc_ncmd); 2209 break; 2210 } 2211 2212 lxri = lpfc_sli4_next_xritag(phba); 2213 if (lxri == NO_XRI) { 2214 dma_pool_free(phba->lpfc_sg_dma_buf_pool, 2215 lpfc_ncmd->data, lpfc_ncmd->dma_handle); 2216 kfree(lpfc_ncmd); 2217 break; 2218 } 2219 pwqeq = &(lpfc_ncmd->cur_iocbq); 2220 wqe = &pwqeq->wqe; 2221 2222 /* Allocate iotag for lpfc_ncmd->cur_iocbq. */ 2223 iotag = lpfc_sli_next_iotag(phba, pwqeq); 2224 if (iotag == 0) { 2225 dma_pool_free(phba->lpfc_sg_dma_buf_pool, 2226 lpfc_ncmd->data, lpfc_ncmd->dma_handle); 2227 kfree(lpfc_ncmd); 2228 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2229 "6121 Failed to allocated IOTAG for" 2230 " XRI:0x%x\n", lxri); 2231 lpfc_sli4_free_xri(phba, lxri); 2232 break; 2233 } 2234 pwqeq->sli4_lxritag = lxri; 2235 pwqeq->sli4_xritag = phba->sli4_hba.xri_ids[lxri]; 2236 pwqeq->iocb_flag |= LPFC_IO_NVME; 2237 pwqeq->context1 = lpfc_ncmd; 2238 pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl; 2239 2240 /* Initialize local short-hand pointers. */ 2241 lpfc_ncmd->nvme_sgl = lpfc_ncmd->data; 2242 sgl = lpfc_ncmd->nvme_sgl; 2243 pdma_phys_sgl = lpfc_ncmd->dma_handle; 2244 lpfc_ncmd->dma_phys_sgl = pdma_phys_sgl; 2245 2246 /* Rsp SGE will be filled in when we rcv an IO 2247 * from the NVME Layer to be sent. 2248 * The cmd is going to be embedded so we need a SKIP SGE. 2249 */ 2250 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2251 bf_set(lpfc_sli4_sge_last, sgl, 0); 2252 sgl->word2 = cpu_to_le32(sgl->word2); 2253 /* Fill in word 3 / sgl_len during cmd submission */ 2254 2255 lpfc_ncmd->cur_iocbq.context1 = lpfc_ncmd; 2256 2257 /* Initialize WQE */ 2258 memset(wqe, 0, sizeof(union lpfc_wqe)); 2259 2260 /* add the nvme buffer to a post list */ 2261 list_add_tail(&lpfc_ncmd->list, &post_nblist); 2262 spin_lock_irq(&phba->nvme_buf_list_get_lock); 2263 phba->sli4_hba.nvme_xri_cnt++; 2264 spin_unlock_irq(&phba->nvme_buf_list_get_lock); 2265 } 2266 lpfc_printf_log(phba, KERN_INFO, LOG_NVME, 2267 "6114 Allocate %d out of %d requested new NVME " 2268 "buffers\n", bcnt, num_to_alloc); 2269 2270 /* post the list of nvme buffer sgls to port if available */ 2271 if (!list_empty(&post_nblist)) 2272 num_posted = lpfc_post_nvme_sgl_list(phba, 2273 &post_nblist, bcnt); 2274 else 2275 num_posted = 0; 2276 2277 return num_posted; 2278 } 2279 2280 static inline struct lpfc_nvme_buf * 2281 lpfc_nvme_buf(struct lpfc_hba *phba) 2282 { 2283 struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; 2284 2285 list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, 2286 &phba->lpfc_nvme_buf_list_get, list) { 2287 list_del_init(&lpfc_ncmd->list); 2288 phba->get_nvme_bufs--; 2289 return lpfc_ncmd; 2290 } 2291 return NULL; 2292 } 2293 2294 /** 2295 * lpfc_get_nvme_buf - Get a nvme buffer from lpfc_nvme_buf_list of the HBA 2296 * @phba: The HBA for which this call is being executed. 2297 * 2298 * This routine removes a nvme buffer from head of @phba lpfc_nvme_buf_list list 2299 * and returns to caller. 2300 * 2301 * Return codes: 2302 * NULL - Error 2303 * Pointer to lpfc_nvme_buf - Success 2304 **/ 2305 static struct lpfc_nvme_buf * 2306 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 2307 int expedite) 2308 { 2309 struct lpfc_nvme_buf *lpfc_ncmd = NULL; 2310 unsigned long iflag = 0; 2311 2312 spin_lock_irqsave(&phba->nvme_buf_list_get_lock, iflag); 2313 if (phba->get_nvme_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite) 2314 lpfc_ncmd = lpfc_nvme_buf(phba); 2315 if (!lpfc_ncmd) { 2316 spin_lock(&phba->nvme_buf_list_put_lock); 2317 list_splice(&phba->lpfc_nvme_buf_list_put, 2318 &phba->lpfc_nvme_buf_list_get); 2319 phba->get_nvme_bufs += phba->put_nvme_bufs; 2320 INIT_LIST_HEAD(&phba->lpfc_nvme_buf_list_put); 2321 phba->put_nvme_bufs = 0; 2322 spin_unlock(&phba->nvme_buf_list_put_lock); 2323 if (phba->get_nvme_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite) 2324 lpfc_ncmd = lpfc_nvme_buf(phba); 2325 } 2326 spin_unlock_irqrestore(&phba->nvme_buf_list_get_lock, iflag); 2327 return lpfc_ncmd; 2328 } 2329 2330 /** 2331 * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list. 2332 * @phba: The Hba for which this call is being executed. 2333 * @lpfc_ncmd: The nvme buffer which is being released. 2334 * 2335 * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba 2336 * lpfc_nvme_buf_list list. For SLI4 XRI's are tied to the nvme buffer 2337 * and cannot be reused for at least RA_TOV amount of time if it was 2338 * aborted. 2339 **/ 2340 static void 2341 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_nvme_buf *lpfc_ncmd) 2342 { 2343 unsigned long iflag = 0; 2344 2345 lpfc_ncmd->nonsg_phys = 0; 2346 if (lpfc_ncmd->flags & LPFC_SBUF_XBUSY) { 2347 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2348 "6310 XB release deferred for " 2349 "ox_id x%x on reqtag x%x\n", 2350 lpfc_ncmd->cur_iocbq.sli4_xritag, 2351 lpfc_ncmd->cur_iocbq.iotag); 2352 2353 spin_lock_irqsave(&phba->sli4_hba.abts_nvme_buf_list_lock, 2354 iflag); 2355 list_add_tail(&lpfc_ncmd->list, 2356 &phba->sli4_hba.lpfc_abts_nvme_buf_list); 2357 spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, 2358 iflag); 2359 } else { 2360 lpfc_ncmd->nvmeCmd = NULL; 2361 lpfc_ncmd->cur_iocbq.iocb_flag = LPFC_IO_NVME; 2362 spin_lock_irqsave(&phba->nvme_buf_list_put_lock, iflag); 2363 list_add_tail(&lpfc_ncmd->list, &phba->lpfc_nvme_buf_list_put); 2364 phba->put_nvme_bufs++; 2365 spin_unlock_irqrestore(&phba->nvme_buf_list_put_lock, iflag); 2366 } 2367 } 2368 2369 /** 2370 * lpfc_nvme_create_localport - Create/Bind an nvme localport instance. 2371 * @pvport - the lpfc_vport instance requesting a localport. 2372 * 2373 * This routine is invoked to create an nvme localport instance to bind 2374 * to the nvme_fc_transport. It is called once during driver load 2375 * like lpfc_create_shost after all other services are initialized. 2376 * It requires a vport, vpi, and wwns at call time. Other localport 2377 * parameters are modified as the driver's FCID and the Fabric WWN 2378 * are established. 2379 * 2380 * Return codes 2381 * 0 - successful 2382 * -ENOMEM - no heap memory available 2383 * other values - from nvme registration upcall 2384 **/ 2385 int 2386 lpfc_nvme_create_localport(struct lpfc_vport *vport) 2387 { 2388 int ret = 0; 2389 struct lpfc_hba *phba = vport->phba; 2390 struct nvme_fc_port_info nfcp_info; 2391 struct nvme_fc_local_port *localport; 2392 struct lpfc_nvme_lport *lport; 2393 int len; 2394 2395 /* Initialize this localport instance. The vport wwn usage ensures 2396 * that NPIV is accounted for. 2397 */ 2398 memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info)); 2399 nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR; 2400 nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn); 2401 nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn); 2402 2403 /* Limit to LPFC_MAX_NVME_SEG_CNT. 2404 * For now need + 1 to get around NVME transport logic. 2405 */ 2406 if (phba->cfg_sg_seg_cnt > LPFC_MAX_NVME_SEG_CNT) { 2407 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_INIT, 2408 "6300 Reducing sg segment cnt to %d\n", 2409 LPFC_MAX_NVME_SEG_CNT); 2410 phba->cfg_nvme_seg_cnt = LPFC_MAX_NVME_SEG_CNT; 2411 } else { 2412 phba->cfg_nvme_seg_cnt = phba->cfg_sg_seg_cnt; 2413 } 2414 lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; 2415 lpfc_nvme_template.max_hw_queues = phba->cfg_nvme_io_channel; 2416 2417 /* localport is allocated from the stack, but the registration 2418 * call allocates heap memory as well as the private area. 2419 */ 2420 #if (IS_ENABLED(CONFIG_NVME_FC)) 2421 ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template, 2422 &vport->phba->pcidev->dev, &localport); 2423 #else 2424 ret = -ENOMEM; 2425 #endif 2426 if (!ret) { 2427 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC, 2428 "6005 Successfully registered local " 2429 "NVME port num %d, localP %p, private %p, " 2430 "sg_seg %d\n", 2431 localport->port_num, localport, 2432 localport->private, 2433 lpfc_nvme_template.max_sgl_segments); 2434 2435 /* Private is our lport size declared in the template. */ 2436 lport = (struct lpfc_nvme_lport *)localport->private; 2437 vport->localport = localport; 2438 lport->vport = vport; 2439 vport->nvmei_support = 1; 2440 2441 atomic_set(&lport->xmt_fcp_noxri, 0); 2442 atomic_set(&lport->xmt_fcp_bad_ndlp, 0); 2443 atomic_set(&lport->xmt_fcp_qdepth, 0); 2444 atomic_set(&lport->xmt_fcp_wqerr, 0); 2445 atomic_set(&lport->xmt_fcp_abort, 0); 2446 atomic_set(&lport->xmt_ls_abort, 0); 2447 atomic_set(&lport->xmt_ls_err, 0); 2448 atomic_set(&lport->cmpl_fcp_xb, 0); 2449 atomic_set(&lport->cmpl_fcp_err, 0); 2450 atomic_set(&lport->cmpl_ls_xb, 0); 2451 atomic_set(&lport->cmpl_ls_err, 0); 2452 2453 /* Don't post more new bufs if repost already recovered 2454 * the nvme sgls. 2455 */ 2456 if (phba->sli4_hba.nvme_xri_cnt == 0) { 2457 len = lpfc_new_nvme_buf(vport, 2458 phba->sli4_hba.nvme_xri_max); 2459 vport->phba->total_nvme_bufs += len; 2460 } 2461 } 2462 2463 return ret; 2464 } 2465 2466 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg. 2467 * 2468 * The driver has to wait for the host nvme transport to callback 2469 * indicating the localport has successfully unregistered all 2470 * resources. Since this is an uninterruptible wait, loop every ten 2471 * seconds and print a message indicating no progress. 2472 * 2473 * An uninterruptible wait is used because of the risk of transport-to- 2474 * driver state mismatch. 2475 */ 2476 void 2477 lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport, 2478 struct lpfc_nvme_lport *lport) 2479 { 2480 #if (IS_ENABLED(CONFIG_NVME_FC)) 2481 u32 wait_tmo; 2482 int ret; 2483 2484 /* Host transport has to clean up and confirm requiring an indefinite 2485 * wait. Print a message if a 10 second wait expires and renew the 2486 * wait. This is unexpected. 2487 */ 2488 wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000); 2489 while (true) { 2490 ret = wait_for_completion_timeout(&lport->lport_unreg_done, 2491 wait_tmo); 2492 if (unlikely(!ret)) { 2493 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 2494 "6176 Lport %p Localport %p wait " 2495 "timed out. Renewing.\n", 2496 lport, vport->localport); 2497 continue; 2498 } 2499 break; 2500 } 2501 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 2502 "6177 Lport %p Localport %p Complete Success\n", 2503 lport, vport->localport); 2504 #endif 2505 } 2506 2507 /** 2508 * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport. 2509 * @pnvme: pointer to lpfc nvme data structure. 2510 * 2511 * This routine is invoked to destroy all lports bound to the phba. 2512 * The lport memory was allocated by the nvme fc transport and is 2513 * released there. This routine ensures all rports bound to the 2514 * lport have been disconnected. 2515 * 2516 **/ 2517 void 2518 lpfc_nvme_destroy_localport(struct lpfc_vport *vport) 2519 { 2520 #if (IS_ENABLED(CONFIG_NVME_FC)) 2521 struct nvme_fc_local_port *localport; 2522 struct lpfc_nvme_lport *lport; 2523 int ret; 2524 2525 if (vport->nvmei_support == 0) 2526 return; 2527 2528 localport = vport->localport; 2529 vport->localport = NULL; 2530 lport = (struct lpfc_nvme_lport *)localport->private; 2531 2532 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 2533 "6011 Destroying NVME localport %p\n", 2534 localport); 2535 2536 /* lport's rport list is clear. Unregister 2537 * lport and release resources. 2538 */ 2539 init_completion(&lport->lport_unreg_done); 2540 ret = nvme_fc_unregister_localport(localport); 2541 2542 /* Wait for completion. This either blocks 2543 * indefinitely or succeeds 2544 */ 2545 lpfc_nvme_lport_unreg_wait(vport, lport); 2546 2547 /* Regardless of the unregister upcall response, clear 2548 * nvmei_support. All rports are unregistered and the 2549 * driver will clean up. 2550 */ 2551 vport->nvmei_support = 0; 2552 if (ret == 0) { 2553 lpfc_printf_vlog(vport, 2554 KERN_INFO, LOG_NVME_DISC, 2555 "6009 Unregistered lport Success\n"); 2556 } else { 2557 lpfc_printf_vlog(vport, 2558 KERN_INFO, LOG_NVME_DISC, 2559 "6010 Unregistered lport " 2560 "Failed, status x%x\n", 2561 ret); 2562 } 2563 #endif 2564 } 2565 2566 void 2567 lpfc_nvme_update_localport(struct lpfc_vport *vport) 2568 { 2569 #if (IS_ENABLED(CONFIG_NVME_FC)) 2570 struct nvme_fc_local_port *localport; 2571 struct lpfc_nvme_lport *lport; 2572 2573 localport = vport->localport; 2574 if (!localport) { 2575 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 2576 "6710 Update NVME fail. No localport\n"); 2577 return; 2578 } 2579 lport = (struct lpfc_nvme_lport *)localport->private; 2580 if (!lport) { 2581 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 2582 "6171 Update NVME fail. localP %p, No lport\n", 2583 localport); 2584 return; 2585 } 2586 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 2587 "6012 Update NVME lport %p did x%x\n", 2588 localport, vport->fc_myDID); 2589 2590 localport->port_id = vport->fc_myDID; 2591 if (localport->port_id == 0) 2592 localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY; 2593 else 2594 localport->port_role = FC_PORT_ROLE_NVME_INITIATOR; 2595 2596 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2597 "6030 bound lport %p to DID x%06x\n", 2598 lport, localport->port_id); 2599 #endif 2600 } 2601 2602 int 2603 lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2604 { 2605 #if (IS_ENABLED(CONFIG_NVME_FC)) 2606 int ret = 0; 2607 struct nvme_fc_local_port *localport; 2608 struct lpfc_nvme_lport *lport; 2609 struct lpfc_nvme_rport *rport; 2610 struct nvme_fc_remote_port *remote_port; 2611 struct nvme_fc_port_info rpinfo; 2612 struct lpfc_nodelist *prev_ndlp; 2613 2614 lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC, 2615 "6006 Register NVME PORT. DID x%06x nlptype x%x\n", 2616 ndlp->nlp_DID, ndlp->nlp_type); 2617 2618 localport = vport->localport; 2619 if (!localport) 2620 return 0; 2621 2622 lport = (struct lpfc_nvme_lport *)localport->private; 2623 2624 /* NVME rports are not preserved across devloss. 2625 * Just register this instance. Note, rpinfo->dev_loss_tmo 2626 * is left 0 to indicate accept transport defaults. The 2627 * driver communicates port role capabilities consistent 2628 * with the PRLI response data. 2629 */ 2630 memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info)); 2631 rpinfo.port_id = ndlp->nlp_DID; 2632 if (ndlp->nlp_type & NLP_NVME_TARGET) 2633 rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET; 2634 if (ndlp->nlp_type & NLP_NVME_INITIATOR) 2635 rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR; 2636 2637 if (ndlp->nlp_type & NLP_NVME_DISCOVERY) 2638 rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY; 2639 2640 rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn); 2641 rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn); 2642 if (!ndlp->nrport) 2643 lpfc_nlp_get(ndlp); 2644 2645 ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port); 2646 if (!ret) { 2647 /* If the ndlp already has an nrport, this is just 2648 * a resume of the existing rport. Else this is a 2649 * new rport. 2650 */ 2651 rport = remote_port->private; 2652 if (ndlp->nrport) { 2653 if (ndlp->nrport == remote_port->private) { 2654 /* Same remoteport. Just reuse. */ 2655 lpfc_printf_vlog(ndlp->vport, KERN_INFO, 2656 LOG_NVME_DISC, 2657 "6014 Rebinding lport to " 2658 "remoteport %p wwpn 0x%llx, " 2659 "Data: x%x x%x %p x%x x%06x\n", 2660 remote_port, 2661 remote_port->port_name, 2662 remote_port->port_id, 2663 remote_port->port_role, 2664 ndlp, 2665 ndlp->nlp_type, 2666 ndlp->nlp_DID); 2667 return 0; 2668 } 2669 prev_ndlp = rport->ndlp; 2670 2671 /* Sever the ndlp<->rport association 2672 * before dropping the ndlp ref from 2673 * register. 2674 */ 2675 spin_lock_irq(&vport->phba->hbalock); 2676 ndlp->nrport = NULL; 2677 spin_unlock_irq(&vport->phba->hbalock); 2678 rport->ndlp = NULL; 2679 rport->remoteport = NULL; 2680 if (prev_ndlp) 2681 lpfc_nlp_put(ndlp); 2682 } 2683 2684 /* Clean bind the rport to the ndlp. */ 2685 rport->remoteport = remote_port; 2686 rport->lport = lport; 2687 rport->ndlp = ndlp; 2688 spin_lock_irq(&vport->phba->hbalock); 2689 ndlp->nrport = rport; 2690 spin_unlock_irq(&vport->phba->hbalock); 2691 lpfc_printf_vlog(vport, KERN_INFO, 2692 LOG_NVME_DISC | LOG_NODE, 2693 "6022 Binding new rport to " 2694 "lport %p Remoteport %p WWNN 0x%llx, " 2695 "Rport WWPN 0x%llx DID " 2696 "x%06x Role x%x, ndlp %p\n", 2697 lport, remote_port, 2698 rpinfo.node_name, rpinfo.port_name, 2699 rpinfo.port_id, rpinfo.port_role, 2700 ndlp); 2701 } else { 2702 lpfc_printf_vlog(vport, KERN_ERR, 2703 LOG_NVME_DISC | LOG_NODE, 2704 "6031 RemotePort Registration failed " 2705 "err: %d, DID x%06x\n", 2706 ret, ndlp->nlp_DID); 2707 } 2708 2709 return ret; 2710 #else 2711 return 0; 2712 #endif 2713 } 2714 2715 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport. 2716 * 2717 * There is no notion of Devloss or rport recovery from the current 2718 * nvme_transport perspective. Loss of an rport just means IO cannot 2719 * be sent and recovery is completely up to the initator. 2720 * For now, the driver just unbinds the DID and port_role so that 2721 * no further IO can be issued. Changes are planned for later. 2722 * 2723 * Notes - the ndlp reference count is not decremented here since 2724 * since there is no nvme_transport api for devloss. Node ref count 2725 * is only adjusted in driver unload. 2726 */ 2727 void 2728 lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2729 { 2730 #if (IS_ENABLED(CONFIG_NVME_FC)) 2731 int ret; 2732 struct nvme_fc_local_port *localport; 2733 struct lpfc_nvme_lport *lport; 2734 struct lpfc_nvme_rport *rport; 2735 struct nvme_fc_remote_port *remoteport; 2736 2737 localport = vport->localport; 2738 2739 /* This is fundamental error. The localport is always 2740 * available until driver unload. Just exit. 2741 */ 2742 if (!localport) 2743 return; 2744 2745 lport = (struct lpfc_nvme_lport *)localport->private; 2746 if (!lport) 2747 goto input_err; 2748 2749 rport = ndlp->nrport; 2750 if (!rport) 2751 goto input_err; 2752 2753 remoteport = rport->remoteport; 2754 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2755 "6033 Unreg nvme remoteport %p, portname x%llx, " 2756 "port_id x%06x, portstate x%x port type x%x\n", 2757 remoteport, remoteport->port_name, 2758 remoteport->port_id, remoteport->port_state, 2759 ndlp->nlp_type); 2760 2761 /* Sanity check ndlp type. Only call for NVME ports. Don't 2762 * clear any rport state until the transport calls back. 2763 */ 2764 2765 if (ndlp->nlp_type & NLP_NVME_TARGET) { 2766 /* No concern about the role change on the nvme remoteport. 2767 * The transport will update it. 2768 */ 2769 ndlp->upcall_flags |= NLP_WAIT_FOR_UNREG; 2770 ret = nvme_fc_unregister_remoteport(remoteport); 2771 if (ret != 0) { 2772 lpfc_nlp_put(ndlp); 2773 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 2774 "6167 NVME unregister failed %d " 2775 "port_state x%x\n", 2776 ret, remoteport->port_state); 2777 } 2778 } 2779 return; 2780 2781 input_err: 2782 #endif 2783 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 2784 "6168 State error: lport %p, rport%p FCID x%06x\n", 2785 vport->localport, ndlp->rport, ndlp->nlp_DID); 2786 } 2787 2788 /** 2789 * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort 2790 * @phba: pointer to lpfc hba data structure. 2791 * @axri: pointer to the fcp xri abort wcqe structure. 2792 * 2793 * This routine is invoked by the worker thread to process a SLI4 fast-path 2794 * NVME aborted xri. Aborted NVME IO commands are completed to the transport 2795 * here. 2796 **/ 2797 void 2798 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, 2799 struct sli4_wcqe_xri_aborted *axri) 2800 { 2801 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); 2802 struct lpfc_nvme_buf *lpfc_ncmd, *next_lpfc_ncmd; 2803 struct nvmefc_fcp_req *nvme_cmd = NULL; 2804 struct lpfc_nodelist *ndlp; 2805 unsigned long iflag = 0; 2806 2807 if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) 2808 return; 2809 spin_lock_irqsave(&phba->hbalock, iflag); 2810 spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); 2811 list_for_each_entry_safe(lpfc_ncmd, next_lpfc_ncmd, 2812 &phba->sli4_hba.lpfc_abts_nvme_buf_list, 2813 list) { 2814 if (lpfc_ncmd->cur_iocbq.sli4_xritag == xri) { 2815 list_del_init(&lpfc_ncmd->list); 2816 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 2817 lpfc_ncmd->status = IOSTAT_SUCCESS; 2818 spin_unlock( 2819 &phba->sli4_hba.abts_nvme_buf_list_lock); 2820 2821 spin_unlock_irqrestore(&phba->hbalock, iflag); 2822 ndlp = lpfc_ncmd->ndlp; 2823 if (ndlp) 2824 lpfc_sli4_abts_err_handler(phba, ndlp, axri); 2825 2826 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2827 "6311 nvme_cmd %p xri x%x tag x%x " 2828 "abort complete and xri released\n", 2829 lpfc_ncmd->nvmeCmd, xri, 2830 lpfc_ncmd->cur_iocbq.iotag); 2831 2832 /* Aborted NVME commands are required to not complete 2833 * before the abort exchange command fully completes. 2834 * Once completed, it is available via the put list. 2835 */ 2836 if (lpfc_ncmd->nvmeCmd) { 2837 nvme_cmd = lpfc_ncmd->nvmeCmd; 2838 nvme_cmd->done(nvme_cmd); 2839 lpfc_ncmd->nvmeCmd = NULL; 2840 } 2841 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2842 return; 2843 } 2844 } 2845 spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); 2846 spin_unlock_irqrestore(&phba->hbalock, iflag); 2847 2848 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2849 "6312 XRI Aborted xri x%x not found\n", xri); 2850 2851 } 2852 2853 /** 2854 * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete 2855 * @phba: Pointer to HBA context object. 2856 * 2857 * This function flushes all wqes in the nvme rings and frees all resources 2858 * in the txcmplq. This function does not issue abort wqes for the IO 2859 * commands in txcmplq, they will just be returned with 2860 * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI 2861 * slot has been permanently disabled. 2862 **/ 2863 void 2864 lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba) 2865 { 2866 struct lpfc_sli_ring *pring; 2867 u32 i, wait_cnt = 0; 2868 2869 if (phba->sli_rev < LPFC_SLI_REV4) 2870 return; 2871 2872 /* Cycle through all NVME rings and make sure all outstanding 2873 * WQEs have been removed from the txcmplqs. 2874 */ 2875 for (i = 0; i < phba->cfg_nvme_io_channel; i++) { 2876 pring = phba->sli4_hba.nvme_wq[i]->pring; 2877 2878 /* Retrieve everything on the txcmplq */ 2879 while (!list_empty(&pring->txcmplq)) { 2880 msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1); 2881 wait_cnt++; 2882 2883 /* The sleep is 10mS. Every ten seconds, 2884 * dump a message. Something is wrong. 2885 */ 2886 if ((wait_cnt % 1000) == 0) { 2887 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2888 "6178 NVME IO not empty, " 2889 "cnt %d\n", wait_cnt); 2890 } 2891 } 2892 } 2893 } 2894