1 /******************************************************************* 2 * This file is part of the Emulex Linux Device Driver for * 3 * Fibre Channsel Host Bus Adapters. * 4 * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * 5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * 6 * Copyright (C) 2004-2016 Emulex. All rights reserved. * 7 * EMULEX and SLI are trademarks of Emulex. * 8 * www.broadcom.com * 9 * Portions Copyright (C) 2004-2005 Christoph Hellwig * 10 * * 11 * This program is free software; you can redistribute it and/or * 12 * modify it under the terms of version 2 of the GNU General * 13 * Public License as published by the Free Software Foundation. * 14 * This program is distributed in the hope that it will be useful. * 15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 19 * TO BE LEGALLY INVALID. See the GNU General Public License for * 20 * more details, a copy of which can be found in the file COPYING * 21 * included with this package. * 22 ********************************************************************/ 23 #include <linux/pci.h> 24 #include <linux/slab.h> 25 #include <linux/interrupt.h> 26 #include <linux/delay.h> 27 #include <asm/unaligned.h> 28 #include <linux/crc-t10dif.h> 29 #include <net/checksum.h> 30 31 #include <scsi/scsi.h> 32 #include <scsi/scsi_device.h> 33 #include <scsi/scsi_eh.h> 34 #include <scsi/scsi_host.h> 35 #include <scsi/scsi_tcq.h> 36 #include <scsi/scsi_transport_fc.h> 37 #include <scsi/fc/fc_fs.h> 38 39 #include "lpfc_version.h" 40 #include "lpfc_hw4.h" 41 #include "lpfc_hw.h" 42 #include "lpfc_sli.h" 43 #include "lpfc_sli4.h" 44 #include "lpfc_nl.h" 45 #include "lpfc_disc.h" 46 #include "lpfc.h" 47 #include "lpfc_scsi.h" 48 #include "lpfc_nvme.h" 49 #include "lpfc_logmsg.h" 50 #include "lpfc_crtn.h" 51 #include "lpfc_vport.h" 52 #include "lpfc_debugfs.h" 53 54 static struct lpfc_iocbq *lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *, 55 struct lpfc_async_xchg_ctx *, 56 dma_addr_t rspbuf, 57 uint16_t rspsize); 58 static struct lpfc_iocbq *lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *, 59 struct lpfc_async_xchg_ctx *); 60 static int lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *, 61 struct lpfc_async_xchg_ctx *, 62 uint32_t, uint16_t); 63 static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *, 64 struct lpfc_async_xchg_ctx *, 65 uint32_t, uint16_t); 66 static void lpfc_nvmet_wqfull_flush(struct lpfc_hba *, struct lpfc_queue *, 67 struct lpfc_async_xchg_ctx *); 68 static void lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *); 69 70 static void lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf); 71 72 static union lpfc_wqe128 lpfc_tsend_cmd_template; 73 static union lpfc_wqe128 lpfc_treceive_cmd_template; 74 static union lpfc_wqe128 lpfc_trsp_cmd_template; 75 76 /* Setup WQE templates for NVME IOs */ 77 void 78 lpfc_nvmet_cmd_template(void) 79 { 80 union lpfc_wqe128 *wqe; 81 82 /* TSEND template */ 83 wqe = &lpfc_tsend_cmd_template; 84 memset(wqe, 0, sizeof(union lpfc_wqe128)); 85 86 /* Word 0, 1, 2 - BDE is variable */ 87 88 /* Word 3 - payload_offset_len is zero */ 89 90 /* Word 4 - relative_offset is variable */ 91 92 /* Word 5 - is zero */ 93 94 /* Word 6 - ctxt_tag, xri_tag is variable */ 95 96 /* Word 7 - wqe_ar is variable */ 97 bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE); 98 bf_set(wqe_pu, &wqe->fcp_tsend.wqe_com, PARM_REL_OFF); 99 bf_set(wqe_class, &wqe->fcp_tsend.wqe_com, CLASS3); 100 bf_set(wqe_ct, &wqe->fcp_tsend.wqe_com, SLI4_CT_RPI); 101 bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 1); 102 103 /* Word 8 - abort_tag is variable */ 104 105 /* Word 9 - reqtag, rcvoxid is variable */ 106 107 /* Word 10 - wqes, xc is variable */ 108 bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1); 109 bf_set(wqe_dbde, &wqe->fcp_tsend.wqe_com, 1); 110 bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0); 111 bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1); 112 bf_set(wqe_iod, &wqe->fcp_tsend.wqe_com, LPFC_WQE_IOD_WRITE); 113 bf_set(wqe_lenloc, &wqe->fcp_tsend.wqe_com, LPFC_WQE_LENLOC_WORD12); 114 115 /* Word 11 - sup, irsp, irsplen is variable */ 116 bf_set(wqe_cmd_type, &wqe->fcp_tsend.wqe_com, FCP_COMMAND_TSEND); 117 bf_set(wqe_cqid, &wqe->fcp_tsend.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 118 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0); 119 bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0); 120 bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0); 121 bf_set(wqe_pbde, &wqe->fcp_tsend.wqe_com, 0); 122 123 /* Word 12 - fcp_data_len is variable */ 124 125 /* Word 13, 14, 15 - PBDE is zero */ 126 127 /* TRECEIVE template */ 128 wqe = &lpfc_treceive_cmd_template; 129 memset(wqe, 0, sizeof(union lpfc_wqe128)); 130 131 /* Word 0, 1, 2 - BDE is variable */ 132 133 /* Word 3 */ 134 wqe->fcp_treceive.payload_offset_len = TXRDY_PAYLOAD_LEN; 135 136 /* Word 4 - relative_offset is variable */ 137 138 /* Word 5 - is zero */ 139 140 /* Word 6 - ctxt_tag, xri_tag is variable */ 141 142 /* Word 7 */ 143 bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com, CMD_FCP_TRECEIVE64_WQE); 144 bf_set(wqe_pu, &wqe->fcp_treceive.wqe_com, PARM_REL_OFF); 145 bf_set(wqe_class, &wqe->fcp_treceive.wqe_com, CLASS3); 146 bf_set(wqe_ct, &wqe->fcp_treceive.wqe_com, SLI4_CT_RPI); 147 bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0); 148 149 /* Word 8 - abort_tag is variable */ 150 151 /* Word 9 - reqtag, rcvoxid is variable */ 152 153 /* Word 10 - xc is variable */ 154 bf_set(wqe_dbde, &wqe->fcp_treceive.wqe_com, 1); 155 bf_set(wqe_wqes, &wqe->fcp_treceive.wqe_com, 0); 156 bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1); 157 bf_set(wqe_iod, &wqe->fcp_treceive.wqe_com, LPFC_WQE_IOD_READ); 158 bf_set(wqe_lenloc, &wqe->fcp_treceive.wqe_com, LPFC_WQE_LENLOC_WORD12); 159 bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1); 160 161 /* Word 11 - pbde is variable */ 162 bf_set(wqe_cmd_type, &wqe->fcp_treceive.wqe_com, FCP_COMMAND_TRECEIVE); 163 bf_set(wqe_cqid, &wqe->fcp_treceive.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 164 bf_set(wqe_sup, &wqe->fcp_treceive.wqe_com, 0); 165 bf_set(wqe_irsp, &wqe->fcp_treceive.wqe_com, 0); 166 bf_set(wqe_irsplen, &wqe->fcp_treceive.wqe_com, 0); 167 bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 1); 168 169 /* Word 12 - fcp_data_len is variable */ 170 171 /* Word 13, 14, 15 - PBDE is variable */ 172 173 /* TRSP template */ 174 wqe = &lpfc_trsp_cmd_template; 175 memset(wqe, 0, sizeof(union lpfc_wqe128)); 176 177 /* Word 0, 1, 2 - BDE is variable */ 178 179 /* Word 3 - response_len is variable */ 180 181 /* Word 4, 5 - is zero */ 182 183 /* Word 6 - ctxt_tag, xri_tag is variable */ 184 185 /* Word 7 */ 186 bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE); 187 bf_set(wqe_pu, &wqe->fcp_trsp.wqe_com, PARM_UNUSED); 188 bf_set(wqe_class, &wqe->fcp_trsp.wqe_com, CLASS3); 189 bf_set(wqe_ct, &wqe->fcp_trsp.wqe_com, SLI4_CT_RPI); 190 bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1); /* wqe_ar */ 191 192 /* Word 8 - abort_tag is variable */ 193 194 /* Word 9 - reqtag is variable */ 195 196 /* Word 10 wqes, xc is variable */ 197 bf_set(wqe_dbde, &wqe->fcp_trsp.wqe_com, 1); 198 bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1); 199 bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 0); 200 bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 0); 201 bf_set(wqe_iod, &wqe->fcp_trsp.wqe_com, LPFC_WQE_IOD_NONE); 202 bf_set(wqe_lenloc, &wqe->fcp_trsp.wqe_com, LPFC_WQE_LENLOC_WORD3); 203 204 /* Word 11 irsp, irsplen is variable */ 205 bf_set(wqe_cmd_type, &wqe->fcp_trsp.wqe_com, FCP_COMMAND_TRSP); 206 bf_set(wqe_cqid, &wqe->fcp_trsp.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 207 bf_set(wqe_sup, &wqe->fcp_trsp.wqe_com, 0); 208 bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 0); 209 bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 0); 210 bf_set(wqe_pbde, &wqe->fcp_trsp.wqe_com, 0); 211 212 /* Word 12, 13, 14, 15 - is zero */ 213 } 214 215 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 216 static struct lpfc_async_xchg_ctx * 217 lpfc_nvmet_get_ctx_for_xri(struct lpfc_hba *phba, u16 xri) 218 { 219 struct lpfc_async_xchg_ctx *ctxp; 220 unsigned long iflag; 221 bool found = false; 222 223 spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag); 224 list_for_each_entry(ctxp, &phba->sli4_hba.t_active_ctx_list, list) { 225 if (ctxp->ctxbuf->sglq->sli4_xritag != xri) 226 continue; 227 228 found = true; 229 break; 230 } 231 spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag); 232 if (found) 233 return ctxp; 234 235 return NULL; 236 } 237 238 static struct lpfc_async_xchg_ctx * 239 lpfc_nvmet_get_ctx_for_oxid(struct lpfc_hba *phba, u16 oxid, u32 sid) 240 { 241 struct lpfc_async_xchg_ctx *ctxp; 242 unsigned long iflag; 243 bool found = false; 244 245 spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag); 246 list_for_each_entry(ctxp, &phba->sli4_hba.t_active_ctx_list, list) { 247 if (ctxp->oxid != oxid || ctxp->sid != sid) 248 continue; 249 250 found = true; 251 break; 252 } 253 spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag); 254 if (found) 255 return ctxp; 256 257 return NULL; 258 } 259 #endif 260 261 static void 262 lpfc_nvmet_defer_release(struct lpfc_hba *phba, 263 struct lpfc_async_xchg_ctx *ctxp) 264 { 265 lockdep_assert_held(&ctxp->ctxlock); 266 267 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 268 "6313 NVMET Defer ctx release oxid x%x flg x%x\n", 269 ctxp->oxid, ctxp->flag); 270 271 if (ctxp->flag & LPFC_NVME_CTX_RLS) 272 return; 273 274 ctxp->flag |= LPFC_NVME_CTX_RLS; 275 spin_lock(&phba->sli4_hba.t_active_list_lock); 276 list_del(&ctxp->list); 277 spin_unlock(&phba->sli4_hba.t_active_list_lock); 278 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 279 list_add_tail(&ctxp->list, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list); 280 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 281 } 282 283 /** 284 * __lpfc_nvme_xmt_ls_rsp_cmp - Generic completion handler for the 285 * transmission of an NVME LS response. 286 * @phba: Pointer to HBA context object. 287 * @cmdwqe: Pointer to driver command WQE object. 288 * @wcqe: Pointer to driver response CQE object. 289 * 290 * The function is called from SLI ring event handler with no 291 * lock held. The function frees memory resources used for the command 292 * used to send the NVME LS RSP. 293 **/ 294 void 295 __lpfc_nvme_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 296 struct lpfc_wcqe_complete *wcqe) 297 { 298 struct lpfc_async_xchg_ctx *axchg = cmdwqe->context2; 299 struct nvmefc_ls_rsp *ls_rsp = &axchg->ls_rsp; 300 uint32_t status, result; 301 302 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 303 result = wcqe->parameter; 304 305 if (axchg->state != LPFC_NVME_STE_LS_RSP || axchg->entry_cnt != 2) { 306 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC | LOG_NVME_IOERR, 307 "6410 NVMEx LS cmpl state mismatch IO x%x: " 308 "%d %d\n", 309 axchg->oxid, axchg->state, axchg->entry_cnt); 310 } 311 312 lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x result x%x\n", 313 axchg->oxid, status, result); 314 315 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 316 "6038 NVMEx LS rsp cmpl: %d %d oxid x%x\n", 317 status, result, axchg->oxid); 318 319 lpfc_nlp_put(cmdwqe->context1); 320 cmdwqe->context2 = NULL; 321 cmdwqe->context3 = NULL; 322 lpfc_sli_release_iocbq(phba, cmdwqe); 323 ls_rsp->done(ls_rsp); 324 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 325 "6200 NVMEx LS rsp cmpl done status %d oxid x%x\n", 326 status, axchg->oxid); 327 kfree(axchg); 328 } 329 330 /** 331 * lpfc_nvmet_xmt_ls_rsp_cmp - Completion handler for LS Response 332 * @phba: Pointer to HBA context object. 333 * @cmdwqe: Pointer to driver command WQE object. 334 * @wcqe: Pointer to driver response CQE object. 335 * 336 * The function is called from SLI ring event handler with no 337 * lock held. This function is the completion handler for NVME LS commands 338 * The function updates any states and statistics, then calls the 339 * generic completion handler to free resources. 340 **/ 341 static void 342 lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 343 struct lpfc_wcqe_complete *wcqe) 344 { 345 struct lpfc_nvmet_tgtport *tgtp; 346 uint32_t status, result; 347 348 if (!phba->targetport) 349 goto finish; 350 351 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 352 result = wcqe->parameter; 353 354 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 355 if (tgtp) { 356 if (status) { 357 atomic_inc(&tgtp->xmt_ls_rsp_error); 358 if (result == IOERR_ABORT_REQUESTED) 359 atomic_inc(&tgtp->xmt_ls_rsp_aborted); 360 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 361 atomic_inc(&tgtp->xmt_ls_rsp_xb_set); 362 } else { 363 atomic_inc(&tgtp->xmt_ls_rsp_cmpl); 364 } 365 } 366 367 finish: 368 __lpfc_nvme_xmt_ls_rsp_cmp(phba, cmdwqe, wcqe); 369 } 370 371 /** 372 * lpfc_nvmet_ctxbuf_post - Repost a NVMET RQ DMA buffer and clean up context 373 * @phba: HBA buffer is associated with 374 * @ctxp: context to clean up 375 * @mp: Buffer to free 376 * 377 * Description: Frees the given DMA buffer in the appropriate way given by 378 * reposting it to its associated RQ so it can be reused. 379 * 380 * Notes: Takes phba->hbalock. Can be called with or without other locks held. 381 * 382 * Returns: None 383 **/ 384 void 385 lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) 386 { 387 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 388 struct lpfc_async_xchg_ctx *ctxp = ctx_buf->context; 389 struct lpfc_nvmet_tgtport *tgtp; 390 struct fc_frame_header *fc_hdr; 391 struct rqb_dmabuf *nvmebuf; 392 struct lpfc_nvmet_ctx_info *infop; 393 uint32_t size, oxid, sid; 394 int cpu; 395 unsigned long iflag; 396 397 if (ctxp->state == LPFC_NVME_STE_FREE) { 398 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 399 "6411 NVMET free, already free IO x%x: %d %d\n", 400 ctxp->oxid, ctxp->state, ctxp->entry_cnt); 401 } 402 403 if (ctxp->rqb_buffer) { 404 spin_lock_irqsave(&ctxp->ctxlock, iflag); 405 nvmebuf = ctxp->rqb_buffer; 406 /* check if freed in another path whilst acquiring lock */ 407 if (nvmebuf) { 408 ctxp->rqb_buffer = NULL; 409 if (ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) { 410 ctxp->flag &= ~LPFC_NVME_CTX_REUSE_WQ; 411 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 412 nvmebuf->hrq->rqbp->rqb_free_buffer(phba, 413 nvmebuf); 414 } else { 415 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 416 /* repost */ 417 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); 418 } 419 } else { 420 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 421 } 422 } 423 ctxp->state = LPFC_NVME_STE_FREE; 424 425 spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag); 426 if (phba->sli4_hba.nvmet_io_wait_cnt) { 427 list_remove_head(&phba->sli4_hba.lpfc_nvmet_io_wait_list, 428 nvmebuf, struct rqb_dmabuf, 429 hbuf.list); 430 phba->sli4_hba.nvmet_io_wait_cnt--; 431 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, 432 iflag); 433 434 fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt); 435 oxid = be16_to_cpu(fc_hdr->fh_ox_id); 436 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 437 size = nvmebuf->bytes_recv; 438 sid = sli4_sid_from_fc_hdr(fc_hdr); 439 440 ctxp = (struct lpfc_async_xchg_ctx *)ctx_buf->context; 441 ctxp->wqeq = NULL; 442 ctxp->offset = 0; 443 ctxp->phba = phba; 444 ctxp->size = size; 445 ctxp->oxid = oxid; 446 ctxp->sid = sid; 447 ctxp->state = LPFC_NVME_STE_RCV; 448 ctxp->entry_cnt = 1; 449 ctxp->flag = 0; 450 ctxp->ctxbuf = ctx_buf; 451 ctxp->rqb_buffer = (void *)nvmebuf; 452 spin_lock_init(&ctxp->ctxlock); 453 454 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 455 /* NOTE: isr time stamp is stale when context is re-assigned*/ 456 if (ctxp->ts_isr_cmd) { 457 ctxp->ts_cmd_nvme = 0; 458 ctxp->ts_nvme_data = 0; 459 ctxp->ts_data_wqput = 0; 460 ctxp->ts_isr_data = 0; 461 ctxp->ts_data_nvme = 0; 462 ctxp->ts_nvme_status = 0; 463 ctxp->ts_status_wqput = 0; 464 ctxp->ts_isr_status = 0; 465 ctxp->ts_status_nvme = 0; 466 } 467 #endif 468 atomic_inc(&tgtp->rcv_fcp_cmd_in); 469 470 /* Indicate that a replacement buffer has been posted */ 471 spin_lock_irqsave(&ctxp->ctxlock, iflag); 472 ctxp->flag |= LPFC_NVME_CTX_REUSE_WQ; 473 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 474 475 if (!queue_work(phba->wq, &ctx_buf->defer_work)) { 476 atomic_inc(&tgtp->rcv_fcp_cmd_drop); 477 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 478 "6181 Unable to queue deferred work " 479 "for oxid x%x. " 480 "FCP Drop IO [x%x x%x x%x]\n", 481 ctxp->oxid, 482 atomic_read(&tgtp->rcv_fcp_cmd_in), 483 atomic_read(&tgtp->rcv_fcp_cmd_out), 484 atomic_read(&tgtp->xmt_fcp_release)); 485 486 spin_lock_irqsave(&ctxp->ctxlock, iflag); 487 lpfc_nvmet_defer_release(phba, ctxp); 488 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 489 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid); 490 } 491 return; 492 } 493 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, iflag); 494 495 /* 496 * Use the CPU context list, from the MRQ the IO was received on 497 * (ctxp->idx), to save context structure. 498 */ 499 spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag); 500 list_del_init(&ctxp->list); 501 spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag); 502 cpu = raw_smp_processor_id(); 503 infop = lpfc_get_ctx_list(phba, cpu, ctxp->idx); 504 spin_lock_irqsave(&infop->nvmet_ctx_list_lock, iflag); 505 list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list); 506 infop->nvmet_ctx_list_cnt++; 507 spin_unlock_irqrestore(&infop->nvmet_ctx_list_lock, iflag); 508 #endif 509 } 510 511 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 512 static void 513 lpfc_nvmet_ktime(struct lpfc_hba *phba, 514 struct lpfc_async_xchg_ctx *ctxp) 515 { 516 uint64_t seg1, seg2, seg3, seg4, seg5; 517 uint64_t seg6, seg7, seg8, seg9, seg10; 518 uint64_t segsum; 519 520 if (!ctxp->ts_isr_cmd || !ctxp->ts_cmd_nvme || 521 !ctxp->ts_nvme_data || !ctxp->ts_data_wqput || 522 !ctxp->ts_isr_data || !ctxp->ts_data_nvme || 523 !ctxp->ts_nvme_status || !ctxp->ts_status_wqput || 524 !ctxp->ts_isr_status || !ctxp->ts_status_nvme) 525 return; 526 527 if (ctxp->ts_status_nvme < ctxp->ts_isr_cmd) 528 return; 529 if (ctxp->ts_isr_cmd > ctxp->ts_cmd_nvme) 530 return; 531 if (ctxp->ts_cmd_nvme > ctxp->ts_nvme_data) 532 return; 533 if (ctxp->ts_nvme_data > ctxp->ts_data_wqput) 534 return; 535 if (ctxp->ts_data_wqput > ctxp->ts_isr_data) 536 return; 537 if (ctxp->ts_isr_data > ctxp->ts_data_nvme) 538 return; 539 if (ctxp->ts_data_nvme > ctxp->ts_nvme_status) 540 return; 541 if (ctxp->ts_nvme_status > ctxp->ts_status_wqput) 542 return; 543 if (ctxp->ts_status_wqput > ctxp->ts_isr_status) 544 return; 545 if (ctxp->ts_isr_status > ctxp->ts_status_nvme) 546 return; 547 /* 548 * Segment 1 - Time from FCP command received by MSI-X ISR 549 * to FCP command is passed to NVME Layer. 550 * Segment 2 - Time from FCP command payload handed 551 * off to NVME Layer to Driver receives a Command op 552 * from NVME Layer. 553 * Segment 3 - Time from Driver receives a Command op 554 * from NVME Layer to Command is put on WQ. 555 * Segment 4 - Time from Driver WQ put is done 556 * to MSI-X ISR for Command cmpl. 557 * Segment 5 - Time from MSI-X ISR for Command cmpl to 558 * Command cmpl is passed to NVME Layer. 559 * Segment 6 - Time from Command cmpl is passed to NVME 560 * Layer to Driver receives a RSP op from NVME Layer. 561 * Segment 7 - Time from Driver receives a RSP op from 562 * NVME Layer to WQ put is done on TRSP FCP Status. 563 * Segment 8 - Time from Driver WQ put is done on TRSP 564 * FCP Status to MSI-X ISR for TRSP cmpl. 565 * Segment 9 - Time from MSI-X ISR for TRSP cmpl to 566 * TRSP cmpl is passed to NVME Layer. 567 * Segment 10 - Time from FCP command received by 568 * MSI-X ISR to command is completed on wire. 569 * (Segments 1 thru 8) for READDATA / WRITEDATA 570 * (Segments 1 thru 4) for READDATA_RSP 571 */ 572 seg1 = ctxp->ts_cmd_nvme - ctxp->ts_isr_cmd; 573 segsum = seg1; 574 575 seg2 = ctxp->ts_nvme_data - ctxp->ts_isr_cmd; 576 if (segsum > seg2) 577 return; 578 seg2 -= segsum; 579 segsum += seg2; 580 581 seg3 = ctxp->ts_data_wqput - ctxp->ts_isr_cmd; 582 if (segsum > seg3) 583 return; 584 seg3 -= segsum; 585 segsum += seg3; 586 587 seg4 = ctxp->ts_isr_data - ctxp->ts_isr_cmd; 588 if (segsum > seg4) 589 return; 590 seg4 -= segsum; 591 segsum += seg4; 592 593 seg5 = ctxp->ts_data_nvme - ctxp->ts_isr_cmd; 594 if (segsum > seg5) 595 return; 596 seg5 -= segsum; 597 segsum += seg5; 598 599 600 /* For auto rsp commands seg6 thru seg10 will be 0 */ 601 if (ctxp->ts_nvme_status > ctxp->ts_data_nvme) { 602 seg6 = ctxp->ts_nvme_status - ctxp->ts_isr_cmd; 603 if (segsum > seg6) 604 return; 605 seg6 -= segsum; 606 segsum += seg6; 607 608 seg7 = ctxp->ts_status_wqput - ctxp->ts_isr_cmd; 609 if (segsum > seg7) 610 return; 611 seg7 -= segsum; 612 segsum += seg7; 613 614 seg8 = ctxp->ts_isr_status - ctxp->ts_isr_cmd; 615 if (segsum > seg8) 616 return; 617 seg8 -= segsum; 618 segsum += seg8; 619 620 seg9 = ctxp->ts_status_nvme - ctxp->ts_isr_cmd; 621 if (segsum > seg9) 622 return; 623 seg9 -= segsum; 624 segsum += seg9; 625 626 if (ctxp->ts_isr_status < ctxp->ts_isr_cmd) 627 return; 628 seg10 = (ctxp->ts_isr_status - 629 ctxp->ts_isr_cmd); 630 } else { 631 if (ctxp->ts_isr_data < ctxp->ts_isr_cmd) 632 return; 633 seg6 = 0; 634 seg7 = 0; 635 seg8 = 0; 636 seg9 = 0; 637 seg10 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd); 638 } 639 640 phba->ktime_seg1_total += seg1; 641 if (seg1 < phba->ktime_seg1_min) 642 phba->ktime_seg1_min = seg1; 643 else if (seg1 > phba->ktime_seg1_max) 644 phba->ktime_seg1_max = seg1; 645 646 phba->ktime_seg2_total += seg2; 647 if (seg2 < phba->ktime_seg2_min) 648 phba->ktime_seg2_min = seg2; 649 else if (seg2 > phba->ktime_seg2_max) 650 phba->ktime_seg2_max = seg2; 651 652 phba->ktime_seg3_total += seg3; 653 if (seg3 < phba->ktime_seg3_min) 654 phba->ktime_seg3_min = seg3; 655 else if (seg3 > phba->ktime_seg3_max) 656 phba->ktime_seg3_max = seg3; 657 658 phba->ktime_seg4_total += seg4; 659 if (seg4 < phba->ktime_seg4_min) 660 phba->ktime_seg4_min = seg4; 661 else if (seg4 > phba->ktime_seg4_max) 662 phba->ktime_seg4_max = seg4; 663 664 phba->ktime_seg5_total += seg5; 665 if (seg5 < phba->ktime_seg5_min) 666 phba->ktime_seg5_min = seg5; 667 else if (seg5 > phba->ktime_seg5_max) 668 phba->ktime_seg5_max = seg5; 669 670 phba->ktime_data_samples++; 671 if (!seg6) 672 goto out; 673 674 phba->ktime_seg6_total += seg6; 675 if (seg6 < phba->ktime_seg6_min) 676 phba->ktime_seg6_min = seg6; 677 else if (seg6 > phba->ktime_seg6_max) 678 phba->ktime_seg6_max = seg6; 679 680 phba->ktime_seg7_total += seg7; 681 if (seg7 < phba->ktime_seg7_min) 682 phba->ktime_seg7_min = seg7; 683 else if (seg7 > phba->ktime_seg7_max) 684 phba->ktime_seg7_max = seg7; 685 686 phba->ktime_seg8_total += seg8; 687 if (seg8 < phba->ktime_seg8_min) 688 phba->ktime_seg8_min = seg8; 689 else if (seg8 > phba->ktime_seg8_max) 690 phba->ktime_seg8_max = seg8; 691 692 phba->ktime_seg9_total += seg9; 693 if (seg9 < phba->ktime_seg9_min) 694 phba->ktime_seg9_min = seg9; 695 else if (seg9 > phba->ktime_seg9_max) 696 phba->ktime_seg9_max = seg9; 697 out: 698 phba->ktime_seg10_total += seg10; 699 if (seg10 < phba->ktime_seg10_min) 700 phba->ktime_seg10_min = seg10; 701 else if (seg10 > phba->ktime_seg10_max) 702 phba->ktime_seg10_max = seg10; 703 phba->ktime_status_samples++; 704 } 705 #endif 706 707 /** 708 * lpfc_nvmet_xmt_fcp_op_cmp - Completion handler for FCP Response 709 * @phba: Pointer to HBA context object. 710 * @cmdwqe: Pointer to driver command WQE object. 711 * @wcqe: Pointer to driver response CQE object. 712 * 713 * The function is called from SLI ring event handler with no 714 * lock held. This function is the completion handler for NVME FCP commands 715 * The function frees memory resources used for the NVME commands. 716 **/ 717 static void 718 lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 719 struct lpfc_wcqe_complete *wcqe) 720 { 721 struct lpfc_nvmet_tgtport *tgtp; 722 struct nvmefc_tgt_fcp_req *rsp; 723 struct lpfc_async_xchg_ctx *ctxp; 724 uint32_t status, result, op, start_clean, logerr; 725 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 726 int id; 727 #endif 728 729 ctxp = cmdwqe->context2; 730 ctxp->flag &= ~LPFC_NVME_IO_INP; 731 732 rsp = &ctxp->hdlrctx.fcp_req; 733 op = rsp->op; 734 735 status = bf_get(lpfc_wcqe_c_status, wcqe); 736 result = wcqe->parameter; 737 738 if (phba->targetport) 739 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 740 else 741 tgtp = NULL; 742 743 lpfc_nvmeio_data(phba, "NVMET FCP CMPL: xri x%x op x%x status x%x\n", 744 ctxp->oxid, op, status); 745 746 if (status) { 747 rsp->fcp_error = NVME_SC_DATA_XFER_ERROR; 748 rsp->transferred_length = 0; 749 if (tgtp) { 750 atomic_inc(&tgtp->xmt_fcp_rsp_error); 751 if (result == IOERR_ABORT_REQUESTED) 752 atomic_inc(&tgtp->xmt_fcp_rsp_aborted); 753 } 754 755 logerr = LOG_NVME_IOERR; 756 757 /* pick up SLI4 exhange busy condition */ 758 if (bf_get(lpfc_wcqe_c_xb, wcqe)) { 759 ctxp->flag |= LPFC_NVME_XBUSY; 760 logerr |= LOG_NVME_ABTS; 761 if (tgtp) 762 atomic_inc(&tgtp->xmt_fcp_rsp_xb_set); 763 764 } else { 765 ctxp->flag &= ~LPFC_NVME_XBUSY; 766 } 767 768 lpfc_printf_log(phba, KERN_INFO, logerr, 769 "6315 IO Error Cmpl oxid: x%x xri: x%x %x/%x " 770 "XBUSY:x%x\n", 771 ctxp->oxid, ctxp->ctxbuf->sglq->sli4_xritag, 772 status, result, ctxp->flag); 773 774 } else { 775 rsp->fcp_error = NVME_SC_SUCCESS; 776 if (op == NVMET_FCOP_RSP) 777 rsp->transferred_length = rsp->rsplen; 778 else 779 rsp->transferred_length = rsp->transfer_length; 780 if (tgtp) 781 atomic_inc(&tgtp->xmt_fcp_rsp_cmpl); 782 } 783 784 if ((op == NVMET_FCOP_READDATA_RSP) || 785 (op == NVMET_FCOP_RSP)) { 786 /* Sanity check */ 787 ctxp->state = LPFC_NVME_STE_DONE; 788 ctxp->entry_cnt++; 789 790 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 791 if (ctxp->ts_cmd_nvme) { 792 if (rsp->op == NVMET_FCOP_READDATA_RSP) { 793 ctxp->ts_isr_data = 794 cmdwqe->isr_timestamp; 795 ctxp->ts_data_nvme = 796 ktime_get_ns(); 797 ctxp->ts_nvme_status = 798 ctxp->ts_data_nvme; 799 ctxp->ts_status_wqput = 800 ctxp->ts_data_nvme; 801 ctxp->ts_isr_status = 802 ctxp->ts_data_nvme; 803 ctxp->ts_status_nvme = 804 ctxp->ts_data_nvme; 805 } else { 806 ctxp->ts_isr_status = 807 cmdwqe->isr_timestamp; 808 ctxp->ts_status_nvme = 809 ktime_get_ns(); 810 } 811 } 812 #endif 813 rsp->done(rsp); 814 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 815 if (ctxp->ts_cmd_nvme) 816 lpfc_nvmet_ktime(phba, ctxp); 817 #endif 818 /* lpfc_nvmet_xmt_fcp_release() will recycle the context */ 819 } else { 820 ctxp->entry_cnt++; 821 start_clean = offsetof(struct lpfc_iocbq, iocb_flag); 822 memset(((char *)cmdwqe) + start_clean, 0, 823 (sizeof(struct lpfc_iocbq) - start_clean)); 824 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 825 if (ctxp->ts_cmd_nvme) { 826 ctxp->ts_isr_data = cmdwqe->isr_timestamp; 827 ctxp->ts_data_nvme = ktime_get_ns(); 828 } 829 #endif 830 rsp->done(rsp); 831 } 832 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 833 if (phba->hdwqstat_on & LPFC_CHECK_NVMET_IO) { 834 id = raw_smp_processor_id(); 835 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io); 836 if (ctxp->cpu != id) 837 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 838 "6704 CPU Check cmdcmpl: " 839 "cpu %d expect %d\n", 840 id, ctxp->cpu); 841 } 842 #endif 843 } 844 845 /** 846 * __lpfc_nvme_xmt_ls_rsp - Generic service routine to issue transmit 847 * an NVME LS rsp for a prior NVME LS request that was received. 848 * @axchg: pointer to exchange context for the NVME LS request the response 849 * is for. 850 * @ls_rsp: pointer to the transport LS RSP that is to be sent 851 * @xmt_ls_rsp_cmp: completion routine to call upon RSP transmit done 852 * 853 * This routine is used to format and send a WQE to transmit a NVME LS 854 * Response. The response is for a prior NVME LS request that was 855 * received and posted to the transport. 856 * 857 * Returns: 858 * 0 : if response successfully transmit 859 * non-zero : if response failed to transmit, of the form -Exxx. 860 **/ 861 int 862 __lpfc_nvme_xmt_ls_rsp(struct lpfc_async_xchg_ctx *axchg, 863 struct nvmefc_ls_rsp *ls_rsp, 864 void (*xmt_ls_rsp_cmp)(struct lpfc_hba *phba, 865 struct lpfc_iocbq *cmdwqe, 866 struct lpfc_wcqe_complete *wcqe)) 867 { 868 struct lpfc_hba *phba = axchg->phba; 869 struct hbq_dmabuf *nvmebuf = (struct hbq_dmabuf *)axchg->rqb_buffer; 870 struct lpfc_iocbq *nvmewqeq; 871 struct lpfc_dmabuf dmabuf; 872 struct ulp_bde64 bpl; 873 int rc; 874 875 if (phba->pport->load_flag & FC_UNLOADING) 876 return -ENODEV; 877 878 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 879 "6023 NVMEx LS rsp oxid x%x\n", axchg->oxid); 880 881 if (axchg->state != LPFC_NVME_STE_LS_RCV || axchg->entry_cnt != 1) { 882 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC | LOG_NVME_IOERR, 883 "6412 NVMEx LS rsp state mismatch " 884 "oxid x%x: %d %d\n", 885 axchg->oxid, axchg->state, axchg->entry_cnt); 886 return -EALREADY; 887 } 888 axchg->state = LPFC_NVME_STE_LS_RSP; 889 axchg->entry_cnt++; 890 891 nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, axchg, ls_rsp->rspdma, 892 ls_rsp->rsplen); 893 if (nvmewqeq == NULL) { 894 lpfc_printf_log(phba, KERN_ERR, 895 LOG_NVME_DISC | LOG_NVME_IOERR | LOG_NVME_ABTS, 896 "6150 NVMEx LS Drop Rsp x%x: Prep\n", 897 axchg->oxid); 898 rc = -ENOMEM; 899 goto out_free_buf; 900 } 901 902 /* Save numBdes for bpl2sgl */ 903 nvmewqeq->rsvd2 = 1; 904 nvmewqeq->hba_wqidx = 0; 905 nvmewqeq->context3 = &dmabuf; 906 dmabuf.virt = &bpl; 907 bpl.addrLow = nvmewqeq->wqe.xmit_sequence.bde.addrLow; 908 bpl.addrHigh = nvmewqeq->wqe.xmit_sequence.bde.addrHigh; 909 bpl.tus.f.bdeSize = ls_rsp->rsplen; 910 bpl.tus.f.bdeFlags = 0; 911 bpl.tus.w = le32_to_cpu(bpl.tus.w); 912 /* 913 * Note: although we're using stack space for the dmabuf, the 914 * call to lpfc_sli4_issue_wqe is synchronous, so it will not 915 * be referenced after it returns back to this routine. 916 */ 917 918 nvmewqeq->wqe_cmpl = xmt_ls_rsp_cmp; 919 nvmewqeq->iocb_cmpl = NULL; 920 nvmewqeq->context2 = axchg; 921 922 lpfc_nvmeio_data(phba, "NVMEx LS RSP: xri x%x wqidx x%x len x%x\n", 923 axchg->oxid, nvmewqeq->hba_wqidx, ls_rsp->rsplen); 924 925 rc = lpfc_sli4_issue_wqe(phba, axchg->hdwq, nvmewqeq); 926 927 /* clear to be sure there's no reference */ 928 nvmewqeq->context3 = NULL; 929 930 if (rc == WQE_SUCCESS) { 931 /* 932 * Okay to repost buffer here, but wait till cmpl 933 * before freeing ctxp and iocbq. 934 */ 935 lpfc_in_buf_free(phba, &nvmebuf->dbuf); 936 return 0; 937 } 938 939 lpfc_printf_log(phba, KERN_ERR, 940 LOG_NVME_DISC | LOG_NVME_IOERR | LOG_NVME_ABTS, 941 "6151 NVMEx LS RSP x%x: failed to transmit %d\n", 942 axchg->oxid, rc); 943 944 rc = -ENXIO; 945 946 lpfc_nlp_put(nvmewqeq->context1); 947 948 out_free_buf: 949 /* Give back resources */ 950 lpfc_in_buf_free(phba, &nvmebuf->dbuf); 951 952 /* 953 * As transport doesn't track completions of responses, if the rsp 954 * fails to send, the transport will effectively ignore the rsp 955 * and consider the LS done. However, the driver has an active 956 * exchange open for the LS - so be sure to abort the exchange 957 * if the response isn't sent. 958 */ 959 lpfc_nvme_unsol_ls_issue_abort(phba, axchg, axchg->sid, axchg->oxid); 960 return rc; 961 } 962 963 /** 964 * lpfc_nvmet_xmt_ls_rsp - Transmit NVME LS response 965 * @tgtport: pointer to target port that NVME LS is to be transmit from. 966 * @ls_rsp: pointer to the transport LS RSP that is to be sent 967 * 968 * Driver registers this routine to transmit responses for received NVME 969 * LS requests. 970 * 971 * This routine is used to format and send a WQE to transmit a NVME LS 972 * Response. The ls_rsp is used to reverse-map the LS to the original 973 * NVME LS request sequence, which provides addressing information for 974 * the remote port the LS to be sent to, as well as the exchange id 975 * that is the LS is bound to. 976 * 977 * Returns: 978 * 0 : if response successfully transmit 979 * non-zero : if response failed to transmit, of the form -Exxx. 980 **/ 981 static int 982 lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport, 983 struct nvmefc_ls_rsp *ls_rsp) 984 { 985 struct lpfc_async_xchg_ctx *axchg = 986 container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp); 987 struct lpfc_nvmet_tgtport *nvmep = tgtport->private; 988 int rc; 989 990 if (axchg->phba->pport->load_flag & FC_UNLOADING) 991 return -ENODEV; 992 993 rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, lpfc_nvmet_xmt_ls_rsp_cmp); 994 995 if (rc) { 996 atomic_inc(&nvmep->xmt_ls_drop); 997 /* 998 * unless the failure is due to having already sent 999 * the response, an abort will be generated for the 1000 * exchange if the rsp can't be sent. 1001 */ 1002 if (rc != -EALREADY) 1003 atomic_inc(&nvmep->xmt_ls_abort); 1004 return rc; 1005 } 1006 1007 atomic_inc(&nvmep->xmt_ls_rsp); 1008 return 0; 1009 } 1010 1011 static int 1012 lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport, 1013 struct nvmefc_tgt_fcp_req *rsp) 1014 { 1015 struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private; 1016 struct lpfc_async_xchg_ctx *ctxp = 1017 container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req); 1018 struct lpfc_hba *phba = ctxp->phba; 1019 struct lpfc_queue *wq; 1020 struct lpfc_iocbq *nvmewqeq; 1021 struct lpfc_sli_ring *pring; 1022 unsigned long iflags; 1023 int rc; 1024 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1025 int id; 1026 #endif 1027 1028 if (phba->pport->load_flag & FC_UNLOADING) { 1029 rc = -ENODEV; 1030 goto aerr; 1031 } 1032 1033 if (phba->pport->load_flag & FC_UNLOADING) { 1034 rc = -ENODEV; 1035 goto aerr; 1036 } 1037 1038 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1039 if (ctxp->ts_cmd_nvme) { 1040 if (rsp->op == NVMET_FCOP_RSP) 1041 ctxp->ts_nvme_status = ktime_get_ns(); 1042 else 1043 ctxp->ts_nvme_data = ktime_get_ns(); 1044 } 1045 1046 /* Setup the hdw queue if not already set */ 1047 if (!ctxp->hdwq) 1048 ctxp->hdwq = &phba->sli4_hba.hdwq[rsp->hwqid]; 1049 1050 if (phba->hdwqstat_on & LPFC_CHECK_NVMET_IO) { 1051 id = raw_smp_processor_id(); 1052 this_cpu_inc(phba->sli4_hba.c_stat->xmt_io); 1053 if (rsp->hwqid != id) 1054 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 1055 "6705 CPU Check OP: " 1056 "cpu %d expect %d\n", 1057 id, rsp->hwqid); 1058 ctxp->cpu = id; /* Setup cpu for cmpl check */ 1059 } 1060 #endif 1061 1062 /* Sanity check */ 1063 if ((ctxp->flag & LPFC_NVME_ABTS_RCV) || 1064 (ctxp->state == LPFC_NVME_STE_ABORT)) { 1065 atomic_inc(&lpfc_nvmep->xmt_fcp_drop); 1066 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1067 "6102 IO oxid x%x aborted\n", 1068 ctxp->oxid); 1069 rc = -ENXIO; 1070 goto aerr; 1071 } 1072 1073 nvmewqeq = lpfc_nvmet_prep_fcp_wqe(phba, ctxp); 1074 if (nvmewqeq == NULL) { 1075 atomic_inc(&lpfc_nvmep->xmt_fcp_drop); 1076 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1077 "6152 FCP Drop IO x%x: Prep\n", 1078 ctxp->oxid); 1079 rc = -ENXIO; 1080 goto aerr; 1081 } 1082 1083 nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_fcp_op_cmp; 1084 nvmewqeq->iocb_cmpl = NULL; 1085 nvmewqeq->context2 = ctxp; 1086 nvmewqeq->iocb_flag |= LPFC_IO_NVMET; 1087 ctxp->wqeq->hba_wqidx = rsp->hwqid; 1088 1089 lpfc_nvmeio_data(phba, "NVMET FCP CMND: xri x%x op x%x len x%x\n", 1090 ctxp->oxid, rsp->op, rsp->rsplen); 1091 1092 ctxp->flag |= LPFC_NVME_IO_INP; 1093 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq); 1094 if (rc == WQE_SUCCESS) { 1095 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1096 if (!ctxp->ts_cmd_nvme) 1097 return 0; 1098 if (rsp->op == NVMET_FCOP_RSP) 1099 ctxp->ts_status_wqput = ktime_get_ns(); 1100 else 1101 ctxp->ts_data_wqput = ktime_get_ns(); 1102 #endif 1103 return 0; 1104 } 1105 1106 if (rc == -EBUSY) { 1107 /* 1108 * WQ was full, so queue nvmewqeq to be sent after 1109 * WQE release CQE 1110 */ 1111 ctxp->flag |= LPFC_NVME_DEFER_WQFULL; 1112 wq = ctxp->hdwq->io_wq; 1113 pring = wq->pring; 1114 spin_lock_irqsave(&pring->ring_lock, iflags); 1115 list_add_tail(&nvmewqeq->list, &wq->wqfull_list); 1116 wq->q_flag |= HBA_NVMET_WQFULL; 1117 spin_unlock_irqrestore(&pring->ring_lock, iflags); 1118 atomic_inc(&lpfc_nvmep->defer_wqfull); 1119 return 0; 1120 } 1121 1122 /* Give back resources */ 1123 atomic_inc(&lpfc_nvmep->xmt_fcp_drop); 1124 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1125 "6153 FCP Drop IO x%x: Issue: %d\n", 1126 ctxp->oxid, rc); 1127 1128 ctxp->wqeq->hba_wqidx = 0; 1129 nvmewqeq->context2 = NULL; 1130 nvmewqeq->context3 = NULL; 1131 rc = -EBUSY; 1132 aerr: 1133 return rc; 1134 } 1135 1136 static void 1137 lpfc_nvmet_targetport_delete(struct nvmet_fc_target_port *targetport) 1138 { 1139 struct lpfc_nvmet_tgtport *tport = targetport->private; 1140 1141 /* release any threads waiting for the unreg to complete */ 1142 if (tport->phba->targetport) 1143 complete(tport->tport_unreg_cmp); 1144 } 1145 1146 static void 1147 lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport, 1148 struct nvmefc_tgt_fcp_req *req) 1149 { 1150 struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private; 1151 struct lpfc_async_xchg_ctx *ctxp = 1152 container_of(req, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req); 1153 struct lpfc_hba *phba = ctxp->phba; 1154 struct lpfc_queue *wq; 1155 unsigned long flags; 1156 1157 if (phba->pport->load_flag & FC_UNLOADING) 1158 return; 1159 1160 if (phba->pport->load_flag & FC_UNLOADING) 1161 return; 1162 1163 if (!ctxp->hdwq) 1164 ctxp->hdwq = &phba->sli4_hba.hdwq[0]; 1165 1166 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1167 "6103 NVMET Abort op: oxid x%x flg x%x ste %d\n", 1168 ctxp->oxid, ctxp->flag, ctxp->state); 1169 1170 lpfc_nvmeio_data(phba, "NVMET FCP ABRT: xri x%x flg x%x ste x%x\n", 1171 ctxp->oxid, ctxp->flag, ctxp->state); 1172 1173 atomic_inc(&lpfc_nvmep->xmt_fcp_abort); 1174 1175 spin_lock_irqsave(&ctxp->ctxlock, flags); 1176 1177 /* Since iaab/iaar are NOT set, we need to check 1178 * if the firmware is in process of aborting IO 1179 */ 1180 if (ctxp->flag & (LPFC_NVME_XBUSY | LPFC_NVME_ABORT_OP)) { 1181 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 1182 return; 1183 } 1184 ctxp->flag |= LPFC_NVME_ABORT_OP; 1185 1186 if (ctxp->flag & LPFC_NVME_DEFER_WQFULL) { 1187 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 1188 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, 1189 ctxp->oxid); 1190 wq = ctxp->hdwq->io_wq; 1191 lpfc_nvmet_wqfull_flush(phba, wq, ctxp); 1192 return; 1193 } 1194 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 1195 1196 /* A state of LPFC_NVME_STE_RCV means we have just received 1197 * the NVME command and have not started processing it. 1198 * (by issuing any IO WQEs on this exchange yet) 1199 */ 1200 if (ctxp->state == LPFC_NVME_STE_RCV) 1201 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, 1202 ctxp->oxid); 1203 else 1204 lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid, 1205 ctxp->oxid); 1206 } 1207 1208 static void 1209 lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport, 1210 struct nvmefc_tgt_fcp_req *rsp) 1211 { 1212 struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private; 1213 struct lpfc_async_xchg_ctx *ctxp = 1214 container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req); 1215 struct lpfc_hba *phba = ctxp->phba; 1216 unsigned long flags; 1217 bool aborting = false; 1218 1219 spin_lock_irqsave(&ctxp->ctxlock, flags); 1220 if (ctxp->flag & LPFC_NVME_XBUSY) 1221 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 1222 "6027 NVMET release with XBUSY flag x%x" 1223 " oxid x%x\n", 1224 ctxp->flag, ctxp->oxid); 1225 else if (ctxp->state != LPFC_NVME_STE_DONE && 1226 ctxp->state != LPFC_NVME_STE_ABORT) 1227 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1228 "6413 NVMET release bad state %d %d oxid x%x\n", 1229 ctxp->state, ctxp->entry_cnt, ctxp->oxid); 1230 1231 if ((ctxp->flag & LPFC_NVME_ABORT_OP) || 1232 (ctxp->flag & LPFC_NVME_XBUSY)) { 1233 aborting = true; 1234 /* let the abort path do the real release */ 1235 lpfc_nvmet_defer_release(phba, ctxp); 1236 } 1237 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 1238 1239 lpfc_nvmeio_data(phba, "NVMET FCP FREE: xri x%x ste %d abt %d\n", ctxp->oxid, 1240 ctxp->state, aborting); 1241 1242 atomic_inc(&lpfc_nvmep->xmt_fcp_release); 1243 ctxp->flag &= ~LPFC_NVME_TNOTIFY; 1244 1245 if (aborting) 1246 return; 1247 1248 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); 1249 } 1250 1251 static void 1252 lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport, 1253 struct nvmefc_tgt_fcp_req *rsp) 1254 { 1255 struct lpfc_nvmet_tgtport *tgtp; 1256 struct lpfc_async_xchg_ctx *ctxp = 1257 container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req); 1258 struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer; 1259 struct lpfc_hba *phba = ctxp->phba; 1260 unsigned long iflag; 1261 1262 1263 lpfc_nvmeio_data(phba, "NVMET DEFERRCV: xri x%x sz %d CPU %02x\n", 1264 ctxp->oxid, ctxp->size, raw_smp_processor_id()); 1265 1266 if (!nvmebuf) { 1267 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 1268 "6425 Defer rcv: no buffer oxid x%x: " 1269 "flg %x ste %x\n", 1270 ctxp->oxid, ctxp->flag, ctxp->state); 1271 return; 1272 } 1273 1274 tgtp = phba->targetport->private; 1275 if (tgtp) 1276 atomic_inc(&tgtp->rcv_fcp_cmd_defer); 1277 1278 /* Free the nvmebuf since a new buffer already replaced it */ 1279 nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); 1280 spin_lock_irqsave(&ctxp->ctxlock, iflag); 1281 ctxp->rqb_buffer = NULL; 1282 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 1283 } 1284 1285 /** 1286 * lpfc_nvmet_ls_req_cmp - completion handler for a nvme ls request 1287 * @phba: Pointer to HBA context object 1288 * @cmdwqe: Pointer to driver command WQE object. 1289 * @wcqe: Pointer to driver response CQE object. 1290 * 1291 * This function is the completion handler for NVME LS requests. 1292 * The function updates any states and statistics, then calls the 1293 * generic completion handler to finish completion of the request. 1294 **/ 1295 static void 1296 lpfc_nvmet_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 1297 struct lpfc_wcqe_complete *wcqe) 1298 { 1299 __lpfc_nvme_ls_req_cmp(phba, cmdwqe->vport, cmdwqe, wcqe); 1300 } 1301 1302 /** 1303 * lpfc_nvmet_ls_req - Issue an Link Service request 1304 * @targetport - pointer to target instance registered with nvmet transport. 1305 * @hosthandle - hosthandle set by the driver in a prior ls_rqst_rcv. 1306 * Driver sets this value to the ndlp pointer. 1307 * @pnvme_lsreq - the transport nvme_ls_req structure for the LS 1308 * 1309 * Driver registers this routine to handle any link service request 1310 * from the nvme_fc transport to a remote nvme-aware port. 1311 * 1312 * Return value : 1313 * 0 - Success 1314 * non-zero: various error codes, in form of -Exxx 1315 **/ 1316 static int 1317 lpfc_nvmet_ls_req(struct nvmet_fc_target_port *targetport, 1318 void *hosthandle, 1319 struct nvmefc_ls_req *pnvme_lsreq) 1320 { 1321 struct lpfc_nvmet_tgtport *lpfc_nvmet = targetport->private; 1322 struct lpfc_hba *phba; 1323 struct lpfc_nodelist *ndlp; 1324 int ret; 1325 u32 hstate; 1326 1327 if (!lpfc_nvmet) 1328 return -EINVAL; 1329 1330 phba = lpfc_nvmet->phba; 1331 if (phba->pport->load_flag & FC_UNLOADING) 1332 return -EINVAL; 1333 1334 hstate = atomic_read(&lpfc_nvmet->state); 1335 if (hstate == LPFC_NVMET_INV_HOST_ACTIVE) 1336 return -EACCES; 1337 1338 ndlp = (struct lpfc_nodelist *)hosthandle; 1339 1340 ret = __lpfc_nvme_ls_req(phba->pport, ndlp, pnvme_lsreq, 1341 lpfc_nvmet_ls_req_cmp); 1342 1343 return ret; 1344 } 1345 1346 /** 1347 * lpfc_nvmet_ls_abort - Abort a prior NVME LS request 1348 * @targetport: Transport targetport, that LS was issued from. 1349 * @hosthandle - hosthandle set by the driver in a prior ls_rqst_rcv. 1350 * Driver sets this value to the ndlp pointer. 1351 * @pnvme_lsreq - the transport nvme_ls_req structure for LS to be aborted 1352 * 1353 * Driver registers this routine to abort an NVME LS request that is 1354 * in progress (from the transports perspective). 1355 **/ 1356 static void 1357 lpfc_nvmet_ls_abort(struct nvmet_fc_target_port *targetport, 1358 void *hosthandle, 1359 struct nvmefc_ls_req *pnvme_lsreq) 1360 { 1361 struct lpfc_nvmet_tgtport *lpfc_nvmet = targetport->private; 1362 struct lpfc_hba *phba; 1363 struct lpfc_nodelist *ndlp; 1364 int ret; 1365 1366 phba = lpfc_nvmet->phba; 1367 if (phba->pport->load_flag & FC_UNLOADING) 1368 return; 1369 1370 ndlp = (struct lpfc_nodelist *)hosthandle; 1371 1372 ret = __lpfc_nvme_ls_abort(phba->pport, ndlp, pnvme_lsreq); 1373 if (!ret) 1374 atomic_inc(&lpfc_nvmet->xmt_ls_abort); 1375 } 1376 1377 static void 1378 lpfc_nvmet_host_release(void *hosthandle) 1379 { 1380 struct lpfc_nodelist *ndlp = hosthandle; 1381 struct lpfc_hba *phba = NULL; 1382 struct lpfc_nvmet_tgtport *tgtp; 1383 1384 phba = ndlp->phba; 1385 if (!phba->targetport || !phba->targetport->private) 1386 return; 1387 1388 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 1389 "6202 NVMET XPT releasing hosthandle x%px\n", 1390 hosthandle); 1391 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 1392 atomic_set(&tgtp->state, 0); 1393 } 1394 1395 static void 1396 lpfc_nvmet_discovery_event(struct nvmet_fc_target_port *tgtport) 1397 { 1398 struct lpfc_nvmet_tgtport *tgtp; 1399 struct lpfc_hba *phba; 1400 uint32_t rc; 1401 1402 tgtp = tgtport->private; 1403 phba = tgtp->phba; 1404 1405 rc = lpfc_issue_els_rscn(phba->pport, 0); 1406 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 1407 "6420 NVMET subsystem change: Notification %s\n", 1408 (rc) ? "Failed" : "Sent"); 1409 } 1410 1411 static struct nvmet_fc_target_template lpfc_tgttemplate = { 1412 .targetport_delete = lpfc_nvmet_targetport_delete, 1413 .xmt_ls_rsp = lpfc_nvmet_xmt_ls_rsp, 1414 .fcp_op = lpfc_nvmet_xmt_fcp_op, 1415 .fcp_abort = lpfc_nvmet_xmt_fcp_abort, 1416 .fcp_req_release = lpfc_nvmet_xmt_fcp_release, 1417 .defer_rcv = lpfc_nvmet_defer_rcv, 1418 .discovery_event = lpfc_nvmet_discovery_event, 1419 .ls_req = lpfc_nvmet_ls_req, 1420 .ls_abort = lpfc_nvmet_ls_abort, 1421 .host_release = lpfc_nvmet_host_release, 1422 1423 .max_hw_queues = 1, 1424 .max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS, 1425 .max_dif_sgl_segments = LPFC_NVMET_DEFAULT_SEGS, 1426 .dma_boundary = 0xFFFFFFFF, 1427 1428 /* optional features */ 1429 .target_features = 0, 1430 /* sizes of additional private data for data structures */ 1431 .target_priv_sz = sizeof(struct lpfc_nvmet_tgtport), 1432 .lsrqst_priv_sz = 0, 1433 }; 1434 1435 static void 1436 __lpfc_nvmet_clean_io_for_cpu(struct lpfc_hba *phba, 1437 struct lpfc_nvmet_ctx_info *infop) 1438 { 1439 struct lpfc_nvmet_ctxbuf *ctx_buf, *next_ctx_buf; 1440 unsigned long flags; 1441 1442 spin_lock_irqsave(&infop->nvmet_ctx_list_lock, flags); 1443 list_for_each_entry_safe(ctx_buf, next_ctx_buf, 1444 &infop->nvmet_ctx_list, list) { 1445 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1446 list_del_init(&ctx_buf->list); 1447 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1448 1449 __lpfc_clear_active_sglq(phba, ctx_buf->sglq->sli4_lxritag); 1450 ctx_buf->sglq->state = SGL_FREED; 1451 ctx_buf->sglq->ndlp = NULL; 1452 1453 spin_lock(&phba->sli4_hba.sgl_list_lock); 1454 list_add_tail(&ctx_buf->sglq->list, 1455 &phba->sli4_hba.lpfc_nvmet_sgl_list); 1456 spin_unlock(&phba->sli4_hba.sgl_list_lock); 1457 1458 lpfc_sli_release_iocbq(phba, ctx_buf->iocbq); 1459 kfree(ctx_buf->context); 1460 } 1461 spin_unlock_irqrestore(&infop->nvmet_ctx_list_lock, flags); 1462 } 1463 1464 static void 1465 lpfc_nvmet_cleanup_io_context(struct lpfc_hba *phba) 1466 { 1467 struct lpfc_nvmet_ctx_info *infop; 1468 int i, j; 1469 1470 /* The first context list, MRQ 0 CPU 0 */ 1471 infop = phba->sli4_hba.nvmet_ctx_info; 1472 if (!infop) 1473 return; 1474 1475 /* Cycle the the entire CPU context list for every MRQ */ 1476 for (i = 0; i < phba->cfg_nvmet_mrq; i++) { 1477 for_each_present_cpu(j) { 1478 infop = lpfc_get_ctx_list(phba, j, i); 1479 __lpfc_nvmet_clean_io_for_cpu(phba, infop); 1480 } 1481 } 1482 kfree(phba->sli4_hba.nvmet_ctx_info); 1483 phba->sli4_hba.nvmet_ctx_info = NULL; 1484 } 1485 1486 static int 1487 lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) 1488 { 1489 struct lpfc_nvmet_ctxbuf *ctx_buf; 1490 struct lpfc_iocbq *nvmewqe; 1491 union lpfc_wqe128 *wqe; 1492 struct lpfc_nvmet_ctx_info *last_infop; 1493 struct lpfc_nvmet_ctx_info *infop; 1494 int i, j, idx, cpu; 1495 1496 lpfc_printf_log(phba, KERN_INFO, LOG_NVME, 1497 "6403 Allocate NVMET resources for %d XRIs\n", 1498 phba->sli4_hba.nvmet_xri_cnt); 1499 1500 phba->sli4_hba.nvmet_ctx_info = kcalloc( 1501 phba->sli4_hba.num_possible_cpu * phba->cfg_nvmet_mrq, 1502 sizeof(struct lpfc_nvmet_ctx_info), GFP_KERNEL); 1503 if (!phba->sli4_hba.nvmet_ctx_info) { 1504 lpfc_printf_log(phba, KERN_ERR, LOG_INIT, 1505 "6419 Failed allocate memory for " 1506 "nvmet context lists\n"); 1507 return -ENOMEM; 1508 } 1509 1510 /* 1511 * Assuming X CPUs in the system, and Y MRQs, allocate some 1512 * lpfc_nvmet_ctx_info structures as follows: 1513 * 1514 * cpu0/mrq0 cpu1/mrq0 ... cpuX/mrq0 1515 * cpu0/mrq1 cpu1/mrq1 ... cpuX/mrq1 1516 * ... 1517 * cpuX/mrqY cpuX/mrqY ... cpuX/mrqY 1518 * 1519 * Each line represents a MRQ "silo" containing an entry for 1520 * every CPU. 1521 * 1522 * MRQ X is initially assumed to be associated with CPU X, thus 1523 * contexts are initially distributed across all MRQs using 1524 * the MRQ index (N) as follows cpuN/mrqN. When contexts are 1525 * freed, the are freed to the MRQ silo based on the CPU number 1526 * of the IO completion. Thus a context that was allocated for MRQ A 1527 * whose IO completed on CPU B will be freed to cpuB/mrqA. 1528 */ 1529 for_each_possible_cpu(i) { 1530 for (j = 0; j < phba->cfg_nvmet_mrq; j++) { 1531 infop = lpfc_get_ctx_list(phba, i, j); 1532 INIT_LIST_HEAD(&infop->nvmet_ctx_list); 1533 spin_lock_init(&infop->nvmet_ctx_list_lock); 1534 infop->nvmet_ctx_list_cnt = 0; 1535 } 1536 } 1537 1538 /* 1539 * Setup the next CPU context info ptr for each MRQ. 1540 * MRQ 0 will cycle thru CPUs 0 - X separately from 1541 * MRQ 1 cycling thru CPUs 0 - X, and so on. 1542 */ 1543 for (j = 0; j < phba->cfg_nvmet_mrq; j++) { 1544 last_infop = lpfc_get_ctx_list(phba, 1545 cpumask_first(cpu_present_mask), 1546 j); 1547 for (i = phba->sli4_hba.num_possible_cpu - 1; i >= 0; i--) { 1548 infop = lpfc_get_ctx_list(phba, i, j); 1549 infop->nvmet_ctx_next_cpu = last_infop; 1550 last_infop = infop; 1551 } 1552 } 1553 1554 /* For all nvmet xris, allocate resources needed to process a 1555 * received command on a per xri basis. 1556 */ 1557 idx = 0; 1558 cpu = cpumask_first(cpu_present_mask); 1559 for (i = 0; i < phba->sli4_hba.nvmet_xri_cnt; i++) { 1560 ctx_buf = kzalloc(sizeof(*ctx_buf), GFP_KERNEL); 1561 if (!ctx_buf) { 1562 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 1563 "6404 Ran out of memory for NVMET\n"); 1564 return -ENOMEM; 1565 } 1566 1567 ctx_buf->context = kzalloc(sizeof(*ctx_buf->context), 1568 GFP_KERNEL); 1569 if (!ctx_buf->context) { 1570 kfree(ctx_buf); 1571 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 1572 "6405 Ran out of NVMET " 1573 "context memory\n"); 1574 return -ENOMEM; 1575 } 1576 ctx_buf->context->ctxbuf = ctx_buf; 1577 ctx_buf->context->state = LPFC_NVME_STE_FREE; 1578 1579 ctx_buf->iocbq = lpfc_sli_get_iocbq(phba); 1580 if (!ctx_buf->iocbq) { 1581 kfree(ctx_buf->context); 1582 kfree(ctx_buf); 1583 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 1584 "6406 Ran out of NVMET iocb/WQEs\n"); 1585 return -ENOMEM; 1586 } 1587 ctx_buf->iocbq->iocb_flag = LPFC_IO_NVMET; 1588 nvmewqe = ctx_buf->iocbq; 1589 wqe = &nvmewqe->wqe; 1590 1591 /* Initialize WQE */ 1592 memset(wqe, 0, sizeof(union lpfc_wqe)); 1593 1594 ctx_buf->iocbq->context1 = NULL; 1595 spin_lock(&phba->sli4_hba.sgl_list_lock); 1596 ctx_buf->sglq = __lpfc_sli_get_nvmet_sglq(phba, ctx_buf->iocbq); 1597 spin_unlock(&phba->sli4_hba.sgl_list_lock); 1598 if (!ctx_buf->sglq) { 1599 lpfc_sli_release_iocbq(phba, ctx_buf->iocbq); 1600 kfree(ctx_buf->context); 1601 kfree(ctx_buf); 1602 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 1603 "6407 Ran out of NVMET XRIs\n"); 1604 return -ENOMEM; 1605 } 1606 INIT_WORK(&ctx_buf->defer_work, lpfc_nvmet_fcp_rqst_defer_work); 1607 1608 /* 1609 * Add ctx to MRQidx context list. Our initial assumption 1610 * is MRQidx will be associated with CPUidx. This association 1611 * can change on the fly. 1612 */ 1613 infop = lpfc_get_ctx_list(phba, cpu, idx); 1614 spin_lock(&infop->nvmet_ctx_list_lock); 1615 list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list); 1616 infop->nvmet_ctx_list_cnt++; 1617 spin_unlock(&infop->nvmet_ctx_list_lock); 1618 1619 /* Spread ctx structures evenly across all MRQs */ 1620 idx++; 1621 if (idx >= phba->cfg_nvmet_mrq) { 1622 idx = 0; 1623 cpu = cpumask_first(cpu_present_mask); 1624 continue; 1625 } 1626 cpu = cpumask_next(cpu, cpu_present_mask); 1627 if (cpu == nr_cpu_ids) 1628 cpu = cpumask_first(cpu_present_mask); 1629 1630 } 1631 1632 for_each_present_cpu(i) { 1633 for (j = 0; j < phba->cfg_nvmet_mrq; j++) { 1634 infop = lpfc_get_ctx_list(phba, i, j); 1635 lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT, 1636 "6408 TOTAL NVMET ctx for CPU %d " 1637 "MRQ %d: cnt %d nextcpu x%px\n", 1638 i, j, infop->nvmet_ctx_list_cnt, 1639 infop->nvmet_ctx_next_cpu); 1640 } 1641 } 1642 return 0; 1643 } 1644 1645 int 1646 lpfc_nvmet_create_targetport(struct lpfc_hba *phba) 1647 { 1648 struct lpfc_vport *vport = phba->pport; 1649 struct lpfc_nvmet_tgtport *tgtp; 1650 struct nvmet_fc_port_info pinfo; 1651 int error; 1652 1653 if (phba->targetport) 1654 return 0; 1655 1656 error = lpfc_nvmet_setup_io_context(phba); 1657 if (error) 1658 return error; 1659 1660 memset(&pinfo, 0, sizeof(struct nvmet_fc_port_info)); 1661 pinfo.node_name = wwn_to_u64(vport->fc_nodename.u.wwn); 1662 pinfo.port_name = wwn_to_u64(vport->fc_portname.u.wwn); 1663 pinfo.port_id = vport->fc_myDID; 1664 1665 /* We need to tell the transport layer + 1 because it takes page 1666 * alignment into account. When space for the SGL is allocated we 1667 * allocate + 3, one for cmd, one for rsp and one for this alignment 1668 */ 1669 lpfc_tgttemplate.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; 1670 lpfc_tgttemplate.max_hw_queues = phba->cfg_hdw_queue; 1671 lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP; 1672 1673 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1674 error = nvmet_fc_register_targetport(&pinfo, &lpfc_tgttemplate, 1675 &phba->pcidev->dev, 1676 &phba->targetport); 1677 #else 1678 error = -ENOENT; 1679 #endif 1680 if (error) { 1681 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, 1682 "6025 Cannot register NVME targetport x%x: " 1683 "portnm %llx nodenm %llx segs %d qs %d\n", 1684 error, 1685 pinfo.port_name, pinfo.node_name, 1686 lpfc_tgttemplate.max_sgl_segments, 1687 lpfc_tgttemplate.max_hw_queues); 1688 phba->targetport = NULL; 1689 phba->nvmet_support = 0; 1690 1691 lpfc_nvmet_cleanup_io_context(phba); 1692 1693 } else { 1694 tgtp = (struct lpfc_nvmet_tgtport *) 1695 phba->targetport->private; 1696 tgtp->phba = phba; 1697 1698 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 1699 "6026 Registered NVME " 1700 "targetport: x%px, private x%px " 1701 "portnm %llx nodenm %llx segs %d qs %d\n", 1702 phba->targetport, tgtp, 1703 pinfo.port_name, pinfo.node_name, 1704 lpfc_tgttemplate.max_sgl_segments, 1705 lpfc_tgttemplate.max_hw_queues); 1706 1707 atomic_set(&tgtp->rcv_ls_req_in, 0); 1708 atomic_set(&tgtp->rcv_ls_req_out, 0); 1709 atomic_set(&tgtp->rcv_ls_req_drop, 0); 1710 atomic_set(&tgtp->xmt_ls_abort, 0); 1711 atomic_set(&tgtp->xmt_ls_abort_cmpl, 0); 1712 atomic_set(&tgtp->xmt_ls_rsp, 0); 1713 atomic_set(&tgtp->xmt_ls_drop, 0); 1714 atomic_set(&tgtp->xmt_ls_rsp_error, 0); 1715 atomic_set(&tgtp->xmt_ls_rsp_xb_set, 0); 1716 atomic_set(&tgtp->xmt_ls_rsp_aborted, 0); 1717 atomic_set(&tgtp->xmt_ls_rsp_cmpl, 0); 1718 atomic_set(&tgtp->rcv_fcp_cmd_in, 0); 1719 atomic_set(&tgtp->rcv_fcp_cmd_out, 0); 1720 atomic_set(&tgtp->rcv_fcp_cmd_drop, 0); 1721 atomic_set(&tgtp->xmt_fcp_drop, 0); 1722 atomic_set(&tgtp->xmt_fcp_read_rsp, 0); 1723 atomic_set(&tgtp->xmt_fcp_read, 0); 1724 atomic_set(&tgtp->xmt_fcp_write, 0); 1725 atomic_set(&tgtp->xmt_fcp_rsp, 0); 1726 atomic_set(&tgtp->xmt_fcp_release, 0); 1727 atomic_set(&tgtp->xmt_fcp_rsp_cmpl, 0); 1728 atomic_set(&tgtp->xmt_fcp_rsp_error, 0); 1729 atomic_set(&tgtp->xmt_fcp_rsp_xb_set, 0); 1730 atomic_set(&tgtp->xmt_fcp_rsp_aborted, 0); 1731 atomic_set(&tgtp->xmt_fcp_rsp_drop, 0); 1732 atomic_set(&tgtp->xmt_fcp_xri_abort_cqe, 0); 1733 atomic_set(&tgtp->xmt_fcp_abort, 0); 1734 atomic_set(&tgtp->xmt_fcp_abort_cmpl, 0); 1735 atomic_set(&tgtp->xmt_abort_unsol, 0); 1736 atomic_set(&tgtp->xmt_abort_sol, 0); 1737 atomic_set(&tgtp->xmt_abort_rsp, 0); 1738 atomic_set(&tgtp->xmt_abort_rsp_error, 0); 1739 atomic_set(&tgtp->defer_ctx, 0); 1740 atomic_set(&tgtp->defer_fod, 0); 1741 atomic_set(&tgtp->defer_wqfull, 0); 1742 } 1743 return error; 1744 } 1745 1746 int 1747 lpfc_nvmet_update_targetport(struct lpfc_hba *phba) 1748 { 1749 struct lpfc_vport *vport = phba->pport; 1750 1751 if (!phba->targetport) 1752 return 0; 1753 1754 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 1755 "6007 Update NVMET port x%px did x%x\n", 1756 phba->targetport, vport->fc_myDID); 1757 1758 phba->targetport->port_id = vport->fc_myDID; 1759 return 0; 1760 } 1761 1762 /** 1763 * lpfc_sli4_nvmet_xri_aborted - Fast-path process of nvmet xri abort 1764 * @phba: pointer to lpfc hba data structure. 1765 * @axri: pointer to the nvmet xri abort wcqe structure. 1766 * 1767 * This routine is invoked by the worker thread to process a SLI4 fast-path 1768 * NVMET aborted xri. 1769 **/ 1770 void 1771 lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, 1772 struct sli4_wcqe_xri_aborted *axri) 1773 { 1774 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1775 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); 1776 uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri); 1777 struct lpfc_async_xchg_ctx *ctxp, *next_ctxp; 1778 struct lpfc_nvmet_tgtport *tgtp; 1779 struct nvmefc_tgt_fcp_req *req = NULL; 1780 struct lpfc_nodelist *ndlp; 1781 unsigned long iflag = 0; 1782 int rrq_empty = 0; 1783 bool released = false; 1784 1785 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1786 "6317 XB aborted xri x%x rxid x%x\n", xri, rxid); 1787 1788 if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) 1789 return; 1790 1791 if (phba->targetport) { 1792 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 1793 atomic_inc(&tgtp->xmt_fcp_xri_abort_cqe); 1794 } 1795 1796 spin_lock_irqsave(&phba->hbalock, iflag); 1797 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1798 list_for_each_entry_safe(ctxp, next_ctxp, 1799 &phba->sli4_hba.lpfc_abts_nvmet_ctx_list, 1800 list) { 1801 if (ctxp->ctxbuf->sglq->sli4_xritag != xri) 1802 continue; 1803 1804 spin_lock(&ctxp->ctxlock); 1805 /* Check if we already received a free context call 1806 * and we have completed processing an abort situation. 1807 */ 1808 if (ctxp->flag & LPFC_NVME_CTX_RLS && 1809 !(ctxp->flag & LPFC_NVME_ABORT_OP)) { 1810 list_del_init(&ctxp->list); 1811 released = true; 1812 } 1813 ctxp->flag &= ~LPFC_NVME_XBUSY; 1814 spin_unlock(&ctxp->ctxlock); 1815 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1816 1817 rrq_empty = list_empty(&phba->active_rrq_list); 1818 spin_unlock_irqrestore(&phba->hbalock, iflag); 1819 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid); 1820 if (ndlp && NLP_CHK_NODE_ACT(ndlp) && 1821 (ndlp->nlp_state == NLP_STE_UNMAPPED_NODE || 1822 ndlp->nlp_state == NLP_STE_MAPPED_NODE)) { 1823 lpfc_set_rrq_active(phba, ndlp, 1824 ctxp->ctxbuf->sglq->sli4_lxritag, 1825 rxid, 1); 1826 lpfc_sli4_abts_err_handler(phba, ndlp, axri); 1827 } 1828 1829 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1830 "6318 XB aborted oxid x%x flg x%x (%x)\n", 1831 ctxp->oxid, ctxp->flag, released); 1832 if (released) 1833 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); 1834 1835 if (rrq_empty) 1836 lpfc_worker_wake_up(phba); 1837 return; 1838 } 1839 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1840 spin_unlock_irqrestore(&phba->hbalock, iflag); 1841 1842 ctxp = lpfc_nvmet_get_ctx_for_xri(phba, xri); 1843 if (ctxp) { 1844 /* 1845 * Abort already done by FW, so BA_ACC sent. 1846 * However, the transport may be unaware. 1847 */ 1848 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1849 "6323 NVMET Rcv ABTS xri x%x ctxp state x%x " 1850 "flag x%x oxid x%x rxid x%x\n", 1851 xri, ctxp->state, ctxp->flag, ctxp->oxid, 1852 rxid); 1853 1854 spin_lock_irqsave(&ctxp->ctxlock, iflag); 1855 ctxp->flag |= LPFC_NVME_ABTS_RCV; 1856 ctxp->state = LPFC_NVME_STE_ABORT; 1857 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 1858 1859 lpfc_nvmeio_data(phba, 1860 "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n", 1861 xri, raw_smp_processor_id(), 0); 1862 1863 req = &ctxp->hdlrctx.fcp_req; 1864 if (req) 1865 nvmet_fc_rcv_fcp_abort(phba->targetport, req); 1866 } 1867 #endif 1868 } 1869 1870 int 1871 lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, 1872 struct fc_frame_header *fc_hdr) 1873 { 1874 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1875 struct lpfc_hba *phba = vport->phba; 1876 struct lpfc_async_xchg_ctx *ctxp, *next_ctxp; 1877 struct nvmefc_tgt_fcp_req *rsp; 1878 uint32_t sid; 1879 uint16_t oxid, xri; 1880 unsigned long iflag = 0; 1881 1882 sid = sli4_sid_from_fc_hdr(fc_hdr); 1883 oxid = be16_to_cpu(fc_hdr->fh_ox_id); 1884 1885 spin_lock_irqsave(&phba->hbalock, iflag); 1886 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1887 list_for_each_entry_safe(ctxp, next_ctxp, 1888 &phba->sli4_hba.lpfc_abts_nvmet_ctx_list, 1889 list) { 1890 if (ctxp->oxid != oxid || ctxp->sid != sid) 1891 continue; 1892 1893 xri = ctxp->ctxbuf->sglq->sli4_xritag; 1894 1895 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1896 spin_unlock_irqrestore(&phba->hbalock, iflag); 1897 1898 spin_lock_irqsave(&ctxp->ctxlock, iflag); 1899 ctxp->flag |= LPFC_NVME_ABTS_RCV; 1900 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 1901 1902 lpfc_nvmeio_data(phba, 1903 "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n", 1904 xri, raw_smp_processor_id(), 0); 1905 1906 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1907 "6319 NVMET Rcv ABTS:acc xri x%x\n", xri); 1908 1909 rsp = &ctxp->hdlrctx.fcp_req; 1910 nvmet_fc_rcv_fcp_abort(phba->targetport, rsp); 1911 1912 /* Respond with BA_ACC accordingly */ 1913 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1); 1914 return 0; 1915 } 1916 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1917 spin_unlock_irqrestore(&phba->hbalock, iflag); 1918 1919 /* check the wait list */ 1920 if (phba->sli4_hba.nvmet_io_wait_cnt) { 1921 struct rqb_dmabuf *nvmebuf; 1922 struct fc_frame_header *fc_hdr_tmp; 1923 u32 sid_tmp; 1924 u16 oxid_tmp; 1925 bool found = false; 1926 1927 spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag); 1928 1929 /* match by oxid and s_id */ 1930 list_for_each_entry(nvmebuf, 1931 &phba->sli4_hba.lpfc_nvmet_io_wait_list, 1932 hbuf.list) { 1933 fc_hdr_tmp = (struct fc_frame_header *) 1934 (nvmebuf->hbuf.virt); 1935 oxid_tmp = be16_to_cpu(fc_hdr_tmp->fh_ox_id); 1936 sid_tmp = sli4_sid_from_fc_hdr(fc_hdr_tmp); 1937 if (oxid_tmp != oxid || sid_tmp != sid) 1938 continue; 1939 1940 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1941 "6321 NVMET Rcv ABTS oxid x%x from x%x " 1942 "is waiting for a ctxp\n", 1943 oxid, sid); 1944 1945 list_del_init(&nvmebuf->hbuf.list); 1946 phba->sli4_hba.nvmet_io_wait_cnt--; 1947 found = true; 1948 break; 1949 } 1950 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, 1951 iflag); 1952 1953 /* free buffer since already posted a new DMA buffer to RQ */ 1954 if (found) { 1955 nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); 1956 /* Respond with BA_ACC accordingly */ 1957 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1); 1958 return 0; 1959 } 1960 } 1961 1962 /* check active list */ 1963 ctxp = lpfc_nvmet_get_ctx_for_oxid(phba, oxid, sid); 1964 if (ctxp) { 1965 xri = ctxp->ctxbuf->sglq->sli4_xritag; 1966 1967 spin_lock_irqsave(&ctxp->ctxlock, iflag); 1968 ctxp->flag |= (LPFC_NVME_ABTS_RCV | LPFC_NVME_ABORT_OP); 1969 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 1970 1971 lpfc_nvmeio_data(phba, 1972 "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n", 1973 xri, raw_smp_processor_id(), 0); 1974 1975 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1976 "6322 NVMET Rcv ABTS:acc oxid x%x xri x%x " 1977 "flag x%x state x%x\n", 1978 ctxp->oxid, xri, ctxp->flag, ctxp->state); 1979 1980 if (ctxp->flag & LPFC_NVME_TNOTIFY) { 1981 /* Notify the transport */ 1982 nvmet_fc_rcv_fcp_abort(phba->targetport, 1983 &ctxp->hdlrctx.fcp_req); 1984 } else { 1985 cancel_work_sync(&ctxp->ctxbuf->defer_work); 1986 spin_lock_irqsave(&ctxp->ctxlock, iflag); 1987 lpfc_nvmet_defer_release(phba, ctxp); 1988 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 1989 } 1990 lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid, 1991 ctxp->oxid); 1992 1993 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1); 1994 return 0; 1995 } 1996 1997 lpfc_nvmeio_data(phba, "NVMET ABTS RCV: oxid x%x CPU %02x rjt %d\n", 1998 oxid, raw_smp_processor_id(), 1); 1999 2000 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2001 "6320 NVMET Rcv ABTS:rjt oxid x%x\n", oxid); 2002 2003 /* Respond with BA_RJT accordingly */ 2004 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 0); 2005 #endif 2006 return 0; 2007 } 2008 2009 static void 2010 lpfc_nvmet_wqfull_flush(struct lpfc_hba *phba, struct lpfc_queue *wq, 2011 struct lpfc_async_xchg_ctx *ctxp) 2012 { 2013 struct lpfc_sli_ring *pring; 2014 struct lpfc_iocbq *nvmewqeq; 2015 struct lpfc_iocbq *next_nvmewqeq; 2016 unsigned long iflags; 2017 struct lpfc_wcqe_complete wcqe; 2018 struct lpfc_wcqe_complete *wcqep; 2019 2020 pring = wq->pring; 2021 wcqep = &wcqe; 2022 2023 /* Fake an ABORT error code back to cmpl routine */ 2024 memset(wcqep, 0, sizeof(struct lpfc_wcqe_complete)); 2025 bf_set(lpfc_wcqe_c_status, wcqep, IOSTAT_LOCAL_REJECT); 2026 wcqep->parameter = IOERR_ABORT_REQUESTED; 2027 2028 spin_lock_irqsave(&pring->ring_lock, iflags); 2029 list_for_each_entry_safe(nvmewqeq, next_nvmewqeq, 2030 &wq->wqfull_list, list) { 2031 if (ctxp) { 2032 /* Checking for a specific IO to flush */ 2033 if (nvmewqeq->context2 == ctxp) { 2034 list_del(&nvmewqeq->list); 2035 spin_unlock_irqrestore(&pring->ring_lock, 2036 iflags); 2037 lpfc_nvmet_xmt_fcp_op_cmp(phba, nvmewqeq, 2038 wcqep); 2039 return; 2040 } 2041 continue; 2042 } else { 2043 /* Flush all IOs */ 2044 list_del(&nvmewqeq->list); 2045 spin_unlock_irqrestore(&pring->ring_lock, iflags); 2046 lpfc_nvmet_xmt_fcp_op_cmp(phba, nvmewqeq, wcqep); 2047 spin_lock_irqsave(&pring->ring_lock, iflags); 2048 } 2049 } 2050 if (!ctxp) 2051 wq->q_flag &= ~HBA_NVMET_WQFULL; 2052 spin_unlock_irqrestore(&pring->ring_lock, iflags); 2053 } 2054 2055 void 2056 lpfc_nvmet_wqfull_process(struct lpfc_hba *phba, 2057 struct lpfc_queue *wq) 2058 { 2059 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 2060 struct lpfc_sli_ring *pring; 2061 struct lpfc_iocbq *nvmewqeq; 2062 struct lpfc_async_xchg_ctx *ctxp; 2063 unsigned long iflags; 2064 int rc; 2065 2066 /* 2067 * Some WQE slots are available, so try to re-issue anything 2068 * on the WQ wqfull_list. 2069 */ 2070 pring = wq->pring; 2071 spin_lock_irqsave(&pring->ring_lock, iflags); 2072 while (!list_empty(&wq->wqfull_list)) { 2073 list_remove_head(&wq->wqfull_list, nvmewqeq, struct lpfc_iocbq, 2074 list); 2075 spin_unlock_irqrestore(&pring->ring_lock, iflags); 2076 ctxp = (struct lpfc_async_xchg_ctx *)nvmewqeq->context2; 2077 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq); 2078 spin_lock_irqsave(&pring->ring_lock, iflags); 2079 if (rc == -EBUSY) { 2080 /* WQ was full again, so put it back on the list */ 2081 list_add(&nvmewqeq->list, &wq->wqfull_list); 2082 spin_unlock_irqrestore(&pring->ring_lock, iflags); 2083 return; 2084 } 2085 if (rc == WQE_SUCCESS) { 2086 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 2087 if (ctxp->ts_cmd_nvme) { 2088 if (ctxp->hdlrctx.fcp_req.op == NVMET_FCOP_RSP) 2089 ctxp->ts_status_wqput = ktime_get_ns(); 2090 else 2091 ctxp->ts_data_wqput = ktime_get_ns(); 2092 } 2093 #endif 2094 } else { 2095 WARN_ON(rc); 2096 } 2097 } 2098 wq->q_flag &= ~HBA_NVMET_WQFULL; 2099 spin_unlock_irqrestore(&pring->ring_lock, iflags); 2100 2101 #endif 2102 } 2103 2104 void 2105 lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba) 2106 { 2107 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 2108 struct lpfc_nvmet_tgtport *tgtp; 2109 struct lpfc_queue *wq; 2110 uint32_t qidx; 2111 DECLARE_COMPLETION_ONSTACK(tport_unreg_cmp); 2112 2113 if (phba->nvmet_support == 0) 2114 return; 2115 if (phba->targetport) { 2116 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2117 for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { 2118 wq = phba->sli4_hba.hdwq[qidx].io_wq; 2119 lpfc_nvmet_wqfull_flush(phba, wq, NULL); 2120 } 2121 tgtp->tport_unreg_cmp = &tport_unreg_cmp; 2122 nvmet_fc_unregister_targetport(phba->targetport); 2123 if (!wait_for_completion_timeout(tgtp->tport_unreg_cmp, 2124 msecs_to_jiffies(LPFC_NVMET_WAIT_TMO))) 2125 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 2126 "6179 Unreg targetport x%px timeout " 2127 "reached.\n", phba->targetport); 2128 lpfc_nvmet_cleanup_io_context(phba); 2129 } 2130 phba->targetport = NULL; 2131 #endif 2132 } 2133 2134 /** 2135 * lpfc_nvmet_handle_lsreq - Process an NVME LS request 2136 * @phba: pointer to lpfc hba data structure. 2137 * @axchg: pointer to exchange context for the NVME LS request 2138 * 2139 * This routine is used for processing an asychronously received NVME LS 2140 * request. Any remaining validation is done and the LS is then forwarded 2141 * to the nvmet-fc transport via nvmet_fc_rcv_ls_req(). 2142 * 2143 * The calling sequence should be: nvmet_fc_rcv_ls_req() -> (processing) 2144 * -> lpfc_nvmet_xmt_ls_rsp/cmp -> req->done. 2145 * lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg. 2146 * 2147 * Returns 0 if LS was handled and delivered to the transport 2148 * Returns 1 if LS failed to be handled and should be dropped 2149 */ 2150 int 2151 lpfc_nvmet_handle_lsreq(struct lpfc_hba *phba, 2152 struct lpfc_async_xchg_ctx *axchg) 2153 { 2154 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 2155 struct lpfc_nvmet_tgtport *tgtp = phba->targetport->private; 2156 uint32_t *payload = axchg->payload; 2157 int rc; 2158 2159 atomic_inc(&tgtp->rcv_ls_req_in); 2160 2161 /* 2162 * Driver passes the ndlp as the hosthandle argument allowing 2163 * the transport to generate LS requests for any associateions 2164 * that are created. 2165 */ 2166 rc = nvmet_fc_rcv_ls_req(phba->targetport, axchg->ndlp, &axchg->ls_rsp, 2167 axchg->payload, axchg->size); 2168 2169 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 2170 "6037 NVMET Unsol rcv: sz %d rc %d: %08x %08x %08x " 2171 "%08x %08x %08x\n", axchg->size, rc, 2172 *payload, *(payload+1), *(payload+2), 2173 *(payload+3), *(payload+4), *(payload+5)); 2174 2175 if (!rc) { 2176 atomic_inc(&tgtp->rcv_ls_req_out); 2177 return 0; 2178 } 2179 2180 atomic_inc(&tgtp->rcv_ls_req_drop); 2181 #endif 2182 return 1; 2183 } 2184 2185 static void 2186 lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf) 2187 { 2188 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 2189 struct lpfc_async_xchg_ctx *ctxp = ctx_buf->context; 2190 struct lpfc_hba *phba = ctxp->phba; 2191 struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer; 2192 struct lpfc_nvmet_tgtport *tgtp; 2193 uint32_t *payload, qno; 2194 uint32_t rc; 2195 unsigned long iflags; 2196 2197 if (!nvmebuf) { 2198 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2199 "6159 process_rcv_fcp_req, nvmebuf is NULL, " 2200 "oxid: x%x flg: x%x state: x%x\n", 2201 ctxp->oxid, ctxp->flag, ctxp->state); 2202 spin_lock_irqsave(&ctxp->ctxlock, iflags); 2203 lpfc_nvmet_defer_release(phba, ctxp); 2204 spin_unlock_irqrestore(&ctxp->ctxlock, iflags); 2205 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, 2206 ctxp->oxid); 2207 return; 2208 } 2209 2210 if (ctxp->flag & LPFC_NVME_ABTS_RCV) { 2211 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2212 "6324 IO oxid x%x aborted\n", 2213 ctxp->oxid); 2214 return; 2215 } 2216 2217 payload = (uint32_t *)(nvmebuf->dbuf.virt); 2218 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2219 ctxp->flag |= LPFC_NVME_TNOTIFY; 2220 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 2221 if (ctxp->ts_isr_cmd) 2222 ctxp->ts_cmd_nvme = ktime_get_ns(); 2223 #endif 2224 /* 2225 * The calling sequence should be: 2226 * nvmet_fc_rcv_fcp_req->lpfc_nvmet_xmt_fcp_op/cmp- req->done 2227 * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp. 2228 * When we return from nvmet_fc_rcv_fcp_req, all relevant info 2229 * the NVME command / FC header is stored. 2230 * A buffer has already been reposted for this IO, so just free 2231 * the nvmebuf. 2232 */ 2233 rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->hdlrctx.fcp_req, 2234 payload, ctxp->size); 2235 /* Process FCP command */ 2236 if (rc == 0) { 2237 atomic_inc(&tgtp->rcv_fcp_cmd_out); 2238 spin_lock_irqsave(&ctxp->ctxlock, iflags); 2239 if ((ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) || 2240 (nvmebuf != ctxp->rqb_buffer)) { 2241 spin_unlock_irqrestore(&ctxp->ctxlock, iflags); 2242 return; 2243 } 2244 ctxp->rqb_buffer = NULL; 2245 spin_unlock_irqrestore(&ctxp->ctxlock, iflags); 2246 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */ 2247 return; 2248 } 2249 2250 /* Processing of FCP command is deferred */ 2251 if (rc == -EOVERFLOW) { 2252 lpfc_nvmeio_data(phba, "NVMET RCV BUSY: xri x%x sz %d " 2253 "from %06x\n", 2254 ctxp->oxid, ctxp->size, ctxp->sid); 2255 atomic_inc(&tgtp->rcv_fcp_cmd_out); 2256 atomic_inc(&tgtp->defer_fod); 2257 spin_lock_irqsave(&ctxp->ctxlock, iflags); 2258 if (ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) { 2259 spin_unlock_irqrestore(&ctxp->ctxlock, iflags); 2260 return; 2261 } 2262 spin_unlock_irqrestore(&ctxp->ctxlock, iflags); 2263 /* 2264 * Post a replacement DMA buffer to RQ and defer 2265 * freeing rcv buffer till .defer_rcv callback 2266 */ 2267 qno = nvmebuf->idx; 2268 lpfc_post_rq_buffer( 2269 phba, phba->sli4_hba.nvmet_mrq_hdr[qno], 2270 phba->sli4_hba.nvmet_mrq_data[qno], 1, qno); 2271 return; 2272 } 2273 ctxp->flag &= ~LPFC_NVME_TNOTIFY; 2274 atomic_inc(&tgtp->rcv_fcp_cmd_drop); 2275 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2276 "2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n", 2277 ctxp->oxid, rc, 2278 atomic_read(&tgtp->rcv_fcp_cmd_in), 2279 atomic_read(&tgtp->rcv_fcp_cmd_out), 2280 atomic_read(&tgtp->xmt_fcp_release)); 2281 lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n", 2282 ctxp->oxid, ctxp->size, ctxp->sid); 2283 spin_lock_irqsave(&ctxp->ctxlock, iflags); 2284 lpfc_nvmet_defer_release(phba, ctxp); 2285 spin_unlock_irqrestore(&ctxp->ctxlock, iflags); 2286 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); 2287 #endif 2288 } 2289 2290 static void 2291 lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *work) 2292 { 2293 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 2294 struct lpfc_nvmet_ctxbuf *ctx_buf = 2295 container_of(work, struct lpfc_nvmet_ctxbuf, defer_work); 2296 2297 lpfc_nvmet_process_rcv_fcp_req(ctx_buf); 2298 #endif 2299 } 2300 2301 static struct lpfc_nvmet_ctxbuf * 2302 lpfc_nvmet_replenish_context(struct lpfc_hba *phba, 2303 struct lpfc_nvmet_ctx_info *current_infop) 2304 { 2305 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 2306 struct lpfc_nvmet_ctxbuf *ctx_buf = NULL; 2307 struct lpfc_nvmet_ctx_info *get_infop; 2308 int i; 2309 2310 /* 2311 * The current_infop for the MRQ a NVME command IU was received 2312 * on is empty. Our goal is to replenish this MRQs context 2313 * list from a another CPUs. 2314 * 2315 * First we need to pick a context list to start looking on. 2316 * nvmet_ctx_start_cpu has available context the last time 2317 * we needed to replenish this CPU where nvmet_ctx_next_cpu 2318 * is just the next sequential CPU for this MRQ. 2319 */ 2320 if (current_infop->nvmet_ctx_start_cpu) 2321 get_infop = current_infop->nvmet_ctx_start_cpu; 2322 else 2323 get_infop = current_infop->nvmet_ctx_next_cpu; 2324 2325 for (i = 0; i < phba->sli4_hba.num_possible_cpu; i++) { 2326 if (get_infop == current_infop) { 2327 get_infop = get_infop->nvmet_ctx_next_cpu; 2328 continue; 2329 } 2330 spin_lock(&get_infop->nvmet_ctx_list_lock); 2331 2332 /* Just take the entire context list, if there are any */ 2333 if (get_infop->nvmet_ctx_list_cnt) { 2334 list_splice_init(&get_infop->nvmet_ctx_list, 2335 ¤t_infop->nvmet_ctx_list); 2336 current_infop->nvmet_ctx_list_cnt = 2337 get_infop->nvmet_ctx_list_cnt - 1; 2338 get_infop->nvmet_ctx_list_cnt = 0; 2339 spin_unlock(&get_infop->nvmet_ctx_list_lock); 2340 2341 current_infop->nvmet_ctx_start_cpu = get_infop; 2342 list_remove_head(¤t_infop->nvmet_ctx_list, 2343 ctx_buf, struct lpfc_nvmet_ctxbuf, 2344 list); 2345 return ctx_buf; 2346 } 2347 2348 /* Otherwise, move on to the next CPU for this MRQ */ 2349 spin_unlock(&get_infop->nvmet_ctx_list_lock); 2350 get_infop = get_infop->nvmet_ctx_next_cpu; 2351 } 2352 2353 #endif 2354 /* Nothing found, all contexts for the MRQ are in-flight */ 2355 return NULL; 2356 } 2357 2358 /** 2359 * lpfc_nvmet_unsol_fcp_buffer - Process an unsolicited event data buffer 2360 * @phba: pointer to lpfc hba data structure. 2361 * @idx: relative index of MRQ vector 2362 * @nvmebuf: pointer to lpfc nvme command HBQ data structure. 2363 * @isr_timestamp: in jiffies. 2364 * @cqflag: cq processing information regarding workload. 2365 * 2366 * This routine is used for processing the WQE associated with a unsolicited 2367 * event. It first determines whether there is an existing ndlp that matches 2368 * the DID from the unsolicited WQE. If not, it will create a new one with 2369 * the DID from the unsolicited WQE. The ELS command from the unsolicited 2370 * WQE is then used to invoke the proper routine and to set up proper state 2371 * of the discovery state machine. 2372 **/ 2373 static void 2374 lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, 2375 uint32_t idx, 2376 struct rqb_dmabuf *nvmebuf, 2377 uint64_t isr_timestamp, 2378 uint8_t cqflag) 2379 { 2380 struct lpfc_async_xchg_ctx *ctxp; 2381 struct lpfc_nvmet_tgtport *tgtp; 2382 struct fc_frame_header *fc_hdr; 2383 struct lpfc_nvmet_ctxbuf *ctx_buf; 2384 struct lpfc_nvmet_ctx_info *current_infop; 2385 uint32_t size, oxid, sid, qno; 2386 unsigned long iflag; 2387 int current_cpu; 2388 2389 if (!IS_ENABLED(CONFIG_NVME_TARGET_FC)) 2390 return; 2391 2392 ctx_buf = NULL; 2393 if (!nvmebuf || !phba->targetport) { 2394 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2395 "6157 NVMET FCP Drop IO\n"); 2396 if (nvmebuf) 2397 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); 2398 return; 2399 } 2400 2401 /* 2402 * Get a pointer to the context list for this MRQ based on 2403 * the CPU this MRQ IRQ is associated with. If the CPU association 2404 * changes from our initial assumption, the context list could 2405 * be empty, thus it would need to be replenished with the 2406 * context list from another CPU for this MRQ. 2407 */ 2408 current_cpu = raw_smp_processor_id(); 2409 current_infop = lpfc_get_ctx_list(phba, current_cpu, idx); 2410 spin_lock_irqsave(¤t_infop->nvmet_ctx_list_lock, iflag); 2411 if (current_infop->nvmet_ctx_list_cnt) { 2412 list_remove_head(¤t_infop->nvmet_ctx_list, 2413 ctx_buf, struct lpfc_nvmet_ctxbuf, list); 2414 current_infop->nvmet_ctx_list_cnt--; 2415 } else { 2416 ctx_buf = lpfc_nvmet_replenish_context(phba, current_infop); 2417 } 2418 spin_unlock_irqrestore(¤t_infop->nvmet_ctx_list_lock, iflag); 2419 2420 fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt); 2421 oxid = be16_to_cpu(fc_hdr->fh_ox_id); 2422 size = nvmebuf->bytes_recv; 2423 2424 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 2425 if (phba->hdwqstat_on & LPFC_CHECK_NVMET_IO) { 2426 this_cpu_inc(phba->sli4_hba.c_stat->rcv_io); 2427 if (idx != current_cpu) 2428 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 2429 "6703 CPU Check rcv: " 2430 "cpu %d expect %d\n", 2431 current_cpu, idx); 2432 } 2433 #endif 2434 2435 lpfc_nvmeio_data(phba, "NVMET FCP RCV: xri x%x sz %d CPU %02x\n", 2436 oxid, size, raw_smp_processor_id()); 2437 2438 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2439 2440 if (!ctx_buf) { 2441 /* Queue this NVME IO to process later */ 2442 spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag); 2443 list_add_tail(&nvmebuf->hbuf.list, 2444 &phba->sli4_hba.lpfc_nvmet_io_wait_list); 2445 phba->sli4_hba.nvmet_io_wait_cnt++; 2446 phba->sli4_hba.nvmet_io_wait_total++; 2447 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, 2448 iflag); 2449 2450 /* Post a brand new DMA buffer to RQ */ 2451 qno = nvmebuf->idx; 2452 lpfc_post_rq_buffer( 2453 phba, phba->sli4_hba.nvmet_mrq_hdr[qno], 2454 phba->sli4_hba.nvmet_mrq_data[qno], 1, qno); 2455 2456 atomic_inc(&tgtp->defer_ctx); 2457 return; 2458 } 2459 2460 sid = sli4_sid_from_fc_hdr(fc_hdr); 2461 2462 ctxp = (struct lpfc_async_xchg_ctx *)ctx_buf->context; 2463 spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag); 2464 list_add_tail(&ctxp->list, &phba->sli4_hba.t_active_ctx_list); 2465 spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag); 2466 if (ctxp->state != LPFC_NVME_STE_FREE) { 2467 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2468 "6414 NVMET Context corrupt %d %d oxid x%x\n", 2469 ctxp->state, ctxp->entry_cnt, ctxp->oxid); 2470 } 2471 ctxp->wqeq = NULL; 2472 ctxp->offset = 0; 2473 ctxp->phba = phba; 2474 ctxp->size = size; 2475 ctxp->oxid = oxid; 2476 ctxp->sid = sid; 2477 ctxp->idx = idx; 2478 ctxp->state = LPFC_NVME_STE_RCV; 2479 ctxp->entry_cnt = 1; 2480 ctxp->flag = 0; 2481 ctxp->ctxbuf = ctx_buf; 2482 ctxp->rqb_buffer = (void *)nvmebuf; 2483 ctxp->hdwq = NULL; 2484 spin_lock_init(&ctxp->ctxlock); 2485 2486 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 2487 if (isr_timestamp) 2488 ctxp->ts_isr_cmd = isr_timestamp; 2489 ctxp->ts_cmd_nvme = 0; 2490 ctxp->ts_nvme_data = 0; 2491 ctxp->ts_data_wqput = 0; 2492 ctxp->ts_isr_data = 0; 2493 ctxp->ts_data_nvme = 0; 2494 ctxp->ts_nvme_status = 0; 2495 ctxp->ts_status_wqput = 0; 2496 ctxp->ts_isr_status = 0; 2497 ctxp->ts_status_nvme = 0; 2498 #endif 2499 2500 atomic_inc(&tgtp->rcv_fcp_cmd_in); 2501 /* check for cq processing load */ 2502 if (!cqflag) { 2503 lpfc_nvmet_process_rcv_fcp_req(ctx_buf); 2504 return; 2505 } 2506 2507 if (!queue_work(phba->wq, &ctx_buf->defer_work)) { 2508 atomic_inc(&tgtp->rcv_fcp_cmd_drop); 2509 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 2510 "6325 Unable to queue work for oxid x%x. " 2511 "FCP Drop IO [x%x x%x x%x]\n", 2512 ctxp->oxid, 2513 atomic_read(&tgtp->rcv_fcp_cmd_in), 2514 atomic_read(&tgtp->rcv_fcp_cmd_out), 2515 atomic_read(&tgtp->xmt_fcp_release)); 2516 2517 spin_lock_irqsave(&ctxp->ctxlock, iflag); 2518 lpfc_nvmet_defer_release(phba, ctxp); 2519 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 2520 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid); 2521 } 2522 } 2523 2524 /** 2525 * lpfc_nvmet_unsol_fcp_event - Process an unsolicited event from an nvme nport 2526 * @phba: pointer to lpfc hba data structure. 2527 * @idx: relative index of MRQ vector 2528 * @nvmebuf: pointer to received nvme data structure. 2529 * @isr_timestamp: in jiffies. 2530 * @cqflag: cq processing information regarding workload. 2531 * 2532 * This routine is used to process an unsolicited event received from a SLI 2533 * (Service Level Interface) ring. The actual processing of the data buffer 2534 * associated with the unsolicited event is done by invoking the routine 2535 * lpfc_nvmet_unsol_fcp_buffer() after properly set up the buffer from the 2536 * SLI RQ on which the unsolicited event was received. 2537 **/ 2538 void 2539 lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba, 2540 uint32_t idx, 2541 struct rqb_dmabuf *nvmebuf, 2542 uint64_t isr_timestamp, 2543 uint8_t cqflag) 2544 { 2545 if (!nvmebuf) { 2546 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2547 "3167 NVMET FCP Drop IO\n"); 2548 return; 2549 } 2550 if (phba->nvmet_support == 0) { 2551 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); 2552 return; 2553 } 2554 lpfc_nvmet_unsol_fcp_buffer(phba, idx, nvmebuf, isr_timestamp, cqflag); 2555 } 2556 2557 /** 2558 * lpfc_nvmet_prep_ls_wqe - Allocate and prepare a lpfc wqe data structure 2559 * @phba: pointer to a host N_Port data structure. 2560 * @ctxp: Context info for NVME LS Request 2561 * @rspbuf: DMA buffer of NVME command. 2562 * @rspsize: size of the NVME command. 2563 * 2564 * This routine is used for allocating a lpfc-WQE data structure from 2565 * the driver lpfc-WQE free-list and prepare the WQE with the parameters 2566 * passed into the routine for discovery state machine to issue an Extended 2567 * Link Service (NVME) commands. It is a generic lpfc-WQE allocation 2568 * and preparation routine that is used by all the discovery state machine 2569 * routines and the NVME command-specific fields will be later set up by 2570 * the individual discovery machine routines after calling this routine 2571 * allocating and preparing a generic WQE data structure. It fills in the 2572 * Buffer Descriptor Entries (BDEs), allocates buffers for both command 2573 * payload and response payload (if expected). The reference count on the 2574 * ndlp is incremented by 1 and the reference to the ndlp is put into 2575 * context1 of the WQE data structure for this WQE to hold the ndlp 2576 * reference for the command's callback function to access later. 2577 * 2578 * Return code 2579 * Pointer to the newly allocated/prepared nvme wqe data structure 2580 * NULL - when nvme wqe data structure allocation/preparation failed 2581 **/ 2582 static struct lpfc_iocbq * 2583 lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *phba, 2584 struct lpfc_async_xchg_ctx *ctxp, 2585 dma_addr_t rspbuf, uint16_t rspsize) 2586 { 2587 struct lpfc_nodelist *ndlp; 2588 struct lpfc_iocbq *nvmewqe; 2589 union lpfc_wqe128 *wqe; 2590 2591 if (!lpfc_is_link_up(phba)) { 2592 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, 2593 "6104 NVMET prep LS wqe: link err: " 2594 "NPORT x%x oxid:x%x ste %d\n", 2595 ctxp->sid, ctxp->oxid, ctxp->state); 2596 return NULL; 2597 } 2598 2599 /* Allocate buffer for command wqe */ 2600 nvmewqe = lpfc_sli_get_iocbq(phba); 2601 if (nvmewqe == NULL) { 2602 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, 2603 "6105 NVMET prep LS wqe: No WQE: " 2604 "NPORT x%x oxid x%x ste %d\n", 2605 ctxp->sid, ctxp->oxid, ctxp->state); 2606 return NULL; 2607 } 2608 2609 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid); 2610 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || 2611 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && 2612 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { 2613 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, 2614 "6106 NVMET prep LS wqe: No ndlp: " 2615 "NPORT x%x oxid x%x ste %d\n", 2616 ctxp->sid, ctxp->oxid, ctxp->state); 2617 goto nvme_wqe_free_wqeq_exit; 2618 } 2619 ctxp->wqeq = nvmewqe; 2620 2621 /* prevent preparing wqe with NULL ndlp reference */ 2622 nvmewqe->context1 = lpfc_nlp_get(ndlp); 2623 if (nvmewqe->context1 == NULL) 2624 goto nvme_wqe_free_wqeq_exit; 2625 nvmewqe->context2 = ctxp; 2626 2627 wqe = &nvmewqe->wqe; 2628 memset(wqe, 0, sizeof(union lpfc_wqe)); 2629 2630 /* Words 0 - 2 */ 2631 wqe->xmit_sequence.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 2632 wqe->xmit_sequence.bde.tus.f.bdeSize = rspsize; 2633 wqe->xmit_sequence.bde.addrLow = le32_to_cpu(putPaddrLow(rspbuf)); 2634 wqe->xmit_sequence.bde.addrHigh = le32_to_cpu(putPaddrHigh(rspbuf)); 2635 2636 /* Word 3 */ 2637 2638 /* Word 4 */ 2639 2640 /* Word 5 */ 2641 bf_set(wqe_dfctl, &wqe->xmit_sequence.wge_ctl, 0); 2642 bf_set(wqe_ls, &wqe->xmit_sequence.wge_ctl, 1); 2643 bf_set(wqe_la, &wqe->xmit_sequence.wge_ctl, 0); 2644 bf_set(wqe_rctl, &wqe->xmit_sequence.wge_ctl, FC_RCTL_ELS4_REP); 2645 bf_set(wqe_type, &wqe->xmit_sequence.wge_ctl, FC_TYPE_NVME); 2646 2647 /* Word 6 */ 2648 bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com, 2649 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 2650 bf_set(wqe_xri_tag, &wqe->xmit_sequence.wqe_com, nvmewqe->sli4_xritag); 2651 2652 /* Word 7 */ 2653 bf_set(wqe_cmnd, &wqe->xmit_sequence.wqe_com, 2654 CMD_XMIT_SEQUENCE64_WQE); 2655 bf_set(wqe_ct, &wqe->xmit_sequence.wqe_com, SLI4_CT_RPI); 2656 bf_set(wqe_class, &wqe->xmit_sequence.wqe_com, CLASS3); 2657 bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0); 2658 2659 /* Word 8 */ 2660 wqe->xmit_sequence.wqe_com.abort_tag = nvmewqe->iotag; 2661 2662 /* Word 9 */ 2663 bf_set(wqe_reqtag, &wqe->xmit_sequence.wqe_com, nvmewqe->iotag); 2664 /* Needs to be set by caller */ 2665 bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, ctxp->oxid); 2666 2667 /* Word 10 */ 2668 bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1); 2669 bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE); 2670 bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com, 2671 LPFC_WQE_LENLOC_WORD12); 2672 bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0); 2673 2674 /* Word 11 */ 2675 bf_set(wqe_cqid, &wqe->xmit_sequence.wqe_com, 2676 LPFC_WQE_CQ_ID_DEFAULT); 2677 bf_set(wqe_cmd_type, &wqe->xmit_sequence.wqe_com, 2678 OTHER_COMMAND); 2679 2680 /* Word 12 */ 2681 wqe->xmit_sequence.xmit_len = rspsize; 2682 2683 nvmewqe->retry = 1; 2684 nvmewqe->vport = phba->pport; 2685 nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT; 2686 nvmewqe->iocb_flag |= LPFC_IO_NVME_LS; 2687 2688 /* Xmit NVMET response to remote NPORT <did> */ 2689 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 2690 "6039 Xmit NVMET LS response to remote " 2691 "NPORT x%x iotag:x%x oxid:x%x size:x%x\n", 2692 ndlp->nlp_DID, nvmewqe->iotag, ctxp->oxid, 2693 rspsize); 2694 return nvmewqe; 2695 2696 nvme_wqe_free_wqeq_exit: 2697 nvmewqe->context2 = NULL; 2698 nvmewqe->context3 = NULL; 2699 lpfc_sli_release_iocbq(phba, nvmewqe); 2700 return NULL; 2701 } 2702 2703 2704 static struct lpfc_iocbq * 2705 lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba, 2706 struct lpfc_async_xchg_ctx *ctxp) 2707 { 2708 struct nvmefc_tgt_fcp_req *rsp = &ctxp->hdlrctx.fcp_req; 2709 struct lpfc_nvmet_tgtport *tgtp; 2710 struct sli4_sge *sgl; 2711 struct lpfc_nodelist *ndlp; 2712 struct lpfc_iocbq *nvmewqe; 2713 struct scatterlist *sgel; 2714 union lpfc_wqe128 *wqe; 2715 struct ulp_bde64 *bde; 2716 dma_addr_t physaddr; 2717 int i, cnt, nsegs; 2718 int do_pbde; 2719 int xc = 1; 2720 2721 if (!lpfc_is_link_up(phba)) { 2722 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2723 "6107 NVMET prep FCP wqe: link err:" 2724 "NPORT x%x oxid x%x ste %d\n", 2725 ctxp->sid, ctxp->oxid, ctxp->state); 2726 return NULL; 2727 } 2728 2729 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid); 2730 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || 2731 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && 2732 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { 2733 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2734 "6108 NVMET prep FCP wqe: no ndlp: " 2735 "NPORT x%x oxid x%x ste %d\n", 2736 ctxp->sid, ctxp->oxid, ctxp->state); 2737 return NULL; 2738 } 2739 2740 if (rsp->sg_cnt > lpfc_tgttemplate.max_sgl_segments) { 2741 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2742 "6109 NVMET prep FCP wqe: seg cnt err: " 2743 "NPORT x%x oxid x%x ste %d cnt %d\n", 2744 ctxp->sid, ctxp->oxid, ctxp->state, 2745 phba->cfg_nvme_seg_cnt); 2746 return NULL; 2747 } 2748 nsegs = rsp->sg_cnt; 2749 2750 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2751 nvmewqe = ctxp->wqeq; 2752 if (nvmewqe == NULL) { 2753 /* Allocate buffer for command wqe */ 2754 nvmewqe = ctxp->ctxbuf->iocbq; 2755 if (nvmewqe == NULL) { 2756 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2757 "6110 NVMET prep FCP wqe: No " 2758 "WQE: NPORT x%x oxid x%x ste %d\n", 2759 ctxp->sid, ctxp->oxid, ctxp->state); 2760 return NULL; 2761 } 2762 ctxp->wqeq = nvmewqe; 2763 xc = 0; /* create new XRI */ 2764 nvmewqe->sli4_lxritag = NO_XRI; 2765 nvmewqe->sli4_xritag = NO_XRI; 2766 } 2767 2768 /* Sanity check */ 2769 if (((ctxp->state == LPFC_NVME_STE_RCV) && 2770 (ctxp->entry_cnt == 1)) || 2771 (ctxp->state == LPFC_NVME_STE_DATA)) { 2772 wqe = &nvmewqe->wqe; 2773 } else { 2774 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2775 "6111 Wrong state NVMET FCP: %d cnt %d\n", 2776 ctxp->state, ctxp->entry_cnt); 2777 return NULL; 2778 } 2779 2780 sgl = (struct sli4_sge *)ctxp->ctxbuf->sglq->sgl; 2781 switch (rsp->op) { 2782 case NVMET_FCOP_READDATA: 2783 case NVMET_FCOP_READDATA_RSP: 2784 /* From the tsend template, initialize words 7 - 11 */ 2785 memcpy(&wqe->words[7], 2786 &lpfc_tsend_cmd_template.words[7], 2787 sizeof(uint32_t) * 5); 2788 2789 /* Words 0 - 2 : The first sg segment */ 2790 sgel = &rsp->sg[0]; 2791 physaddr = sg_dma_address(sgel); 2792 wqe->fcp_tsend.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 2793 wqe->fcp_tsend.bde.tus.f.bdeSize = sg_dma_len(sgel); 2794 wqe->fcp_tsend.bde.addrLow = cpu_to_le32(putPaddrLow(physaddr)); 2795 wqe->fcp_tsend.bde.addrHigh = 2796 cpu_to_le32(putPaddrHigh(physaddr)); 2797 2798 /* Word 3 */ 2799 wqe->fcp_tsend.payload_offset_len = 0; 2800 2801 /* Word 4 */ 2802 wqe->fcp_tsend.relative_offset = ctxp->offset; 2803 2804 /* Word 5 */ 2805 wqe->fcp_tsend.reserved = 0; 2806 2807 /* Word 6 */ 2808 bf_set(wqe_ctxt_tag, &wqe->fcp_tsend.wqe_com, 2809 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 2810 bf_set(wqe_xri_tag, &wqe->fcp_tsend.wqe_com, 2811 nvmewqe->sli4_xritag); 2812 2813 /* Word 7 - set ar later */ 2814 2815 /* Word 8 */ 2816 wqe->fcp_tsend.wqe_com.abort_tag = nvmewqe->iotag; 2817 2818 /* Word 9 */ 2819 bf_set(wqe_reqtag, &wqe->fcp_tsend.wqe_com, nvmewqe->iotag); 2820 bf_set(wqe_rcvoxid, &wqe->fcp_tsend.wqe_com, ctxp->oxid); 2821 2822 /* Word 10 - set wqes later, in template xc=1 */ 2823 if (!xc) 2824 bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 0); 2825 2826 /* Word 11 - set sup, irsp, irsplen later */ 2827 do_pbde = 0; 2828 2829 /* Word 12 */ 2830 wqe->fcp_tsend.fcp_data_len = rsp->transfer_length; 2831 2832 /* Setup 2 SKIP SGEs */ 2833 sgl->addr_hi = 0; 2834 sgl->addr_lo = 0; 2835 sgl->word2 = 0; 2836 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2837 sgl->word2 = cpu_to_le32(sgl->word2); 2838 sgl->sge_len = 0; 2839 sgl++; 2840 sgl->addr_hi = 0; 2841 sgl->addr_lo = 0; 2842 sgl->word2 = 0; 2843 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2844 sgl->word2 = cpu_to_le32(sgl->word2); 2845 sgl->sge_len = 0; 2846 sgl++; 2847 if (rsp->op == NVMET_FCOP_READDATA_RSP) { 2848 atomic_inc(&tgtp->xmt_fcp_read_rsp); 2849 2850 /* In template ar=1 wqes=0 sup=0 irsp=0 irsplen=0 */ 2851 2852 if (rsp->rsplen == LPFC_NVMET_SUCCESS_LEN) { 2853 if (ndlp->nlp_flag & NLP_SUPPRESS_RSP) 2854 bf_set(wqe_sup, 2855 &wqe->fcp_tsend.wqe_com, 1); 2856 } else { 2857 bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 1); 2858 bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 1); 2859 bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 2860 ((rsp->rsplen >> 2) - 1)); 2861 memcpy(&wqe->words[16], rsp->rspaddr, 2862 rsp->rsplen); 2863 } 2864 } else { 2865 atomic_inc(&tgtp->xmt_fcp_read); 2866 2867 /* In template ar=1 wqes=0 sup=0 irsp=0 irsplen=0 */ 2868 bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 0); 2869 } 2870 break; 2871 2872 case NVMET_FCOP_WRITEDATA: 2873 /* From the treceive template, initialize words 3 - 11 */ 2874 memcpy(&wqe->words[3], 2875 &lpfc_treceive_cmd_template.words[3], 2876 sizeof(uint32_t) * 9); 2877 2878 /* Words 0 - 2 : First SGE is skipped, set invalid BDE type */ 2879 wqe->fcp_treceive.bde.tus.f.bdeFlags = LPFC_SGE_TYPE_SKIP; 2880 wqe->fcp_treceive.bde.tus.f.bdeSize = 0; 2881 wqe->fcp_treceive.bde.addrLow = 0; 2882 wqe->fcp_treceive.bde.addrHigh = 0; 2883 2884 /* Word 4 */ 2885 wqe->fcp_treceive.relative_offset = ctxp->offset; 2886 2887 /* Word 6 */ 2888 bf_set(wqe_ctxt_tag, &wqe->fcp_treceive.wqe_com, 2889 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 2890 bf_set(wqe_xri_tag, &wqe->fcp_treceive.wqe_com, 2891 nvmewqe->sli4_xritag); 2892 2893 /* Word 7 */ 2894 2895 /* Word 8 */ 2896 wqe->fcp_treceive.wqe_com.abort_tag = nvmewqe->iotag; 2897 2898 /* Word 9 */ 2899 bf_set(wqe_reqtag, &wqe->fcp_treceive.wqe_com, nvmewqe->iotag); 2900 bf_set(wqe_rcvoxid, &wqe->fcp_treceive.wqe_com, ctxp->oxid); 2901 2902 /* Word 10 - in template xc=1 */ 2903 if (!xc) 2904 bf_set(wqe_xc, &wqe->fcp_treceive.wqe_com, 0); 2905 2906 /* Word 11 - set pbde later */ 2907 if (phba->cfg_enable_pbde) { 2908 do_pbde = 1; 2909 } else { 2910 bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 0); 2911 do_pbde = 0; 2912 } 2913 2914 /* Word 12 */ 2915 wqe->fcp_tsend.fcp_data_len = rsp->transfer_length; 2916 2917 /* Setup 2 SKIP SGEs */ 2918 sgl->addr_hi = 0; 2919 sgl->addr_lo = 0; 2920 sgl->word2 = 0; 2921 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2922 sgl->word2 = cpu_to_le32(sgl->word2); 2923 sgl->sge_len = 0; 2924 sgl++; 2925 sgl->addr_hi = 0; 2926 sgl->addr_lo = 0; 2927 sgl->word2 = 0; 2928 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2929 sgl->word2 = cpu_to_le32(sgl->word2); 2930 sgl->sge_len = 0; 2931 sgl++; 2932 atomic_inc(&tgtp->xmt_fcp_write); 2933 break; 2934 2935 case NVMET_FCOP_RSP: 2936 /* From the treceive template, initialize words 4 - 11 */ 2937 memcpy(&wqe->words[4], 2938 &lpfc_trsp_cmd_template.words[4], 2939 sizeof(uint32_t) * 8); 2940 2941 /* Words 0 - 2 */ 2942 physaddr = rsp->rspdma; 2943 wqe->fcp_trsp.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 2944 wqe->fcp_trsp.bde.tus.f.bdeSize = rsp->rsplen; 2945 wqe->fcp_trsp.bde.addrLow = 2946 cpu_to_le32(putPaddrLow(physaddr)); 2947 wqe->fcp_trsp.bde.addrHigh = 2948 cpu_to_le32(putPaddrHigh(physaddr)); 2949 2950 /* Word 3 */ 2951 wqe->fcp_trsp.response_len = rsp->rsplen; 2952 2953 /* Word 6 */ 2954 bf_set(wqe_ctxt_tag, &wqe->fcp_trsp.wqe_com, 2955 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 2956 bf_set(wqe_xri_tag, &wqe->fcp_trsp.wqe_com, 2957 nvmewqe->sli4_xritag); 2958 2959 /* Word 7 */ 2960 2961 /* Word 8 */ 2962 wqe->fcp_trsp.wqe_com.abort_tag = nvmewqe->iotag; 2963 2964 /* Word 9 */ 2965 bf_set(wqe_reqtag, &wqe->fcp_trsp.wqe_com, nvmewqe->iotag); 2966 bf_set(wqe_rcvoxid, &wqe->fcp_trsp.wqe_com, ctxp->oxid); 2967 2968 /* Word 10 */ 2969 if (xc) 2970 bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 1); 2971 2972 /* Word 11 */ 2973 /* In template wqes=0 irsp=0 irsplen=0 - good response */ 2974 if (rsp->rsplen != LPFC_NVMET_SUCCESS_LEN) { 2975 /* Bad response - embed it */ 2976 bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 1); 2977 bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 1); 2978 bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 2979 ((rsp->rsplen >> 2) - 1)); 2980 memcpy(&wqe->words[16], rsp->rspaddr, rsp->rsplen); 2981 } 2982 do_pbde = 0; 2983 2984 /* Word 12 */ 2985 wqe->fcp_trsp.rsvd_12_15[0] = 0; 2986 2987 /* Use rspbuf, NOT sg list */ 2988 nsegs = 0; 2989 sgl->word2 = 0; 2990 atomic_inc(&tgtp->xmt_fcp_rsp); 2991 break; 2992 2993 default: 2994 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 2995 "6064 Unknown Rsp Op %d\n", 2996 rsp->op); 2997 return NULL; 2998 } 2999 3000 nvmewqe->retry = 1; 3001 nvmewqe->vport = phba->pport; 3002 nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT; 3003 nvmewqe->context1 = ndlp; 3004 3005 for_each_sg(rsp->sg, sgel, nsegs, i) { 3006 physaddr = sg_dma_address(sgel); 3007 cnt = sg_dma_len(sgel); 3008 sgl->addr_hi = putPaddrHigh(physaddr); 3009 sgl->addr_lo = putPaddrLow(physaddr); 3010 sgl->word2 = 0; 3011 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA); 3012 bf_set(lpfc_sli4_sge_offset, sgl, ctxp->offset); 3013 if ((i+1) == rsp->sg_cnt) 3014 bf_set(lpfc_sli4_sge_last, sgl, 1); 3015 sgl->word2 = cpu_to_le32(sgl->word2); 3016 sgl->sge_len = cpu_to_le32(cnt); 3017 if (i == 0) { 3018 bde = (struct ulp_bde64 *)&wqe->words[13]; 3019 if (do_pbde) { 3020 /* Words 13-15 (PBDE) */ 3021 bde->addrLow = sgl->addr_lo; 3022 bde->addrHigh = sgl->addr_hi; 3023 bde->tus.f.bdeSize = 3024 le32_to_cpu(sgl->sge_len); 3025 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64; 3026 bde->tus.w = cpu_to_le32(bde->tus.w); 3027 } else { 3028 memset(bde, 0, sizeof(struct ulp_bde64)); 3029 } 3030 } 3031 sgl++; 3032 ctxp->offset += cnt; 3033 } 3034 ctxp->state = LPFC_NVME_STE_DATA; 3035 ctxp->entry_cnt++; 3036 return nvmewqe; 3037 } 3038 3039 /** 3040 * lpfc_nvmet_sol_fcp_abort_cmp - Completion handler for ABTS 3041 * @phba: Pointer to HBA context object. 3042 * @cmdwqe: Pointer to driver command WQE object. 3043 * @wcqe: Pointer to driver response CQE object. 3044 * 3045 * The function is called from SLI ring event handler with no 3046 * lock held. This function is the completion handler for NVME ABTS for FCP cmds 3047 * The function frees memory resources used for the NVME commands. 3048 **/ 3049 static void 3050 lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 3051 struct lpfc_wcqe_complete *wcqe) 3052 { 3053 struct lpfc_async_xchg_ctx *ctxp; 3054 struct lpfc_nvmet_tgtport *tgtp; 3055 uint32_t result; 3056 unsigned long flags; 3057 bool released = false; 3058 3059 ctxp = cmdwqe->context2; 3060 result = wcqe->parameter; 3061 3062 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 3063 if (ctxp->flag & LPFC_NVME_ABORT_OP) 3064 atomic_inc(&tgtp->xmt_fcp_abort_cmpl); 3065 3066 spin_lock_irqsave(&ctxp->ctxlock, flags); 3067 ctxp->state = LPFC_NVME_STE_DONE; 3068 3069 /* Check if we already received a free context call 3070 * and we have completed processing an abort situation. 3071 */ 3072 if ((ctxp->flag & LPFC_NVME_CTX_RLS) && 3073 !(ctxp->flag & LPFC_NVME_XBUSY)) { 3074 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 3075 list_del_init(&ctxp->list); 3076 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 3077 released = true; 3078 } 3079 ctxp->flag &= ~LPFC_NVME_ABORT_OP; 3080 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3081 atomic_inc(&tgtp->xmt_abort_rsp); 3082 3083 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 3084 "6165 ABORT cmpl: oxid x%x flg x%x (%d) " 3085 "WCQE: %08x %08x %08x %08x\n", 3086 ctxp->oxid, ctxp->flag, released, 3087 wcqe->word0, wcqe->total_data_placed, 3088 result, wcqe->word3); 3089 3090 cmdwqe->context2 = NULL; 3091 cmdwqe->context3 = NULL; 3092 /* 3093 * if transport has released ctx, then can reuse it. Otherwise, 3094 * will be recycled by transport release call. 3095 */ 3096 if (released) 3097 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); 3098 3099 /* This is the iocbq for the abort, not the command */ 3100 lpfc_sli_release_iocbq(phba, cmdwqe); 3101 3102 /* Since iaab/iaar are NOT set, there is no work left. 3103 * For LPFC_NVME_XBUSY, lpfc_sli4_nvmet_xri_aborted 3104 * should have been called already. 3105 */ 3106 } 3107 3108 /** 3109 * lpfc_nvmet_unsol_fcp_abort_cmp - Completion handler for ABTS 3110 * @phba: Pointer to HBA context object. 3111 * @cmdwqe: Pointer to driver command WQE object. 3112 * @wcqe: Pointer to driver response CQE object. 3113 * 3114 * The function is called from SLI ring event handler with no 3115 * lock held. This function is the completion handler for NVME ABTS for FCP cmds 3116 * The function frees memory resources used for the NVME commands. 3117 **/ 3118 static void 3119 lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 3120 struct lpfc_wcqe_complete *wcqe) 3121 { 3122 struct lpfc_async_xchg_ctx *ctxp; 3123 struct lpfc_nvmet_tgtport *tgtp; 3124 unsigned long flags; 3125 uint32_t result; 3126 bool released = false; 3127 3128 ctxp = cmdwqe->context2; 3129 result = wcqe->parameter; 3130 3131 if (!ctxp) { 3132 /* if context is clear, related io alrady complete */ 3133 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 3134 "6070 ABTS cmpl: WCQE: %08x %08x %08x %08x\n", 3135 wcqe->word0, wcqe->total_data_placed, 3136 result, wcqe->word3); 3137 return; 3138 } 3139 3140 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 3141 spin_lock_irqsave(&ctxp->ctxlock, flags); 3142 if (ctxp->flag & LPFC_NVME_ABORT_OP) 3143 atomic_inc(&tgtp->xmt_fcp_abort_cmpl); 3144 3145 /* Sanity check */ 3146 if (ctxp->state != LPFC_NVME_STE_ABORT) { 3147 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3148 "6112 ABTS Wrong state:%d oxid x%x\n", 3149 ctxp->state, ctxp->oxid); 3150 } 3151 3152 /* Check if we already received a free context call 3153 * and we have completed processing an abort situation. 3154 */ 3155 ctxp->state = LPFC_NVME_STE_DONE; 3156 if ((ctxp->flag & LPFC_NVME_CTX_RLS) && 3157 !(ctxp->flag & LPFC_NVME_XBUSY)) { 3158 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 3159 list_del_init(&ctxp->list); 3160 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 3161 released = true; 3162 } 3163 ctxp->flag &= ~LPFC_NVME_ABORT_OP; 3164 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3165 atomic_inc(&tgtp->xmt_abort_rsp); 3166 3167 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 3168 "6316 ABTS cmpl oxid x%x flg x%x (%x) " 3169 "WCQE: %08x %08x %08x %08x\n", 3170 ctxp->oxid, ctxp->flag, released, 3171 wcqe->word0, wcqe->total_data_placed, 3172 result, wcqe->word3); 3173 3174 cmdwqe->context2 = NULL; 3175 cmdwqe->context3 = NULL; 3176 /* 3177 * if transport has released ctx, then can reuse it. Otherwise, 3178 * will be recycled by transport release call. 3179 */ 3180 if (released) 3181 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); 3182 3183 /* Since iaab/iaar are NOT set, there is no work left. 3184 * For LPFC_NVME_XBUSY, lpfc_sli4_nvmet_xri_aborted 3185 * should have been called already. 3186 */ 3187 } 3188 3189 /** 3190 * lpfc_nvmet_xmt_ls_abort_cmp - Completion handler for ABTS 3191 * @phba: Pointer to HBA context object. 3192 * @cmdwqe: Pointer to driver command WQE object. 3193 * @wcqe: Pointer to driver response CQE object. 3194 * 3195 * The function is called from SLI ring event handler with no 3196 * lock held. This function is the completion handler for NVME ABTS for LS cmds 3197 * The function frees memory resources used for the NVME commands. 3198 **/ 3199 static void 3200 lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 3201 struct lpfc_wcqe_complete *wcqe) 3202 { 3203 struct lpfc_async_xchg_ctx *ctxp; 3204 struct lpfc_nvmet_tgtport *tgtp; 3205 uint32_t result; 3206 3207 ctxp = cmdwqe->context2; 3208 result = wcqe->parameter; 3209 3210 if (phba->nvmet_support) { 3211 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 3212 atomic_inc(&tgtp->xmt_ls_abort_cmpl); 3213 } 3214 3215 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 3216 "6083 Abort cmpl: ctx x%px WCQE:%08x %08x %08x %08x\n", 3217 ctxp, wcqe->word0, wcqe->total_data_placed, 3218 result, wcqe->word3); 3219 3220 if (!ctxp) { 3221 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3222 "6415 NVMET LS Abort No ctx: WCQE: " 3223 "%08x %08x %08x %08x\n", 3224 wcqe->word0, wcqe->total_data_placed, 3225 result, wcqe->word3); 3226 3227 lpfc_sli_release_iocbq(phba, cmdwqe); 3228 return; 3229 } 3230 3231 if (ctxp->state != LPFC_NVME_STE_LS_ABORT) { 3232 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 3233 "6416 NVMET LS abort cmpl state mismatch: " 3234 "oxid x%x: %d %d\n", 3235 ctxp->oxid, ctxp->state, ctxp->entry_cnt); 3236 } 3237 3238 cmdwqe->context2 = NULL; 3239 cmdwqe->context3 = NULL; 3240 lpfc_sli_release_iocbq(phba, cmdwqe); 3241 kfree(ctxp); 3242 } 3243 3244 static int 3245 lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba, 3246 struct lpfc_async_xchg_ctx *ctxp, 3247 uint32_t sid, uint16_t xri) 3248 { 3249 struct lpfc_nvmet_tgtport *tgtp = NULL; 3250 struct lpfc_iocbq *abts_wqeq; 3251 union lpfc_wqe128 *wqe_abts; 3252 struct lpfc_nodelist *ndlp; 3253 3254 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 3255 "6067 ABTS: sid %x xri x%x/x%x\n", 3256 sid, xri, ctxp->wqeq->sli4_xritag); 3257 3258 if (phba->nvmet_support && phba->targetport) 3259 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 3260 3261 ndlp = lpfc_findnode_did(phba->pport, sid); 3262 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || 3263 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && 3264 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { 3265 if (tgtp) 3266 atomic_inc(&tgtp->xmt_abort_rsp_error); 3267 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3268 "6134 Drop ABTS - wrong NDLP state x%x.\n", 3269 (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE); 3270 3271 /* No failure to an ABTS request. */ 3272 return 0; 3273 } 3274 3275 abts_wqeq = ctxp->wqeq; 3276 wqe_abts = &abts_wqeq->wqe; 3277 3278 /* 3279 * Since we zero the whole WQE, we need to ensure we set the WQE fields 3280 * that were initialized in lpfc_sli4_nvmet_alloc. 3281 */ 3282 memset(wqe_abts, 0, sizeof(union lpfc_wqe)); 3283 3284 /* Word 5 */ 3285 bf_set(wqe_dfctl, &wqe_abts->xmit_sequence.wge_ctl, 0); 3286 bf_set(wqe_ls, &wqe_abts->xmit_sequence.wge_ctl, 1); 3287 bf_set(wqe_la, &wqe_abts->xmit_sequence.wge_ctl, 0); 3288 bf_set(wqe_rctl, &wqe_abts->xmit_sequence.wge_ctl, FC_RCTL_BA_ABTS); 3289 bf_set(wqe_type, &wqe_abts->xmit_sequence.wge_ctl, FC_TYPE_BLS); 3290 3291 /* Word 6 */ 3292 bf_set(wqe_ctxt_tag, &wqe_abts->xmit_sequence.wqe_com, 3293 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 3294 bf_set(wqe_xri_tag, &wqe_abts->xmit_sequence.wqe_com, 3295 abts_wqeq->sli4_xritag); 3296 3297 /* Word 7 */ 3298 bf_set(wqe_cmnd, &wqe_abts->xmit_sequence.wqe_com, 3299 CMD_XMIT_SEQUENCE64_WQE); 3300 bf_set(wqe_ct, &wqe_abts->xmit_sequence.wqe_com, SLI4_CT_RPI); 3301 bf_set(wqe_class, &wqe_abts->xmit_sequence.wqe_com, CLASS3); 3302 bf_set(wqe_pu, &wqe_abts->xmit_sequence.wqe_com, 0); 3303 3304 /* Word 8 */ 3305 wqe_abts->xmit_sequence.wqe_com.abort_tag = abts_wqeq->iotag; 3306 3307 /* Word 9 */ 3308 bf_set(wqe_reqtag, &wqe_abts->xmit_sequence.wqe_com, abts_wqeq->iotag); 3309 /* Needs to be set by caller */ 3310 bf_set(wqe_rcvoxid, &wqe_abts->xmit_sequence.wqe_com, xri); 3311 3312 /* Word 10 */ 3313 bf_set(wqe_dbde, &wqe_abts->xmit_sequence.wqe_com, 1); 3314 bf_set(wqe_iod, &wqe_abts->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE); 3315 bf_set(wqe_lenloc, &wqe_abts->xmit_sequence.wqe_com, 3316 LPFC_WQE_LENLOC_WORD12); 3317 bf_set(wqe_ebde_cnt, &wqe_abts->xmit_sequence.wqe_com, 0); 3318 bf_set(wqe_qosd, &wqe_abts->xmit_sequence.wqe_com, 0); 3319 3320 /* Word 11 */ 3321 bf_set(wqe_cqid, &wqe_abts->xmit_sequence.wqe_com, 3322 LPFC_WQE_CQ_ID_DEFAULT); 3323 bf_set(wqe_cmd_type, &wqe_abts->xmit_sequence.wqe_com, 3324 OTHER_COMMAND); 3325 3326 abts_wqeq->vport = phba->pport; 3327 abts_wqeq->context1 = ndlp; 3328 abts_wqeq->context2 = ctxp; 3329 abts_wqeq->context3 = NULL; 3330 abts_wqeq->rsvd2 = 0; 3331 /* hba_wqidx should already be setup from command we are aborting */ 3332 abts_wqeq->iocb.ulpCommand = CMD_XMIT_SEQUENCE64_CR; 3333 abts_wqeq->iocb.ulpLe = 1; 3334 3335 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 3336 "6069 Issue ABTS to xri x%x reqtag x%x\n", 3337 xri, abts_wqeq->iotag); 3338 return 1; 3339 } 3340 3341 static int 3342 lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, 3343 struct lpfc_async_xchg_ctx *ctxp, 3344 uint32_t sid, uint16_t xri) 3345 { 3346 struct lpfc_nvmet_tgtport *tgtp; 3347 struct lpfc_iocbq *abts_wqeq; 3348 struct lpfc_nodelist *ndlp; 3349 unsigned long flags; 3350 u8 opt; 3351 int rc; 3352 3353 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 3354 if (!ctxp->wqeq) { 3355 ctxp->wqeq = ctxp->ctxbuf->iocbq; 3356 ctxp->wqeq->hba_wqidx = 0; 3357 } 3358 3359 ndlp = lpfc_findnode_did(phba->pport, sid); 3360 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || 3361 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && 3362 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { 3363 atomic_inc(&tgtp->xmt_abort_rsp_error); 3364 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3365 "6160 Drop ABORT - wrong NDLP state x%x.\n", 3366 (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE); 3367 3368 /* No failure to an ABTS request. */ 3369 spin_lock_irqsave(&ctxp->ctxlock, flags); 3370 ctxp->flag &= ~LPFC_NVME_ABORT_OP; 3371 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3372 return 0; 3373 } 3374 3375 /* Issue ABTS for this WQE based on iotag */ 3376 ctxp->abort_wqeq = lpfc_sli_get_iocbq(phba); 3377 spin_lock_irqsave(&ctxp->ctxlock, flags); 3378 if (!ctxp->abort_wqeq) { 3379 atomic_inc(&tgtp->xmt_abort_rsp_error); 3380 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3381 "6161 ABORT failed: No wqeqs: " 3382 "xri: x%x\n", ctxp->oxid); 3383 /* No failure to an ABTS request. */ 3384 ctxp->flag &= ~LPFC_NVME_ABORT_OP; 3385 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3386 return 0; 3387 } 3388 abts_wqeq = ctxp->abort_wqeq; 3389 ctxp->state = LPFC_NVME_STE_ABORT; 3390 opt = (ctxp->flag & LPFC_NVME_ABTS_RCV) ? INHIBIT_ABORT : 0; 3391 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3392 3393 /* Announce entry to new IO submit field. */ 3394 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 3395 "6162 ABORT Request to rport DID x%06x " 3396 "for xri x%x x%x\n", 3397 ctxp->sid, ctxp->oxid, ctxp->wqeq->sli4_xritag); 3398 3399 /* If the hba is getting reset, this flag is set. It is 3400 * cleared when the reset is complete and rings reestablished. 3401 */ 3402 spin_lock_irqsave(&phba->hbalock, flags); 3403 /* driver queued commands are in process of being flushed */ 3404 if (phba->hba_flag & HBA_IOQ_FLUSH) { 3405 spin_unlock_irqrestore(&phba->hbalock, flags); 3406 atomic_inc(&tgtp->xmt_abort_rsp_error); 3407 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 3408 "6163 Driver in reset cleanup - flushing " 3409 "NVME Req now. hba_flag x%x oxid x%x\n", 3410 phba->hba_flag, ctxp->oxid); 3411 lpfc_sli_release_iocbq(phba, abts_wqeq); 3412 spin_lock_irqsave(&ctxp->ctxlock, flags); 3413 ctxp->flag &= ~LPFC_NVME_ABORT_OP; 3414 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3415 return 0; 3416 } 3417 3418 /* Outstanding abort is in progress */ 3419 if (abts_wqeq->iocb_flag & LPFC_DRIVER_ABORTED) { 3420 spin_unlock_irqrestore(&phba->hbalock, flags); 3421 atomic_inc(&tgtp->xmt_abort_rsp_error); 3422 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 3423 "6164 Outstanding NVME I/O Abort Request " 3424 "still pending on oxid x%x\n", 3425 ctxp->oxid); 3426 lpfc_sli_release_iocbq(phba, abts_wqeq); 3427 spin_lock_irqsave(&ctxp->ctxlock, flags); 3428 ctxp->flag &= ~LPFC_NVME_ABORT_OP; 3429 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3430 return 0; 3431 } 3432 3433 /* Ready - mark outstanding as aborted by driver. */ 3434 abts_wqeq->iocb_flag |= LPFC_DRIVER_ABORTED; 3435 3436 lpfc_nvme_prep_abort_wqe(abts_wqeq, ctxp->wqeq->sli4_xritag, opt); 3437 3438 /* ABTS WQE must go to the same WQ as the WQE to be aborted */ 3439 abts_wqeq->hba_wqidx = ctxp->wqeq->hba_wqidx; 3440 abts_wqeq->wqe_cmpl = lpfc_nvmet_sol_fcp_abort_cmp; 3441 abts_wqeq->iocb_cmpl = NULL; 3442 abts_wqeq->iocb_flag |= LPFC_IO_NVME; 3443 abts_wqeq->context2 = ctxp; 3444 abts_wqeq->vport = phba->pport; 3445 if (!ctxp->hdwq) 3446 ctxp->hdwq = &phba->sli4_hba.hdwq[abts_wqeq->hba_wqidx]; 3447 3448 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq); 3449 spin_unlock_irqrestore(&phba->hbalock, flags); 3450 if (rc == WQE_SUCCESS) { 3451 atomic_inc(&tgtp->xmt_abort_sol); 3452 return 0; 3453 } 3454 3455 atomic_inc(&tgtp->xmt_abort_rsp_error); 3456 spin_lock_irqsave(&ctxp->ctxlock, flags); 3457 ctxp->flag &= ~LPFC_NVME_ABORT_OP; 3458 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3459 lpfc_sli_release_iocbq(phba, abts_wqeq); 3460 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3461 "6166 Failed ABORT issue_wqe with status x%x " 3462 "for oxid x%x.\n", 3463 rc, ctxp->oxid); 3464 return 1; 3465 } 3466 3467 static int 3468 lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba, 3469 struct lpfc_async_xchg_ctx *ctxp, 3470 uint32_t sid, uint16_t xri) 3471 { 3472 struct lpfc_nvmet_tgtport *tgtp; 3473 struct lpfc_iocbq *abts_wqeq; 3474 unsigned long flags; 3475 bool released = false; 3476 int rc; 3477 3478 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 3479 if (!ctxp->wqeq) { 3480 ctxp->wqeq = ctxp->ctxbuf->iocbq; 3481 ctxp->wqeq->hba_wqidx = 0; 3482 } 3483 3484 if (ctxp->state == LPFC_NVME_STE_FREE) { 3485 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 3486 "6417 NVMET ABORT ctx freed %d %d oxid x%x\n", 3487 ctxp->state, ctxp->entry_cnt, ctxp->oxid); 3488 rc = WQE_BUSY; 3489 goto aerr; 3490 } 3491 ctxp->state = LPFC_NVME_STE_ABORT; 3492 ctxp->entry_cnt++; 3493 rc = lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri); 3494 if (rc == 0) 3495 goto aerr; 3496 3497 spin_lock_irqsave(&phba->hbalock, flags); 3498 abts_wqeq = ctxp->wqeq; 3499 abts_wqeq->wqe_cmpl = lpfc_nvmet_unsol_fcp_abort_cmp; 3500 abts_wqeq->iocb_cmpl = NULL; 3501 abts_wqeq->iocb_flag |= LPFC_IO_NVMET; 3502 if (!ctxp->hdwq) 3503 ctxp->hdwq = &phba->sli4_hba.hdwq[abts_wqeq->hba_wqidx]; 3504 3505 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq); 3506 spin_unlock_irqrestore(&phba->hbalock, flags); 3507 if (rc == WQE_SUCCESS) { 3508 return 0; 3509 } 3510 3511 aerr: 3512 spin_lock_irqsave(&ctxp->ctxlock, flags); 3513 if (ctxp->flag & LPFC_NVME_CTX_RLS) { 3514 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 3515 list_del_init(&ctxp->list); 3516 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 3517 released = true; 3518 } 3519 ctxp->flag &= ~(LPFC_NVME_ABORT_OP | LPFC_NVME_CTX_RLS); 3520 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3521 3522 atomic_inc(&tgtp->xmt_abort_rsp_error); 3523 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3524 "6135 Failed to Issue ABTS for oxid x%x. Status x%x " 3525 "(%x)\n", 3526 ctxp->oxid, rc, released); 3527 if (released) 3528 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); 3529 return 1; 3530 } 3531 3532 /** 3533 * lpfc_nvme_unsol_ls_issue_abort - issue ABTS on an exchange received 3534 * via async frame receive where the frame is not handled. 3535 * @phba: pointer to adapter structure 3536 * @ctxp: pointer to the asynchronously received received sequence 3537 * @sid: address of the remote port to send the ABTS to 3538 * @xri: oxid value to for the ABTS (other side's exchange id). 3539 **/ 3540 int 3541 lpfc_nvme_unsol_ls_issue_abort(struct lpfc_hba *phba, 3542 struct lpfc_async_xchg_ctx *ctxp, 3543 uint32_t sid, uint16_t xri) 3544 { 3545 struct lpfc_nvmet_tgtport *tgtp = NULL; 3546 struct lpfc_iocbq *abts_wqeq; 3547 unsigned long flags; 3548 int rc; 3549 3550 if ((ctxp->state == LPFC_NVME_STE_LS_RCV && ctxp->entry_cnt == 1) || 3551 (ctxp->state == LPFC_NVME_STE_LS_RSP && ctxp->entry_cnt == 2)) { 3552 ctxp->state = LPFC_NVME_STE_LS_ABORT; 3553 ctxp->entry_cnt++; 3554 } else { 3555 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 3556 "6418 NVMET LS abort state mismatch " 3557 "IO x%x: %d %d\n", 3558 ctxp->oxid, ctxp->state, ctxp->entry_cnt); 3559 ctxp->state = LPFC_NVME_STE_LS_ABORT; 3560 } 3561 3562 if (phba->nvmet_support && phba->targetport) 3563 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 3564 3565 if (!ctxp->wqeq) { 3566 /* Issue ABTS for this WQE based on iotag */ 3567 ctxp->wqeq = lpfc_sli_get_iocbq(phba); 3568 if (!ctxp->wqeq) { 3569 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3570 "6068 Abort failed: No wqeqs: " 3571 "xri: x%x\n", xri); 3572 /* No failure to an ABTS request. */ 3573 kfree(ctxp); 3574 return 0; 3575 } 3576 } 3577 abts_wqeq = ctxp->wqeq; 3578 3579 if (lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri) == 0) { 3580 rc = WQE_BUSY; 3581 goto out; 3582 } 3583 3584 spin_lock_irqsave(&phba->hbalock, flags); 3585 abts_wqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_abort_cmp; 3586 abts_wqeq->iocb_cmpl = NULL; 3587 abts_wqeq->iocb_flag |= LPFC_IO_NVME_LS; 3588 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq); 3589 spin_unlock_irqrestore(&phba->hbalock, flags); 3590 if (rc == WQE_SUCCESS) { 3591 if (tgtp) 3592 atomic_inc(&tgtp->xmt_abort_unsol); 3593 return 0; 3594 } 3595 out: 3596 if (tgtp) 3597 atomic_inc(&tgtp->xmt_abort_rsp_error); 3598 abts_wqeq->context2 = NULL; 3599 abts_wqeq->context3 = NULL; 3600 lpfc_sli_release_iocbq(phba, abts_wqeq); 3601 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3602 "6056 Failed to Issue ABTS. Status x%x\n", rc); 3603 return 1; 3604 } 3605 3606 /** 3607 * lpfc_nvmet_invalidate_host 3608 * 3609 * @phba - pointer to the driver instance bound to an adapter port. 3610 * @ndlp - pointer to an lpfc_nodelist type 3611 * 3612 * This routine upcalls the nvmet transport to invalidate an NVME 3613 * host to which this target instance had active connections. 3614 */ 3615 void 3616 lpfc_nvmet_invalidate_host(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) 3617 { 3618 struct lpfc_nvmet_tgtport *tgtp; 3619 3620 lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_NVME_ABTS, 3621 "6203 Invalidating hosthandle x%px\n", 3622 ndlp); 3623 3624 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 3625 atomic_set(&tgtp->state, LPFC_NVMET_INV_HOST_ACTIVE); 3626 3627 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 3628 /* Need to get the nvmet_fc_target_port pointer here.*/ 3629 nvmet_fc_invalidate_host(phba->targetport, ndlp); 3630 #endif 3631 } 3632