1 /******************************************************************* 2 * This file is part of the Emulex Linux Device Driver for * 3 * Fibre Channsel Host Bus Adapters. * 4 * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * 5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * 6 * Copyright (C) 2004-2016 Emulex. All rights reserved. * 7 * EMULEX and SLI are trademarks of Emulex. * 8 * www.broadcom.com * 9 * Portions Copyright (C) 2004-2005 Christoph Hellwig * 10 * * 11 * This program is free software; you can redistribute it and/or * 12 * modify it under the terms of version 2 of the GNU General * 13 * Public License as published by the Free Software Foundation. * 14 * This program is distributed in the hope that it will be useful. * 15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 19 * TO BE LEGALLY INVALID. See the GNU General Public License for * 20 * more details, a copy of which can be found in the file COPYING * 21 * included with this package. * 22 ********************************************************************/ 23 #include <linux/pci.h> 24 #include <linux/slab.h> 25 #include <linux/interrupt.h> 26 #include <linux/delay.h> 27 #include <asm/unaligned.h> 28 #include <linux/crc-t10dif.h> 29 #include <net/checksum.h> 30 31 #include <scsi/scsi.h> 32 #include <scsi/scsi_device.h> 33 #include <scsi/scsi_eh.h> 34 #include <scsi/scsi_host.h> 35 #include <scsi/scsi_tcq.h> 36 #include <scsi/scsi_transport_fc.h> 37 #include <scsi/fc/fc_fs.h> 38 39 #include <linux/nvme.h> 40 #include <linux/nvme-fc-driver.h> 41 #include <linux/nvme-fc.h> 42 43 #include "lpfc_version.h" 44 #include "lpfc_hw4.h" 45 #include "lpfc_hw.h" 46 #include "lpfc_sli.h" 47 #include "lpfc_sli4.h" 48 #include "lpfc_nl.h" 49 #include "lpfc_disc.h" 50 #include "lpfc.h" 51 #include "lpfc_scsi.h" 52 #include "lpfc_nvme.h" 53 #include "lpfc_nvmet.h" 54 #include "lpfc_logmsg.h" 55 #include "lpfc_crtn.h" 56 #include "lpfc_vport.h" 57 #include "lpfc_debugfs.h" 58 59 static struct lpfc_iocbq *lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *, 60 struct lpfc_nvmet_rcv_ctx *, 61 dma_addr_t rspbuf, 62 uint16_t rspsize); 63 static struct lpfc_iocbq *lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *, 64 struct lpfc_nvmet_rcv_ctx *); 65 static int lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *, 66 struct lpfc_nvmet_rcv_ctx *, 67 uint32_t, uint16_t); 68 static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *, 69 struct lpfc_nvmet_rcv_ctx *, 70 uint32_t, uint16_t); 71 static int lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *, 72 struct lpfc_nvmet_rcv_ctx *, 73 uint32_t, uint16_t); 74 static void lpfc_nvmet_wqfull_flush(struct lpfc_hba *, struct lpfc_queue *, 75 struct lpfc_nvmet_rcv_ctx *); 76 static void lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *); 77 78 static void lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf); 79 80 static union lpfc_wqe128 lpfc_tsend_cmd_template; 81 static union lpfc_wqe128 lpfc_treceive_cmd_template; 82 static union lpfc_wqe128 lpfc_trsp_cmd_template; 83 84 /* Setup WQE templates for NVME IOs */ 85 void 86 lpfc_nvmet_cmd_template(void) 87 { 88 union lpfc_wqe128 *wqe; 89 90 /* TSEND template */ 91 wqe = &lpfc_tsend_cmd_template; 92 memset(wqe, 0, sizeof(union lpfc_wqe128)); 93 94 /* Word 0, 1, 2 - BDE is variable */ 95 96 /* Word 3 - payload_offset_len is zero */ 97 98 /* Word 4 - relative_offset is variable */ 99 100 /* Word 5 - is zero */ 101 102 /* Word 6 - ctxt_tag, xri_tag is variable */ 103 104 /* Word 7 - wqe_ar is variable */ 105 bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE); 106 bf_set(wqe_pu, &wqe->fcp_tsend.wqe_com, PARM_REL_OFF); 107 bf_set(wqe_class, &wqe->fcp_tsend.wqe_com, CLASS3); 108 bf_set(wqe_ct, &wqe->fcp_tsend.wqe_com, SLI4_CT_RPI); 109 bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 1); 110 111 /* Word 8 - abort_tag is variable */ 112 113 /* Word 9 - reqtag, rcvoxid is variable */ 114 115 /* Word 10 - wqes, xc is variable */ 116 bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1); 117 bf_set(wqe_dbde, &wqe->fcp_tsend.wqe_com, 1); 118 bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0); 119 bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1); 120 bf_set(wqe_iod, &wqe->fcp_tsend.wqe_com, LPFC_WQE_IOD_WRITE); 121 bf_set(wqe_lenloc, &wqe->fcp_tsend.wqe_com, LPFC_WQE_LENLOC_WORD12); 122 123 /* Word 11 - sup, irsp, irsplen is variable */ 124 bf_set(wqe_cmd_type, &wqe->fcp_tsend.wqe_com, FCP_COMMAND_TSEND); 125 bf_set(wqe_cqid, &wqe->fcp_tsend.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 126 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0); 127 bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0); 128 bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0); 129 bf_set(wqe_pbde, &wqe->fcp_tsend.wqe_com, 0); 130 131 /* Word 12 - fcp_data_len is variable */ 132 133 /* Word 13, 14, 15 - PBDE is zero */ 134 135 /* TRECEIVE template */ 136 wqe = &lpfc_treceive_cmd_template; 137 memset(wqe, 0, sizeof(union lpfc_wqe128)); 138 139 /* Word 0, 1, 2 - BDE is variable */ 140 141 /* Word 3 */ 142 wqe->fcp_treceive.payload_offset_len = TXRDY_PAYLOAD_LEN; 143 144 /* Word 4 - relative_offset is variable */ 145 146 /* Word 5 - is zero */ 147 148 /* Word 6 - ctxt_tag, xri_tag is variable */ 149 150 /* Word 7 */ 151 bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com, CMD_FCP_TRECEIVE64_WQE); 152 bf_set(wqe_pu, &wqe->fcp_treceive.wqe_com, PARM_REL_OFF); 153 bf_set(wqe_class, &wqe->fcp_treceive.wqe_com, CLASS3); 154 bf_set(wqe_ct, &wqe->fcp_treceive.wqe_com, SLI4_CT_RPI); 155 bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0); 156 157 /* Word 8 - abort_tag is variable */ 158 159 /* Word 9 - reqtag, rcvoxid is variable */ 160 161 /* Word 10 - xc is variable */ 162 bf_set(wqe_dbde, &wqe->fcp_treceive.wqe_com, 1); 163 bf_set(wqe_wqes, &wqe->fcp_treceive.wqe_com, 0); 164 bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1); 165 bf_set(wqe_iod, &wqe->fcp_treceive.wqe_com, LPFC_WQE_IOD_READ); 166 bf_set(wqe_lenloc, &wqe->fcp_treceive.wqe_com, LPFC_WQE_LENLOC_WORD12); 167 bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1); 168 169 /* Word 11 - pbde is variable */ 170 bf_set(wqe_cmd_type, &wqe->fcp_treceive.wqe_com, FCP_COMMAND_TRECEIVE); 171 bf_set(wqe_cqid, &wqe->fcp_treceive.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 172 bf_set(wqe_sup, &wqe->fcp_treceive.wqe_com, 0); 173 bf_set(wqe_irsp, &wqe->fcp_treceive.wqe_com, 0); 174 bf_set(wqe_irsplen, &wqe->fcp_treceive.wqe_com, 0); 175 bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 1); 176 177 /* Word 12 - fcp_data_len is variable */ 178 179 /* Word 13, 14, 15 - PBDE is variable */ 180 181 /* TRSP template */ 182 wqe = &lpfc_trsp_cmd_template; 183 memset(wqe, 0, sizeof(union lpfc_wqe128)); 184 185 /* Word 0, 1, 2 - BDE is variable */ 186 187 /* Word 3 - response_len is variable */ 188 189 /* Word 4, 5 - is zero */ 190 191 /* Word 6 - ctxt_tag, xri_tag is variable */ 192 193 /* Word 7 */ 194 bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE); 195 bf_set(wqe_pu, &wqe->fcp_trsp.wqe_com, PARM_UNUSED); 196 bf_set(wqe_class, &wqe->fcp_trsp.wqe_com, CLASS3); 197 bf_set(wqe_ct, &wqe->fcp_trsp.wqe_com, SLI4_CT_RPI); 198 bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1); /* wqe_ar */ 199 200 /* Word 8 - abort_tag is variable */ 201 202 /* Word 9 - reqtag is variable */ 203 204 /* Word 10 wqes, xc is variable */ 205 bf_set(wqe_dbde, &wqe->fcp_trsp.wqe_com, 1); 206 bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1); 207 bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 0); 208 bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 0); 209 bf_set(wqe_iod, &wqe->fcp_trsp.wqe_com, LPFC_WQE_IOD_NONE); 210 bf_set(wqe_lenloc, &wqe->fcp_trsp.wqe_com, LPFC_WQE_LENLOC_WORD3); 211 212 /* Word 11 irsp, irsplen is variable */ 213 bf_set(wqe_cmd_type, &wqe->fcp_trsp.wqe_com, FCP_COMMAND_TRSP); 214 bf_set(wqe_cqid, &wqe->fcp_trsp.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 215 bf_set(wqe_sup, &wqe->fcp_trsp.wqe_com, 0); 216 bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 0); 217 bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 0); 218 bf_set(wqe_pbde, &wqe->fcp_trsp.wqe_com, 0); 219 220 /* Word 12, 13, 14, 15 - is zero */ 221 } 222 223 void 224 lpfc_nvmet_defer_release(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp) 225 { 226 lockdep_assert_held(&ctxp->ctxlock); 227 228 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 229 "6313 NVMET Defer ctx release xri x%x flg x%x\n", 230 ctxp->oxid, ctxp->flag); 231 232 if (ctxp->flag & LPFC_NVMET_CTX_RLS) 233 return; 234 235 ctxp->flag |= LPFC_NVMET_CTX_RLS; 236 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 237 list_add_tail(&ctxp->list, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list); 238 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 239 } 240 241 /** 242 * lpfc_nvmet_xmt_ls_rsp_cmp - Completion handler for LS Response 243 * @phba: Pointer to HBA context object. 244 * @cmdwqe: Pointer to driver command WQE object. 245 * @wcqe: Pointer to driver response CQE object. 246 * 247 * The function is called from SLI ring event handler with no 248 * lock held. This function is the completion handler for NVME LS commands 249 * The function frees memory resources used for the NVME commands. 250 **/ 251 static void 252 lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 253 struct lpfc_wcqe_complete *wcqe) 254 { 255 struct lpfc_nvmet_tgtport *tgtp; 256 struct nvmefc_tgt_ls_req *rsp; 257 struct lpfc_nvmet_rcv_ctx *ctxp; 258 uint32_t status, result; 259 260 status = bf_get(lpfc_wcqe_c_status, wcqe); 261 result = wcqe->parameter; 262 ctxp = cmdwqe->context2; 263 264 if (ctxp->state != LPFC_NVMET_STE_LS_RSP || ctxp->entry_cnt != 2) { 265 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 266 "6410 NVMET LS cmpl state mismatch IO x%x: " 267 "%d %d\n", 268 ctxp->oxid, ctxp->state, ctxp->entry_cnt); 269 } 270 271 if (!phba->targetport) 272 goto out; 273 274 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 275 276 if (tgtp) { 277 if (status) { 278 atomic_inc(&tgtp->xmt_ls_rsp_error); 279 if (result == IOERR_ABORT_REQUESTED) 280 atomic_inc(&tgtp->xmt_ls_rsp_aborted); 281 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 282 atomic_inc(&tgtp->xmt_ls_rsp_xb_set); 283 } else { 284 atomic_inc(&tgtp->xmt_ls_rsp_cmpl); 285 } 286 } 287 288 out: 289 rsp = &ctxp->ctx.ls_req; 290 291 lpfc_nvmeio_data(phba, "NVMET LS CMPL: xri x%x stat x%x result x%x\n", 292 ctxp->oxid, status, result); 293 294 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 295 "6038 NVMET LS rsp cmpl: %d %d oxid x%x\n", 296 status, result, ctxp->oxid); 297 298 lpfc_nlp_put(cmdwqe->context1); 299 cmdwqe->context2 = NULL; 300 cmdwqe->context3 = NULL; 301 lpfc_sli_release_iocbq(phba, cmdwqe); 302 rsp->done(rsp); 303 kfree(ctxp); 304 } 305 306 /** 307 * lpfc_nvmet_ctxbuf_post - Repost a NVMET RQ DMA buffer and clean up context 308 * @phba: HBA buffer is associated with 309 * @ctxp: context to clean up 310 * @mp: Buffer to free 311 * 312 * Description: Frees the given DMA buffer in the appropriate way given by 313 * reposting it to its associated RQ so it can be reused. 314 * 315 * Notes: Takes phba->hbalock. Can be called with or without other locks held. 316 * 317 * Returns: None 318 **/ 319 void 320 lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) 321 { 322 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 323 struct lpfc_nvmet_rcv_ctx *ctxp = ctx_buf->context; 324 struct lpfc_nvmet_tgtport *tgtp; 325 struct fc_frame_header *fc_hdr; 326 struct rqb_dmabuf *nvmebuf; 327 struct lpfc_nvmet_ctx_info *infop; 328 uint32_t *payload; 329 uint32_t size, oxid, sid; 330 int cpu; 331 unsigned long iflag; 332 333 if (ctxp->txrdy) { 334 dma_pool_free(phba->txrdy_payload_pool, ctxp->txrdy, 335 ctxp->txrdy_phys); 336 ctxp->txrdy = NULL; 337 ctxp->txrdy_phys = 0; 338 } 339 340 if (ctxp->state == LPFC_NVMET_STE_FREE) { 341 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 342 "6411 NVMET free, already free IO x%x: %d %d\n", 343 ctxp->oxid, ctxp->state, ctxp->entry_cnt); 344 } 345 346 if (ctxp->rqb_buffer) { 347 nvmebuf = ctxp->rqb_buffer; 348 spin_lock_irqsave(&ctxp->ctxlock, iflag); 349 ctxp->rqb_buffer = NULL; 350 if (ctxp->flag & LPFC_NVMET_CTX_REUSE_WQ) { 351 ctxp->flag &= ~LPFC_NVMET_CTX_REUSE_WQ; 352 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 353 nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); 354 } else { 355 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 356 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */ 357 } 358 } 359 ctxp->state = LPFC_NVMET_STE_FREE; 360 361 spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag); 362 if (phba->sli4_hba.nvmet_io_wait_cnt) { 363 list_remove_head(&phba->sli4_hba.lpfc_nvmet_io_wait_list, 364 nvmebuf, struct rqb_dmabuf, 365 hbuf.list); 366 phba->sli4_hba.nvmet_io_wait_cnt--; 367 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, 368 iflag); 369 370 fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt); 371 oxid = be16_to_cpu(fc_hdr->fh_ox_id); 372 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 373 payload = (uint32_t *)(nvmebuf->dbuf.virt); 374 size = nvmebuf->bytes_recv; 375 sid = sli4_sid_from_fc_hdr(fc_hdr); 376 377 ctxp = (struct lpfc_nvmet_rcv_ctx *)ctx_buf->context; 378 ctxp->wqeq = NULL; 379 ctxp->txrdy = NULL; 380 ctxp->offset = 0; 381 ctxp->phba = phba; 382 ctxp->size = size; 383 ctxp->oxid = oxid; 384 ctxp->sid = sid; 385 ctxp->state = LPFC_NVMET_STE_RCV; 386 ctxp->entry_cnt = 1; 387 ctxp->flag = 0; 388 ctxp->ctxbuf = ctx_buf; 389 ctxp->rqb_buffer = (void *)nvmebuf; 390 spin_lock_init(&ctxp->ctxlock); 391 392 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 393 if (ctxp->ts_cmd_nvme) { 394 ctxp->ts_cmd_nvme = ktime_get_ns(); 395 ctxp->ts_nvme_data = 0; 396 ctxp->ts_data_wqput = 0; 397 ctxp->ts_isr_data = 0; 398 ctxp->ts_data_nvme = 0; 399 ctxp->ts_nvme_status = 0; 400 ctxp->ts_status_wqput = 0; 401 ctxp->ts_isr_status = 0; 402 ctxp->ts_status_nvme = 0; 403 } 404 #endif 405 atomic_inc(&tgtp->rcv_fcp_cmd_in); 406 407 /* flag new work queued, replacement buffer has already 408 * been reposted 409 */ 410 spin_lock_irqsave(&ctxp->ctxlock, iflag); 411 ctxp->flag |= LPFC_NVMET_CTX_REUSE_WQ; 412 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 413 414 if (!queue_work(phba->wq, &ctx_buf->defer_work)) { 415 atomic_inc(&tgtp->rcv_fcp_cmd_drop); 416 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 417 "6181 Unable to queue deferred work " 418 "for oxid x%x. " 419 "FCP Drop IO [x%x x%x x%x]\n", 420 ctxp->oxid, 421 atomic_read(&tgtp->rcv_fcp_cmd_in), 422 atomic_read(&tgtp->rcv_fcp_cmd_out), 423 atomic_read(&tgtp->xmt_fcp_release)); 424 425 spin_lock_irqsave(&ctxp->ctxlock, iflag); 426 lpfc_nvmet_defer_release(phba, ctxp); 427 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 428 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid); 429 } 430 return; 431 } 432 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, iflag); 433 434 /* 435 * Use the CPU context list, from the MRQ the IO was received on 436 * (ctxp->idx), to save context structure. 437 */ 438 cpu = smp_processor_id(); 439 infop = lpfc_get_ctx_list(phba, cpu, ctxp->idx); 440 spin_lock_irqsave(&infop->nvmet_ctx_list_lock, iflag); 441 list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list); 442 infop->nvmet_ctx_list_cnt++; 443 spin_unlock_irqrestore(&infop->nvmet_ctx_list_lock, iflag); 444 #endif 445 } 446 447 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 448 static void 449 lpfc_nvmet_ktime(struct lpfc_hba *phba, 450 struct lpfc_nvmet_rcv_ctx *ctxp) 451 { 452 uint64_t seg1, seg2, seg3, seg4, seg5; 453 uint64_t seg6, seg7, seg8, seg9, seg10; 454 uint64_t segsum; 455 456 if (!ctxp->ts_isr_cmd || !ctxp->ts_cmd_nvme || 457 !ctxp->ts_nvme_data || !ctxp->ts_data_wqput || 458 !ctxp->ts_isr_data || !ctxp->ts_data_nvme || 459 !ctxp->ts_nvme_status || !ctxp->ts_status_wqput || 460 !ctxp->ts_isr_status || !ctxp->ts_status_nvme) 461 return; 462 463 if (ctxp->ts_status_nvme < ctxp->ts_isr_cmd) 464 return; 465 if (ctxp->ts_isr_cmd > ctxp->ts_cmd_nvme) 466 return; 467 if (ctxp->ts_cmd_nvme > ctxp->ts_nvme_data) 468 return; 469 if (ctxp->ts_nvme_data > ctxp->ts_data_wqput) 470 return; 471 if (ctxp->ts_data_wqput > ctxp->ts_isr_data) 472 return; 473 if (ctxp->ts_isr_data > ctxp->ts_data_nvme) 474 return; 475 if (ctxp->ts_data_nvme > ctxp->ts_nvme_status) 476 return; 477 if (ctxp->ts_nvme_status > ctxp->ts_status_wqput) 478 return; 479 if (ctxp->ts_status_wqput > ctxp->ts_isr_status) 480 return; 481 if (ctxp->ts_isr_status > ctxp->ts_status_nvme) 482 return; 483 /* 484 * Segment 1 - Time from FCP command received by MSI-X ISR 485 * to FCP command is passed to NVME Layer. 486 * Segment 2 - Time from FCP command payload handed 487 * off to NVME Layer to Driver receives a Command op 488 * from NVME Layer. 489 * Segment 3 - Time from Driver receives a Command op 490 * from NVME Layer to Command is put on WQ. 491 * Segment 4 - Time from Driver WQ put is done 492 * to MSI-X ISR for Command cmpl. 493 * Segment 5 - Time from MSI-X ISR for Command cmpl to 494 * Command cmpl is passed to NVME Layer. 495 * Segment 6 - Time from Command cmpl is passed to NVME 496 * Layer to Driver receives a RSP op from NVME Layer. 497 * Segment 7 - Time from Driver receives a RSP op from 498 * NVME Layer to WQ put is done on TRSP FCP Status. 499 * Segment 8 - Time from Driver WQ put is done on TRSP 500 * FCP Status to MSI-X ISR for TRSP cmpl. 501 * Segment 9 - Time from MSI-X ISR for TRSP cmpl to 502 * TRSP cmpl is passed to NVME Layer. 503 * Segment 10 - Time from FCP command received by 504 * MSI-X ISR to command is completed on wire. 505 * (Segments 1 thru 8) for READDATA / WRITEDATA 506 * (Segments 1 thru 4) for READDATA_RSP 507 */ 508 seg1 = ctxp->ts_cmd_nvme - ctxp->ts_isr_cmd; 509 segsum = seg1; 510 511 seg2 = ctxp->ts_nvme_data - ctxp->ts_isr_cmd; 512 if (segsum > seg2) 513 return; 514 seg2 -= segsum; 515 segsum += seg2; 516 517 seg3 = ctxp->ts_data_wqput - ctxp->ts_isr_cmd; 518 if (segsum > seg3) 519 return; 520 seg3 -= segsum; 521 segsum += seg3; 522 523 seg4 = ctxp->ts_isr_data - ctxp->ts_isr_cmd; 524 if (segsum > seg4) 525 return; 526 seg4 -= segsum; 527 segsum += seg4; 528 529 seg5 = ctxp->ts_data_nvme - ctxp->ts_isr_cmd; 530 if (segsum > seg5) 531 return; 532 seg5 -= segsum; 533 segsum += seg5; 534 535 536 /* For auto rsp commands seg6 thru seg10 will be 0 */ 537 if (ctxp->ts_nvme_status > ctxp->ts_data_nvme) { 538 seg6 = ctxp->ts_nvme_status - ctxp->ts_isr_cmd; 539 if (segsum > seg6) 540 return; 541 seg6 -= segsum; 542 segsum += seg6; 543 544 seg7 = ctxp->ts_status_wqput - ctxp->ts_isr_cmd; 545 if (segsum > seg7) 546 return; 547 seg7 -= segsum; 548 segsum += seg7; 549 550 seg8 = ctxp->ts_isr_status - ctxp->ts_isr_cmd; 551 if (segsum > seg8) 552 return; 553 seg8 -= segsum; 554 segsum += seg8; 555 556 seg9 = ctxp->ts_status_nvme - ctxp->ts_isr_cmd; 557 if (segsum > seg9) 558 return; 559 seg9 -= segsum; 560 segsum += seg9; 561 562 if (ctxp->ts_isr_status < ctxp->ts_isr_cmd) 563 return; 564 seg10 = (ctxp->ts_isr_status - 565 ctxp->ts_isr_cmd); 566 } else { 567 if (ctxp->ts_isr_data < ctxp->ts_isr_cmd) 568 return; 569 seg6 = 0; 570 seg7 = 0; 571 seg8 = 0; 572 seg9 = 0; 573 seg10 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd); 574 } 575 576 phba->ktime_seg1_total += seg1; 577 if (seg1 < phba->ktime_seg1_min) 578 phba->ktime_seg1_min = seg1; 579 else if (seg1 > phba->ktime_seg1_max) 580 phba->ktime_seg1_max = seg1; 581 582 phba->ktime_seg2_total += seg2; 583 if (seg2 < phba->ktime_seg2_min) 584 phba->ktime_seg2_min = seg2; 585 else if (seg2 > phba->ktime_seg2_max) 586 phba->ktime_seg2_max = seg2; 587 588 phba->ktime_seg3_total += seg3; 589 if (seg3 < phba->ktime_seg3_min) 590 phba->ktime_seg3_min = seg3; 591 else if (seg3 > phba->ktime_seg3_max) 592 phba->ktime_seg3_max = seg3; 593 594 phba->ktime_seg4_total += seg4; 595 if (seg4 < phba->ktime_seg4_min) 596 phba->ktime_seg4_min = seg4; 597 else if (seg4 > phba->ktime_seg4_max) 598 phba->ktime_seg4_max = seg4; 599 600 phba->ktime_seg5_total += seg5; 601 if (seg5 < phba->ktime_seg5_min) 602 phba->ktime_seg5_min = seg5; 603 else if (seg5 > phba->ktime_seg5_max) 604 phba->ktime_seg5_max = seg5; 605 606 phba->ktime_data_samples++; 607 if (!seg6) 608 goto out; 609 610 phba->ktime_seg6_total += seg6; 611 if (seg6 < phba->ktime_seg6_min) 612 phba->ktime_seg6_min = seg6; 613 else if (seg6 > phba->ktime_seg6_max) 614 phba->ktime_seg6_max = seg6; 615 616 phba->ktime_seg7_total += seg7; 617 if (seg7 < phba->ktime_seg7_min) 618 phba->ktime_seg7_min = seg7; 619 else if (seg7 > phba->ktime_seg7_max) 620 phba->ktime_seg7_max = seg7; 621 622 phba->ktime_seg8_total += seg8; 623 if (seg8 < phba->ktime_seg8_min) 624 phba->ktime_seg8_min = seg8; 625 else if (seg8 > phba->ktime_seg8_max) 626 phba->ktime_seg8_max = seg8; 627 628 phba->ktime_seg9_total += seg9; 629 if (seg9 < phba->ktime_seg9_min) 630 phba->ktime_seg9_min = seg9; 631 else if (seg9 > phba->ktime_seg9_max) 632 phba->ktime_seg9_max = seg9; 633 out: 634 phba->ktime_seg10_total += seg10; 635 if (seg10 < phba->ktime_seg10_min) 636 phba->ktime_seg10_min = seg10; 637 else if (seg10 > phba->ktime_seg10_max) 638 phba->ktime_seg10_max = seg10; 639 phba->ktime_status_samples++; 640 } 641 #endif 642 643 /** 644 * lpfc_nvmet_xmt_fcp_op_cmp - Completion handler for FCP Response 645 * @phba: Pointer to HBA context object. 646 * @cmdwqe: Pointer to driver command WQE object. 647 * @wcqe: Pointer to driver response CQE object. 648 * 649 * The function is called from SLI ring event handler with no 650 * lock held. This function is the completion handler for NVME FCP commands 651 * The function frees memory resources used for the NVME commands. 652 **/ 653 static void 654 lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 655 struct lpfc_wcqe_complete *wcqe) 656 { 657 struct lpfc_nvmet_tgtport *tgtp; 658 struct nvmefc_tgt_fcp_req *rsp; 659 struct lpfc_nvmet_rcv_ctx *ctxp; 660 uint32_t status, result, op, start_clean, logerr; 661 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 662 uint32_t id; 663 #endif 664 665 ctxp = cmdwqe->context2; 666 ctxp->flag &= ~LPFC_NVMET_IO_INP; 667 668 rsp = &ctxp->ctx.fcp_req; 669 op = rsp->op; 670 671 status = bf_get(lpfc_wcqe_c_status, wcqe); 672 result = wcqe->parameter; 673 674 if (phba->targetport) 675 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 676 else 677 tgtp = NULL; 678 679 lpfc_nvmeio_data(phba, "NVMET FCP CMPL: xri x%x op x%x status x%x\n", 680 ctxp->oxid, op, status); 681 682 if (status) { 683 rsp->fcp_error = NVME_SC_DATA_XFER_ERROR; 684 rsp->transferred_length = 0; 685 if (tgtp) { 686 atomic_inc(&tgtp->xmt_fcp_rsp_error); 687 if (result == IOERR_ABORT_REQUESTED) 688 atomic_inc(&tgtp->xmt_fcp_rsp_aborted); 689 } 690 691 logerr = LOG_NVME_IOERR; 692 693 /* pick up SLI4 exhange busy condition */ 694 if (bf_get(lpfc_wcqe_c_xb, wcqe)) { 695 ctxp->flag |= LPFC_NVMET_XBUSY; 696 logerr |= LOG_NVME_ABTS; 697 if (tgtp) 698 atomic_inc(&tgtp->xmt_fcp_rsp_xb_set); 699 700 } else { 701 ctxp->flag &= ~LPFC_NVMET_XBUSY; 702 } 703 704 lpfc_printf_log(phba, KERN_INFO, logerr, 705 "6315 IO Error Cmpl xri x%x: %x/%x XBUSY:x%x\n", 706 ctxp->oxid, status, result, ctxp->flag); 707 708 } else { 709 rsp->fcp_error = NVME_SC_SUCCESS; 710 if (op == NVMET_FCOP_RSP) 711 rsp->transferred_length = rsp->rsplen; 712 else 713 rsp->transferred_length = rsp->transfer_length; 714 if (tgtp) 715 atomic_inc(&tgtp->xmt_fcp_rsp_cmpl); 716 } 717 718 if ((op == NVMET_FCOP_READDATA_RSP) || 719 (op == NVMET_FCOP_RSP)) { 720 /* Sanity check */ 721 ctxp->state = LPFC_NVMET_STE_DONE; 722 ctxp->entry_cnt++; 723 724 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 725 if (ctxp->ts_cmd_nvme) { 726 if (rsp->op == NVMET_FCOP_READDATA_RSP) { 727 ctxp->ts_isr_data = 728 cmdwqe->isr_timestamp; 729 ctxp->ts_data_nvme = 730 ktime_get_ns(); 731 ctxp->ts_nvme_status = 732 ctxp->ts_data_nvme; 733 ctxp->ts_status_wqput = 734 ctxp->ts_data_nvme; 735 ctxp->ts_isr_status = 736 ctxp->ts_data_nvme; 737 ctxp->ts_status_nvme = 738 ctxp->ts_data_nvme; 739 } else { 740 ctxp->ts_isr_status = 741 cmdwqe->isr_timestamp; 742 ctxp->ts_status_nvme = 743 ktime_get_ns(); 744 } 745 } 746 #endif 747 rsp->done(rsp); 748 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 749 if (ctxp->ts_cmd_nvme) 750 lpfc_nvmet_ktime(phba, ctxp); 751 #endif 752 /* lpfc_nvmet_xmt_fcp_release() will recycle the context */ 753 } else { 754 ctxp->entry_cnt++; 755 start_clean = offsetof(struct lpfc_iocbq, iocb_flag); 756 memset(((char *)cmdwqe) + start_clean, 0, 757 (sizeof(struct lpfc_iocbq) - start_clean)); 758 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 759 if (ctxp->ts_cmd_nvme) { 760 ctxp->ts_isr_data = cmdwqe->isr_timestamp; 761 ctxp->ts_data_nvme = ktime_get_ns(); 762 } 763 #endif 764 rsp->done(rsp); 765 } 766 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 767 if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) { 768 id = smp_processor_id(); 769 if (id < LPFC_CHECK_CPU_CNT) { 770 if (ctxp->cpu != id) 771 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 772 "6704 CPU Check cmdcmpl: " 773 "cpu %d expect %d\n", 774 id, ctxp->cpu); 775 phba->sli4_hba.hdwq[rsp->hwqid].cpucheck_cmpl_io[id]++; 776 } 777 } 778 #endif 779 } 780 781 static int 782 lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport, 783 struct nvmefc_tgt_ls_req *rsp) 784 { 785 struct lpfc_nvmet_rcv_ctx *ctxp = 786 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.ls_req); 787 struct lpfc_hba *phba = ctxp->phba; 788 struct hbq_dmabuf *nvmebuf = 789 (struct hbq_dmabuf *)ctxp->rqb_buffer; 790 struct lpfc_iocbq *nvmewqeq; 791 struct lpfc_nvmet_tgtport *nvmep = tgtport->private; 792 struct lpfc_dmabuf dmabuf; 793 struct ulp_bde64 bpl; 794 int rc; 795 796 if (phba->pport->load_flag & FC_UNLOADING) 797 return -ENODEV; 798 799 if (phba->pport->load_flag & FC_UNLOADING) 800 return -ENODEV; 801 802 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 803 "6023 NVMET LS rsp oxid x%x\n", ctxp->oxid); 804 805 if ((ctxp->state != LPFC_NVMET_STE_LS_RCV) || 806 (ctxp->entry_cnt != 1)) { 807 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 808 "6412 NVMET LS rsp state mismatch " 809 "oxid x%x: %d %d\n", 810 ctxp->oxid, ctxp->state, ctxp->entry_cnt); 811 } 812 ctxp->state = LPFC_NVMET_STE_LS_RSP; 813 ctxp->entry_cnt++; 814 815 nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, ctxp, rsp->rspdma, 816 rsp->rsplen); 817 if (nvmewqeq == NULL) { 818 atomic_inc(&nvmep->xmt_ls_drop); 819 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 820 "6150 LS Drop IO x%x: Prep\n", 821 ctxp->oxid); 822 lpfc_in_buf_free(phba, &nvmebuf->dbuf); 823 atomic_inc(&nvmep->xmt_ls_abort); 824 lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, 825 ctxp->sid, ctxp->oxid); 826 return -ENOMEM; 827 } 828 829 /* Save numBdes for bpl2sgl */ 830 nvmewqeq->rsvd2 = 1; 831 nvmewqeq->hba_wqidx = 0; 832 nvmewqeq->context3 = &dmabuf; 833 dmabuf.virt = &bpl; 834 bpl.addrLow = nvmewqeq->wqe.xmit_sequence.bde.addrLow; 835 bpl.addrHigh = nvmewqeq->wqe.xmit_sequence.bde.addrHigh; 836 bpl.tus.f.bdeSize = rsp->rsplen; 837 bpl.tus.f.bdeFlags = 0; 838 bpl.tus.w = le32_to_cpu(bpl.tus.w); 839 840 nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_rsp_cmp; 841 nvmewqeq->iocb_cmpl = NULL; 842 nvmewqeq->context2 = ctxp; 843 844 lpfc_nvmeio_data(phba, "NVMET LS RESP: xri x%x wqidx x%x len x%x\n", 845 ctxp->oxid, nvmewqeq->hba_wqidx, rsp->rsplen); 846 847 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq); 848 if (rc == WQE_SUCCESS) { 849 /* 850 * Okay to repost buffer here, but wait till cmpl 851 * before freeing ctxp and iocbq. 852 */ 853 lpfc_in_buf_free(phba, &nvmebuf->dbuf); 854 ctxp->rqb_buffer = 0; 855 atomic_inc(&nvmep->xmt_ls_rsp); 856 return 0; 857 } 858 /* Give back resources */ 859 atomic_inc(&nvmep->xmt_ls_drop); 860 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 861 "6151 LS Drop IO x%x: Issue %d\n", 862 ctxp->oxid, rc); 863 864 lpfc_nlp_put(nvmewqeq->context1); 865 866 lpfc_in_buf_free(phba, &nvmebuf->dbuf); 867 atomic_inc(&nvmep->xmt_ls_abort); 868 lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); 869 return -ENXIO; 870 } 871 872 static int 873 lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport, 874 struct nvmefc_tgt_fcp_req *rsp) 875 { 876 struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private; 877 struct lpfc_nvmet_rcv_ctx *ctxp = 878 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req); 879 struct lpfc_hba *phba = ctxp->phba; 880 struct lpfc_queue *wq; 881 struct lpfc_iocbq *nvmewqeq; 882 struct lpfc_sli_ring *pring; 883 unsigned long iflags; 884 int rc; 885 886 if (phba->pport->load_flag & FC_UNLOADING) { 887 rc = -ENODEV; 888 goto aerr; 889 } 890 891 if (phba->pport->load_flag & FC_UNLOADING) { 892 rc = -ENODEV; 893 goto aerr; 894 } 895 896 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 897 if (ctxp->ts_cmd_nvme) { 898 if (rsp->op == NVMET_FCOP_RSP) 899 ctxp->ts_nvme_status = ktime_get_ns(); 900 else 901 ctxp->ts_nvme_data = ktime_get_ns(); 902 } 903 904 /* Setup the hdw queue if not already set */ 905 if (!ctxp->hdwq) 906 ctxp->hdwq = &phba->sli4_hba.hdwq[rsp->hwqid]; 907 908 if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) { 909 int id = smp_processor_id(); 910 if (id < LPFC_CHECK_CPU_CNT) { 911 if (rsp->hwqid != id) 912 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 913 "6705 CPU Check OP: " 914 "cpu %d expect %d\n", 915 id, rsp->hwqid); 916 phba->sli4_hba.hdwq[rsp->hwqid].cpucheck_xmt_io[id]++; 917 } 918 ctxp->cpu = id; /* Setup cpu for cmpl check */ 919 } 920 #endif 921 922 /* Sanity check */ 923 if ((ctxp->flag & LPFC_NVMET_ABTS_RCV) || 924 (ctxp->state == LPFC_NVMET_STE_ABORT)) { 925 atomic_inc(&lpfc_nvmep->xmt_fcp_drop); 926 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 927 "6102 IO xri x%x aborted\n", 928 ctxp->oxid); 929 rc = -ENXIO; 930 goto aerr; 931 } 932 933 nvmewqeq = lpfc_nvmet_prep_fcp_wqe(phba, ctxp); 934 if (nvmewqeq == NULL) { 935 atomic_inc(&lpfc_nvmep->xmt_fcp_drop); 936 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 937 "6152 FCP Drop IO x%x: Prep\n", 938 ctxp->oxid); 939 rc = -ENXIO; 940 goto aerr; 941 } 942 943 nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_fcp_op_cmp; 944 nvmewqeq->iocb_cmpl = NULL; 945 nvmewqeq->context2 = ctxp; 946 nvmewqeq->iocb_flag |= LPFC_IO_NVMET; 947 ctxp->wqeq->hba_wqidx = rsp->hwqid; 948 949 lpfc_nvmeio_data(phba, "NVMET FCP CMND: xri x%x op x%x len x%x\n", 950 ctxp->oxid, rsp->op, rsp->rsplen); 951 952 ctxp->flag |= LPFC_NVMET_IO_INP; 953 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq); 954 if (rc == WQE_SUCCESS) { 955 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 956 if (!ctxp->ts_cmd_nvme) 957 return 0; 958 if (rsp->op == NVMET_FCOP_RSP) 959 ctxp->ts_status_wqput = ktime_get_ns(); 960 else 961 ctxp->ts_data_wqput = ktime_get_ns(); 962 #endif 963 return 0; 964 } 965 966 if (rc == -EBUSY) { 967 /* 968 * WQ was full, so queue nvmewqeq to be sent after 969 * WQE release CQE 970 */ 971 ctxp->flag |= LPFC_NVMET_DEFER_WQFULL; 972 wq = ctxp->hdwq->nvme_wq; 973 pring = wq->pring; 974 spin_lock_irqsave(&pring->ring_lock, iflags); 975 list_add_tail(&nvmewqeq->list, &wq->wqfull_list); 976 wq->q_flag |= HBA_NVMET_WQFULL; 977 spin_unlock_irqrestore(&pring->ring_lock, iflags); 978 atomic_inc(&lpfc_nvmep->defer_wqfull); 979 return 0; 980 } 981 982 /* Give back resources */ 983 atomic_inc(&lpfc_nvmep->xmt_fcp_drop); 984 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 985 "6153 FCP Drop IO x%x: Issue: %d\n", 986 ctxp->oxid, rc); 987 988 ctxp->wqeq->hba_wqidx = 0; 989 nvmewqeq->context2 = NULL; 990 nvmewqeq->context3 = NULL; 991 rc = -EBUSY; 992 aerr: 993 return rc; 994 } 995 996 static void 997 lpfc_nvmet_targetport_delete(struct nvmet_fc_target_port *targetport) 998 { 999 struct lpfc_nvmet_tgtport *tport = targetport->private; 1000 1001 /* release any threads waiting for the unreg to complete */ 1002 if (tport->phba->targetport) 1003 complete(tport->tport_unreg_cmp); 1004 } 1005 1006 static void 1007 lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport, 1008 struct nvmefc_tgt_fcp_req *req) 1009 { 1010 struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private; 1011 struct lpfc_nvmet_rcv_ctx *ctxp = 1012 container_of(req, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req); 1013 struct lpfc_hba *phba = ctxp->phba; 1014 struct lpfc_queue *wq; 1015 unsigned long flags; 1016 1017 if (phba->pport->load_flag & FC_UNLOADING) 1018 return; 1019 1020 if (phba->pport->load_flag & FC_UNLOADING) 1021 return; 1022 1023 if (!ctxp->hdwq) 1024 ctxp->hdwq = &phba->sli4_hba.hdwq[0]; 1025 1026 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1027 "6103 NVMET Abort op: oxri x%x flg x%x ste %d\n", 1028 ctxp->oxid, ctxp->flag, ctxp->state); 1029 1030 lpfc_nvmeio_data(phba, "NVMET FCP ABRT: xri x%x flg x%x ste x%x\n", 1031 ctxp->oxid, ctxp->flag, ctxp->state); 1032 1033 atomic_inc(&lpfc_nvmep->xmt_fcp_abort); 1034 1035 spin_lock_irqsave(&ctxp->ctxlock, flags); 1036 1037 /* Since iaab/iaar are NOT set, we need to check 1038 * if the firmware is in process of aborting IO 1039 */ 1040 if (ctxp->flag & LPFC_NVMET_XBUSY) { 1041 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 1042 return; 1043 } 1044 ctxp->flag |= LPFC_NVMET_ABORT_OP; 1045 1046 if (ctxp->flag & LPFC_NVMET_DEFER_WQFULL) { 1047 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 1048 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, 1049 ctxp->oxid); 1050 wq = ctxp->hdwq->nvme_wq; 1051 lpfc_nvmet_wqfull_flush(phba, wq, ctxp); 1052 return; 1053 } 1054 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 1055 1056 /* An state of LPFC_NVMET_STE_RCV means we have just received 1057 * the NVME command and have not started processing it. 1058 * (by issuing any IO WQEs on this exchange yet) 1059 */ 1060 if (ctxp->state == LPFC_NVMET_STE_RCV) 1061 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, 1062 ctxp->oxid); 1063 else 1064 lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid, 1065 ctxp->oxid); 1066 } 1067 1068 static void 1069 lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport, 1070 struct nvmefc_tgt_fcp_req *rsp) 1071 { 1072 struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private; 1073 struct lpfc_nvmet_rcv_ctx *ctxp = 1074 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req); 1075 struct lpfc_hba *phba = ctxp->phba; 1076 unsigned long flags; 1077 bool aborting = false; 1078 1079 spin_lock_irqsave(&ctxp->ctxlock, flags); 1080 if (ctxp->flag & LPFC_NVMET_XBUSY) 1081 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 1082 "6027 NVMET release with XBUSY flag x%x" 1083 " oxid x%x\n", 1084 ctxp->flag, ctxp->oxid); 1085 else if (ctxp->state != LPFC_NVMET_STE_DONE && 1086 ctxp->state != LPFC_NVMET_STE_ABORT) 1087 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1088 "6413 NVMET release bad state %d %d oxid x%x\n", 1089 ctxp->state, ctxp->entry_cnt, ctxp->oxid); 1090 1091 if ((ctxp->flag & LPFC_NVMET_ABORT_OP) || 1092 (ctxp->flag & LPFC_NVMET_XBUSY)) { 1093 aborting = true; 1094 /* let the abort path do the real release */ 1095 lpfc_nvmet_defer_release(phba, ctxp); 1096 } 1097 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 1098 1099 lpfc_nvmeio_data(phba, "NVMET FCP FREE: xri x%x ste %d abt %d\n", ctxp->oxid, 1100 ctxp->state, aborting); 1101 1102 atomic_inc(&lpfc_nvmep->xmt_fcp_release); 1103 1104 if (aborting) 1105 return; 1106 1107 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); 1108 } 1109 1110 static void 1111 lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport, 1112 struct nvmefc_tgt_fcp_req *rsp) 1113 { 1114 struct lpfc_nvmet_tgtport *tgtp; 1115 struct lpfc_nvmet_rcv_ctx *ctxp = 1116 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req); 1117 struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer; 1118 struct lpfc_hba *phba = ctxp->phba; 1119 unsigned long iflag; 1120 1121 1122 lpfc_nvmeio_data(phba, "NVMET DEFERRCV: xri x%x sz %d CPU %02x\n", 1123 ctxp->oxid, ctxp->size, smp_processor_id()); 1124 1125 if (!nvmebuf) { 1126 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 1127 "6425 Defer rcv: no buffer xri x%x: " 1128 "flg %x ste %x\n", 1129 ctxp->oxid, ctxp->flag, ctxp->state); 1130 return; 1131 } 1132 1133 tgtp = phba->targetport->private; 1134 if (tgtp) 1135 atomic_inc(&tgtp->rcv_fcp_cmd_defer); 1136 1137 /* Free the nvmebuf since a new buffer already replaced it */ 1138 nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); 1139 spin_lock_irqsave(&ctxp->ctxlock, iflag); 1140 ctxp->rqb_buffer = NULL; 1141 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 1142 } 1143 1144 static struct nvmet_fc_target_template lpfc_tgttemplate = { 1145 .targetport_delete = lpfc_nvmet_targetport_delete, 1146 .xmt_ls_rsp = lpfc_nvmet_xmt_ls_rsp, 1147 .fcp_op = lpfc_nvmet_xmt_fcp_op, 1148 .fcp_abort = lpfc_nvmet_xmt_fcp_abort, 1149 .fcp_req_release = lpfc_nvmet_xmt_fcp_release, 1150 .defer_rcv = lpfc_nvmet_defer_rcv, 1151 1152 .max_hw_queues = 1, 1153 .max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS, 1154 .max_dif_sgl_segments = LPFC_NVMET_DEFAULT_SEGS, 1155 .dma_boundary = 0xFFFFFFFF, 1156 1157 /* optional features */ 1158 .target_features = 0, 1159 /* sizes of additional private data for data structures */ 1160 .target_priv_sz = sizeof(struct lpfc_nvmet_tgtport), 1161 }; 1162 1163 static void 1164 __lpfc_nvmet_clean_io_for_cpu(struct lpfc_hba *phba, 1165 struct lpfc_nvmet_ctx_info *infop) 1166 { 1167 struct lpfc_nvmet_ctxbuf *ctx_buf, *next_ctx_buf; 1168 unsigned long flags; 1169 1170 spin_lock_irqsave(&infop->nvmet_ctx_list_lock, flags); 1171 list_for_each_entry_safe(ctx_buf, next_ctx_buf, 1172 &infop->nvmet_ctx_list, list) { 1173 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1174 list_del_init(&ctx_buf->list); 1175 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1176 1177 __lpfc_clear_active_sglq(phba, ctx_buf->sglq->sli4_lxritag); 1178 ctx_buf->sglq->state = SGL_FREED; 1179 ctx_buf->sglq->ndlp = NULL; 1180 1181 spin_lock(&phba->sli4_hba.sgl_list_lock); 1182 list_add_tail(&ctx_buf->sglq->list, 1183 &phba->sli4_hba.lpfc_nvmet_sgl_list); 1184 spin_unlock(&phba->sli4_hba.sgl_list_lock); 1185 1186 lpfc_sli_release_iocbq(phba, ctx_buf->iocbq); 1187 kfree(ctx_buf->context); 1188 } 1189 spin_unlock_irqrestore(&infop->nvmet_ctx_list_lock, flags); 1190 } 1191 1192 static void 1193 lpfc_nvmet_cleanup_io_context(struct lpfc_hba *phba) 1194 { 1195 struct lpfc_nvmet_ctx_info *infop; 1196 int i, j; 1197 1198 /* The first context list, MRQ 0 CPU 0 */ 1199 infop = phba->sli4_hba.nvmet_ctx_info; 1200 if (!infop) 1201 return; 1202 1203 /* Cycle the the entire CPU context list for every MRQ */ 1204 for (i = 0; i < phba->cfg_nvmet_mrq; i++) { 1205 for_each_present_cpu(j) { 1206 infop = lpfc_get_ctx_list(phba, j, i); 1207 __lpfc_nvmet_clean_io_for_cpu(phba, infop); 1208 } 1209 } 1210 kfree(phba->sli4_hba.nvmet_ctx_info); 1211 phba->sli4_hba.nvmet_ctx_info = NULL; 1212 } 1213 1214 static int 1215 lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) 1216 { 1217 struct lpfc_nvmet_ctxbuf *ctx_buf; 1218 struct lpfc_iocbq *nvmewqe; 1219 union lpfc_wqe128 *wqe; 1220 struct lpfc_nvmet_ctx_info *last_infop; 1221 struct lpfc_nvmet_ctx_info *infop; 1222 int i, j, idx, cpu; 1223 1224 lpfc_printf_log(phba, KERN_INFO, LOG_NVME, 1225 "6403 Allocate NVMET resources for %d XRIs\n", 1226 phba->sli4_hba.nvmet_xri_cnt); 1227 1228 phba->sli4_hba.nvmet_ctx_info = kcalloc( 1229 phba->sli4_hba.num_possible_cpu * phba->cfg_nvmet_mrq, 1230 sizeof(struct lpfc_nvmet_ctx_info), GFP_KERNEL); 1231 if (!phba->sli4_hba.nvmet_ctx_info) { 1232 lpfc_printf_log(phba, KERN_ERR, LOG_INIT, 1233 "6419 Failed allocate memory for " 1234 "nvmet context lists\n"); 1235 return -ENOMEM; 1236 } 1237 1238 /* 1239 * Assuming X CPUs in the system, and Y MRQs, allocate some 1240 * lpfc_nvmet_ctx_info structures as follows: 1241 * 1242 * cpu0/mrq0 cpu1/mrq0 ... cpuX/mrq0 1243 * cpu0/mrq1 cpu1/mrq1 ... cpuX/mrq1 1244 * ... 1245 * cpuX/mrqY cpuX/mrqY ... cpuX/mrqY 1246 * 1247 * Each line represents a MRQ "silo" containing an entry for 1248 * every CPU. 1249 * 1250 * MRQ X is initially assumed to be associated with CPU X, thus 1251 * contexts are initially distributed across all MRQs using 1252 * the MRQ index (N) as follows cpuN/mrqN. When contexts are 1253 * freed, the are freed to the MRQ silo based on the CPU number 1254 * of the IO completion. Thus a context that was allocated for MRQ A 1255 * whose IO completed on CPU B will be freed to cpuB/mrqA. 1256 */ 1257 for_each_possible_cpu(i) { 1258 for (j = 0; j < phba->cfg_nvmet_mrq; j++) { 1259 infop = lpfc_get_ctx_list(phba, i, j); 1260 INIT_LIST_HEAD(&infop->nvmet_ctx_list); 1261 spin_lock_init(&infop->nvmet_ctx_list_lock); 1262 infop->nvmet_ctx_list_cnt = 0; 1263 } 1264 } 1265 1266 /* 1267 * Setup the next CPU context info ptr for each MRQ. 1268 * MRQ 0 will cycle thru CPUs 0 - X separately from 1269 * MRQ 1 cycling thru CPUs 0 - X, and so on. 1270 */ 1271 for (j = 0; j < phba->cfg_nvmet_mrq; j++) { 1272 last_infop = lpfc_get_ctx_list(phba, 1273 cpumask_first(cpu_present_mask), 1274 j); 1275 for (i = phba->sli4_hba.num_possible_cpu - 1; i >= 0; i--) { 1276 infop = lpfc_get_ctx_list(phba, i, j); 1277 infop->nvmet_ctx_next_cpu = last_infop; 1278 last_infop = infop; 1279 } 1280 } 1281 1282 /* For all nvmet xris, allocate resources needed to process a 1283 * received command on a per xri basis. 1284 */ 1285 idx = 0; 1286 cpu = cpumask_first(cpu_present_mask); 1287 for (i = 0; i < phba->sli4_hba.nvmet_xri_cnt; i++) { 1288 ctx_buf = kzalloc(sizeof(*ctx_buf), GFP_KERNEL); 1289 if (!ctx_buf) { 1290 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 1291 "6404 Ran out of memory for NVMET\n"); 1292 return -ENOMEM; 1293 } 1294 1295 ctx_buf->context = kzalloc(sizeof(*ctx_buf->context), 1296 GFP_KERNEL); 1297 if (!ctx_buf->context) { 1298 kfree(ctx_buf); 1299 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 1300 "6405 Ran out of NVMET " 1301 "context memory\n"); 1302 return -ENOMEM; 1303 } 1304 ctx_buf->context->ctxbuf = ctx_buf; 1305 ctx_buf->context->state = LPFC_NVMET_STE_FREE; 1306 1307 ctx_buf->iocbq = lpfc_sli_get_iocbq(phba); 1308 if (!ctx_buf->iocbq) { 1309 kfree(ctx_buf->context); 1310 kfree(ctx_buf); 1311 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 1312 "6406 Ran out of NVMET iocb/WQEs\n"); 1313 return -ENOMEM; 1314 } 1315 ctx_buf->iocbq->iocb_flag = LPFC_IO_NVMET; 1316 nvmewqe = ctx_buf->iocbq; 1317 wqe = &nvmewqe->wqe; 1318 1319 /* Initialize WQE */ 1320 memset(wqe, 0, sizeof(union lpfc_wqe)); 1321 1322 ctx_buf->iocbq->context1 = NULL; 1323 spin_lock(&phba->sli4_hba.sgl_list_lock); 1324 ctx_buf->sglq = __lpfc_sli_get_nvmet_sglq(phba, ctx_buf->iocbq); 1325 spin_unlock(&phba->sli4_hba.sgl_list_lock); 1326 if (!ctx_buf->sglq) { 1327 lpfc_sli_release_iocbq(phba, ctx_buf->iocbq); 1328 kfree(ctx_buf->context); 1329 kfree(ctx_buf); 1330 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 1331 "6407 Ran out of NVMET XRIs\n"); 1332 return -ENOMEM; 1333 } 1334 INIT_WORK(&ctx_buf->defer_work, lpfc_nvmet_fcp_rqst_defer_work); 1335 1336 /* 1337 * Add ctx to MRQidx context list. Our initial assumption 1338 * is MRQidx will be associated with CPUidx. This association 1339 * can change on the fly. 1340 */ 1341 infop = lpfc_get_ctx_list(phba, cpu, idx); 1342 spin_lock(&infop->nvmet_ctx_list_lock); 1343 list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list); 1344 infop->nvmet_ctx_list_cnt++; 1345 spin_unlock(&infop->nvmet_ctx_list_lock); 1346 1347 /* Spread ctx structures evenly across all MRQs */ 1348 idx++; 1349 if (idx >= phba->cfg_nvmet_mrq) { 1350 idx = 0; 1351 cpu = cpumask_first(cpu_present_mask); 1352 continue; 1353 } 1354 cpu = cpumask_next(cpu, cpu_present_mask); 1355 if (cpu == nr_cpu_ids) 1356 cpu = cpumask_first(cpu_present_mask); 1357 1358 } 1359 1360 for_each_present_cpu(i) { 1361 for (j = 0; j < phba->cfg_nvmet_mrq; j++) { 1362 infop = lpfc_get_ctx_list(phba, i, j); 1363 lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT, 1364 "6408 TOTAL NVMET ctx for CPU %d " 1365 "MRQ %d: cnt %d nextcpu %p\n", 1366 i, j, infop->nvmet_ctx_list_cnt, 1367 infop->nvmet_ctx_next_cpu); 1368 } 1369 } 1370 return 0; 1371 } 1372 1373 int 1374 lpfc_nvmet_create_targetport(struct lpfc_hba *phba) 1375 { 1376 struct lpfc_vport *vport = phba->pport; 1377 struct lpfc_nvmet_tgtport *tgtp; 1378 struct nvmet_fc_port_info pinfo; 1379 int error; 1380 1381 if (phba->targetport) 1382 return 0; 1383 1384 error = lpfc_nvmet_setup_io_context(phba); 1385 if (error) 1386 return error; 1387 1388 memset(&pinfo, 0, sizeof(struct nvmet_fc_port_info)); 1389 pinfo.node_name = wwn_to_u64(vport->fc_nodename.u.wwn); 1390 pinfo.port_name = wwn_to_u64(vport->fc_portname.u.wwn); 1391 pinfo.port_id = vport->fc_myDID; 1392 1393 /* We need to tell the transport layer + 1 because it takes page 1394 * alignment into account. When space for the SGL is allocated we 1395 * allocate + 3, one for cmd, one for rsp and one for this alignment 1396 */ 1397 lpfc_tgttemplate.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; 1398 lpfc_tgttemplate.max_hw_queues = phba->cfg_hdw_queue; 1399 lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP; 1400 1401 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1402 error = nvmet_fc_register_targetport(&pinfo, &lpfc_tgttemplate, 1403 &phba->pcidev->dev, 1404 &phba->targetport); 1405 #else 1406 error = -ENOENT; 1407 #endif 1408 if (error) { 1409 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, 1410 "6025 Cannot register NVME targetport x%x: " 1411 "portnm %llx nodenm %llx segs %d qs %d\n", 1412 error, 1413 pinfo.port_name, pinfo.node_name, 1414 lpfc_tgttemplate.max_sgl_segments, 1415 lpfc_tgttemplate.max_hw_queues); 1416 phba->targetport = NULL; 1417 phba->nvmet_support = 0; 1418 1419 lpfc_nvmet_cleanup_io_context(phba); 1420 1421 } else { 1422 tgtp = (struct lpfc_nvmet_tgtport *) 1423 phba->targetport->private; 1424 tgtp->phba = phba; 1425 1426 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 1427 "6026 Registered NVME " 1428 "targetport: %p, private %p " 1429 "portnm %llx nodenm %llx segs %d qs %d\n", 1430 phba->targetport, tgtp, 1431 pinfo.port_name, pinfo.node_name, 1432 lpfc_tgttemplate.max_sgl_segments, 1433 lpfc_tgttemplate.max_hw_queues); 1434 1435 atomic_set(&tgtp->rcv_ls_req_in, 0); 1436 atomic_set(&tgtp->rcv_ls_req_out, 0); 1437 atomic_set(&tgtp->rcv_ls_req_drop, 0); 1438 atomic_set(&tgtp->xmt_ls_abort, 0); 1439 atomic_set(&tgtp->xmt_ls_abort_cmpl, 0); 1440 atomic_set(&tgtp->xmt_ls_rsp, 0); 1441 atomic_set(&tgtp->xmt_ls_drop, 0); 1442 atomic_set(&tgtp->xmt_ls_rsp_error, 0); 1443 atomic_set(&tgtp->xmt_ls_rsp_xb_set, 0); 1444 atomic_set(&tgtp->xmt_ls_rsp_aborted, 0); 1445 atomic_set(&tgtp->xmt_ls_rsp_cmpl, 0); 1446 atomic_set(&tgtp->rcv_fcp_cmd_in, 0); 1447 atomic_set(&tgtp->rcv_fcp_cmd_out, 0); 1448 atomic_set(&tgtp->rcv_fcp_cmd_drop, 0); 1449 atomic_set(&tgtp->xmt_fcp_drop, 0); 1450 atomic_set(&tgtp->xmt_fcp_read_rsp, 0); 1451 atomic_set(&tgtp->xmt_fcp_read, 0); 1452 atomic_set(&tgtp->xmt_fcp_write, 0); 1453 atomic_set(&tgtp->xmt_fcp_rsp, 0); 1454 atomic_set(&tgtp->xmt_fcp_release, 0); 1455 atomic_set(&tgtp->xmt_fcp_rsp_cmpl, 0); 1456 atomic_set(&tgtp->xmt_fcp_rsp_error, 0); 1457 atomic_set(&tgtp->xmt_fcp_rsp_xb_set, 0); 1458 atomic_set(&tgtp->xmt_fcp_rsp_aborted, 0); 1459 atomic_set(&tgtp->xmt_fcp_rsp_drop, 0); 1460 atomic_set(&tgtp->xmt_fcp_xri_abort_cqe, 0); 1461 atomic_set(&tgtp->xmt_fcp_abort, 0); 1462 atomic_set(&tgtp->xmt_fcp_abort_cmpl, 0); 1463 atomic_set(&tgtp->xmt_abort_unsol, 0); 1464 atomic_set(&tgtp->xmt_abort_sol, 0); 1465 atomic_set(&tgtp->xmt_abort_rsp, 0); 1466 atomic_set(&tgtp->xmt_abort_rsp_error, 0); 1467 atomic_set(&tgtp->defer_ctx, 0); 1468 atomic_set(&tgtp->defer_fod, 0); 1469 atomic_set(&tgtp->defer_wqfull, 0); 1470 } 1471 return error; 1472 } 1473 1474 int 1475 lpfc_nvmet_update_targetport(struct lpfc_hba *phba) 1476 { 1477 struct lpfc_vport *vport = phba->pport; 1478 1479 if (!phba->targetport) 1480 return 0; 1481 1482 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 1483 "6007 Update NVMET port %p did x%x\n", 1484 phba->targetport, vport->fc_myDID); 1485 1486 phba->targetport->port_id = vport->fc_myDID; 1487 return 0; 1488 } 1489 1490 /** 1491 * lpfc_sli4_nvmet_xri_aborted - Fast-path process of nvmet xri abort 1492 * @phba: pointer to lpfc hba data structure. 1493 * @axri: pointer to the nvmet xri abort wcqe structure. 1494 * 1495 * This routine is invoked by the worker thread to process a SLI4 fast-path 1496 * NVMET aborted xri. 1497 **/ 1498 void 1499 lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, 1500 struct sli4_wcqe_xri_aborted *axri) 1501 { 1502 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); 1503 uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri); 1504 struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp; 1505 struct lpfc_nvmet_tgtport *tgtp; 1506 struct lpfc_nodelist *ndlp; 1507 unsigned long iflag = 0; 1508 int rrq_empty = 0; 1509 bool released = false; 1510 1511 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1512 "6317 XB aborted xri x%x rxid x%x\n", xri, rxid); 1513 1514 if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) 1515 return; 1516 1517 if (phba->targetport) { 1518 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 1519 atomic_inc(&tgtp->xmt_fcp_xri_abort_cqe); 1520 } 1521 1522 spin_lock_irqsave(&phba->hbalock, iflag); 1523 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1524 list_for_each_entry_safe(ctxp, next_ctxp, 1525 &phba->sli4_hba.lpfc_abts_nvmet_ctx_list, 1526 list) { 1527 if (ctxp->ctxbuf->sglq->sli4_xritag != xri) 1528 continue; 1529 1530 spin_lock(&ctxp->ctxlock); 1531 /* Check if we already received a free context call 1532 * and we have completed processing an abort situation. 1533 */ 1534 if (ctxp->flag & LPFC_NVMET_CTX_RLS && 1535 !(ctxp->flag & LPFC_NVMET_ABORT_OP)) { 1536 list_del(&ctxp->list); 1537 released = true; 1538 } 1539 ctxp->flag &= ~LPFC_NVMET_XBUSY; 1540 spin_unlock(&ctxp->ctxlock); 1541 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1542 1543 rrq_empty = list_empty(&phba->active_rrq_list); 1544 spin_unlock_irqrestore(&phba->hbalock, iflag); 1545 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid); 1546 if (ndlp && NLP_CHK_NODE_ACT(ndlp) && 1547 (ndlp->nlp_state == NLP_STE_UNMAPPED_NODE || 1548 ndlp->nlp_state == NLP_STE_MAPPED_NODE)) { 1549 lpfc_set_rrq_active(phba, ndlp, 1550 ctxp->ctxbuf->sglq->sli4_lxritag, 1551 rxid, 1); 1552 lpfc_sli4_abts_err_handler(phba, ndlp, axri); 1553 } 1554 1555 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1556 "6318 XB aborted oxid %x flg x%x (%x)\n", 1557 ctxp->oxid, ctxp->flag, released); 1558 if (released) 1559 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); 1560 1561 if (rrq_empty) 1562 lpfc_worker_wake_up(phba); 1563 return; 1564 } 1565 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1566 spin_unlock_irqrestore(&phba->hbalock, iflag); 1567 } 1568 1569 int 1570 lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, 1571 struct fc_frame_header *fc_hdr) 1572 { 1573 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1574 struct lpfc_hba *phba = vport->phba; 1575 struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp; 1576 struct nvmefc_tgt_fcp_req *rsp; 1577 uint16_t xri; 1578 unsigned long iflag = 0; 1579 1580 xri = be16_to_cpu(fc_hdr->fh_ox_id); 1581 1582 spin_lock_irqsave(&phba->hbalock, iflag); 1583 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1584 list_for_each_entry_safe(ctxp, next_ctxp, 1585 &phba->sli4_hba.lpfc_abts_nvmet_ctx_list, 1586 list) { 1587 if (ctxp->ctxbuf->sglq->sli4_xritag != xri) 1588 continue; 1589 1590 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1591 spin_unlock_irqrestore(&phba->hbalock, iflag); 1592 1593 spin_lock_irqsave(&ctxp->ctxlock, iflag); 1594 ctxp->flag |= LPFC_NVMET_ABTS_RCV; 1595 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 1596 1597 lpfc_nvmeio_data(phba, 1598 "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n", 1599 xri, smp_processor_id(), 0); 1600 1601 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1602 "6319 NVMET Rcv ABTS:acc xri x%x\n", xri); 1603 1604 rsp = &ctxp->ctx.fcp_req; 1605 nvmet_fc_rcv_fcp_abort(phba->targetport, rsp); 1606 1607 /* Respond with BA_ACC accordingly */ 1608 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1); 1609 return 0; 1610 } 1611 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 1612 spin_unlock_irqrestore(&phba->hbalock, iflag); 1613 1614 lpfc_nvmeio_data(phba, "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n", 1615 xri, smp_processor_id(), 1); 1616 1617 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1618 "6320 NVMET Rcv ABTS:rjt xri x%x\n", xri); 1619 1620 /* Respond with BA_RJT accordingly */ 1621 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 0); 1622 #endif 1623 return 0; 1624 } 1625 1626 static void 1627 lpfc_nvmet_wqfull_flush(struct lpfc_hba *phba, struct lpfc_queue *wq, 1628 struct lpfc_nvmet_rcv_ctx *ctxp) 1629 { 1630 struct lpfc_sli_ring *pring; 1631 struct lpfc_iocbq *nvmewqeq; 1632 struct lpfc_iocbq *next_nvmewqeq; 1633 unsigned long iflags; 1634 struct lpfc_wcqe_complete wcqe; 1635 struct lpfc_wcqe_complete *wcqep; 1636 1637 pring = wq->pring; 1638 wcqep = &wcqe; 1639 1640 /* Fake an ABORT error code back to cmpl routine */ 1641 memset(wcqep, 0, sizeof(struct lpfc_wcqe_complete)); 1642 bf_set(lpfc_wcqe_c_status, wcqep, IOSTAT_LOCAL_REJECT); 1643 wcqep->parameter = IOERR_ABORT_REQUESTED; 1644 1645 spin_lock_irqsave(&pring->ring_lock, iflags); 1646 list_for_each_entry_safe(nvmewqeq, next_nvmewqeq, 1647 &wq->wqfull_list, list) { 1648 if (ctxp) { 1649 /* Checking for a specific IO to flush */ 1650 if (nvmewqeq->context2 == ctxp) { 1651 list_del(&nvmewqeq->list); 1652 spin_unlock_irqrestore(&pring->ring_lock, 1653 iflags); 1654 lpfc_nvmet_xmt_fcp_op_cmp(phba, nvmewqeq, 1655 wcqep); 1656 return; 1657 } 1658 continue; 1659 } else { 1660 /* Flush all IOs */ 1661 list_del(&nvmewqeq->list); 1662 spin_unlock_irqrestore(&pring->ring_lock, iflags); 1663 lpfc_nvmet_xmt_fcp_op_cmp(phba, nvmewqeq, wcqep); 1664 spin_lock_irqsave(&pring->ring_lock, iflags); 1665 } 1666 } 1667 if (!ctxp) 1668 wq->q_flag &= ~HBA_NVMET_WQFULL; 1669 spin_unlock_irqrestore(&pring->ring_lock, iflags); 1670 } 1671 1672 void 1673 lpfc_nvmet_wqfull_process(struct lpfc_hba *phba, 1674 struct lpfc_queue *wq) 1675 { 1676 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1677 struct lpfc_sli_ring *pring; 1678 struct lpfc_iocbq *nvmewqeq; 1679 struct lpfc_nvmet_rcv_ctx *ctxp; 1680 unsigned long iflags; 1681 int rc; 1682 1683 /* 1684 * Some WQE slots are available, so try to re-issue anything 1685 * on the WQ wqfull_list. 1686 */ 1687 pring = wq->pring; 1688 spin_lock_irqsave(&pring->ring_lock, iflags); 1689 while (!list_empty(&wq->wqfull_list)) { 1690 list_remove_head(&wq->wqfull_list, nvmewqeq, struct lpfc_iocbq, 1691 list); 1692 spin_unlock_irqrestore(&pring->ring_lock, iflags); 1693 ctxp = (struct lpfc_nvmet_rcv_ctx *)nvmewqeq->context2; 1694 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq); 1695 spin_lock_irqsave(&pring->ring_lock, iflags); 1696 if (rc == -EBUSY) { 1697 /* WQ was full again, so put it back on the list */ 1698 list_add(&nvmewqeq->list, &wq->wqfull_list); 1699 spin_unlock_irqrestore(&pring->ring_lock, iflags); 1700 return; 1701 } 1702 } 1703 wq->q_flag &= ~HBA_NVMET_WQFULL; 1704 spin_unlock_irqrestore(&pring->ring_lock, iflags); 1705 1706 #endif 1707 } 1708 1709 void 1710 lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba) 1711 { 1712 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1713 struct lpfc_nvmet_tgtport *tgtp; 1714 struct lpfc_queue *wq; 1715 uint32_t qidx; 1716 DECLARE_COMPLETION_ONSTACK(tport_unreg_cmp); 1717 1718 if (phba->nvmet_support == 0) 1719 return; 1720 if (phba->targetport) { 1721 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 1722 for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { 1723 wq = phba->sli4_hba.hdwq[qidx].nvme_wq; 1724 lpfc_nvmet_wqfull_flush(phba, wq, NULL); 1725 } 1726 tgtp->tport_unreg_cmp = &tport_unreg_cmp; 1727 nvmet_fc_unregister_targetport(phba->targetport); 1728 wait_for_completion_timeout(&tport_unreg_cmp, 5); 1729 lpfc_nvmet_cleanup_io_context(phba); 1730 } 1731 phba->targetport = NULL; 1732 #endif 1733 } 1734 1735 /** 1736 * lpfc_nvmet_unsol_ls_buffer - Process an unsolicited event data buffer 1737 * @phba: pointer to lpfc hba data structure. 1738 * @pring: pointer to a SLI ring. 1739 * @nvmebuf: pointer to lpfc nvme command HBQ data structure. 1740 * 1741 * This routine is used for processing the WQE associated with a unsolicited 1742 * event. It first determines whether there is an existing ndlp that matches 1743 * the DID from the unsolicited WQE. If not, it will create a new one with 1744 * the DID from the unsolicited WQE. The ELS command from the unsolicited 1745 * WQE is then used to invoke the proper routine and to set up proper state 1746 * of the discovery state machine. 1747 **/ 1748 static void 1749 lpfc_nvmet_unsol_ls_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, 1750 struct hbq_dmabuf *nvmebuf) 1751 { 1752 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1753 struct lpfc_nvmet_tgtport *tgtp; 1754 struct fc_frame_header *fc_hdr; 1755 struct lpfc_nvmet_rcv_ctx *ctxp; 1756 uint32_t *payload; 1757 uint32_t size, oxid, sid, rc; 1758 1759 fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt); 1760 oxid = be16_to_cpu(fc_hdr->fh_ox_id); 1761 1762 if (!phba->targetport) { 1763 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1764 "6154 LS Drop IO x%x\n", oxid); 1765 oxid = 0; 1766 size = 0; 1767 sid = 0; 1768 ctxp = NULL; 1769 goto dropit; 1770 } 1771 1772 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 1773 payload = (uint32_t *)(nvmebuf->dbuf.virt); 1774 size = bf_get(lpfc_rcqe_length, &nvmebuf->cq_event.cqe.rcqe_cmpl); 1775 sid = sli4_sid_from_fc_hdr(fc_hdr); 1776 1777 ctxp = kzalloc(sizeof(struct lpfc_nvmet_rcv_ctx), GFP_ATOMIC); 1778 if (ctxp == NULL) { 1779 atomic_inc(&tgtp->rcv_ls_req_drop); 1780 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1781 "6155 LS Drop IO x%x: Alloc\n", 1782 oxid); 1783 dropit: 1784 lpfc_nvmeio_data(phba, "NVMET LS DROP: " 1785 "xri x%x sz %d from %06x\n", 1786 oxid, size, sid); 1787 lpfc_in_buf_free(phba, &nvmebuf->dbuf); 1788 return; 1789 } 1790 ctxp->phba = phba; 1791 ctxp->size = size; 1792 ctxp->oxid = oxid; 1793 ctxp->sid = sid; 1794 ctxp->wqeq = NULL; 1795 ctxp->state = LPFC_NVMET_STE_LS_RCV; 1796 ctxp->entry_cnt = 1; 1797 ctxp->rqb_buffer = (void *)nvmebuf; 1798 ctxp->hdwq = &phba->sli4_hba.hdwq[0]; 1799 1800 lpfc_nvmeio_data(phba, "NVMET LS RCV: xri x%x sz %d from %06x\n", 1801 oxid, size, sid); 1802 /* 1803 * The calling sequence should be: 1804 * nvmet_fc_rcv_ls_req -> lpfc_nvmet_xmt_ls_rsp/cmp ->_req->done 1805 * lpfc_nvmet_xmt_ls_rsp_cmp should free the allocated ctxp. 1806 */ 1807 atomic_inc(&tgtp->rcv_ls_req_in); 1808 rc = nvmet_fc_rcv_ls_req(phba->targetport, &ctxp->ctx.ls_req, 1809 payload, size); 1810 1811 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 1812 "6037 NVMET Unsol rcv: sz %d rc %d: %08x %08x %08x " 1813 "%08x %08x %08x\n", size, rc, 1814 *payload, *(payload+1), *(payload+2), 1815 *(payload+3), *(payload+4), *(payload+5)); 1816 1817 if (rc == 0) { 1818 atomic_inc(&tgtp->rcv_ls_req_out); 1819 return; 1820 } 1821 1822 lpfc_nvmeio_data(phba, "NVMET LS DROP: xri x%x sz %d from %06x\n", 1823 oxid, size, sid); 1824 1825 atomic_inc(&tgtp->rcv_ls_req_drop); 1826 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1827 "6156 LS Drop IO x%x: nvmet_fc_rcv_ls_req %d\n", 1828 ctxp->oxid, rc); 1829 1830 /* We assume a rcv'ed cmd ALWAYs fits into 1 buffer */ 1831 lpfc_in_buf_free(phba, &nvmebuf->dbuf); 1832 1833 atomic_inc(&tgtp->xmt_ls_abort); 1834 lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, sid, oxid); 1835 #endif 1836 } 1837 1838 static void 1839 lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf) 1840 { 1841 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1842 struct lpfc_nvmet_rcv_ctx *ctxp = ctx_buf->context; 1843 struct lpfc_hba *phba = ctxp->phba; 1844 struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer; 1845 struct lpfc_nvmet_tgtport *tgtp; 1846 uint32_t *payload; 1847 uint32_t rc; 1848 unsigned long iflags; 1849 1850 if (!nvmebuf) { 1851 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1852 "6159 process_rcv_fcp_req, nvmebuf is NULL, " 1853 "oxid: x%x flg: x%x state: x%x\n", 1854 ctxp->oxid, ctxp->flag, ctxp->state); 1855 spin_lock_irqsave(&ctxp->ctxlock, iflags); 1856 lpfc_nvmet_defer_release(phba, ctxp); 1857 spin_unlock_irqrestore(&ctxp->ctxlock, iflags); 1858 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, 1859 ctxp->oxid); 1860 return; 1861 } 1862 1863 payload = (uint32_t *)(nvmebuf->dbuf.virt); 1864 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 1865 /* 1866 * The calling sequence should be: 1867 * nvmet_fc_rcv_fcp_req->lpfc_nvmet_xmt_fcp_op/cmp- req->done 1868 * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp. 1869 * When we return from nvmet_fc_rcv_fcp_req, all relevant info 1870 * the NVME command / FC header is stored. 1871 * A buffer has already been reposted for this IO, so just free 1872 * the nvmebuf. 1873 */ 1874 rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req, 1875 payload, ctxp->size); 1876 /* Process FCP command */ 1877 if (rc == 0) { 1878 atomic_inc(&tgtp->rcv_fcp_cmd_out); 1879 return; 1880 } 1881 1882 /* Processing of FCP command is deferred */ 1883 if (rc == -EOVERFLOW) { 1884 lpfc_nvmeio_data(phba, "NVMET RCV BUSY: xri x%x sz %d " 1885 "from %06x\n", 1886 ctxp->oxid, ctxp->size, ctxp->sid); 1887 atomic_inc(&tgtp->rcv_fcp_cmd_out); 1888 atomic_inc(&tgtp->defer_fod); 1889 return; 1890 } 1891 atomic_inc(&tgtp->rcv_fcp_cmd_drop); 1892 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1893 "2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n", 1894 ctxp->oxid, rc, 1895 atomic_read(&tgtp->rcv_fcp_cmd_in), 1896 atomic_read(&tgtp->rcv_fcp_cmd_out), 1897 atomic_read(&tgtp->xmt_fcp_release)); 1898 lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n", 1899 ctxp->oxid, ctxp->size, ctxp->sid); 1900 spin_lock_irqsave(&ctxp->ctxlock, iflags); 1901 lpfc_nvmet_defer_release(phba, ctxp); 1902 spin_unlock_irqrestore(&ctxp->ctxlock, iflags); 1903 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); 1904 #endif 1905 } 1906 1907 static void 1908 lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *work) 1909 { 1910 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1911 struct lpfc_nvmet_ctxbuf *ctx_buf = 1912 container_of(work, struct lpfc_nvmet_ctxbuf, defer_work); 1913 1914 lpfc_nvmet_process_rcv_fcp_req(ctx_buf); 1915 #endif 1916 } 1917 1918 static struct lpfc_nvmet_ctxbuf * 1919 lpfc_nvmet_replenish_context(struct lpfc_hba *phba, 1920 struct lpfc_nvmet_ctx_info *current_infop) 1921 { 1922 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1923 struct lpfc_nvmet_ctxbuf *ctx_buf = NULL; 1924 struct lpfc_nvmet_ctx_info *get_infop; 1925 int i; 1926 1927 /* 1928 * The current_infop for the MRQ a NVME command IU was received 1929 * on is empty. Our goal is to replenish this MRQs context 1930 * list from a another CPUs. 1931 * 1932 * First we need to pick a context list to start looking on. 1933 * nvmet_ctx_start_cpu has available context the last time 1934 * we needed to replenish this CPU where nvmet_ctx_next_cpu 1935 * is just the next sequential CPU for this MRQ. 1936 */ 1937 if (current_infop->nvmet_ctx_start_cpu) 1938 get_infop = current_infop->nvmet_ctx_start_cpu; 1939 else 1940 get_infop = current_infop->nvmet_ctx_next_cpu; 1941 1942 for (i = 0; i < phba->sli4_hba.num_possible_cpu; i++) { 1943 if (get_infop == current_infop) { 1944 get_infop = get_infop->nvmet_ctx_next_cpu; 1945 continue; 1946 } 1947 spin_lock(&get_infop->nvmet_ctx_list_lock); 1948 1949 /* Just take the entire context list, if there are any */ 1950 if (get_infop->nvmet_ctx_list_cnt) { 1951 list_splice_init(&get_infop->nvmet_ctx_list, 1952 ¤t_infop->nvmet_ctx_list); 1953 current_infop->nvmet_ctx_list_cnt = 1954 get_infop->nvmet_ctx_list_cnt - 1; 1955 get_infop->nvmet_ctx_list_cnt = 0; 1956 spin_unlock(&get_infop->nvmet_ctx_list_lock); 1957 1958 current_infop->nvmet_ctx_start_cpu = get_infop; 1959 list_remove_head(¤t_infop->nvmet_ctx_list, 1960 ctx_buf, struct lpfc_nvmet_ctxbuf, 1961 list); 1962 return ctx_buf; 1963 } 1964 1965 /* Otherwise, move on to the next CPU for this MRQ */ 1966 spin_unlock(&get_infop->nvmet_ctx_list_lock); 1967 get_infop = get_infop->nvmet_ctx_next_cpu; 1968 } 1969 1970 #endif 1971 /* Nothing found, all contexts for the MRQ are in-flight */ 1972 return NULL; 1973 } 1974 1975 /** 1976 * lpfc_nvmet_unsol_fcp_buffer - Process an unsolicited event data buffer 1977 * @phba: pointer to lpfc hba data structure. 1978 * @idx: relative index of MRQ vector 1979 * @nvmebuf: pointer to lpfc nvme command HBQ data structure. 1980 * 1981 * This routine is used for processing the WQE associated with a unsolicited 1982 * event. It first determines whether there is an existing ndlp that matches 1983 * the DID from the unsolicited WQE. If not, it will create a new one with 1984 * the DID from the unsolicited WQE. The ELS command from the unsolicited 1985 * WQE is then used to invoke the proper routine and to set up proper state 1986 * of the discovery state machine. 1987 **/ 1988 static void 1989 lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, 1990 uint32_t idx, 1991 struct rqb_dmabuf *nvmebuf, 1992 uint64_t isr_timestamp) 1993 { 1994 struct lpfc_nvmet_rcv_ctx *ctxp; 1995 struct lpfc_nvmet_tgtport *tgtp; 1996 struct fc_frame_header *fc_hdr; 1997 struct lpfc_nvmet_ctxbuf *ctx_buf; 1998 struct lpfc_nvmet_ctx_info *current_infop; 1999 uint32_t *payload; 2000 uint32_t size, oxid, sid, qno; 2001 unsigned long iflag; 2002 int current_cpu; 2003 2004 if (!IS_ENABLED(CONFIG_NVME_TARGET_FC)) 2005 return; 2006 2007 ctx_buf = NULL; 2008 if (!nvmebuf || !phba->targetport) { 2009 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2010 "6157 NVMET FCP Drop IO\n"); 2011 if (nvmebuf) 2012 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); 2013 return; 2014 } 2015 2016 /* 2017 * Get a pointer to the context list for this MRQ based on 2018 * the CPU this MRQ IRQ is associated with. If the CPU association 2019 * changes from our initial assumption, the context list could 2020 * be empty, thus it would need to be replenished with the 2021 * context list from another CPU for this MRQ. 2022 */ 2023 current_cpu = smp_processor_id(); 2024 current_infop = lpfc_get_ctx_list(phba, current_cpu, idx); 2025 spin_lock_irqsave(¤t_infop->nvmet_ctx_list_lock, iflag); 2026 if (current_infop->nvmet_ctx_list_cnt) { 2027 list_remove_head(¤t_infop->nvmet_ctx_list, 2028 ctx_buf, struct lpfc_nvmet_ctxbuf, list); 2029 current_infop->nvmet_ctx_list_cnt--; 2030 } else { 2031 ctx_buf = lpfc_nvmet_replenish_context(phba, current_infop); 2032 } 2033 spin_unlock_irqrestore(¤t_infop->nvmet_ctx_list_lock, iflag); 2034 2035 fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt); 2036 oxid = be16_to_cpu(fc_hdr->fh_ox_id); 2037 size = nvmebuf->bytes_recv; 2038 2039 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 2040 if (phba->cpucheck_on & LPFC_CHECK_NVMET_RCV) { 2041 if (current_cpu < LPFC_CHECK_CPU_CNT) { 2042 if (idx != current_cpu) 2043 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 2044 "6703 CPU Check rcv: " 2045 "cpu %d expect %d\n", 2046 current_cpu, idx); 2047 phba->sli4_hba.hdwq[idx].cpucheck_rcv_io[current_cpu]++; 2048 } 2049 } 2050 #endif 2051 2052 lpfc_nvmeio_data(phba, "NVMET FCP RCV: xri x%x sz %d CPU %02x\n", 2053 oxid, size, smp_processor_id()); 2054 2055 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2056 2057 if (!ctx_buf) { 2058 /* Queue this NVME IO to process later */ 2059 spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag); 2060 list_add_tail(&nvmebuf->hbuf.list, 2061 &phba->sli4_hba.lpfc_nvmet_io_wait_list); 2062 phba->sli4_hba.nvmet_io_wait_cnt++; 2063 phba->sli4_hba.nvmet_io_wait_total++; 2064 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, 2065 iflag); 2066 2067 /* Post a brand new DMA buffer to RQ */ 2068 qno = nvmebuf->idx; 2069 lpfc_post_rq_buffer( 2070 phba, phba->sli4_hba.nvmet_mrq_hdr[qno], 2071 phba->sli4_hba.nvmet_mrq_data[qno], 1, qno); 2072 2073 atomic_inc(&tgtp->defer_ctx); 2074 return; 2075 } 2076 2077 payload = (uint32_t *)(nvmebuf->dbuf.virt); 2078 sid = sli4_sid_from_fc_hdr(fc_hdr); 2079 2080 ctxp = (struct lpfc_nvmet_rcv_ctx *)ctx_buf->context; 2081 if (ctxp->state != LPFC_NVMET_STE_FREE) { 2082 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2083 "6414 NVMET Context corrupt %d %d oxid x%x\n", 2084 ctxp->state, ctxp->entry_cnt, ctxp->oxid); 2085 } 2086 ctxp->wqeq = NULL; 2087 ctxp->txrdy = NULL; 2088 ctxp->offset = 0; 2089 ctxp->phba = phba; 2090 ctxp->size = size; 2091 ctxp->oxid = oxid; 2092 ctxp->sid = sid; 2093 ctxp->idx = idx; 2094 ctxp->state = LPFC_NVMET_STE_RCV; 2095 ctxp->entry_cnt = 1; 2096 ctxp->flag = 0; 2097 ctxp->ctxbuf = ctx_buf; 2098 ctxp->rqb_buffer = (void *)nvmebuf; 2099 ctxp->hdwq = NULL; 2100 spin_lock_init(&ctxp->ctxlock); 2101 2102 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 2103 if (isr_timestamp) { 2104 ctxp->ts_isr_cmd = isr_timestamp; 2105 ctxp->ts_cmd_nvme = ktime_get_ns(); 2106 ctxp->ts_nvme_data = 0; 2107 ctxp->ts_data_wqput = 0; 2108 ctxp->ts_isr_data = 0; 2109 ctxp->ts_data_nvme = 0; 2110 ctxp->ts_nvme_status = 0; 2111 ctxp->ts_status_wqput = 0; 2112 ctxp->ts_isr_status = 0; 2113 ctxp->ts_status_nvme = 0; 2114 } else { 2115 ctxp->ts_cmd_nvme = 0; 2116 } 2117 #endif 2118 2119 atomic_inc(&tgtp->rcv_fcp_cmd_in); 2120 lpfc_nvmet_process_rcv_fcp_req(ctx_buf); 2121 } 2122 2123 /** 2124 * lpfc_nvmet_unsol_ls_event - Process an unsolicited event from an nvme nport 2125 * @phba: pointer to lpfc hba data structure. 2126 * @pring: pointer to a SLI ring. 2127 * @nvmebuf: pointer to received nvme data structure. 2128 * 2129 * This routine is used to process an unsolicited event received from a SLI 2130 * (Service Level Interface) ring. The actual processing of the data buffer 2131 * associated with the unsolicited event is done by invoking the routine 2132 * lpfc_nvmet_unsol_ls_buffer() after properly set up the buffer from the 2133 * SLI RQ on which the unsolicited event was received. 2134 **/ 2135 void 2136 lpfc_nvmet_unsol_ls_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, 2137 struct lpfc_iocbq *piocb) 2138 { 2139 struct lpfc_dmabuf *d_buf; 2140 struct hbq_dmabuf *nvmebuf; 2141 2142 d_buf = piocb->context2; 2143 nvmebuf = container_of(d_buf, struct hbq_dmabuf, dbuf); 2144 2145 if (phba->nvmet_support == 0) { 2146 lpfc_in_buf_free(phba, &nvmebuf->dbuf); 2147 return; 2148 } 2149 lpfc_nvmet_unsol_ls_buffer(phba, pring, nvmebuf); 2150 } 2151 2152 /** 2153 * lpfc_nvmet_unsol_fcp_event - Process an unsolicited event from an nvme nport 2154 * @phba: pointer to lpfc hba data structure. 2155 * @idx: relative index of MRQ vector 2156 * @nvmebuf: pointer to received nvme data structure. 2157 * 2158 * This routine is used to process an unsolicited event received from a SLI 2159 * (Service Level Interface) ring. The actual processing of the data buffer 2160 * associated with the unsolicited event is done by invoking the routine 2161 * lpfc_nvmet_unsol_fcp_buffer() after properly set up the buffer from the 2162 * SLI RQ on which the unsolicited event was received. 2163 **/ 2164 void 2165 lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba, 2166 uint32_t idx, 2167 struct rqb_dmabuf *nvmebuf, 2168 uint64_t isr_timestamp) 2169 { 2170 if (phba->nvmet_support == 0) { 2171 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); 2172 return; 2173 } 2174 lpfc_nvmet_unsol_fcp_buffer(phba, idx, nvmebuf, 2175 isr_timestamp); 2176 } 2177 2178 /** 2179 * lpfc_nvmet_prep_ls_wqe - Allocate and prepare a lpfc wqe data structure 2180 * @phba: pointer to a host N_Port data structure. 2181 * @ctxp: Context info for NVME LS Request 2182 * @rspbuf: DMA buffer of NVME command. 2183 * @rspsize: size of the NVME command. 2184 * 2185 * This routine is used for allocating a lpfc-WQE data structure from 2186 * the driver lpfc-WQE free-list and prepare the WQE with the parameters 2187 * passed into the routine for discovery state machine to issue an Extended 2188 * Link Service (NVME) commands. It is a generic lpfc-WQE allocation 2189 * and preparation routine that is used by all the discovery state machine 2190 * routines and the NVME command-specific fields will be later set up by 2191 * the individual discovery machine routines after calling this routine 2192 * allocating and preparing a generic WQE data structure. It fills in the 2193 * Buffer Descriptor Entries (BDEs), allocates buffers for both command 2194 * payload and response payload (if expected). The reference count on the 2195 * ndlp is incremented by 1 and the reference to the ndlp is put into 2196 * context1 of the WQE data structure for this WQE to hold the ndlp 2197 * reference for the command's callback function to access later. 2198 * 2199 * Return code 2200 * Pointer to the newly allocated/prepared nvme wqe data structure 2201 * NULL - when nvme wqe data structure allocation/preparation failed 2202 **/ 2203 static struct lpfc_iocbq * 2204 lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *phba, 2205 struct lpfc_nvmet_rcv_ctx *ctxp, 2206 dma_addr_t rspbuf, uint16_t rspsize) 2207 { 2208 struct lpfc_nodelist *ndlp; 2209 struct lpfc_iocbq *nvmewqe; 2210 union lpfc_wqe128 *wqe; 2211 2212 if (!lpfc_is_link_up(phba)) { 2213 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, 2214 "6104 NVMET prep LS wqe: link err: " 2215 "NPORT x%x oxid:x%x ste %d\n", 2216 ctxp->sid, ctxp->oxid, ctxp->state); 2217 return NULL; 2218 } 2219 2220 /* Allocate buffer for command wqe */ 2221 nvmewqe = lpfc_sli_get_iocbq(phba); 2222 if (nvmewqe == NULL) { 2223 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, 2224 "6105 NVMET prep LS wqe: No WQE: " 2225 "NPORT x%x oxid x%x ste %d\n", 2226 ctxp->sid, ctxp->oxid, ctxp->state); 2227 return NULL; 2228 } 2229 2230 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid); 2231 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || 2232 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && 2233 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { 2234 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, 2235 "6106 NVMET prep LS wqe: No ndlp: " 2236 "NPORT x%x oxid x%x ste %d\n", 2237 ctxp->sid, ctxp->oxid, ctxp->state); 2238 goto nvme_wqe_free_wqeq_exit; 2239 } 2240 ctxp->wqeq = nvmewqe; 2241 2242 /* prevent preparing wqe with NULL ndlp reference */ 2243 nvmewqe->context1 = lpfc_nlp_get(ndlp); 2244 if (nvmewqe->context1 == NULL) 2245 goto nvme_wqe_free_wqeq_exit; 2246 nvmewqe->context2 = ctxp; 2247 2248 wqe = &nvmewqe->wqe; 2249 memset(wqe, 0, sizeof(union lpfc_wqe)); 2250 2251 /* Words 0 - 2 */ 2252 wqe->xmit_sequence.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 2253 wqe->xmit_sequence.bde.tus.f.bdeSize = rspsize; 2254 wqe->xmit_sequence.bde.addrLow = le32_to_cpu(putPaddrLow(rspbuf)); 2255 wqe->xmit_sequence.bde.addrHigh = le32_to_cpu(putPaddrHigh(rspbuf)); 2256 2257 /* Word 3 */ 2258 2259 /* Word 4 */ 2260 2261 /* Word 5 */ 2262 bf_set(wqe_dfctl, &wqe->xmit_sequence.wge_ctl, 0); 2263 bf_set(wqe_ls, &wqe->xmit_sequence.wge_ctl, 1); 2264 bf_set(wqe_la, &wqe->xmit_sequence.wge_ctl, 0); 2265 bf_set(wqe_rctl, &wqe->xmit_sequence.wge_ctl, FC_RCTL_ELS4_REP); 2266 bf_set(wqe_type, &wqe->xmit_sequence.wge_ctl, FC_TYPE_NVME); 2267 2268 /* Word 6 */ 2269 bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com, 2270 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 2271 bf_set(wqe_xri_tag, &wqe->xmit_sequence.wqe_com, nvmewqe->sli4_xritag); 2272 2273 /* Word 7 */ 2274 bf_set(wqe_cmnd, &wqe->xmit_sequence.wqe_com, 2275 CMD_XMIT_SEQUENCE64_WQE); 2276 bf_set(wqe_ct, &wqe->xmit_sequence.wqe_com, SLI4_CT_RPI); 2277 bf_set(wqe_class, &wqe->xmit_sequence.wqe_com, CLASS3); 2278 bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0); 2279 2280 /* Word 8 */ 2281 wqe->xmit_sequence.wqe_com.abort_tag = nvmewqe->iotag; 2282 2283 /* Word 9 */ 2284 bf_set(wqe_reqtag, &wqe->xmit_sequence.wqe_com, nvmewqe->iotag); 2285 /* Needs to be set by caller */ 2286 bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, ctxp->oxid); 2287 2288 /* Word 10 */ 2289 bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1); 2290 bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE); 2291 bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com, 2292 LPFC_WQE_LENLOC_WORD12); 2293 bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0); 2294 2295 /* Word 11 */ 2296 bf_set(wqe_cqid, &wqe->xmit_sequence.wqe_com, 2297 LPFC_WQE_CQ_ID_DEFAULT); 2298 bf_set(wqe_cmd_type, &wqe->xmit_sequence.wqe_com, 2299 OTHER_COMMAND); 2300 2301 /* Word 12 */ 2302 wqe->xmit_sequence.xmit_len = rspsize; 2303 2304 nvmewqe->retry = 1; 2305 nvmewqe->vport = phba->pport; 2306 nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT; 2307 nvmewqe->iocb_flag |= LPFC_IO_NVME_LS; 2308 2309 /* Xmit NVMET response to remote NPORT <did> */ 2310 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 2311 "6039 Xmit NVMET LS response to remote " 2312 "NPORT x%x iotag:x%x oxid:x%x size:x%x\n", 2313 ndlp->nlp_DID, nvmewqe->iotag, ctxp->oxid, 2314 rspsize); 2315 return nvmewqe; 2316 2317 nvme_wqe_free_wqeq_exit: 2318 nvmewqe->context2 = NULL; 2319 nvmewqe->context3 = NULL; 2320 lpfc_sli_release_iocbq(phba, nvmewqe); 2321 return NULL; 2322 } 2323 2324 2325 static struct lpfc_iocbq * 2326 lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba, 2327 struct lpfc_nvmet_rcv_ctx *ctxp) 2328 { 2329 struct nvmefc_tgt_fcp_req *rsp = &ctxp->ctx.fcp_req; 2330 struct lpfc_nvmet_tgtport *tgtp; 2331 struct sli4_sge *sgl; 2332 struct lpfc_nodelist *ndlp; 2333 struct lpfc_iocbq *nvmewqe; 2334 struct scatterlist *sgel; 2335 union lpfc_wqe128 *wqe; 2336 struct ulp_bde64 *bde; 2337 uint32_t *txrdy; 2338 dma_addr_t physaddr; 2339 int i, cnt; 2340 int do_pbde; 2341 int xc = 1; 2342 2343 if (!lpfc_is_link_up(phba)) { 2344 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2345 "6107 NVMET prep FCP wqe: link err:" 2346 "NPORT x%x oxid x%x ste %d\n", 2347 ctxp->sid, ctxp->oxid, ctxp->state); 2348 return NULL; 2349 } 2350 2351 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid); 2352 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || 2353 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && 2354 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { 2355 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2356 "6108 NVMET prep FCP wqe: no ndlp: " 2357 "NPORT x%x oxid x%x ste %d\n", 2358 ctxp->sid, ctxp->oxid, ctxp->state); 2359 return NULL; 2360 } 2361 2362 if (rsp->sg_cnt > lpfc_tgttemplate.max_sgl_segments) { 2363 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2364 "6109 NVMET prep FCP wqe: seg cnt err: " 2365 "NPORT x%x oxid x%x ste %d cnt %d\n", 2366 ctxp->sid, ctxp->oxid, ctxp->state, 2367 phba->cfg_nvme_seg_cnt); 2368 return NULL; 2369 } 2370 2371 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2372 nvmewqe = ctxp->wqeq; 2373 if (nvmewqe == NULL) { 2374 /* Allocate buffer for command wqe */ 2375 nvmewqe = ctxp->ctxbuf->iocbq; 2376 if (nvmewqe == NULL) { 2377 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2378 "6110 NVMET prep FCP wqe: No " 2379 "WQE: NPORT x%x oxid x%x ste %d\n", 2380 ctxp->sid, ctxp->oxid, ctxp->state); 2381 return NULL; 2382 } 2383 ctxp->wqeq = nvmewqe; 2384 xc = 0; /* create new XRI */ 2385 nvmewqe->sli4_lxritag = NO_XRI; 2386 nvmewqe->sli4_xritag = NO_XRI; 2387 } 2388 2389 /* Sanity check */ 2390 if (((ctxp->state == LPFC_NVMET_STE_RCV) && 2391 (ctxp->entry_cnt == 1)) || 2392 (ctxp->state == LPFC_NVMET_STE_DATA)) { 2393 wqe = &nvmewqe->wqe; 2394 } else { 2395 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2396 "6111 Wrong state NVMET FCP: %d cnt %d\n", 2397 ctxp->state, ctxp->entry_cnt); 2398 return NULL; 2399 } 2400 2401 sgl = (struct sli4_sge *)ctxp->ctxbuf->sglq->sgl; 2402 switch (rsp->op) { 2403 case NVMET_FCOP_READDATA: 2404 case NVMET_FCOP_READDATA_RSP: 2405 /* From the tsend template, initialize words 7 - 11 */ 2406 memcpy(&wqe->words[7], 2407 &lpfc_tsend_cmd_template.words[7], 2408 sizeof(uint32_t) * 5); 2409 2410 /* Words 0 - 2 : The first sg segment */ 2411 sgel = &rsp->sg[0]; 2412 physaddr = sg_dma_address(sgel); 2413 wqe->fcp_tsend.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 2414 wqe->fcp_tsend.bde.tus.f.bdeSize = sg_dma_len(sgel); 2415 wqe->fcp_tsend.bde.addrLow = cpu_to_le32(putPaddrLow(physaddr)); 2416 wqe->fcp_tsend.bde.addrHigh = 2417 cpu_to_le32(putPaddrHigh(physaddr)); 2418 2419 /* Word 3 */ 2420 wqe->fcp_tsend.payload_offset_len = 0; 2421 2422 /* Word 4 */ 2423 wqe->fcp_tsend.relative_offset = ctxp->offset; 2424 2425 /* Word 5 */ 2426 wqe->fcp_tsend.reserved = 0; 2427 2428 /* Word 6 */ 2429 bf_set(wqe_ctxt_tag, &wqe->fcp_tsend.wqe_com, 2430 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 2431 bf_set(wqe_xri_tag, &wqe->fcp_tsend.wqe_com, 2432 nvmewqe->sli4_xritag); 2433 2434 /* Word 7 - set ar later */ 2435 2436 /* Word 8 */ 2437 wqe->fcp_tsend.wqe_com.abort_tag = nvmewqe->iotag; 2438 2439 /* Word 9 */ 2440 bf_set(wqe_reqtag, &wqe->fcp_tsend.wqe_com, nvmewqe->iotag); 2441 bf_set(wqe_rcvoxid, &wqe->fcp_tsend.wqe_com, ctxp->oxid); 2442 2443 /* Word 10 - set wqes later, in template xc=1 */ 2444 if (!xc) 2445 bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 0); 2446 2447 /* Word 11 - set sup, irsp, irsplen later */ 2448 do_pbde = 0; 2449 2450 /* Word 12 */ 2451 wqe->fcp_tsend.fcp_data_len = rsp->transfer_length; 2452 2453 /* Setup 2 SKIP SGEs */ 2454 sgl->addr_hi = 0; 2455 sgl->addr_lo = 0; 2456 sgl->word2 = 0; 2457 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2458 sgl->word2 = cpu_to_le32(sgl->word2); 2459 sgl->sge_len = 0; 2460 sgl++; 2461 sgl->addr_hi = 0; 2462 sgl->addr_lo = 0; 2463 sgl->word2 = 0; 2464 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2465 sgl->word2 = cpu_to_le32(sgl->word2); 2466 sgl->sge_len = 0; 2467 sgl++; 2468 if (rsp->op == NVMET_FCOP_READDATA_RSP) { 2469 atomic_inc(&tgtp->xmt_fcp_read_rsp); 2470 2471 /* In template ar=1 wqes=0 sup=0 irsp=0 irsplen=0 */ 2472 2473 if (rsp->rsplen == LPFC_NVMET_SUCCESS_LEN) { 2474 if (ndlp->nlp_flag & NLP_SUPPRESS_RSP) 2475 bf_set(wqe_sup, 2476 &wqe->fcp_tsend.wqe_com, 1); 2477 } else { 2478 bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 1); 2479 bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 1); 2480 bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 2481 ((rsp->rsplen >> 2) - 1)); 2482 memcpy(&wqe->words[16], rsp->rspaddr, 2483 rsp->rsplen); 2484 } 2485 } else { 2486 atomic_inc(&tgtp->xmt_fcp_read); 2487 2488 /* In template ar=1 wqes=0 sup=0 irsp=0 irsplen=0 */ 2489 bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 0); 2490 } 2491 break; 2492 2493 case NVMET_FCOP_WRITEDATA: 2494 /* From the treceive template, initialize words 3 - 11 */ 2495 memcpy(&wqe->words[3], 2496 &lpfc_treceive_cmd_template.words[3], 2497 sizeof(uint32_t) * 9); 2498 2499 /* Words 0 - 2 : The first sg segment */ 2500 txrdy = dma_pool_alloc(phba->txrdy_payload_pool, 2501 GFP_KERNEL, &physaddr); 2502 if (!txrdy) { 2503 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2504 "6041 Bad txrdy buffer: oxid x%x\n", 2505 ctxp->oxid); 2506 return NULL; 2507 } 2508 ctxp->txrdy = txrdy; 2509 ctxp->txrdy_phys = physaddr; 2510 wqe->fcp_treceive.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 2511 wqe->fcp_treceive.bde.tus.f.bdeSize = TXRDY_PAYLOAD_LEN; 2512 wqe->fcp_treceive.bde.addrLow = 2513 cpu_to_le32(putPaddrLow(physaddr)); 2514 wqe->fcp_treceive.bde.addrHigh = 2515 cpu_to_le32(putPaddrHigh(physaddr)); 2516 2517 /* Word 4 */ 2518 wqe->fcp_treceive.relative_offset = ctxp->offset; 2519 2520 /* Word 6 */ 2521 bf_set(wqe_ctxt_tag, &wqe->fcp_treceive.wqe_com, 2522 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 2523 bf_set(wqe_xri_tag, &wqe->fcp_treceive.wqe_com, 2524 nvmewqe->sli4_xritag); 2525 2526 /* Word 7 */ 2527 2528 /* Word 8 */ 2529 wqe->fcp_treceive.wqe_com.abort_tag = nvmewqe->iotag; 2530 2531 /* Word 9 */ 2532 bf_set(wqe_reqtag, &wqe->fcp_treceive.wqe_com, nvmewqe->iotag); 2533 bf_set(wqe_rcvoxid, &wqe->fcp_treceive.wqe_com, ctxp->oxid); 2534 2535 /* Word 10 - in template xc=1 */ 2536 if (!xc) 2537 bf_set(wqe_xc, &wqe->fcp_treceive.wqe_com, 0); 2538 2539 /* Word 11 - set pbde later */ 2540 if (phba->cfg_enable_pbde) { 2541 do_pbde = 1; 2542 } else { 2543 bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 0); 2544 do_pbde = 0; 2545 } 2546 2547 /* Word 12 */ 2548 wqe->fcp_tsend.fcp_data_len = rsp->transfer_length; 2549 2550 /* Setup 1 TXRDY and 1 SKIP SGE */ 2551 txrdy[0] = 0; 2552 txrdy[1] = cpu_to_be32(rsp->transfer_length); 2553 txrdy[2] = 0; 2554 2555 sgl->addr_hi = putPaddrHigh(physaddr); 2556 sgl->addr_lo = putPaddrLow(physaddr); 2557 sgl->word2 = 0; 2558 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA); 2559 sgl->word2 = cpu_to_le32(sgl->word2); 2560 sgl->sge_len = cpu_to_le32(TXRDY_PAYLOAD_LEN); 2561 sgl++; 2562 sgl->addr_hi = 0; 2563 sgl->addr_lo = 0; 2564 sgl->word2 = 0; 2565 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2566 sgl->word2 = cpu_to_le32(sgl->word2); 2567 sgl->sge_len = 0; 2568 sgl++; 2569 atomic_inc(&tgtp->xmt_fcp_write); 2570 break; 2571 2572 case NVMET_FCOP_RSP: 2573 /* From the treceive template, initialize words 4 - 11 */ 2574 memcpy(&wqe->words[4], 2575 &lpfc_trsp_cmd_template.words[4], 2576 sizeof(uint32_t) * 8); 2577 2578 /* Words 0 - 2 */ 2579 physaddr = rsp->rspdma; 2580 wqe->fcp_trsp.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 2581 wqe->fcp_trsp.bde.tus.f.bdeSize = rsp->rsplen; 2582 wqe->fcp_trsp.bde.addrLow = 2583 cpu_to_le32(putPaddrLow(physaddr)); 2584 wqe->fcp_trsp.bde.addrHigh = 2585 cpu_to_le32(putPaddrHigh(physaddr)); 2586 2587 /* Word 3 */ 2588 wqe->fcp_trsp.response_len = rsp->rsplen; 2589 2590 /* Word 6 */ 2591 bf_set(wqe_ctxt_tag, &wqe->fcp_trsp.wqe_com, 2592 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 2593 bf_set(wqe_xri_tag, &wqe->fcp_trsp.wqe_com, 2594 nvmewqe->sli4_xritag); 2595 2596 /* Word 7 */ 2597 2598 /* Word 8 */ 2599 wqe->fcp_trsp.wqe_com.abort_tag = nvmewqe->iotag; 2600 2601 /* Word 9 */ 2602 bf_set(wqe_reqtag, &wqe->fcp_trsp.wqe_com, nvmewqe->iotag); 2603 bf_set(wqe_rcvoxid, &wqe->fcp_trsp.wqe_com, ctxp->oxid); 2604 2605 /* Word 10 */ 2606 if (xc) 2607 bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 1); 2608 2609 /* Word 11 */ 2610 /* In template wqes=0 irsp=0 irsplen=0 - good response */ 2611 if (rsp->rsplen != LPFC_NVMET_SUCCESS_LEN) { 2612 /* Bad response - embed it */ 2613 bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 1); 2614 bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 1); 2615 bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 2616 ((rsp->rsplen >> 2) - 1)); 2617 memcpy(&wqe->words[16], rsp->rspaddr, rsp->rsplen); 2618 } 2619 do_pbde = 0; 2620 2621 /* Word 12 */ 2622 wqe->fcp_trsp.rsvd_12_15[0] = 0; 2623 2624 /* Use rspbuf, NOT sg list */ 2625 rsp->sg_cnt = 0; 2626 sgl->word2 = 0; 2627 atomic_inc(&tgtp->xmt_fcp_rsp); 2628 break; 2629 2630 default: 2631 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 2632 "6064 Unknown Rsp Op %d\n", 2633 rsp->op); 2634 return NULL; 2635 } 2636 2637 nvmewqe->retry = 1; 2638 nvmewqe->vport = phba->pport; 2639 nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT; 2640 nvmewqe->context1 = ndlp; 2641 2642 for (i = 0; i < rsp->sg_cnt; i++) { 2643 sgel = &rsp->sg[i]; 2644 physaddr = sg_dma_address(sgel); 2645 cnt = sg_dma_len(sgel); 2646 sgl->addr_hi = putPaddrHigh(physaddr); 2647 sgl->addr_lo = putPaddrLow(physaddr); 2648 sgl->word2 = 0; 2649 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA); 2650 bf_set(lpfc_sli4_sge_offset, sgl, ctxp->offset); 2651 if ((i+1) == rsp->sg_cnt) 2652 bf_set(lpfc_sli4_sge_last, sgl, 1); 2653 sgl->word2 = cpu_to_le32(sgl->word2); 2654 sgl->sge_len = cpu_to_le32(cnt); 2655 if (i == 0) { 2656 bde = (struct ulp_bde64 *)&wqe->words[13]; 2657 if (do_pbde) { 2658 /* Words 13-15 (PBDE) */ 2659 bde->addrLow = sgl->addr_lo; 2660 bde->addrHigh = sgl->addr_hi; 2661 bde->tus.f.bdeSize = 2662 le32_to_cpu(sgl->sge_len); 2663 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64; 2664 bde->tus.w = cpu_to_le32(bde->tus.w); 2665 } else { 2666 memset(bde, 0, sizeof(struct ulp_bde64)); 2667 } 2668 } 2669 sgl++; 2670 ctxp->offset += cnt; 2671 } 2672 ctxp->state = LPFC_NVMET_STE_DATA; 2673 ctxp->entry_cnt++; 2674 return nvmewqe; 2675 } 2676 2677 /** 2678 * lpfc_nvmet_sol_fcp_abort_cmp - Completion handler for ABTS 2679 * @phba: Pointer to HBA context object. 2680 * @cmdwqe: Pointer to driver command WQE object. 2681 * @wcqe: Pointer to driver response CQE object. 2682 * 2683 * The function is called from SLI ring event handler with no 2684 * lock held. This function is the completion handler for NVME ABTS for FCP cmds 2685 * The function frees memory resources used for the NVME commands. 2686 **/ 2687 static void 2688 lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 2689 struct lpfc_wcqe_complete *wcqe) 2690 { 2691 struct lpfc_nvmet_rcv_ctx *ctxp; 2692 struct lpfc_nvmet_tgtport *tgtp; 2693 uint32_t status, result; 2694 unsigned long flags; 2695 bool released = false; 2696 2697 ctxp = cmdwqe->context2; 2698 status = bf_get(lpfc_wcqe_c_status, wcqe); 2699 result = wcqe->parameter; 2700 2701 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2702 if (ctxp->flag & LPFC_NVMET_ABORT_OP) 2703 atomic_inc(&tgtp->xmt_fcp_abort_cmpl); 2704 2705 spin_lock_irqsave(&ctxp->ctxlock, flags); 2706 ctxp->state = LPFC_NVMET_STE_DONE; 2707 2708 /* Check if we already received a free context call 2709 * and we have completed processing an abort situation. 2710 */ 2711 if ((ctxp->flag & LPFC_NVMET_CTX_RLS) && 2712 !(ctxp->flag & LPFC_NVMET_XBUSY)) { 2713 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 2714 list_del(&ctxp->list); 2715 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 2716 released = true; 2717 } 2718 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 2719 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 2720 atomic_inc(&tgtp->xmt_abort_rsp); 2721 2722 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2723 "6165 ABORT cmpl: xri x%x flg x%x (%d) " 2724 "WCQE: %08x %08x %08x %08x\n", 2725 ctxp->oxid, ctxp->flag, released, 2726 wcqe->word0, wcqe->total_data_placed, 2727 result, wcqe->word3); 2728 2729 cmdwqe->context2 = NULL; 2730 cmdwqe->context3 = NULL; 2731 /* 2732 * if transport has released ctx, then can reuse it. Otherwise, 2733 * will be recycled by transport release call. 2734 */ 2735 if (released) 2736 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); 2737 2738 /* This is the iocbq for the abort, not the command */ 2739 lpfc_sli_release_iocbq(phba, cmdwqe); 2740 2741 /* Since iaab/iaar are NOT set, there is no work left. 2742 * For LPFC_NVMET_XBUSY, lpfc_sli4_nvmet_xri_aborted 2743 * should have been called already. 2744 */ 2745 } 2746 2747 /** 2748 * lpfc_nvmet_unsol_fcp_abort_cmp - Completion handler for ABTS 2749 * @phba: Pointer to HBA context object. 2750 * @cmdwqe: Pointer to driver command WQE object. 2751 * @wcqe: Pointer to driver response CQE object. 2752 * 2753 * The function is called from SLI ring event handler with no 2754 * lock held. This function is the completion handler for NVME ABTS for FCP cmds 2755 * The function frees memory resources used for the NVME commands. 2756 **/ 2757 static void 2758 lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 2759 struct lpfc_wcqe_complete *wcqe) 2760 { 2761 struct lpfc_nvmet_rcv_ctx *ctxp; 2762 struct lpfc_nvmet_tgtport *tgtp; 2763 unsigned long flags; 2764 uint32_t status, result; 2765 bool released = false; 2766 2767 ctxp = cmdwqe->context2; 2768 status = bf_get(lpfc_wcqe_c_status, wcqe); 2769 result = wcqe->parameter; 2770 2771 if (!ctxp) { 2772 /* if context is clear, related io alrady complete */ 2773 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2774 "6070 ABTS cmpl: WCQE: %08x %08x %08x %08x\n", 2775 wcqe->word0, wcqe->total_data_placed, 2776 result, wcqe->word3); 2777 return; 2778 } 2779 2780 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2781 spin_lock_irqsave(&ctxp->ctxlock, flags); 2782 if (ctxp->flag & LPFC_NVMET_ABORT_OP) 2783 atomic_inc(&tgtp->xmt_fcp_abort_cmpl); 2784 2785 /* Sanity check */ 2786 if (ctxp->state != LPFC_NVMET_STE_ABORT) { 2787 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 2788 "6112 ABTS Wrong state:%d oxid x%x\n", 2789 ctxp->state, ctxp->oxid); 2790 } 2791 2792 /* Check if we already received a free context call 2793 * and we have completed processing an abort situation. 2794 */ 2795 ctxp->state = LPFC_NVMET_STE_DONE; 2796 if ((ctxp->flag & LPFC_NVMET_CTX_RLS) && 2797 !(ctxp->flag & LPFC_NVMET_XBUSY)) { 2798 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 2799 list_del(&ctxp->list); 2800 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 2801 released = true; 2802 } 2803 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 2804 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 2805 atomic_inc(&tgtp->xmt_abort_rsp); 2806 2807 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2808 "6316 ABTS cmpl xri x%x flg x%x (%x) " 2809 "WCQE: %08x %08x %08x %08x\n", 2810 ctxp->oxid, ctxp->flag, released, 2811 wcqe->word0, wcqe->total_data_placed, 2812 result, wcqe->word3); 2813 2814 cmdwqe->context2 = NULL; 2815 cmdwqe->context3 = NULL; 2816 /* 2817 * if transport has released ctx, then can reuse it. Otherwise, 2818 * will be recycled by transport release call. 2819 */ 2820 if (released) 2821 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); 2822 2823 /* Since iaab/iaar are NOT set, there is no work left. 2824 * For LPFC_NVMET_XBUSY, lpfc_sli4_nvmet_xri_aborted 2825 * should have been called already. 2826 */ 2827 } 2828 2829 /** 2830 * lpfc_nvmet_xmt_ls_abort_cmp - Completion handler for ABTS 2831 * @phba: Pointer to HBA context object. 2832 * @cmdwqe: Pointer to driver command WQE object. 2833 * @wcqe: Pointer to driver response CQE object. 2834 * 2835 * The function is called from SLI ring event handler with no 2836 * lock held. This function is the completion handler for NVME ABTS for LS cmds 2837 * The function frees memory resources used for the NVME commands. 2838 **/ 2839 static void 2840 lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 2841 struct lpfc_wcqe_complete *wcqe) 2842 { 2843 struct lpfc_nvmet_rcv_ctx *ctxp; 2844 struct lpfc_nvmet_tgtport *tgtp; 2845 uint32_t status, result; 2846 2847 ctxp = cmdwqe->context2; 2848 status = bf_get(lpfc_wcqe_c_status, wcqe); 2849 result = wcqe->parameter; 2850 2851 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2852 atomic_inc(&tgtp->xmt_ls_abort_cmpl); 2853 2854 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2855 "6083 Abort cmpl: ctx %p WCQE:%08x %08x %08x %08x\n", 2856 ctxp, wcqe->word0, wcqe->total_data_placed, 2857 result, wcqe->word3); 2858 2859 if (!ctxp) { 2860 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 2861 "6415 NVMET LS Abort No ctx: WCQE: " 2862 "%08x %08x %08x %08x\n", 2863 wcqe->word0, wcqe->total_data_placed, 2864 result, wcqe->word3); 2865 2866 lpfc_sli_release_iocbq(phba, cmdwqe); 2867 return; 2868 } 2869 2870 if (ctxp->state != LPFC_NVMET_STE_LS_ABORT) { 2871 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2872 "6416 NVMET LS abort cmpl state mismatch: " 2873 "oxid x%x: %d %d\n", 2874 ctxp->oxid, ctxp->state, ctxp->entry_cnt); 2875 } 2876 2877 cmdwqe->context2 = NULL; 2878 cmdwqe->context3 = NULL; 2879 lpfc_sli_release_iocbq(phba, cmdwqe); 2880 kfree(ctxp); 2881 } 2882 2883 static int 2884 lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba, 2885 struct lpfc_nvmet_rcv_ctx *ctxp, 2886 uint32_t sid, uint16_t xri) 2887 { 2888 struct lpfc_nvmet_tgtport *tgtp; 2889 struct lpfc_iocbq *abts_wqeq; 2890 union lpfc_wqe128 *wqe_abts; 2891 struct lpfc_nodelist *ndlp; 2892 2893 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2894 "6067 ABTS: sid %x xri x%x/x%x\n", 2895 sid, xri, ctxp->wqeq->sli4_xritag); 2896 2897 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2898 2899 ndlp = lpfc_findnode_did(phba->pport, sid); 2900 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || 2901 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && 2902 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { 2903 atomic_inc(&tgtp->xmt_abort_rsp_error); 2904 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 2905 "6134 Drop ABTS - wrong NDLP state x%x.\n", 2906 (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE); 2907 2908 /* No failure to an ABTS request. */ 2909 return 0; 2910 } 2911 2912 abts_wqeq = ctxp->wqeq; 2913 wqe_abts = &abts_wqeq->wqe; 2914 2915 /* 2916 * Since we zero the whole WQE, we need to ensure we set the WQE fields 2917 * that were initialized in lpfc_sli4_nvmet_alloc. 2918 */ 2919 memset(wqe_abts, 0, sizeof(union lpfc_wqe)); 2920 2921 /* Word 5 */ 2922 bf_set(wqe_dfctl, &wqe_abts->xmit_sequence.wge_ctl, 0); 2923 bf_set(wqe_ls, &wqe_abts->xmit_sequence.wge_ctl, 1); 2924 bf_set(wqe_la, &wqe_abts->xmit_sequence.wge_ctl, 0); 2925 bf_set(wqe_rctl, &wqe_abts->xmit_sequence.wge_ctl, FC_RCTL_BA_ABTS); 2926 bf_set(wqe_type, &wqe_abts->xmit_sequence.wge_ctl, FC_TYPE_BLS); 2927 2928 /* Word 6 */ 2929 bf_set(wqe_ctxt_tag, &wqe_abts->xmit_sequence.wqe_com, 2930 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 2931 bf_set(wqe_xri_tag, &wqe_abts->xmit_sequence.wqe_com, 2932 abts_wqeq->sli4_xritag); 2933 2934 /* Word 7 */ 2935 bf_set(wqe_cmnd, &wqe_abts->xmit_sequence.wqe_com, 2936 CMD_XMIT_SEQUENCE64_WQE); 2937 bf_set(wqe_ct, &wqe_abts->xmit_sequence.wqe_com, SLI4_CT_RPI); 2938 bf_set(wqe_class, &wqe_abts->xmit_sequence.wqe_com, CLASS3); 2939 bf_set(wqe_pu, &wqe_abts->xmit_sequence.wqe_com, 0); 2940 2941 /* Word 8 */ 2942 wqe_abts->xmit_sequence.wqe_com.abort_tag = abts_wqeq->iotag; 2943 2944 /* Word 9 */ 2945 bf_set(wqe_reqtag, &wqe_abts->xmit_sequence.wqe_com, abts_wqeq->iotag); 2946 /* Needs to be set by caller */ 2947 bf_set(wqe_rcvoxid, &wqe_abts->xmit_sequence.wqe_com, xri); 2948 2949 /* Word 10 */ 2950 bf_set(wqe_dbde, &wqe_abts->xmit_sequence.wqe_com, 1); 2951 bf_set(wqe_iod, &wqe_abts->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE); 2952 bf_set(wqe_lenloc, &wqe_abts->xmit_sequence.wqe_com, 2953 LPFC_WQE_LENLOC_WORD12); 2954 bf_set(wqe_ebde_cnt, &wqe_abts->xmit_sequence.wqe_com, 0); 2955 bf_set(wqe_qosd, &wqe_abts->xmit_sequence.wqe_com, 0); 2956 2957 /* Word 11 */ 2958 bf_set(wqe_cqid, &wqe_abts->xmit_sequence.wqe_com, 2959 LPFC_WQE_CQ_ID_DEFAULT); 2960 bf_set(wqe_cmd_type, &wqe_abts->xmit_sequence.wqe_com, 2961 OTHER_COMMAND); 2962 2963 abts_wqeq->vport = phba->pport; 2964 abts_wqeq->context1 = ndlp; 2965 abts_wqeq->context2 = ctxp; 2966 abts_wqeq->context3 = NULL; 2967 abts_wqeq->rsvd2 = 0; 2968 /* hba_wqidx should already be setup from command we are aborting */ 2969 abts_wqeq->iocb.ulpCommand = CMD_XMIT_SEQUENCE64_CR; 2970 abts_wqeq->iocb.ulpLe = 1; 2971 2972 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2973 "6069 Issue ABTS to xri x%x reqtag x%x\n", 2974 xri, abts_wqeq->iotag); 2975 return 1; 2976 } 2977 2978 static int 2979 lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, 2980 struct lpfc_nvmet_rcv_ctx *ctxp, 2981 uint32_t sid, uint16_t xri) 2982 { 2983 struct lpfc_nvmet_tgtport *tgtp; 2984 struct lpfc_iocbq *abts_wqeq; 2985 union lpfc_wqe128 *abts_wqe; 2986 struct lpfc_nodelist *ndlp; 2987 unsigned long flags; 2988 int rc; 2989 2990 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2991 if (!ctxp->wqeq) { 2992 ctxp->wqeq = ctxp->ctxbuf->iocbq; 2993 ctxp->wqeq->hba_wqidx = 0; 2994 } 2995 2996 ndlp = lpfc_findnode_did(phba->pport, sid); 2997 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || 2998 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && 2999 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { 3000 atomic_inc(&tgtp->xmt_abort_rsp_error); 3001 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3002 "6160 Drop ABORT - wrong NDLP state x%x.\n", 3003 (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE); 3004 3005 /* No failure to an ABTS request. */ 3006 spin_lock_irqsave(&ctxp->ctxlock, flags); 3007 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 3008 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3009 return 0; 3010 } 3011 3012 /* Issue ABTS for this WQE based on iotag */ 3013 ctxp->abort_wqeq = lpfc_sli_get_iocbq(phba); 3014 spin_lock_irqsave(&ctxp->ctxlock, flags); 3015 if (!ctxp->abort_wqeq) { 3016 atomic_inc(&tgtp->xmt_abort_rsp_error); 3017 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3018 "6161 ABORT failed: No wqeqs: " 3019 "xri: x%x\n", ctxp->oxid); 3020 /* No failure to an ABTS request. */ 3021 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 3022 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3023 return 0; 3024 } 3025 abts_wqeq = ctxp->abort_wqeq; 3026 abts_wqe = &abts_wqeq->wqe; 3027 ctxp->state = LPFC_NVMET_STE_ABORT; 3028 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3029 3030 /* Announce entry to new IO submit field. */ 3031 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 3032 "6162 ABORT Request to rport DID x%06x " 3033 "for xri x%x x%x\n", 3034 ctxp->sid, ctxp->oxid, ctxp->wqeq->sli4_xritag); 3035 3036 /* If the hba is getting reset, this flag is set. It is 3037 * cleared when the reset is complete and rings reestablished. 3038 */ 3039 spin_lock_irqsave(&phba->hbalock, flags); 3040 /* driver queued commands are in process of being flushed */ 3041 if (phba->hba_flag & HBA_NVME_IOQ_FLUSH) { 3042 spin_unlock_irqrestore(&phba->hbalock, flags); 3043 atomic_inc(&tgtp->xmt_abort_rsp_error); 3044 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 3045 "6163 Driver in reset cleanup - flushing " 3046 "NVME Req now. hba_flag x%x oxid x%x\n", 3047 phba->hba_flag, ctxp->oxid); 3048 lpfc_sli_release_iocbq(phba, abts_wqeq); 3049 spin_lock_irqsave(&ctxp->ctxlock, flags); 3050 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 3051 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3052 return 0; 3053 } 3054 3055 /* Outstanding abort is in progress */ 3056 if (abts_wqeq->iocb_flag & LPFC_DRIVER_ABORTED) { 3057 spin_unlock_irqrestore(&phba->hbalock, flags); 3058 atomic_inc(&tgtp->xmt_abort_rsp_error); 3059 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 3060 "6164 Outstanding NVME I/O Abort Request " 3061 "still pending on oxid x%x\n", 3062 ctxp->oxid); 3063 lpfc_sli_release_iocbq(phba, abts_wqeq); 3064 spin_lock_irqsave(&ctxp->ctxlock, flags); 3065 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 3066 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3067 return 0; 3068 } 3069 3070 /* Ready - mark outstanding as aborted by driver. */ 3071 abts_wqeq->iocb_flag |= LPFC_DRIVER_ABORTED; 3072 3073 /* WQEs are reused. Clear stale data and set key fields to 3074 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag. 3075 */ 3076 memset(abts_wqe, 0, sizeof(union lpfc_wqe)); 3077 3078 /* word 3 */ 3079 bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG); 3080 3081 /* word 7 */ 3082 bf_set(wqe_ct, &abts_wqe->abort_cmd.wqe_com, 0); 3083 bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX); 3084 3085 /* word 8 - tell the FW to abort the IO associated with this 3086 * outstanding exchange ID. 3087 */ 3088 abts_wqe->abort_cmd.wqe_com.abort_tag = ctxp->wqeq->sli4_xritag; 3089 3090 /* word 9 - this is the iotag for the abts_wqe completion. */ 3091 bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com, 3092 abts_wqeq->iotag); 3093 3094 /* word 10 */ 3095 bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1); 3096 bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE); 3097 3098 /* word 11 */ 3099 bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND); 3100 bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1); 3101 bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 3102 3103 /* ABTS WQE must go to the same WQ as the WQE to be aborted */ 3104 abts_wqeq->hba_wqidx = ctxp->wqeq->hba_wqidx; 3105 abts_wqeq->wqe_cmpl = lpfc_nvmet_sol_fcp_abort_cmp; 3106 abts_wqeq->iocb_cmpl = 0; 3107 abts_wqeq->iocb_flag |= LPFC_IO_NVME; 3108 abts_wqeq->context2 = ctxp; 3109 abts_wqeq->vport = phba->pport; 3110 if (!ctxp->hdwq) 3111 ctxp->hdwq = &phba->sli4_hba.hdwq[abts_wqeq->hba_wqidx]; 3112 3113 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq); 3114 spin_unlock_irqrestore(&phba->hbalock, flags); 3115 if (rc == WQE_SUCCESS) { 3116 atomic_inc(&tgtp->xmt_abort_sol); 3117 return 0; 3118 } 3119 3120 atomic_inc(&tgtp->xmt_abort_rsp_error); 3121 spin_lock_irqsave(&ctxp->ctxlock, flags); 3122 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 3123 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3124 lpfc_sli_release_iocbq(phba, abts_wqeq); 3125 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3126 "6166 Failed ABORT issue_wqe with status x%x " 3127 "for oxid x%x.\n", 3128 rc, ctxp->oxid); 3129 return 1; 3130 } 3131 3132 static int 3133 lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba, 3134 struct lpfc_nvmet_rcv_ctx *ctxp, 3135 uint32_t sid, uint16_t xri) 3136 { 3137 struct lpfc_nvmet_tgtport *tgtp; 3138 struct lpfc_iocbq *abts_wqeq; 3139 unsigned long flags; 3140 bool released = false; 3141 int rc; 3142 3143 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 3144 if (!ctxp->wqeq) { 3145 ctxp->wqeq = ctxp->ctxbuf->iocbq; 3146 ctxp->wqeq->hba_wqidx = 0; 3147 } 3148 3149 if (ctxp->state == LPFC_NVMET_STE_FREE) { 3150 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 3151 "6417 NVMET ABORT ctx freed %d %d oxid x%x\n", 3152 ctxp->state, ctxp->entry_cnt, ctxp->oxid); 3153 rc = WQE_BUSY; 3154 goto aerr; 3155 } 3156 ctxp->state = LPFC_NVMET_STE_ABORT; 3157 ctxp->entry_cnt++; 3158 rc = lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri); 3159 if (rc == 0) 3160 goto aerr; 3161 3162 spin_lock_irqsave(&phba->hbalock, flags); 3163 abts_wqeq = ctxp->wqeq; 3164 abts_wqeq->wqe_cmpl = lpfc_nvmet_unsol_fcp_abort_cmp; 3165 abts_wqeq->iocb_cmpl = NULL; 3166 abts_wqeq->iocb_flag |= LPFC_IO_NVMET; 3167 if (!ctxp->hdwq) 3168 ctxp->hdwq = &phba->sli4_hba.hdwq[abts_wqeq->hba_wqidx]; 3169 3170 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq); 3171 spin_unlock_irqrestore(&phba->hbalock, flags); 3172 if (rc == WQE_SUCCESS) { 3173 return 0; 3174 } 3175 3176 aerr: 3177 spin_lock_irqsave(&ctxp->ctxlock, flags); 3178 if (ctxp->flag & LPFC_NVMET_CTX_RLS) { 3179 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 3180 list_del(&ctxp->list); 3181 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); 3182 released = true; 3183 } 3184 ctxp->flag &= ~(LPFC_NVMET_ABORT_OP | LPFC_NVMET_CTX_RLS); 3185 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 3186 3187 atomic_inc(&tgtp->xmt_abort_rsp_error); 3188 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3189 "6135 Failed to Issue ABTS for oxid x%x. Status x%x\n", 3190 ctxp->oxid, rc); 3191 if (released) 3192 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); 3193 return 1; 3194 } 3195 3196 static int 3197 lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *phba, 3198 struct lpfc_nvmet_rcv_ctx *ctxp, 3199 uint32_t sid, uint16_t xri) 3200 { 3201 struct lpfc_nvmet_tgtport *tgtp; 3202 struct lpfc_iocbq *abts_wqeq; 3203 union lpfc_wqe128 *wqe_abts; 3204 unsigned long flags; 3205 int rc; 3206 3207 if ((ctxp->state == LPFC_NVMET_STE_LS_RCV && ctxp->entry_cnt == 1) || 3208 (ctxp->state == LPFC_NVMET_STE_LS_RSP && ctxp->entry_cnt == 2)) { 3209 ctxp->state = LPFC_NVMET_STE_LS_ABORT; 3210 ctxp->entry_cnt++; 3211 } else { 3212 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 3213 "6418 NVMET LS abort state mismatch " 3214 "IO x%x: %d %d\n", 3215 ctxp->oxid, ctxp->state, ctxp->entry_cnt); 3216 ctxp->state = LPFC_NVMET_STE_LS_ABORT; 3217 } 3218 3219 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 3220 if (!ctxp->wqeq) { 3221 /* Issue ABTS for this WQE based on iotag */ 3222 ctxp->wqeq = lpfc_sli_get_iocbq(phba); 3223 if (!ctxp->wqeq) { 3224 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3225 "6068 Abort failed: No wqeqs: " 3226 "xri: x%x\n", xri); 3227 /* No failure to an ABTS request. */ 3228 kfree(ctxp); 3229 return 0; 3230 } 3231 } 3232 abts_wqeq = ctxp->wqeq; 3233 wqe_abts = &abts_wqeq->wqe; 3234 3235 if (lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri) == 0) { 3236 rc = WQE_BUSY; 3237 goto out; 3238 } 3239 3240 spin_lock_irqsave(&phba->hbalock, flags); 3241 abts_wqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_abort_cmp; 3242 abts_wqeq->iocb_cmpl = 0; 3243 abts_wqeq->iocb_flag |= LPFC_IO_NVME_LS; 3244 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq); 3245 spin_unlock_irqrestore(&phba->hbalock, flags); 3246 if (rc == WQE_SUCCESS) { 3247 atomic_inc(&tgtp->xmt_abort_unsol); 3248 return 0; 3249 } 3250 out: 3251 atomic_inc(&tgtp->xmt_abort_rsp_error); 3252 abts_wqeq->context2 = NULL; 3253 abts_wqeq->context3 = NULL; 3254 lpfc_sli_release_iocbq(phba, abts_wqeq); 3255 kfree(ctxp); 3256 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 3257 "6056 Failed to Issue ABTS. Status x%x\n", rc); 3258 return 0; 3259 } 3260