1 /******************************************************************* 2 * This file is part of the Emulex Linux Device Driver for * 3 * Fibre Channsel Host Bus Adapters. * 4 * Copyright (C) 2017 Broadcom. All Rights Reserved. The term * 5 * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. * 6 * Copyright (C) 2004-2016 Emulex. All rights reserved. * 7 * EMULEX and SLI are trademarks of Emulex. * 8 * www.broadcom.com * 9 * Portions Copyright (C) 2004-2005 Christoph Hellwig * 10 * * 11 * This program is free software; you can redistribute it and/or * 12 * modify it under the terms of version 2 of the GNU General * 13 * Public License as published by the Free Software Foundation. * 14 * This program is distributed in the hope that it will be useful. * 15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 19 * TO BE LEGALLY INVALID. See the GNU General Public License for * 20 * more details, a copy of which can be found in the file COPYING * 21 * included with this package. * 22 ********************************************************************/ 23 #include <linux/pci.h> 24 #include <linux/slab.h> 25 #include <linux/interrupt.h> 26 #include <linux/delay.h> 27 #include <asm/unaligned.h> 28 #include <linux/crc-t10dif.h> 29 #include <net/checksum.h> 30 31 #include <scsi/scsi.h> 32 #include <scsi/scsi_device.h> 33 #include <scsi/scsi_eh.h> 34 #include <scsi/scsi_host.h> 35 #include <scsi/scsi_tcq.h> 36 #include <scsi/scsi_transport_fc.h> 37 #include <scsi/fc/fc_fs.h> 38 39 #include <../drivers/nvme/host/nvme.h> 40 #include <linux/nvme-fc-driver.h> 41 #include <linux/nvme-fc.h> 42 43 #include "lpfc_version.h" 44 #include "lpfc_hw4.h" 45 #include "lpfc_hw.h" 46 #include "lpfc_sli.h" 47 #include "lpfc_sli4.h" 48 #include "lpfc_nl.h" 49 #include "lpfc_disc.h" 50 #include "lpfc.h" 51 #include "lpfc_scsi.h" 52 #include "lpfc_nvme.h" 53 #include "lpfc_nvmet.h" 54 #include "lpfc_logmsg.h" 55 #include "lpfc_crtn.h" 56 #include "lpfc_vport.h" 57 #include "lpfc_debugfs.h" 58 59 static struct lpfc_iocbq *lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *, 60 struct lpfc_nvmet_rcv_ctx *, 61 dma_addr_t rspbuf, 62 uint16_t rspsize); 63 static struct lpfc_iocbq *lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *, 64 struct lpfc_nvmet_rcv_ctx *); 65 static int lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *, 66 struct lpfc_nvmet_rcv_ctx *, 67 uint32_t, uint16_t); 68 static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *, 69 struct lpfc_nvmet_rcv_ctx *, 70 uint32_t, uint16_t); 71 static int lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *, 72 struct lpfc_nvmet_rcv_ctx *, 73 uint32_t, uint16_t); 74 75 void 76 lpfc_nvmet_defer_release(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp) 77 { 78 unsigned long iflag; 79 80 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 81 "6313 NVMET Defer ctx release xri x%x flg x%x\n", 82 ctxp->oxid, ctxp->flag); 83 84 spin_lock_irqsave(&phba->sli4_hba.abts_nvme_buf_list_lock, iflag); 85 if (ctxp->flag & LPFC_NVMET_CTX_RLS) { 86 spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, 87 iflag); 88 return; 89 } 90 ctxp->flag |= LPFC_NVMET_CTX_RLS; 91 list_add_tail(&ctxp->list, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list); 92 spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, iflag); 93 } 94 95 /** 96 * lpfc_nvmet_xmt_ls_rsp_cmp - Completion handler for LS Response 97 * @phba: Pointer to HBA context object. 98 * @cmdwqe: Pointer to driver command WQE object. 99 * @wcqe: Pointer to driver response CQE object. 100 * 101 * The function is called from SLI ring event handler with no 102 * lock held. This function is the completion handler for NVME LS commands 103 * The function frees memory resources used for the NVME commands. 104 **/ 105 static void 106 lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 107 struct lpfc_wcqe_complete *wcqe) 108 { 109 struct lpfc_nvmet_tgtport *tgtp; 110 struct nvmefc_tgt_ls_req *rsp; 111 struct lpfc_nvmet_rcv_ctx *ctxp; 112 uint32_t status, result; 113 114 status = bf_get(lpfc_wcqe_c_status, wcqe); 115 result = wcqe->parameter; 116 ctxp = cmdwqe->context2; 117 118 if (ctxp->state != LPFC_NVMET_STE_LS_RSP || ctxp->entry_cnt != 2) { 119 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 120 "6410 NVMET LS cmpl state mismatch IO x%x: " 121 "%d %d\n", 122 ctxp->oxid, ctxp->state, ctxp->entry_cnt); 123 } 124 125 if (!phba->targetport) 126 goto out; 127 128 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 129 130 if (tgtp) { 131 if (status) { 132 atomic_inc(&tgtp->xmt_ls_rsp_error); 133 if (status == IOERR_ABORT_REQUESTED) 134 atomic_inc(&tgtp->xmt_ls_rsp_aborted); 135 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 136 atomic_inc(&tgtp->xmt_ls_rsp_xb_set); 137 } else { 138 atomic_inc(&tgtp->xmt_ls_rsp_cmpl); 139 } 140 } 141 142 out: 143 rsp = &ctxp->ctx.ls_req; 144 145 lpfc_nvmeio_data(phba, "NVMET LS CMPL: xri x%x stat x%x result x%x\n", 146 ctxp->oxid, status, result); 147 148 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 149 "6038 NVMET LS rsp cmpl: %d %d oxid x%x\n", 150 status, result, ctxp->oxid); 151 152 lpfc_nlp_put(cmdwqe->context1); 153 cmdwqe->context2 = NULL; 154 cmdwqe->context3 = NULL; 155 lpfc_sli_release_iocbq(phba, cmdwqe); 156 rsp->done(rsp); 157 kfree(ctxp); 158 } 159 160 /** 161 * lpfc_nvmet_ctxbuf_post - Repost a NVMET RQ DMA buffer and clean up context 162 * @phba: HBA buffer is associated with 163 * @ctxp: context to clean up 164 * @mp: Buffer to free 165 * 166 * Description: Frees the given DMA buffer in the appropriate way given by 167 * reposting it to its associated RQ so it can be reused. 168 * 169 * Notes: Takes phba->hbalock. Can be called with or without other locks held. 170 * 171 * Returns: None 172 **/ 173 void 174 lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) 175 { 176 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 177 struct lpfc_nvmet_rcv_ctx *ctxp = ctx_buf->context; 178 struct lpfc_nvmet_tgtport *tgtp; 179 struct fc_frame_header *fc_hdr; 180 struct rqb_dmabuf *nvmebuf; 181 struct lpfc_nvmet_ctx_info *infop; 182 uint32_t *payload; 183 uint32_t size, oxid, sid, rc; 184 int cpu; 185 unsigned long iflag; 186 187 if (ctxp->txrdy) { 188 dma_pool_free(phba->txrdy_payload_pool, ctxp->txrdy, 189 ctxp->txrdy_phys); 190 ctxp->txrdy = NULL; 191 ctxp->txrdy_phys = 0; 192 } 193 194 if (ctxp->state == LPFC_NVMET_STE_FREE) { 195 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 196 "6411 NVMET free, already free IO x%x: %d %d\n", 197 ctxp->oxid, ctxp->state, ctxp->entry_cnt); 198 } 199 ctxp->state = LPFC_NVMET_STE_FREE; 200 201 spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag); 202 if (phba->sli4_hba.nvmet_io_wait_cnt) { 203 list_remove_head(&phba->sli4_hba.lpfc_nvmet_io_wait_list, 204 nvmebuf, struct rqb_dmabuf, 205 hbuf.list); 206 phba->sli4_hba.nvmet_io_wait_cnt--; 207 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, 208 iflag); 209 210 fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt); 211 oxid = be16_to_cpu(fc_hdr->fh_ox_id); 212 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 213 payload = (uint32_t *)(nvmebuf->dbuf.virt); 214 size = nvmebuf->bytes_recv; 215 sid = sli4_sid_from_fc_hdr(fc_hdr); 216 217 ctxp = (struct lpfc_nvmet_rcv_ctx *)ctx_buf->context; 218 ctxp->wqeq = NULL; 219 ctxp->txrdy = NULL; 220 ctxp->offset = 0; 221 ctxp->phba = phba; 222 ctxp->size = size; 223 ctxp->oxid = oxid; 224 ctxp->sid = sid; 225 ctxp->state = LPFC_NVMET_STE_RCV; 226 ctxp->entry_cnt = 1; 227 ctxp->flag = 0; 228 ctxp->ctxbuf = ctx_buf; 229 ctxp->rqb_buffer = (void *)nvmebuf; 230 spin_lock_init(&ctxp->ctxlock); 231 232 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 233 if (ctxp->ts_cmd_nvme) { 234 ctxp->ts_cmd_nvme = ktime_get_ns(); 235 ctxp->ts_nvme_data = 0; 236 ctxp->ts_data_wqput = 0; 237 ctxp->ts_isr_data = 0; 238 ctxp->ts_data_nvme = 0; 239 ctxp->ts_nvme_status = 0; 240 ctxp->ts_status_wqput = 0; 241 ctxp->ts_isr_status = 0; 242 ctxp->ts_status_nvme = 0; 243 } 244 #endif 245 atomic_inc(&tgtp->rcv_fcp_cmd_in); 246 /* 247 * The calling sequence should be: 248 * nvmet_fc_rcv_fcp_req->lpfc_nvmet_xmt_fcp_op/cmp- req->done 249 * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp. 250 * When we return from nvmet_fc_rcv_fcp_req, all relevant info 251 * the NVME command / FC header is stored. 252 * A buffer has already been reposted for this IO, so just free 253 * the nvmebuf. 254 */ 255 rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req, 256 payload, size); 257 258 /* Process FCP command */ 259 if (rc == 0) { 260 atomic_inc(&tgtp->rcv_fcp_cmd_out); 261 nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); 262 return; 263 } 264 265 /* Processing of FCP command is deferred */ 266 if (rc == -EOVERFLOW) { 267 lpfc_nvmeio_data(phba, 268 "NVMET RCV BUSY: xri x%x sz %d " 269 "from %06x\n", 270 oxid, size, sid); 271 /* defer repost rcv buffer till .defer_rcv callback */ 272 ctxp->flag &= ~LPFC_NVMET_DEFER_RCV_REPOST; 273 atomic_inc(&tgtp->rcv_fcp_cmd_out); 274 return; 275 } 276 atomic_inc(&tgtp->rcv_fcp_cmd_drop); 277 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 278 "2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n", 279 ctxp->oxid, rc, 280 atomic_read(&tgtp->rcv_fcp_cmd_in), 281 atomic_read(&tgtp->rcv_fcp_cmd_out), 282 atomic_read(&tgtp->xmt_fcp_release)); 283 284 lpfc_nvmet_defer_release(phba, ctxp); 285 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid); 286 nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); 287 return; 288 } 289 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, iflag); 290 291 /* 292 * Use the CPU context list, from the MRQ the IO was received on 293 * (ctxp->idx), to save context structure. 294 */ 295 cpu = smp_processor_id(); 296 infop = lpfc_get_ctx_list(phba, cpu, ctxp->idx); 297 spin_lock_irqsave(&infop->nvmet_ctx_list_lock, iflag); 298 list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list); 299 infop->nvmet_ctx_list_cnt++; 300 spin_unlock_irqrestore(&infop->nvmet_ctx_list_lock, iflag); 301 #endif 302 } 303 304 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 305 static void 306 lpfc_nvmet_ktime(struct lpfc_hba *phba, 307 struct lpfc_nvmet_rcv_ctx *ctxp) 308 { 309 uint64_t seg1, seg2, seg3, seg4, seg5; 310 uint64_t seg6, seg7, seg8, seg9, seg10; 311 uint64_t segsum; 312 313 if (!ctxp->ts_isr_cmd || !ctxp->ts_cmd_nvme || 314 !ctxp->ts_nvme_data || !ctxp->ts_data_wqput || 315 !ctxp->ts_isr_data || !ctxp->ts_data_nvme || 316 !ctxp->ts_nvme_status || !ctxp->ts_status_wqput || 317 !ctxp->ts_isr_status || !ctxp->ts_status_nvme) 318 return; 319 320 if (ctxp->ts_status_nvme < ctxp->ts_isr_cmd) 321 return; 322 if (ctxp->ts_isr_cmd > ctxp->ts_cmd_nvme) 323 return; 324 if (ctxp->ts_cmd_nvme > ctxp->ts_nvme_data) 325 return; 326 if (ctxp->ts_nvme_data > ctxp->ts_data_wqput) 327 return; 328 if (ctxp->ts_data_wqput > ctxp->ts_isr_data) 329 return; 330 if (ctxp->ts_isr_data > ctxp->ts_data_nvme) 331 return; 332 if (ctxp->ts_data_nvme > ctxp->ts_nvme_status) 333 return; 334 if (ctxp->ts_nvme_status > ctxp->ts_status_wqput) 335 return; 336 if (ctxp->ts_status_wqput > ctxp->ts_isr_status) 337 return; 338 if (ctxp->ts_isr_status > ctxp->ts_status_nvme) 339 return; 340 /* 341 * Segment 1 - Time from FCP command received by MSI-X ISR 342 * to FCP command is passed to NVME Layer. 343 * Segment 2 - Time from FCP command payload handed 344 * off to NVME Layer to Driver receives a Command op 345 * from NVME Layer. 346 * Segment 3 - Time from Driver receives a Command op 347 * from NVME Layer to Command is put on WQ. 348 * Segment 4 - Time from Driver WQ put is done 349 * to MSI-X ISR for Command cmpl. 350 * Segment 5 - Time from MSI-X ISR for Command cmpl to 351 * Command cmpl is passed to NVME Layer. 352 * Segment 6 - Time from Command cmpl is passed to NVME 353 * Layer to Driver receives a RSP op from NVME Layer. 354 * Segment 7 - Time from Driver receives a RSP op from 355 * NVME Layer to WQ put is done on TRSP FCP Status. 356 * Segment 8 - Time from Driver WQ put is done on TRSP 357 * FCP Status to MSI-X ISR for TRSP cmpl. 358 * Segment 9 - Time from MSI-X ISR for TRSP cmpl to 359 * TRSP cmpl is passed to NVME Layer. 360 * Segment 10 - Time from FCP command received by 361 * MSI-X ISR to command is completed on wire. 362 * (Segments 1 thru 8) for READDATA / WRITEDATA 363 * (Segments 1 thru 4) for READDATA_RSP 364 */ 365 seg1 = ctxp->ts_cmd_nvme - ctxp->ts_isr_cmd; 366 segsum = seg1; 367 368 seg2 = ctxp->ts_nvme_data - ctxp->ts_isr_cmd; 369 if (segsum > seg2) 370 return; 371 seg2 -= segsum; 372 segsum += seg2; 373 374 seg3 = ctxp->ts_data_wqput - ctxp->ts_isr_cmd; 375 if (segsum > seg3) 376 return; 377 seg3 -= segsum; 378 segsum += seg3; 379 380 seg4 = ctxp->ts_isr_data - ctxp->ts_isr_cmd; 381 if (segsum > seg4) 382 return; 383 seg4 -= segsum; 384 segsum += seg4; 385 386 seg5 = ctxp->ts_data_nvme - ctxp->ts_isr_cmd; 387 if (segsum > seg5) 388 return; 389 seg5 -= segsum; 390 segsum += seg5; 391 392 393 /* For auto rsp commands seg6 thru seg10 will be 0 */ 394 if (ctxp->ts_nvme_status > ctxp->ts_data_nvme) { 395 seg6 = ctxp->ts_nvme_status - ctxp->ts_isr_cmd; 396 if (segsum > seg6) 397 return; 398 seg6 -= segsum; 399 segsum += seg6; 400 401 seg7 = ctxp->ts_status_wqput - ctxp->ts_isr_cmd; 402 if (segsum > seg7) 403 return; 404 seg7 -= segsum; 405 segsum += seg7; 406 407 seg8 = ctxp->ts_isr_status - ctxp->ts_isr_cmd; 408 if (segsum > seg8) 409 return; 410 seg8 -= segsum; 411 segsum += seg8; 412 413 seg9 = ctxp->ts_status_nvme - ctxp->ts_isr_cmd; 414 if (segsum > seg9) 415 return; 416 seg9 -= segsum; 417 segsum += seg9; 418 419 if (ctxp->ts_isr_status < ctxp->ts_isr_cmd) 420 return; 421 seg10 = (ctxp->ts_isr_status - 422 ctxp->ts_isr_cmd); 423 } else { 424 if (ctxp->ts_isr_data < ctxp->ts_isr_cmd) 425 return; 426 seg6 = 0; 427 seg7 = 0; 428 seg8 = 0; 429 seg9 = 0; 430 seg10 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd); 431 } 432 433 phba->ktime_seg1_total += seg1; 434 if (seg1 < phba->ktime_seg1_min) 435 phba->ktime_seg1_min = seg1; 436 else if (seg1 > phba->ktime_seg1_max) 437 phba->ktime_seg1_max = seg1; 438 439 phba->ktime_seg2_total += seg2; 440 if (seg2 < phba->ktime_seg2_min) 441 phba->ktime_seg2_min = seg2; 442 else if (seg2 > phba->ktime_seg2_max) 443 phba->ktime_seg2_max = seg2; 444 445 phba->ktime_seg3_total += seg3; 446 if (seg3 < phba->ktime_seg3_min) 447 phba->ktime_seg3_min = seg3; 448 else if (seg3 > phba->ktime_seg3_max) 449 phba->ktime_seg3_max = seg3; 450 451 phba->ktime_seg4_total += seg4; 452 if (seg4 < phba->ktime_seg4_min) 453 phba->ktime_seg4_min = seg4; 454 else if (seg4 > phba->ktime_seg4_max) 455 phba->ktime_seg4_max = seg4; 456 457 phba->ktime_seg5_total += seg5; 458 if (seg5 < phba->ktime_seg5_min) 459 phba->ktime_seg5_min = seg5; 460 else if (seg5 > phba->ktime_seg5_max) 461 phba->ktime_seg5_max = seg5; 462 463 phba->ktime_data_samples++; 464 if (!seg6) 465 goto out; 466 467 phba->ktime_seg6_total += seg6; 468 if (seg6 < phba->ktime_seg6_min) 469 phba->ktime_seg6_min = seg6; 470 else if (seg6 > phba->ktime_seg6_max) 471 phba->ktime_seg6_max = seg6; 472 473 phba->ktime_seg7_total += seg7; 474 if (seg7 < phba->ktime_seg7_min) 475 phba->ktime_seg7_min = seg7; 476 else if (seg7 > phba->ktime_seg7_max) 477 phba->ktime_seg7_max = seg7; 478 479 phba->ktime_seg8_total += seg8; 480 if (seg8 < phba->ktime_seg8_min) 481 phba->ktime_seg8_min = seg8; 482 else if (seg8 > phba->ktime_seg8_max) 483 phba->ktime_seg8_max = seg8; 484 485 phba->ktime_seg9_total += seg9; 486 if (seg9 < phba->ktime_seg9_min) 487 phba->ktime_seg9_min = seg9; 488 else if (seg9 > phba->ktime_seg9_max) 489 phba->ktime_seg9_max = seg9; 490 out: 491 phba->ktime_seg10_total += seg10; 492 if (seg10 < phba->ktime_seg10_min) 493 phba->ktime_seg10_min = seg10; 494 else if (seg10 > phba->ktime_seg10_max) 495 phba->ktime_seg10_max = seg10; 496 phba->ktime_status_samples++; 497 } 498 #endif 499 500 /** 501 * lpfc_nvmet_xmt_fcp_op_cmp - Completion handler for FCP Response 502 * @phba: Pointer to HBA context object. 503 * @cmdwqe: Pointer to driver command WQE object. 504 * @wcqe: Pointer to driver response CQE object. 505 * 506 * The function is called from SLI ring event handler with no 507 * lock held. This function is the completion handler for NVME FCP commands 508 * The function frees memory resources used for the NVME commands. 509 **/ 510 static void 511 lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 512 struct lpfc_wcqe_complete *wcqe) 513 { 514 struct lpfc_nvmet_tgtport *tgtp; 515 struct nvmefc_tgt_fcp_req *rsp; 516 struct lpfc_nvmet_rcv_ctx *ctxp; 517 uint32_t status, result, op, start_clean, logerr; 518 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 519 uint32_t id; 520 #endif 521 522 ctxp = cmdwqe->context2; 523 ctxp->flag &= ~LPFC_NVMET_IO_INP; 524 525 rsp = &ctxp->ctx.fcp_req; 526 op = rsp->op; 527 528 status = bf_get(lpfc_wcqe_c_status, wcqe); 529 result = wcqe->parameter; 530 531 if (phba->targetport) 532 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 533 else 534 tgtp = NULL; 535 536 lpfc_nvmeio_data(phba, "NVMET FCP CMPL: xri x%x op x%x status x%x\n", 537 ctxp->oxid, op, status); 538 539 if (status) { 540 rsp->fcp_error = NVME_SC_DATA_XFER_ERROR; 541 rsp->transferred_length = 0; 542 if (tgtp) { 543 atomic_inc(&tgtp->xmt_fcp_rsp_error); 544 if (status == IOERR_ABORT_REQUESTED) 545 atomic_inc(&tgtp->xmt_fcp_rsp_aborted); 546 } 547 548 logerr = LOG_NVME_IOERR; 549 550 /* pick up SLI4 exhange busy condition */ 551 if (bf_get(lpfc_wcqe_c_xb, wcqe)) { 552 ctxp->flag |= LPFC_NVMET_XBUSY; 553 logerr |= LOG_NVME_ABTS; 554 if (tgtp) 555 atomic_inc(&tgtp->xmt_fcp_rsp_xb_set); 556 557 } else { 558 ctxp->flag &= ~LPFC_NVMET_XBUSY; 559 } 560 561 lpfc_printf_log(phba, KERN_INFO, logerr, 562 "6315 IO Error Cmpl xri x%x: %x/%x XBUSY:x%x\n", 563 ctxp->oxid, status, result, ctxp->flag); 564 565 } else { 566 rsp->fcp_error = NVME_SC_SUCCESS; 567 if (op == NVMET_FCOP_RSP) 568 rsp->transferred_length = rsp->rsplen; 569 else 570 rsp->transferred_length = rsp->transfer_length; 571 if (tgtp) 572 atomic_inc(&tgtp->xmt_fcp_rsp_cmpl); 573 } 574 575 if ((op == NVMET_FCOP_READDATA_RSP) || 576 (op == NVMET_FCOP_RSP)) { 577 /* Sanity check */ 578 ctxp->state = LPFC_NVMET_STE_DONE; 579 ctxp->entry_cnt++; 580 581 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 582 if (ctxp->ts_cmd_nvme) { 583 if (rsp->op == NVMET_FCOP_READDATA_RSP) { 584 ctxp->ts_isr_data = 585 cmdwqe->isr_timestamp; 586 ctxp->ts_data_nvme = 587 ktime_get_ns(); 588 ctxp->ts_nvme_status = 589 ctxp->ts_data_nvme; 590 ctxp->ts_status_wqput = 591 ctxp->ts_data_nvme; 592 ctxp->ts_isr_status = 593 ctxp->ts_data_nvme; 594 ctxp->ts_status_nvme = 595 ctxp->ts_data_nvme; 596 } else { 597 ctxp->ts_isr_status = 598 cmdwqe->isr_timestamp; 599 ctxp->ts_status_nvme = 600 ktime_get_ns(); 601 } 602 } 603 if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) { 604 id = smp_processor_id(); 605 if (ctxp->cpu != id) 606 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 607 "6703 CPU Check cmpl: " 608 "cpu %d expect %d\n", 609 id, ctxp->cpu); 610 if (ctxp->cpu < LPFC_CHECK_CPU_CNT) 611 phba->cpucheck_cmpl_io[id]++; 612 } 613 #endif 614 rsp->done(rsp); 615 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 616 if (ctxp->ts_cmd_nvme) 617 lpfc_nvmet_ktime(phba, ctxp); 618 #endif 619 /* lpfc_nvmet_xmt_fcp_release() will recycle the context */ 620 } else { 621 ctxp->entry_cnt++; 622 start_clean = offsetof(struct lpfc_iocbq, iocb_flag); 623 memset(((char *)cmdwqe) + start_clean, 0, 624 (sizeof(struct lpfc_iocbq) - start_clean)); 625 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 626 if (ctxp->ts_cmd_nvme) { 627 ctxp->ts_isr_data = cmdwqe->isr_timestamp; 628 ctxp->ts_data_nvme = ktime_get_ns(); 629 } 630 if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) { 631 id = smp_processor_id(); 632 if (ctxp->cpu != id) 633 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 634 "6704 CPU Check cmdcmpl: " 635 "cpu %d expect %d\n", 636 id, ctxp->cpu); 637 if (ctxp->cpu < LPFC_CHECK_CPU_CNT) 638 phba->cpucheck_ccmpl_io[id]++; 639 } 640 #endif 641 rsp->done(rsp); 642 } 643 } 644 645 static int 646 lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport, 647 struct nvmefc_tgt_ls_req *rsp) 648 { 649 struct lpfc_nvmet_rcv_ctx *ctxp = 650 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.ls_req); 651 struct lpfc_hba *phba = ctxp->phba; 652 struct hbq_dmabuf *nvmebuf = 653 (struct hbq_dmabuf *)ctxp->rqb_buffer; 654 struct lpfc_iocbq *nvmewqeq; 655 struct lpfc_nvmet_tgtport *nvmep = tgtport->private; 656 struct lpfc_dmabuf dmabuf; 657 struct ulp_bde64 bpl; 658 int rc; 659 660 if (phba->pport->load_flag & FC_UNLOADING) 661 return -ENODEV; 662 663 if (phba->pport->load_flag & FC_UNLOADING) 664 return -ENODEV; 665 666 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 667 "6023 NVMET LS rsp oxid x%x\n", ctxp->oxid); 668 669 if ((ctxp->state != LPFC_NVMET_STE_LS_RCV) || 670 (ctxp->entry_cnt != 1)) { 671 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 672 "6412 NVMET LS rsp state mismatch " 673 "oxid x%x: %d %d\n", 674 ctxp->oxid, ctxp->state, ctxp->entry_cnt); 675 } 676 ctxp->state = LPFC_NVMET_STE_LS_RSP; 677 ctxp->entry_cnt++; 678 679 nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, ctxp, rsp->rspdma, 680 rsp->rsplen); 681 if (nvmewqeq == NULL) { 682 atomic_inc(&nvmep->xmt_ls_drop); 683 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 684 "6150 LS Drop IO x%x: Prep\n", 685 ctxp->oxid); 686 lpfc_in_buf_free(phba, &nvmebuf->dbuf); 687 atomic_inc(&nvmep->xmt_ls_abort); 688 lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, 689 ctxp->sid, ctxp->oxid); 690 return -ENOMEM; 691 } 692 693 /* Save numBdes for bpl2sgl */ 694 nvmewqeq->rsvd2 = 1; 695 nvmewqeq->hba_wqidx = 0; 696 nvmewqeq->context3 = &dmabuf; 697 dmabuf.virt = &bpl; 698 bpl.addrLow = nvmewqeq->wqe.xmit_sequence.bde.addrLow; 699 bpl.addrHigh = nvmewqeq->wqe.xmit_sequence.bde.addrHigh; 700 bpl.tus.f.bdeSize = rsp->rsplen; 701 bpl.tus.f.bdeFlags = 0; 702 bpl.tus.w = le32_to_cpu(bpl.tus.w); 703 704 nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_rsp_cmp; 705 nvmewqeq->iocb_cmpl = NULL; 706 nvmewqeq->context2 = ctxp; 707 708 lpfc_nvmeio_data(phba, "NVMET LS RESP: xri x%x wqidx x%x len x%x\n", 709 ctxp->oxid, nvmewqeq->hba_wqidx, rsp->rsplen); 710 711 rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, nvmewqeq); 712 if (rc == WQE_SUCCESS) { 713 /* 714 * Okay to repost buffer here, but wait till cmpl 715 * before freeing ctxp and iocbq. 716 */ 717 lpfc_in_buf_free(phba, &nvmebuf->dbuf); 718 ctxp->rqb_buffer = 0; 719 atomic_inc(&nvmep->xmt_ls_rsp); 720 return 0; 721 } 722 /* Give back resources */ 723 atomic_inc(&nvmep->xmt_ls_drop); 724 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 725 "6151 LS Drop IO x%x: Issue %d\n", 726 ctxp->oxid, rc); 727 728 lpfc_nlp_put(nvmewqeq->context1); 729 730 lpfc_in_buf_free(phba, &nvmebuf->dbuf); 731 atomic_inc(&nvmep->xmt_ls_abort); 732 lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); 733 return -ENXIO; 734 } 735 736 static int 737 lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport, 738 struct nvmefc_tgt_fcp_req *rsp) 739 { 740 struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private; 741 struct lpfc_nvmet_rcv_ctx *ctxp = 742 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req); 743 struct lpfc_hba *phba = ctxp->phba; 744 struct lpfc_iocbq *nvmewqeq; 745 int rc; 746 747 if (phba->pport->load_flag & FC_UNLOADING) { 748 rc = -ENODEV; 749 goto aerr; 750 } 751 752 if (phba->pport->load_flag & FC_UNLOADING) { 753 rc = -ENODEV; 754 goto aerr; 755 } 756 757 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 758 if (ctxp->ts_cmd_nvme) { 759 if (rsp->op == NVMET_FCOP_RSP) 760 ctxp->ts_nvme_status = ktime_get_ns(); 761 else 762 ctxp->ts_nvme_data = ktime_get_ns(); 763 } 764 if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) { 765 int id = smp_processor_id(); 766 ctxp->cpu = id; 767 if (id < LPFC_CHECK_CPU_CNT) 768 phba->cpucheck_xmt_io[id]++; 769 if (rsp->hwqid != id) { 770 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 771 "6705 CPU Check OP: " 772 "cpu %d expect %d\n", 773 id, rsp->hwqid); 774 ctxp->cpu = rsp->hwqid; 775 } 776 } 777 #endif 778 779 /* Sanity check */ 780 if ((ctxp->flag & LPFC_NVMET_ABTS_RCV) || 781 (ctxp->state == LPFC_NVMET_STE_ABORT)) { 782 atomic_inc(&lpfc_nvmep->xmt_fcp_drop); 783 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 784 "6102 IO xri x%x aborted\n", 785 ctxp->oxid); 786 rc = -ENXIO; 787 goto aerr; 788 } 789 790 nvmewqeq = lpfc_nvmet_prep_fcp_wqe(phba, ctxp); 791 if (nvmewqeq == NULL) { 792 atomic_inc(&lpfc_nvmep->xmt_fcp_drop); 793 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 794 "6152 FCP Drop IO x%x: Prep\n", 795 ctxp->oxid); 796 rc = -ENXIO; 797 goto aerr; 798 } 799 800 nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_fcp_op_cmp; 801 nvmewqeq->iocb_cmpl = NULL; 802 nvmewqeq->context2 = ctxp; 803 nvmewqeq->iocb_flag |= LPFC_IO_NVMET; 804 ctxp->wqeq->hba_wqidx = rsp->hwqid; 805 806 lpfc_nvmeio_data(phba, "NVMET FCP CMND: xri x%x op x%x len x%x\n", 807 ctxp->oxid, rsp->op, rsp->rsplen); 808 809 ctxp->flag |= LPFC_NVMET_IO_INP; 810 rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, nvmewqeq); 811 if (rc == WQE_SUCCESS) { 812 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 813 if (!ctxp->ts_cmd_nvme) 814 return 0; 815 if (rsp->op == NVMET_FCOP_RSP) 816 ctxp->ts_status_wqput = ktime_get_ns(); 817 else 818 ctxp->ts_data_wqput = ktime_get_ns(); 819 #endif 820 return 0; 821 } 822 823 /* Give back resources */ 824 atomic_inc(&lpfc_nvmep->xmt_fcp_drop); 825 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 826 "6153 FCP Drop IO x%x: Issue: %d\n", 827 ctxp->oxid, rc); 828 829 ctxp->wqeq->hba_wqidx = 0; 830 nvmewqeq->context2 = NULL; 831 nvmewqeq->context3 = NULL; 832 rc = -EBUSY; 833 aerr: 834 return rc; 835 } 836 837 static void 838 lpfc_nvmet_targetport_delete(struct nvmet_fc_target_port *targetport) 839 { 840 struct lpfc_nvmet_tgtport *tport = targetport->private; 841 842 /* release any threads waiting for the unreg to complete */ 843 complete(&tport->tport_unreg_done); 844 } 845 846 static void 847 lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport, 848 struct nvmefc_tgt_fcp_req *req) 849 { 850 struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private; 851 struct lpfc_nvmet_rcv_ctx *ctxp = 852 container_of(req, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req); 853 struct lpfc_hba *phba = ctxp->phba; 854 unsigned long flags; 855 856 if (phba->pport->load_flag & FC_UNLOADING) 857 return; 858 859 if (phba->pport->load_flag & FC_UNLOADING) 860 return; 861 862 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 863 "6103 NVMET Abort op: oxri x%x flg x%x ste %d\n", 864 ctxp->oxid, ctxp->flag, ctxp->state); 865 866 lpfc_nvmeio_data(phba, "NVMET FCP ABRT: xri x%x flg x%x ste x%x\n", 867 ctxp->oxid, ctxp->flag, ctxp->state); 868 869 atomic_inc(&lpfc_nvmep->xmt_fcp_abort); 870 871 spin_lock_irqsave(&ctxp->ctxlock, flags); 872 ctxp->state = LPFC_NVMET_STE_ABORT; 873 874 /* Since iaab/iaar are NOT set, we need to check 875 * if the firmware is in process of aborting IO 876 */ 877 if (ctxp->flag & LPFC_NVMET_XBUSY) { 878 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 879 return; 880 } 881 ctxp->flag |= LPFC_NVMET_ABORT_OP; 882 883 /* An state of LPFC_NVMET_STE_RCV means we have just received 884 * the NVME command and have not started processing it. 885 * (by issuing any IO WQEs on this exchange yet) 886 */ 887 if (ctxp->state == LPFC_NVMET_STE_RCV) 888 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, 889 ctxp->oxid); 890 else 891 lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid, 892 ctxp->oxid); 893 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 894 } 895 896 static void 897 lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport, 898 struct nvmefc_tgt_fcp_req *rsp) 899 { 900 struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private; 901 struct lpfc_nvmet_rcv_ctx *ctxp = 902 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req); 903 struct lpfc_hba *phba = ctxp->phba; 904 unsigned long flags; 905 bool aborting = false; 906 907 if (ctxp->state != LPFC_NVMET_STE_DONE && 908 ctxp->state != LPFC_NVMET_STE_ABORT) { 909 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 910 "6413 NVMET release bad state %d %d oxid x%x\n", 911 ctxp->state, ctxp->entry_cnt, ctxp->oxid); 912 } 913 914 spin_lock_irqsave(&ctxp->ctxlock, flags); 915 if ((ctxp->flag & LPFC_NVMET_ABORT_OP) || 916 (ctxp->flag & LPFC_NVMET_XBUSY)) { 917 aborting = true; 918 /* let the abort path do the real release */ 919 lpfc_nvmet_defer_release(phba, ctxp); 920 } 921 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 922 923 lpfc_nvmeio_data(phba, "NVMET FCP FREE: xri x%x ste %d abt %d\n", ctxp->oxid, 924 ctxp->state, aborting); 925 926 atomic_inc(&lpfc_nvmep->xmt_fcp_release); 927 928 if (aborting) 929 return; 930 931 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); 932 } 933 934 static void 935 lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport, 936 struct nvmefc_tgt_fcp_req *rsp) 937 { 938 struct lpfc_nvmet_tgtport *tgtp; 939 struct lpfc_nvmet_rcv_ctx *ctxp = 940 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req); 941 struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer; 942 struct lpfc_hba *phba = ctxp->phba; 943 944 lpfc_nvmeio_data(phba, "NVMET DEFERRCV: xri x%x sz %d CPU %02x\n", 945 ctxp->oxid, ctxp->size, smp_processor_id()); 946 947 tgtp = phba->targetport->private; 948 atomic_inc(&tgtp->rcv_fcp_cmd_defer); 949 if (ctxp->flag & LPFC_NVMET_DEFER_RCV_REPOST) 950 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */ 951 else 952 nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); 953 ctxp->flag &= ~LPFC_NVMET_DEFER_RCV_REPOST; 954 } 955 956 static struct nvmet_fc_target_template lpfc_tgttemplate = { 957 .targetport_delete = lpfc_nvmet_targetport_delete, 958 .xmt_ls_rsp = lpfc_nvmet_xmt_ls_rsp, 959 .fcp_op = lpfc_nvmet_xmt_fcp_op, 960 .fcp_abort = lpfc_nvmet_xmt_fcp_abort, 961 .fcp_req_release = lpfc_nvmet_xmt_fcp_release, 962 .defer_rcv = lpfc_nvmet_defer_rcv, 963 964 .max_hw_queues = 1, 965 .max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS, 966 .max_dif_sgl_segments = LPFC_NVMET_DEFAULT_SEGS, 967 .dma_boundary = 0xFFFFFFFF, 968 969 /* optional features */ 970 .target_features = 0, 971 /* sizes of additional private data for data structures */ 972 .target_priv_sz = sizeof(struct lpfc_nvmet_tgtport), 973 }; 974 975 static void 976 __lpfc_nvmet_clean_io_for_cpu(struct lpfc_hba *phba, 977 struct lpfc_nvmet_ctx_info *infop) 978 { 979 struct lpfc_nvmet_ctxbuf *ctx_buf, *next_ctx_buf; 980 unsigned long flags; 981 982 spin_lock_irqsave(&infop->nvmet_ctx_list_lock, flags); 983 list_for_each_entry_safe(ctx_buf, next_ctx_buf, 984 &infop->nvmet_ctx_list, list) { 985 spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); 986 list_del_init(&ctx_buf->list); 987 spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); 988 989 __lpfc_clear_active_sglq(phba, ctx_buf->sglq->sli4_lxritag); 990 ctx_buf->sglq->state = SGL_FREED; 991 ctx_buf->sglq->ndlp = NULL; 992 993 spin_lock(&phba->sli4_hba.sgl_list_lock); 994 list_add_tail(&ctx_buf->sglq->list, 995 &phba->sli4_hba.lpfc_nvmet_sgl_list); 996 spin_unlock(&phba->sli4_hba.sgl_list_lock); 997 998 lpfc_sli_release_iocbq(phba, ctx_buf->iocbq); 999 kfree(ctx_buf->context); 1000 } 1001 spin_unlock_irqrestore(&infop->nvmet_ctx_list_lock, flags); 1002 } 1003 1004 static void 1005 lpfc_nvmet_cleanup_io_context(struct lpfc_hba *phba) 1006 { 1007 struct lpfc_nvmet_ctx_info *infop; 1008 int i, j; 1009 1010 /* The first context list, MRQ 0 CPU 0 */ 1011 infop = phba->sli4_hba.nvmet_ctx_info; 1012 if (!infop) 1013 return; 1014 1015 /* Cycle the the entire CPU context list for every MRQ */ 1016 for (i = 0; i < phba->cfg_nvmet_mrq; i++) { 1017 for (j = 0; j < phba->sli4_hba.num_present_cpu; j++) { 1018 __lpfc_nvmet_clean_io_for_cpu(phba, infop); 1019 infop++; /* next */ 1020 } 1021 } 1022 kfree(phba->sli4_hba.nvmet_ctx_info); 1023 phba->sli4_hba.nvmet_ctx_info = NULL; 1024 } 1025 1026 static int 1027 lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) 1028 { 1029 struct lpfc_nvmet_ctxbuf *ctx_buf; 1030 struct lpfc_iocbq *nvmewqe; 1031 union lpfc_wqe128 *wqe; 1032 struct lpfc_nvmet_ctx_info *last_infop; 1033 struct lpfc_nvmet_ctx_info *infop; 1034 int i, j, idx; 1035 1036 lpfc_printf_log(phba, KERN_INFO, LOG_NVME, 1037 "6403 Allocate NVMET resources for %d XRIs\n", 1038 phba->sli4_hba.nvmet_xri_cnt); 1039 1040 phba->sli4_hba.nvmet_ctx_info = kcalloc( 1041 phba->sli4_hba.num_present_cpu * phba->cfg_nvmet_mrq, 1042 sizeof(struct lpfc_nvmet_ctx_info), GFP_KERNEL); 1043 if (!phba->sli4_hba.nvmet_ctx_info) { 1044 lpfc_printf_log(phba, KERN_ERR, LOG_INIT, 1045 "6419 Failed allocate memory for " 1046 "nvmet context lists\n"); 1047 return -ENOMEM; 1048 } 1049 1050 /* 1051 * Assuming X CPUs in the system, and Y MRQs, allocate some 1052 * lpfc_nvmet_ctx_info structures as follows: 1053 * 1054 * cpu0/mrq0 cpu1/mrq0 ... cpuX/mrq0 1055 * cpu0/mrq1 cpu1/mrq1 ... cpuX/mrq1 1056 * ... 1057 * cpuX/mrqY cpuX/mrqY ... cpuX/mrqY 1058 * 1059 * Each line represents a MRQ "silo" containing an entry for 1060 * every CPU. 1061 * 1062 * MRQ X is initially assumed to be associated with CPU X, thus 1063 * contexts are initially distributed across all MRQs using 1064 * the MRQ index (N) as follows cpuN/mrqN. When contexts are 1065 * freed, the are freed to the MRQ silo based on the CPU number 1066 * of the IO completion. Thus a context that was allocated for MRQ A 1067 * whose IO completed on CPU B will be freed to cpuB/mrqA. 1068 */ 1069 infop = phba->sli4_hba.nvmet_ctx_info; 1070 for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) { 1071 for (j = 0; j < phba->cfg_nvmet_mrq; j++) { 1072 INIT_LIST_HEAD(&infop->nvmet_ctx_list); 1073 spin_lock_init(&infop->nvmet_ctx_list_lock); 1074 infop->nvmet_ctx_list_cnt = 0; 1075 infop++; 1076 } 1077 } 1078 1079 /* 1080 * Setup the next CPU context info ptr for each MRQ. 1081 * MRQ 0 will cycle thru CPUs 0 - X separately from 1082 * MRQ 1 cycling thru CPUs 0 - X, and so on. 1083 */ 1084 for (j = 0; j < phba->cfg_nvmet_mrq; j++) { 1085 last_infop = lpfc_get_ctx_list(phba, 0, j); 1086 for (i = phba->sli4_hba.num_present_cpu - 1; i >= 0; i--) { 1087 infop = lpfc_get_ctx_list(phba, i, j); 1088 infop->nvmet_ctx_next_cpu = last_infop; 1089 last_infop = infop; 1090 } 1091 } 1092 1093 /* For all nvmet xris, allocate resources needed to process a 1094 * received command on a per xri basis. 1095 */ 1096 idx = 0; 1097 for (i = 0; i < phba->sli4_hba.nvmet_xri_cnt; i++) { 1098 ctx_buf = kzalloc(sizeof(*ctx_buf), GFP_KERNEL); 1099 if (!ctx_buf) { 1100 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 1101 "6404 Ran out of memory for NVMET\n"); 1102 return -ENOMEM; 1103 } 1104 1105 ctx_buf->context = kzalloc(sizeof(*ctx_buf->context), 1106 GFP_KERNEL); 1107 if (!ctx_buf->context) { 1108 kfree(ctx_buf); 1109 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 1110 "6405 Ran out of NVMET " 1111 "context memory\n"); 1112 return -ENOMEM; 1113 } 1114 ctx_buf->context->ctxbuf = ctx_buf; 1115 ctx_buf->context->state = LPFC_NVMET_STE_FREE; 1116 1117 ctx_buf->iocbq = lpfc_sli_get_iocbq(phba); 1118 if (!ctx_buf->iocbq) { 1119 kfree(ctx_buf->context); 1120 kfree(ctx_buf); 1121 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 1122 "6406 Ran out of NVMET iocb/WQEs\n"); 1123 return -ENOMEM; 1124 } 1125 ctx_buf->iocbq->iocb_flag = LPFC_IO_NVMET; 1126 nvmewqe = ctx_buf->iocbq; 1127 wqe = (union lpfc_wqe128 *)&nvmewqe->wqe; 1128 /* Initialize WQE */ 1129 memset(wqe, 0, sizeof(union lpfc_wqe)); 1130 /* Word 7 */ 1131 bf_set(wqe_ct, &wqe->generic.wqe_com, SLI4_CT_RPI); 1132 bf_set(wqe_class, &wqe->generic.wqe_com, CLASS3); 1133 /* Word 10 */ 1134 bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1); 1135 bf_set(wqe_ebde_cnt, &wqe->generic.wqe_com, 0); 1136 bf_set(wqe_qosd, &wqe->generic.wqe_com, 0); 1137 1138 ctx_buf->iocbq->context1 = NULL; 1139 spin_lock(&phba->sli4_hba.sgl_list_lock); 1140 ctx_buf->sglq = __lpfc_sli_get_nvmet_sglq(phba, ctx_buf->iocbq); 1141 spin_unlock(&phba->sli4_hba.sgl_list_lock); 1142 if (!ctx_buf->sglq) { 1143 lpfc_sli_release_iocbq(phba, ctx_buf->iocbq); 1144 kfree(ctx_buf->context); 1145 kfree(ctx_buf); 1146 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 1147 "6407 Ran out of NVMET XRIs\n"); 1148 return -ENOMEM; 1149 } 1150 1151 /* 1152 * Add ctx to MRQidx context list. Our initial assumption 1153 * is MRQidx will be associated with CPUidx. This association 1154 * can change on the fly. 1155 */ 1156 infop = lpfc_get_ctx_list(phba, idx, idx); 1157 spin_lock(&infop->nvmet_ctx_list_lock); 1158 list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list); 1159 infop->nvmet_ctx_list_cnt++; 1160 spin_unlock(&infop->nvmet_ctx_list_lock); 1161 1162 /* Spread ctx structures evenly across all MRQs */ 1163 idx++; 1164 if (idx >= phba->cfg_nvmet_mrq) 1165 idx = 0; 1166 } 1167 1168 infop = phba->sli4_hba.nvmet_ctx_info; 1169 for (j = 0; j < phba->cfg_nvmet_mrq; j++) { 1170 for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) { 1171 lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT, 1172 "6408 TOTAL NVMET ctx for CPU %d " 1173 "MRQ %d: cnt %d nextcpu %p\n", 1174 i, j, infop->nvmet_ctx_list_cnt, 1175 infop->nvmet_ctx_next_cpu); 1176 infop++; 1177 } 1178 } 1179 return 0; 1180 } 1181 1182 int 1183 lpfc_nvmet_create_targetport(struct lpfc_hba *phba) 1184 { 1185 struct lpfc_vport *vport = phba->pport; 1186 struct lpfc_nvmet_tgtport *tgtp; 1187 struct nvmet_fc_port_info pinfo; 1188 int error; 1189 1190 if (phba->targetport) 1191 return 0; 1192 1193 error = lpfc_nvmet_setup_io_context(phba); 1194 if (error) 1195 return error; 1196 1197 memset(&pinfo, 0, sizeof(struct nvmet_fc_port_info)); 1198 pinfo.node_name = wwn_to_u64(vport->fc_nodename.u.wwn); 1199 pinfo.port_name = wwn_to_u64(vport->fc_portname.u.wwn); 1200 pinfo.port_id = vport->fc_myDID; 1201 1202 /* Limit to LPFC_MAX_NVME_SEG_CNT. 1203 * For now need + 1 to get around NVME transport logic. 1204 */ 1205 if (phba->cfg_sg_seg_cnt > LPFC_MAX_NVME_SEG_CNT) { 1206 lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT, 1207 "6400 Reducing sg segment cnt to %d\n", 1208 LPFC_MAX_NVME_SEG_CNT); 1209 phba->cfg_nvme_seg_cnt = LPFC_MAX_NVME_SEG_CNT; 1210 } else { 1211 phba->cfg_nvme_seg_cnt = phba->cfg_sg_seg_cnt; 1212 } 1213 lpfc_tgttemplate.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; 1214 lpfc_tgttemplate.max_hw_queues = phba->cfg_nvme_io_channel; 1215 lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP; 1216 1217 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1218 error = nvmet_fc_register_targetport(&pinfo, &lpfc_tgttemplate, 1219 &phba->pcidev->dev, 1220 &phba->targetport); 1221 #else 1222 error = -ENOENT; 1223 #endif 1224 if (error) { 1225 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, 1226 "6025 Cannot register NVME targetport x%x: " 1227 "portnm %llx nodenm %llx segs %d qs %d\n", 1228 error, 1229 pinfo.port_name, pinfo.node_name, 1230 lpfc_tgttemplate.max_sgl_segments, 1231 lpfc_tgttemplate.max_hw_queues); 1232 phba->targetport = NULL; 1233 phba->nvmet_support = 0; 1234 1235 lpfc_nvmet_cleanup_io_context(phba); 1236 1237 } else { 1238 tgtp = (struct lpfc_nvmet_tgtport *) 1239 phba->targetport->private; 1240 tgtp->phba = phba; 1241 1242 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 1243 "6026 Registered NVME " 1244 "targetport: %p, private %p " 1245 "portnm %llx nodenm %llx segs %d qs %d\n", 1246 phba->targetport, tgtp, 1247 pinfo.port_name, pinfo.node_name, 1248 lpfc_tgttemplate.max_sgl_segments, 1249 lpfc_tgttemplate.max_hw_queues); 1250 1251 atomic_set(&tgtp->rcv_ls_req_in, 0); 1252 atomic_set(&tgtp->rcv_ls_req_out, 0); 1253 atomic_set(&tgtp->rcv_ls_req_drop, 0); 1254 atomic_set(&tgtp->xmt_ls_abort, 0); 1255 atomic_set(&tgtp->xmt_ls_abort_cmpl, 0); 1256 atomic_set(&tgtp->xmt_ls_rsp, 0); 1257 atomic_set(&tgtp->xmt_ls_drop, 0); 1258 atomic_set(&tgtp->xmt_ls_rsp_error, 0); 1259 atomic_set(&tgtp->xmt_ls_rsp_xb_set, 0); 1260 atomic_set(&tgtp->xmt_ls_rsp_aborted, 0); 1261 atomic_set(&tgtp->xmt_ls_rsp_cmpl, 0); 1262 atomic_set(&tgtp->rcv_fcp_cmd_in, 0); 1263 atomic_set(&tgtp->rcv_fcp_cmd_out, 0); 1264 atomic_set(&tgtp->rcv_fcp_cmd_drop, 0); 1265 atomic_set(&tgtp->xmt_fcp_drop, 0); 1266 atomic_set(&tgtp->xmt_fcp_read_rsp, 0); 1267 atomic_set(&tgtp->xmt_fcp_read, 0); 1268 atomic_set(&tgtp->xmt_fcp_write, 0); 1269 atomic_set(&tgtp->xmt_fcp_rsp, 0); 1270 atomic_set(&tgtp->xmt_fcp_release, 0); 1271 atomic_set(&tgtp->xmt_fcp_rsp_cmpl, 0); 1272 atomic_set(&tgtp->xmt_fcp_rsp_error, 0); 1273 atomic_set(&tgtp->xmt_fcp_rsp_xb_set, 0); 1274 atomic_set(&tgtp->xmt_fcp_rsp_aborted, 0); 1275 atomic_set(&tgtp->xmt_fcp_rsp_drop, 0); 1276 atomic_set(&tgtp->xmt_fcp_xri_abort_cqe, 0); 1277 atomic_set(&tgtp->xmt_fcp_abort, 0); 1278 atomic_set(&tgtp->xmt_fcp_abort_cmpl, 0); 1279 atomic_set(&tgtp->xmt_abort_unsol, 0); 1280 atomic_set(&tgtp->xmt_abort_sol, 0); 1281 atomic_set(&tgtp->xmt_abort_rsp, 0); 1282 atomic_set(&tgtp->xmt_abort_rsp_error, 0); 1283 } 1284 return error; 1285 } 1286 1287 int 1288 lpfc_nvmet_update_targetport(struct lpfc_hba *phba) 1289 { 1290 struct lpfc_vport *vport = phba->pport; 1291 1292 if (!phba->targetport) 1293 return 0; 1294 1295 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 1296 "6007 Update NVMET port %p did x%x\n", 1297 phba->targetport, vport->fc_myDID); 1298 1299 phba->targetport->port_id = vport->fc_myDID; 1300 return 0; 1301 } 1302 1303 /** 1304 * lpfc_sli4_nvmet_xri_aborted - Fast-path process of nvmet xri abort 1305 * @phba: pointer to lpfc hba data structure. 1306 * @axri: pointer to the nvmet xri abort wcqe structure. 1307 * 1308 * This routine is invoked by the worker thread to process a SLI4 fast-path 1309 * NVMET aborted xri. 1310 **/ 1311 void 1312 lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, 1313 struct sli4_wcqe_xri_aborted *axri) 1314 { 1315 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); 1316 uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri); 1317 struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp; 1318 struct lpfc_nvmet_tgtport *tgtp; 1319 struct lpfc_nodelist *ndlp; 1320 unsigned long iflag = 0; 1321 int rrq_empty = 0; 1322 bool released = false; 1323 1324 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1325 "6317 XB aborted xri x%x rxid x%x\n", xri, rxid); 1326 1327 if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) 1328 return; 1329 1330 if (phba->targetport) { 1331 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 1332 atomic_inc(&tgtp->xmt_fcp_xri_abort_cqe); 1333 } 1334 1335 spin_lock_irqsave(&phba->hbalock, iflag); 1336 spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); 1337 list_for_each_entry_safe(ctxp, next_ctxp, 1338 &phba->sli4_hba.lpfc_abts_nvmet_ctx_list, 1339 list) { 1340 if (ctxp->ctxbuf->sglq->sli4_xritag != xri) 1341 continue; 1342 1343 /* Check if we already received a free context call 1344 * and we have completed processing an abort situation. 1345 */ 1346 if (ctxp->flag & LPFC_NVMET_CTX_RLS && 1347 !(ctxp->flag & LPFC_NVMET_ABORT_OP)) { 1348 list_del(&ctxp->list); 1349 released = true; 1350 } 1351 ctxp->flag &= ~LPFC_NVMET_XBUSY; 1352 spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); 1353 1354 rrq_empty = list_empty(&phba->active_rrq_list); 1355 spin_unlock_irqrestore(&phba->hbalock, iflag); 1356 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid); 1357 if (ndlp && NLP_CHK_NODE_ACT(ndlp) && 1358 (ndlp->nlp_state == NLP_STE_UNMAPPED_NODE || 1359 ndlp->nlp_state == NLP_STE_MAPPED_NODE)) { 1360 lpfc_set_rrq_active(phba, ndlp, 1361 ctxp->ctxbuf->sglq->sli4_lxritag, 1362 rxid, 1); 1363 lpfc_sli4_abts_err_handler(phba, ndlp, axri); 1364 } 1365 1366 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1367 "6318 XB aborted oxid %x flg x%x (%x)\n", 1368 ctxp->oxid, ctxp->flag, released); 1369 if (released) 1370 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); 1371 1372 if (rrq_empty) 1373 lpfc_worker_wake_up(phba); 1374 return; 1375 } 1376 spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); 1377 spin_unlock_irqrestore(&phba->hbalock, iflag); 1378 } 1379 1380 int 1381 lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, 1382 struct fc_frame_header *fc_hdr) 1383 1384 { 1385 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1386 struct lpfc_hba *phba = vport->phba; 1387 struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp; 1388 struct nvmefc_tgt_fcp_req *rsp; 1389 uint16_t xri; 1390 unsigned long iflag = 0; 1391 1392 xri = be16_to_cpu(fc_hdr->fh_ox_id); 1393 1394 spin_lock_irqsave(&phba->hbalock, iflag); 1395 spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); 1396 list_for_each_entry_safe(ctxp, next_ctxp, 1397 &phba->sli4_hba.lpfc_abts_nvmet_ctx_list, 1398 list) { 1399 if (ctxp->ctxbuf->sglq->sli4_xritag != xri) 1400 continue; 1401 1402 spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); 1403 spin_unlock_irqrestore(&phba->hbalock, iflag); 1404 1405 spin_lock_irqsave(&ctxp->ctxlock, iflag); 1406 ctxp->flag |= LPFC_NVMET_ABTS_RCV; 1407 spin_unlock_irqrestore(&ctxp->ctxlock, iflag); 1408 1409 lpfc_nvmeio_data(phba, 1410 "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n", 1411 xri, smp_processor_id(), 0); 1412 1413 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1414 "6319 NVMET Rcv ABTS:acc xri x%x\n", xri); 1415 1416 rsp = &ctxp->ctx.fcp_req; 1417 nvmet_fc_rcv_fcp_abort(phba->targetport, rsp); 1418 1419 /* Respond with BA_ACC accordingly */ 1420 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1); 1421 return 0; 1422 } 1423 spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); 1424 spin_unlock_irqrestore(&phba->hbalock, iflag); 1425 1426 lpfc_nvmeio_data(phba, "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n", 1427 xri, smp_processor_id(), 1); 1428 1429 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 1430 "6320 NVMET Rcv ABTS:rjt xri x%x\n", xri); 1431 1432 /* Respond with BA_RJT accordingly */ 1433 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 0); 1434 #endif 1435 return 0; 1436 } 1437 1438 void 1439 lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba) 1440 { 1441 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1442 struct lpfc_nvmet_tgtport *tgtp; 1443 1444 if (phba->nvmet_support == 0) 1445 return; 1446 if (phba->targetport) { 1447 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 1448 init_completion(&tgtp->tport_unreg_done); 1449 nvmet_fc_unregister_targetport(phba->targetport); 1450 wait_for_completion_timeout(&tgtp->tport_unreg_done, 5); 1451 lpfc_nvmet_cleanup_io_context(phba); 1452 } 1453 phba->targetport = NULL; 1454 #endif 1455 } 1456 1457 /** 1458 * lpfc_nvmet_unsol_ls_buffer - Process an unsolicited event data buffer 1459 * @phba: pointer to lpfc hba data structure. 1460 * @pring: pointer to a SLI ring. 1461 * @nvmebuf: pointer to lpfc nvme command HBQ data structure. 1462 * 1463 * This routine is used for processing the WQE associated with a unsolicited 1464 * event. It first determines whether there is an existing ndlp that matches 1465 * the DID from the unsolicited WQE. If not, it will create a new one with 1466 * the DID from the unsolicited WQE. The ELS command from the unsolicited 1467 * WQE is then used to invoke the proper routine and to set up proper state 1468 * of the discovery state machine. 1469 **/ 1470 static void 1471 lpfc_nvmet_unsol_ls_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, 1472 struct hbq_dmabuf *nvmebuf) 1473 { 1474 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1475 struct lpfc_nvmet_tgtport *tgtp; 1476 struct fc_frame_header *fc_hdr; 1477 struct lpfc_nvmet_rcv_ctx *ctxp; 1478 uint32_t *payload; 1479 uint32_t size, oxid, sid, rc; 1480 1481 if (!nvmebuf || !phba->targetport) { 1482 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1483 "6154 LS Drop IO\n"); 1484 oxid = 0; 1485 size = 0; 1486 sid = 0; 1487 ctxp = NULL; 1488 goto dropit; 1489 } 1490 1491 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 1492 payload = (uint32_t *)(nvmebuf->dbuf.virt); 1493 fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt); 1494 size = bf_get(lpfc_rcqe_length, &nvmebuf->cq_event.cqe.rcqe_cmpl); 1495 oxid = be16_to_cpu(fc_hdr->fh_ox_id); 1496 sid = sli4_sid_from_fc_hdr(fc_hdr); 1497 1498 ctxp = kzalloc(sizeof(struct lpfc_nvmet_rcv_ctx), GFP_ATOMIC); 1499 if (ctxp == NULL) { 1500 atomic_inc(&tgtp->rcv_ls_req_drop); 1501 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1502 "6155 LS Drop IO x%x: Alloc\n", 1503 oxid); 1504 dropit: 1505 lpfc_nvmeio_data(phba, "NVMET LS DROP: " 1506 "xri x%x sz %d from %06x\n", 1507 oxid, size, sid); 1508 if (nvmebuf) 1509 lpfc_in_buf_free(phba, &nvmebuf->dbuf); 1510 return; 1511 } 1512 ctxp->phba = phba; 1513 ctxp->size = size; 1514 ctxp->oxid = oxid; 1515 ctxp->sid = sid; 1516 ctxp->wqeq = NULL; 1517 ctxp->state = LPFC_NVMET_STE_LS_RCV; 1518 ctxp->entry_cnt = 1; 1519 ctxp->rqb_buffer = (void *)nvmebuf; 1520 1521 lpfc_nvmeio_data(phba, "NVMET LS RCV: xri x%x sz %d from %06x\n", 1522 oxid, size, sid); 1523 /* 1524 * The calling sequence should be: 1525 * nvmet_fc_rcv_ls_req -> lpfc_nvmet_xmt_ls_rsp/cmp ->_req->done 1526 * lpfc_nvmet_xmt_ls_rsp_cmp should free the allocated ctxp. 1527 */ 1528 atomic_inc(&tgtp->rcv_ls_req_in); 1529 rc = nvmet_fc_rcv_ls_req(phba->targetport, &ctxp->ctx.ls_req, 1530 payload, size); 1531 1532 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 1533 "6037 NVMET Unsol rcv: sz %d rc %d: %08x %08x %08x " 1534 "%08x %08x %08x\n", size, rc, 1535 *payload, *(payload+1), *(payload+2), 1536 *(payload+3), *(payload+4), *(payload+5)); 1537 1538 if (rc == 0) { 1539 atomic_inc(&tgtp->rcv_ls_req_out); 1540 return; 1541 } 1542 1543 lpfc_nvmeio_data(phba, "NVMET LS DROP: xri x%x sz %d from %06x\n", 1544 oxid, size, sid); 1545 1546 atomic_inc(&tgtp->rcv_ls_req_drop); 1547 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1548 "6156 LS Drop IO x%x: nvmet_fc_rcv_ls_req %d\n", 1549 ctxp->oxid, rc); 1550 1551 /* We assume a rcv'ed cmd ALWAYs fits into 1 buffer */ 1552 if (nvmebuf) 1553 lpfc_in_buf_free(phba, &nvmebuf->dbuf); 1554 1555 atomic_inc(&tgtp->xmt_ls_abort); 1556 lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, sid, oxid); 1557 #endif 1558 } 1559 1560 static struct lpfc_nvmet_ctxbuf * 1561 lpfc_nvmet_replenish_context(struct lpfc_hba *phba, 1562 struct lpfc_nvmet_ctx_info *current_infop) 1563 { 1564 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1565 struct lpfc_nvmet_ctxbuf *ctx_buf = NULL; 1566 struct lpfc_nvmet_ctx_info *get_infop; 1567 int i; 1568 1569 /* 1570 * The current_infop for the MRQ a NVME command IU was received 1571 * on is empty. Our goal is to replenish this MRQs context 1572 * list from a another CPUs. 1573 * 1574 * First we need to pick a context list to start looking on. 1575 * nvmet_ctx_start_cpu has available context the last time 1576 * we needed to replenish this CPU where nvmet_ctx_next_cpu 1577 * is just the next sequential CPU for this MRQ. 1578 */ 1579 if (current_infop->nvmet_ctx_start_cpu) 1580 get_infop = current_infop->nvmet_ctx_start_cpu; 1581 else 1582 get_infop = current_infop->nvmet_ctx_next_cpu; 1583 1584 for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) { 1585 if (get_infop == current_infop) { 1586 get_infop = get_infop->nvmet_ctx_next_cpu; 1587 continue; 1588 } 1589 spin_lock(&get_infop->nvmet_ctx_list_lock); 1590 1591 /* Just take the entire context list, if there are any */ 1592 if (get_infop->nvmet_ctx_list_cnt) { 1593 list_splice_init(&get_infop->nvmet_ctx_list, 1594 ¤t_infop->nvmet_ctx_list); 1595 current_infop->nvmet_ctx_list_cnt = 1596 get_infop->nvmet_ctx_list_cnt - 1; 1597 get_infop->nvmet_ctx_list_cnt = 0; 1598 spin_unlock(&get_infop->nvmet_ctx_list_lock); 1599 1600 current_infop->nvmet_ctx_start_cpu = get_infop; 1601 list_remove_head(¤t_infop->nvmet_ctx_list, 1602 ctx_buf, struct lpfc_nvmet_ctxbuf, 1603 list); 1604 return ctx_buf; 1605 } 1606 1607 /* Otherwise, move on to the next CPU for this MRQ */ 1608 spin_unlock(&get_infop->nvmet_ctx_list_lock); 1609 get_infop = get_infop->nvmet_ctx_next_cpu; 1610 } 1611 1612 #endif 1613 /* Nothing found, all contexts for the MRQ are in-flight */ 1614 return NULL; 1615 } 1616 1617 /** 1618 * lpfc_nvmet_unsol_fcp_buffer - Process an unsolicited event data buffer 1619 * @phba: pointer to lpfc hba data structure. 1620 * @idx: relative index of MRQ vector 1621 * @nvmebuf: pointer to lpfc nvme command HBQ data structure. 1622 * 1623 * This routine is used for processing the WQE associated with a unsolicited 1624 * event. It first determines whether there is an existing ndlp that matches 1625 * the DID from the unsolicited WQE. If not, it will create a new one with 1626 * the DID from the unsolicited WQE. The ELS command from the unsolicited 1627 * WQE is then used to invoke the proper routine and to set up proper state 1628 * of the discovery state machine. 1629 **/ 1630 static void 1631 lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, 1632 uint32_t idx, 1633 struct rqb_dmabuf *nvmebuf, 1634 uint64_t isr_timestamp) 1635 { 1636 struct lpfc_nvmet_rcv_ctx *ctxp; 1637 struct lpfc_nvmet_tgtport *tgtp; 1638 struct fc_frame_header *fc_hdr; 1639 struct lpfc_nvmet_ctxbuf *ctx_buf; 1640 struct lpfc_nvmet_ctx_info *current_infop; 1641 uint32_t *payload; 1642 uint32_t size, oxid, sid, rc, qno; 1643 unsigned long iflag; 1644 int current_cpu; 1645 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1646 uint32_t id; 1647 #endif 1648 1649 if (!IS_ENABLED(CONFIG_NVME_TARGET_FC)) 1650 return; 1651 1652 ctx_buf = NULL; 1653 if (!nvmebuf || !phba->targetport) { 1654 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1655 "6157 NVMET FCP Drop IO\n"); 1656 oxid = 0; 1657 size = 0; 1658 sid = 0; 1659 ctxp = NULL; 1660 goto dropit; 1661 } 1662 1663 /* 1664 * Get a pointer to the context list for this MRQ based on 1665 * the CPU this MRQ IRQ is associated with. If the CPU association 1666 * changes from our initial assumption, the context list could 1667 * be empty, thus it would need to be replenished with the 1668 * context list from another CPU for this MRQ. 1669 */ 1670 current_cpu = smp_processor_id(); 1671 current_infop = lpfc_get_ctx_list(phba, current_cpu, idx); 1672 spin_lock_irqsave(¤t_infop->nvmet_ctx_list_lock, iflag); 1673 if (current_infop->nvmet_ctx_list_cnt) { 1674 list_remove_head(¤t_infop->nvmet_ctx_list, 1675 ctx_buf, struct lpfc_nvmet_ctxbuf, list); 1676 current_infop->nvmet_ctx_list_cnt--; 1677 } else { 1678 ctx_buf = lpfc_nvmet_replenish_context(phba, current_infop); 1679 } 1680 spin_unlock_irqrestore(¤t_infop->nvmet_ctx_list_lock, iflag); 1681 1682 fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt); 1683 oxid = be16_to_cpu(fc_hdr->fh_ox_id); 1684 size = nvmebuf->bytes_recv; 1685 1686 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1687 if (phba->cpucheck_on & LPFC_CHECK_NVMET_RCV) { 1688 id = smp_processor_id(); 1689 if (id < LPFC_CHECK_CPU_CNT) 1690 phba->cpucheck_rcv_io[id]++; 1691 } 1692 #endif 1693 1694 lpfc_nvmeio_data(phba, "NVMET FCP RCV: xri x%x sz %d CPU %02x\n", 1695 oxid, size, smp_processor_id()); 1696 1697 if (!ctx_buf) { 1698 /* Queue this NVME IO to process later */ 1699 spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag); 1700 list_add_tail(&nvmebuf->hbuf.list, 1701 &phba->sli4_hba.lpfc_nvmet_io_wait_list); 1702 phba->sli4_hba.nvmet_io_wait_cnt++; 1703 phba->sli4_hba.nvmet_io_wait_total++; 1704 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, 1705 iflag); 1706 1707 /* Post a brand new DMA buffer to RQ */ 1708 qno = nvmebuf->idx; 1709 lpfc_post_rq_buffer( 1710 phba, phba->sli4_hba.nvmet_mrq_hdr[qno], 1711 phba->sli4_hba.nvmet_mrq_data[qno], 1, qno); 1712 return; 1713 } 1714 1715 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 1716 payload = (uint32_t *)(nvmebuf->dbuf.virt); 1717 sid = sli4_sid_from_fc_hdr(fc_hdr); 1718 1719 ctxp = (struct lpfc_nvmet_rcv_ctx *)ctx_buf->context; 1720 if (ctxp->state != LPFC_NVMET_STE_FREE) { 1721 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1722 "6414 NVMET Context corrupt %d %d oxid x%x\n", 1723 ctxp->state, ctxp->entry_cnt, ctxp->oxid); 1724 } 1725 ctxp->wqeq = NULL; 1726 ctxp->txrdy = NULL; 1727 ctxp->offset = 0; 1728 ctxp->phba = phba; 1729 ctxp->size = size; 1730 ctxp->oxid = oxid; 1731 ctxp->sid = sid; 1732 ctxp->idx = idx; 1733 ctxp->state = LPFC_NVMET_STE_RCV; 1734 ctxp->entry_cnt = 1; 1735 ctxp->flag = 0; 1736 ctxp->ctxbuf = ctx_buf; 1737 ctxp->rqb_buffer = (void *)nvmebuf; 1738 spin_lock_init(&ctxp->ctxlock); 1739 1740 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1741 if (isr_timestamp) { 1742 ctxp->ts_isr_cmd = isr_timestamp; 1743 ctxp->ts_cmd_nvme = ktime_get_ns(); 1744 ctxp->ts_nvme_data = 0; 1745 ctxp->ts_data_wqput = 0; 1746 ctxp->ts_isr_data = 0; 1747 ctxp->ts_data_nvme = 0; 1748 ctxp->ts_nvme_status = 0; 1749 ctxp->ts_status_wqput = 0; 1750 ctxp->ts_isr_status = 0; 1751 ctxp->ts_status_nvme = 0; 1752 } else { 1753 ctxp->ts_cmd_nvme = 0; 1754 } 1755 #endif 1756 1757 atomic_inc(&tgtp->rcv_fcp_cmd_in); 1758 /* 1759 * The calling sequence should be: 1760 * nvmet_fc_rcv_fcp_req -> lpfc_nvmet_xmt_fcp_op/cmp -> req->done 1761 * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp. 1762 * When we return from nvmet_fc_rcv_fcp_req, all relevant info in 1763 * the NVME command / FC header is stored, so we are free to repost 1764 * the buffer. 1765 */ 1766 rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req, 1767 payload, size); 1768 1769 /* Process FCP command */ 1770 if (rc == 0) { 1771 ctxp->rqb_buffer = NULL; 1772 atomic_inc(&tgtp->rcv_fcp_cmd_out); 1773 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */ 1774 return; 1775 } 1776 1777 /* Processing of FCP command is deferred */ 1778 if (rc == -EOVERFLOW) { 1779 lpfc_nvmeio_data(phba, 1780 "NVMET RCV BUSY: xri x%x sz %d from %06x\n", 1781 oxid, size, sid); 1782 /* defer reposting rcv buffer till .defer_rcv callback */ 1783 ctxp->flag |= LPFC_NVMET_DEFER_RCV_REPOST; 1784 atomic_inc(&tgtp->rcv_fcp_cmd_out); 1785 return; 1786 } 1787 ctxp->rqb_buffer = nvmebuf; 1788 1789 atomic_inc(&tgtp->rcv_fcp_cmd_drop); 1790 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1791 "6159 FCP Drop IO x%x: err x%x: x%x x%x x%x\n", 1792 ctxp->oxid, rc, 1793 atomic_read(&tgtp->rcv_fcp_cmd_in), 1794 atomic_read(&tgtp->rcv_fcp_cmd_out), 1795 atomic_read(&tgtp->xmt_fcp_release)); 1796 dropit: 1797 lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n", 1798 oxid, size, sid); 1799 if (oxid) { 1800 lpfc_nvmet_defer_release(phba, ctxp); 1801 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid); 1802 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */ 1803 return; 1804 } 1805 1806 if (ctx_buf) 1807 lpfc_nvmet_ctxbuf_post(phba, ctx_buf); 1808 1809 if (nvmebuf) 1810 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */ 1811 } 1812 1813 /** 1814 * lpfc_nvmet_unsol_ls_event - Process an unsolicited event from an nvme nport 1815 * @phba: pointer to lpfc hba data structure. 1816 * @pring: pointer to a SLI ring. 1817 * @nvmebuf: pointer to received nvme data structure. 1818 * 1819 * This routine is used to process an unsolicited event received from a SLI 1820 * (Service Level Interface) ring. The actual processing of the data buffer 1821 * associated with the unsolicited event is done by invoking the routine 1822 * lpfc_nvmet_unsol_ls_buffer() after properly set up the buffer from the 1823 * SLI RQ on which the unsolicited event was received. 1824 **/ 1825 void 1826 lpfc_nvmet_unsol_ls_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, 1827 struct lpfc_iocbq *piocb) 1828 { 1829 struct lpfc_dmabuf *d_buf; 1830 struct hbq_dmabuf *nvmebuf; 1831 1832 d_buf = piocb->context2; 1833 nvmebuf = container_of(d_buf, struct hbq_dmabuf, dbuf); 1834 1835 if (phba->nvmet_support == 0) { 1836 lpfc_in_buf_free(phba, &nvmebuf->dbuf); 1837 return; 1838 } 1839 lpfc_nvmet_unsol_ls_buffer(phba, pring, nvmebuf); 1840 } 1841 1842 /** 1843 * lpfc_nvmet_unsol_fcp_event - Process an unsolicited event from an nvme nport 1844 * @phba: pointer to lpfc hba data structure. 1845 * @idx: relative index of MRQ vector 1846 * @nvmebuf: pointer to received nvme data structure. 1847 * 1848 * This routine is used to process an unsolicited event received from a SLI 1849 * (Service Level Interface) ring. The actual processing of the data buffer 1850 * associated with the unsolicited event is done by invoking the routine 1851 * lpfc_nvmet_unsol_fcp_buffer() after properly set up the buffer from the 1852 * SLI RQ on which the unsolicited event was received. 1853 **/ 1854 void 1855 lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba, 1856 uint32_t idx, 1857 struct rqb_dmabuf *nvmebuf, 1858 uint64_t isr_timestamp) 1859 { 1860 if (phba->nvmet_support == 0) { 1861 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); 1862 return; 1863 } 1864 lpfc_nvmet_unsol_fcp_buffer(phba, idx, nvmebuf, 1865 isr_timestamp); 1866 } 1867 1868 /** 1869 * lpfc_nvmet_prep_ls_wqe - Allocate and prepare a lpfc wqe data structure 1870 * @phba: pointer to a host N_Port data structure. 1871 * @ctxp: Context info for NVME LS Request 1872 * @rspbuf: DMA buffer of NVME command. 1873 * @rspsize: size of the NVME command. 1874 * 1875 * This routine is used for allocating a lpfc-WQE data structure from 1876 * the driver lpfc-WQE free-list and prepare the WQE with the parameters 1877 * passed into the routine for discovery state machine to issue an Extended 1878 * Link Service (NVME) commands. It is a generic lpfc-WQE allocation 1879 * and preparation routine that is used by all the discovery state machine 1880 * routines and the NVME command-specific fields will be later set up by 1881 * the individual discovery machine routines after calling this routine 1882 * allocating and preparing a generic WQE data structure. It fills in the 1883 * Buffer Descriptor Entries (BDEs), allocates buffers for both command 1884 * payload and response payload (if expected). The reference count on the 1885 * ndlp is incremented by 1 and the reference to the ndlp is put into 1886 * context1 of the WQE data structure for this WQE to hold the ndlp 1887 * reference for the command's callback function to access later. 1888 * 1889 * Return code 1890 * Pointer to the newly allocated/prepared nvme wqe data structure 1891 * NULL - when nvme wqe data structure allocation/preparation failed 1892 **/ 1893 static struct lpfc_iocbq * 1894 lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *phba, 1895 struct lpfc_nvmet_rcv_ctx *ctxp, 1896 dma_addr_t rspbuf, uint16_t rspsize) 1897 { 1898 struct lpfc_nodelist *ndlp; 1899 struct lpfc_iocbq *nvmewqe; 1900 union lpfc_wqe *wqe; 1901 1902 if (!lpfc_is_link_up(phba)) { 1903 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, 1904 "6104 NVMET prep LS wqe: link err: " 1905 "NPORT x%x oxid:x%x ste %d\n", 1906 ctxp->sid, ctxp->oxid, ctxp->state); 1907 return NULL; 1908 } 1909 1910 /* Allocate buffer for command wqe */ 1911 nvmewqe = lpfc_sli_get_iocbq(phba); 1912 if (nvmewqe == NULL) { 1913 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, 1914 "6105 NVMET prep LS wqe: No WQE: " 1915 "NPORT x%x oxid x%x ste %d\n", 1916 ctxp->sid, ctxp->oxid, ctxp->state); 1917 return NULL; 1918 } 1919 1920 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid); 1921 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || 1922 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && 1923 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { 1924 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, 1925 "6106 NVMET prep LS wqe: No ndlp: " 1926 "NPORT x%x oxid x%x ste %d\n", 1927 ctxp->sid, ctxp->oxid, ctxp->state); 1928 goto nvme_wqe_free_wqeq_exit; 1929 } 1930 ctxp->wqeq = nvmewqe; 1931 1932 /* prevent preparing wqe with NULL ndlp reference */ 1933 nvmewqe->context1 = lpfc_nlp_get(ndlp); 1934 if (nvmewqe->context1 == NULL) 1935 goto nvme_wqe_free_wqeq_exit; 1936 nvmewqe->context2 = ctxp; 1937 1938 wqe = &nvmewqe->wqe; 1939 memset(wqe, 0, sizeof(union lpfc_wqe)); 1940 1941 /* Words 0 - 2 */ 1942 wqe->xmit_sequence.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 1943 wqe->xmit_sequence.bde.tus.f.bdeSize = rspsize; 1944 wqe->xmit_sequence.bde.addrLow = le32_to_cpu(putPaddrLow(rspbuf)); 1945 wqe->xmit_sequence.bde.addrHigh = le32_to_cpu(putPaddrHigh(rspbuf)); 1946 1947 /* Word 3 */ 1948 1949 /* Word 4 */ 1950 1951 /* Word 5 */ 1952 bf_set(wqe_dfctl, &wqe->xmit_sequence.wge_ctl, 0); 1953 bf_set(wqe_ls, &wqe->xmit_sequence.wge_ctl, 1); 1954 bf_set(wqe_la, &wqe->xmit_sequence.wge_ctl, 0); 1955 bf_set(wqe_rctl, &wqe->xmit_sequence.wge_ctl, FC_RCTL_ELS4_REP); 1956 bf_set(wqe_type, &wqe->xmit_sequence.wge_ctl, FC_TYPE_NVME); 1957 1958 /* Word 6 */ 1959 bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com, 1960 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 1961 bf_set(wqe_xri_tag, &wqe->xmit_sequence.wqe_com, nvmewqe->sli4_xritag); 1962 1963 /* Word 7 */ 1964 bf_set(wqe_cmnd, &wqe->xmit_sequence.wqe_com, 1965 CMD_XMIT_SEQUENCE64_WQE); 1966 bf_set(wqe_ct, &wqe->xmit_sequence.wqe_com, SLI4_CT_RPI); 1967 bf_set(wqe_class, &wqe->xmit_sequence.wqe_com, CLASS3); 1968 bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0); 1969 1970 /* Word 8 */ 1971 wqe->xmit_sequence.wqe_com.abort_tag = nvmewqe->iotag; 1972 1973 /* Word 9 */ 1974 bf_set(wqe_reqtag, &wqe->xmit_sequence.wqe_com, nvmewqe->iotag); 1975 /* Needs to be set by caller */ 1976 bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, ctxp->oxid); 1977 1978 /* Word 10 */ 1979 bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1); 1980 bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE); 1981 bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com, 1982 LPFC_WQE_LENLOC_WORD12); 1983 bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0); 1984 1985 /* Word 11 */ 1986 bf_set(wqe_cqid, &wqe->xmit_sequence.wqe_com, 1987 LPFC_WQE_CQ_ID_DEFAULT); 1988 bf_set(wqe_cmd_type, &wqe->xmit_sequence.wqe_com, 1989 OTHER_COMMAND); 1990 1991 /* Word 12 */ 1992 wqe->xmit_sequence.xmit_len = rspsize; 1993 1994 nvmewqe->retry = 1; 1995 nvmewqe->vport = phba->pport; 1996 nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT; 1997 nvmewqe->iocb_flag |= LPFC_IO_NVME_LS; 1998 1999 /* Xmit NVMET response to remote NPORT <did> */ 2000 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 2001 "6039 Xmit NVMET LS response to remote " 2002 "NPORT x%x iotag:x%x oxid:x%x size:x%x\n", 2003 ndlp->nlp_DID, nvmewqe->iotag, ctxp->oxid, 2004 rspsize); 2005 return nvmewqe; 2006 2007 nvme_wqe_free_wqeq_exit: 2008 nvmewqe->context2 = NULL; 2009 nvmewqe->context3 = NULL; 2010 lpfc_sli_release_iocbq(phba, nvmewqe); 2011 return NULL; 2012 } 2013 2014 2015 static struct lpfc_iocbq * 2016 lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba, 2017 struct lpfc_nvmet_rcv_ctx *ctxp) 2018 { 2019 struct nvmefc_tgt_fcp_req *rsp = &ctxp->ctx.fcp_req; 2020 struct lpfc_nvmet_tgtport *tgtp; 2021 struct sli4_sge *sgl; 2022 struct lpfc_nodelist *ndlp; 2023 struct lpfc_iocbq *nvmewqe; 2024 struct scatterlist *sgel; 2025 union lpfc_wqe128 *wqe; 2026 uint32_t *txrdy; 2027 dma_addr_t physaddr; 2028 int i, cnt; 2029 int xc = 1; 2030 2031 if (!lpfc_is_link_up(phba)) { 2032 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2033 "6107 NVMET prep FCP wqe: link err:" 2034 "NPORT x%x oxid x%x ste %d\n", 2035 ctxp->sid, ctxp->oxid, ctxp->state); 2036 return NULL; 2037 } 2038 2039 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid); 2040 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || 2041 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && 2042 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { 2043 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2044 "6108 NVMET prep FCP wqe: no ndlp: " 2045 "NPORT x%x oxid x%x ste %d\n", 2046 ctxp->sid, ctxp->oxid, ctxp->state); 2047 return NULL; 2048 } 2049 2050 if (rsp->sg_cnt > lpfc_tgttemplate.max_sgl_segments) { 2051 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2052 "6109 NVMET prep FCP wqe: seg cnt err: " 2053 "NPORT x%x oxid x%x ste %d cnt %d\n", 2054 ctxp->sid, ctxp->oxid, ctxp->state, 2055 phba->cfg_nvme_seg_cnt); 2056 return NULL; 2057 } 2058 2059 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2060 nvmewqe = ctxp->wqeq; 2061 if (nvmewqe == NULL) { 2062 /* Allocate buffer for command wqe */ 2063 nvmewqe = ctxp->ctxbuf->iocbq; 2064 if (nvmewqe == NULL) { 2065 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2066 "6110 NVMET prep FCP wqe: No " 2067 "WQE: NPORT x%x oxid x%x ste %d\n", 2068 ctxp->sid, ctxp->oxid, ctxp->state); 2069 return NULL; 2070 } 2071 ctxp->wqeq = nvmewqe; 2072 xc = 0; /* create new XRI */ 2073 nvmewqe->sli4_lxritag = NO_XRI; 2074 nvmewqe->sli4_xritag = NO_XRI; 2075 } 2076 2077 /* Sanity check */ 2078 if (((ctxp->state == LPFC_NVMET_STE_RCV) && 2079 (ctxp->entry_cnt == 1)) || 2080 (ctxp->state == LPFC_NVMET_STE_DATA)) { 2081 wqe = (union lpfc_wqe128 *)&nvmewqe->wqe; 2082 } else { 2083 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2084 "6111 Wrong state NVMET FCP: %d cnt %d\n", 2085 ctxp->state, ctxp->entry_cnt); 2086 return NULL; 2087 } 2088 2089 sgl = (struct sli4_sge *)ctxp->ctxbuf->sglq->sgl; 2090 switch (rsp->op) { 2091 case NVMET_FCOP_READDATA: 2092 case NVMET_FCOP_READDATA_RSP: 2093 /* Words 0 - 2 : The first sg segment */ 2094 sgel = &rsp->sg[0]; 2095 physaddr = sg_dma_address(sgel); 2096 wqe->fcp_tsend.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 2097 wqe->fcp_tsend.bde.tus.f.bdeSize = sg_dma_len(sgel); 2098 wqe->fcp_tsend.bde.addrLow = cpu_to_le32(putPaddrLow(physaddr)); 2099 wqe->fcp_tsend.bde.addrHigh = 2100 cpu_to_le32(putPaddrHigh(physaddr)); 2101 2102 /* Word 3 */ 2103 wqe->fcp_tsend.payload_offset_len = 0; 2104 2105 /* Word 4 */ 2106 wqe->fcp_tsend.relative_offset = ctxp->offset; 2107 2108 /* Word 5 */ 2109 2110 /* Word 6 */ 2111 bf_set(wqe_ctxt_tag, &wqe->fcp_tsend.wqe_com, 2112 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 2113 bf_set(wqe_xri_tag, &wqe->fcp_tsend.wqe_com, 2114 nvmewqe->sli4_xritag); 2115 2116 /* Word 7 */ 2117 bf_set(wqe_pu, &wqe->fcp_tsend.wqe_com, 1); 2118 bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE); 2119 2120 /* Word 8 */ 2121 wqe->fcp_tsend.wqe_com.abort_tag = nvmewqe->iotag; 2122 2123 /* Word 9 */ 2124 bf_set(wqe_reqtag, &wqe->fcp_tsend.wqe_com, nvmewqe->iotag); 2125 bf_set(wqe_rcvoxid, &wqe->fcp_tsend.wqe_com, ctxp->oxid); 2126 2127 /* Word 10 */ 2128 bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1); 2129 bf_set(wqe_dbde, &wqe->fcp_tsend.wqe_com, 1); 2130 bf_set(wqe_iod, &wqe->fcp_tsend.wqe_com, LPFC_WQE_IOD_WRITE); 2131 bf_set(wqe_lenloc, &wqe->fcp_tsend.wqe_com, 2132 LPFC_WQE_LENLOC_WORD12); 2133 bf_set(wqe_ebde_cnt, &wqe->fcp_tsend.wqe_com, 0); 2134 bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, xc); 2135 bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1); 2136 if (phba->cfg_nvme_oas) 2137 bf_set(wqe_oas, &wqe->fcp_tsend.wqe_com, 1); 2138 2139 /* Word 11 */ 2140 bf_set(wqe_cqid, &wqe->fcp_tsend.wqe_com, 2141 LPFC_WQE_CQ_ID_DEFAULT); 2142 bf_set(wqe_cmd_type, &wqe->fcp_tsend.wqe_com, 2143 FCP_COMMAND_TSEND); 2144 2145 /* Word 12 */ 2146 wqe->fcp_tsend.fcp_data_len = rsp->transfer_length; 2147 2148 /* Setup 2 SKIP SGEs */ 2149 sgl->addr_hi = 0; 2150 sgl->addr_lo = 0; 2151 sgl->word2 = 0; 2152 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2153 sgl->word2 = cpu_to_le32(sgl->word2); 2154 sgl->sge_len = 0; 2155 sgl++; 2156 sgl->addr_hi = 0; 2157 sgl->addr_lo = 0; 2158 sgl->word2 = 0; 2159 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2160 sgl->word2 = cpu_to_le32(sgl->word2); 2161 sgl->sge_len = 0; 2162 sgl++; 2163 if (rsp->op == NVMET_FCOP_READDATA_RSP) { 2164 atomic_inc(&tgtp->xmt_fcp_read_rsp); 2165 bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 1); 2166 if ((ndlp->nlp_flag & NLP_SUPPRESS_RSP) && 2167 (rsp->rsplen == 12)) { 2168 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 1); 2169 bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0); 2170 bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0); 2171 bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0); 2172 } else { 2173 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0); 2174 bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 1); 2175 bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 1); 2176 bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 2177 ((rsp->rsplen >> 2) - 1)); 2178 memcpy(&wqe->words[16], rsp->rspaddr, 2179 rsp->rsplen); 2180 } 2181 } else { 2182 atomic_inc(&tgtp->xmt_fcp_read); 2183 2184 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0); 2185 bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0); 2186 bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0); 2187 bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 0); 2188 bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0); 2189 } 2190 break; 2191 2192 case NVMET_FCOP_WRITEDATA: 2193 /* Words 0 - 2 : The first sg segment */ 2194 txrdy = dma_pool_alloc(phba->txrdy_payload_pool, 2195 GFP_KERNEL, &physaddr); 2196 if (!txrdy) { 2197 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2198 "6041 Bad txrdy buffer: oxid x%x\n", 2199 ctxp->oxid); 2200 return NULL; 2201 } 2202 ctxp->txrdy = txrdy; 2203 ctxp->txrdy_phys = physaddr; 2204 wqe->fcp_treceive.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 2205 wqe->fcp_treceive.bde.tus.f.bdeSize = TXRDY_PAYLOAD_LEN; 2206 wqe->fcp_treceive.bde.addrLow = 2207 cpu_to_le32(putPaddrLow(physaddr)); 2208 wqe->fcp_treceive.bde.addrHigh = 2209 cpu_to_le32(putPaddrHigh(physaddr)); 2210 2211 /* Word 3 */ 2212 wqe->fcp_treceive.payload_offset_len = TXRDY_PAYLOAD_LEN; 2213 2214 /* Word 4 */ 2215 wqe->fcp_treceive.relative_offset = ctxp->offset; 2216 2217 /* Word 5 */ 2218 2219 /* Word 6 */ 2220 bf_set(wqe_ctxt_tag, &wqe->fcp_treceive.wqe_com, 2221 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 2222 bf_set(wqe_xri_tag, &wqe->fcp_treceive.wqe_com, 2223 nvmewqe->sli4_xritag); 2224 2225 /* Word 7 */ 2226 bf_set(wqe_pu, &wqe->fcp_treceive.wqe_com, 1); 2227 bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0); 2228 bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com, 2229 CMD_FCP_TRECEIVE64_WQE); 2230 2231 /* Word 8 */ 2232 wqe->fcp_treceive.wqe_com.abort_tag = nvmewqe->iotag; 2233 2234 /* Word 9 */ 2235 bf_set(wqe_reqtag, &wqe->fcp_treceive.wqe_com, nvmewqe->iotag); 2236 bf_set(wqe_rcvoxid, &wqe->fcp_treceive.wqe_com, ctxp->oxid); 2237 2238 /* Word 10 */ 2239 bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1); 2240 bf_set(wqe_dbde, &wqe->fcp_treceive.wqe_com, 1); 2241 bf_set(wqe_iod, &wqe->fcp_treceive.wqe_com, LPFC_WQE_IOD_READ); 2242 bf_set(wqe_lenloc, &wqe->fcp_treceive.wqe_com, 2243 LPFC_WQE_LENLOC_WORD12); 2244 bf_set(wqe_xc, &wqe->fcp_treceive.wqe_com, xc); 2245 bf_set(wqe_wqes, &wqe->fcp_treceive.wqe_com, 0); 2246 bf_set(wqe_irsp, &wqe->fcp_treceive.wqe_com, 0); 2247 bf_set(wqe_irsplen, &wqe->fcp_treceive.wqe_com, 0); 2248 bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1); 2249 if (phba->cfg_nvme_oas) 2250 bf_set(wqe_oas, &wqe->fcp_treceive.wqe_com, 1); 2251 2252 /* Word 11 */ 2253 bf_set(wqe_cqid, &wqe->fcp_treceive.wqe_com, 2254 LPFC_WQE_CQ_ID_DEFAULT); 2255 bf_set(wqe_cmd_type, &wqe->fcp_treceive.wqe_com, 2256 FCP_COMMAND_TRECEIVE); 2257 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0); 2258 2259 /* Word 12 */ 2260 wqe->fcp_tsend.fcp_data_len = rsp->transfer_length; 2261 2262 /* Setup 1 TXRDY and 1 SKIP SGE */ 2263 txrdy[0] = 0; 2264 txrdy[1] = cpu_to_be32(rsp->transfer_length); 2265 txrdy[2] = 0; 2266 2267 sgl->addr_hi = putPaddrHigh(physaddr); 2268 sgl->addr_lo = putPaddrLow(physaddr); 2269 sgl->word2 = 0; 2270 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA); 2271 sgl->word2 = cpu_to_le32(sgl->word2); 2272 sgl->sge_len = cpu_to_le32(TXRDY_PAYLOAD_LEN); 2273 sgl++; 2274 sgl->addr_hi = 0; 2275 sgl->addr_lo = 0; 2276 sgl->word2 = 0; 2277 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2278 sgl->word2 = cpu_to_le32(sgl->word2); 2279 sgl->sge_len = 0; 2280 sgl++; 2281 atomic_inc(&tgtp->xmt_fcp_write); 2282 break; 2283 2284 case NVMET_FCOP_RSP: 2285 /* Words 0 - 2 */ 2286 physaddr = rsp->rspdma; 2287 wqe->fcp_trsp.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 2288 wqe->fcp_trsp.bde.tus.f.bdeSize = rsp->rsplen; 2289 wqe->fcp_trsp.bde.addrLow = 2290 cpu_to_le32(putPaddrLow(physaddr)); 2291 wqe->fcp_trsp.bde.addrHigh = 2292 cpu_to_le32(putPaddrHigh(physaddr)); 2293 2294 /* Word 3 */ 2295 wqe->fcp_trsp.response_len = rsp->rsplen; 2296 2297 /* Word 4 */ 2298 wqe->fcp_trsp.rsvd_4_5[0] = 0; 2299 2300 2301 /* Word 5 */ 2302 2303 /* Word 6 */ 2304 bf_set(wqe_ctxt_tag, &wqe->fcp_trsp.wqe_com, 2305 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 2306 bf_set(wqe_xri_tag, &wqe->fcp_trsp.wqe_com, 2307 nvmewqe->sli4_xritag); 2308 2309 /* Word 7 */ 2310 bf_set(wqe_pu, &wqe->fcp_trsp.wqe_com, 0); 2311 bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1); 2312 bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE); 2313 2314 /* Word 8 */ 2315 wqe->fcp_trsp.wqe_com.abort_tag = nvmewqe->iotag; 2316 2317 /* Word 9 */ 2318 bf_set(wqe_reqtag, &wqe->fcp_trsp.wqe_com, nvmewqe->iotag); 2319 bf_set(wqe_rcvoxid, &wqe->fcp_trsp.wqe_com, ctxp->oxid); 2320 2321 /* Word 10 */ 2322 bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1); 2323 bf_set(wqe_dbde, &wqe->fcp_trsp.wqe_com, 0); 2324 bf_set(wqe_iod, &wqe->fcp_trsp.wqe_com, LPFC_WQE_IOD_WRITE); 2325 bf_set(wqe_lenloc, &wqe->fcp_trsp.wqe_com, 2326 LPFC_WQE_LENLOC_WORD3); 2327 bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, xc); 2328 bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1); 2329 if (phba->cfg_nvme_oas) 2330 bf_set(wqe_oas, &wqe->fcp_trsp.wqe_com, 1); 2331 2332 /* Word 11 */ 2333 bf_set(wqe_cqid, &wqe->fcp_trsp.wqe_com, 2334 LPFC_WQE_CQ_ID_DEFAULT); 2335 bf_set(wqe_cmd_type, &wqe->fcp_trsp.wqe_com, 2336 FCP_COMMAND_TRSP); 2337 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0); 2338 2339 if (rsp->rsplen == LPFC_NVMET_SUCCESS_LEN) { 2340 /* Good response - all zero's on wire */ 2341 bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 0); 2342 bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 0); 2343 bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 0); 2344 } else { 2345 bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 1); 2346 bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 1); 2347 bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 2348 ((rsp->rsplen >> 2) - 1)); 2349 memcpy(&wqe->words[16], rsp->rspaddr, rsp->rsplen); 2350 } 2351 2352 /* Use rspbuf, NOT sg list */ 2353 rsp->sg_cnt = 0; 2354 sgl->word2 = 0; 2355 atomic_inc(&tgtp->xmt_fcp_rsp); 2356 break; 2357 2358 default: 2359 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 2360 "6064 Unknown Rsp Op %d\n", 2361 rsp->op); 2362 return NULL; 2363 } 2364 2365 nvmewqe->retry = 1; 2366 nvmewqe->vport = phba->pport; 2367 nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT; 2368 nvmewqe->context1 = ndlp; 2369 2370 for (i = 0; i < rsp->sg_cnt; i++) { 2371 sgel = &rsp->sg[i]; 2372 physaddr = sg_dma_address(sgel); 2373 cnt = sg_dma_len(sgel); 2374 sgl->addr_hi = putPaddrHigh(physaddr); 2375 sgl->addr_lo = putPaddrLow(physaddr); 2376 sgl->word2 = 0; 2377 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA); 2378 bf_set(lpfc_sli4_sge_offset, sgl, ctxp->offset); 2379 if ((i+1) == rsp->sg_cnt) 2380 bf_set(lpfc_sli4_sge_last, sgl, 1); 2381 sgl->word2 = cpu_to_le32(sgl->word2); 2382 sgl->sge_len = cpu_to_le32(cnt); 2383 sgl++; 2384 ctxp->offset += cnt; 2385 } 2386 ctxp->state = LPFC_NVMET_STE_DATA; 2387 ctxp->entry_cnt++; 2388 return nvmewqe; 2389 } 2390 2391 /** 2392 * lpfc_nvmet_sol_fcp_abort_cmp - Completion handler for ABTS 2393 * @phba: Pointer to HBA context object. 2394 * @cmdwqe: Pointer to driver command WQE object. 2395 * @wcqe: Pointer to driver response CQE object. 2396 * 2397 * The function is called from SLI ring event handler with no 2398 * lock held. This function is the completion handler for NVME ABTS for FCP cmds 2399 * The function frees memory resources used for the NVME commands. 2400 **/ 2401 static void 2402 lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 2403 struct lpfc_wcqe_complete *wcqe) 2404 { 2405 struct lpfc_nvmet_rcv_ctx *ctxp; 2406 struct lpfc_nvmet_tgtport *tgtp; 2407 uint32_t status, result; 2408 unsigned long flags; 2409 bool released = false; 2410 2411 ctxp = cmdwqe->context2; 2412 status = bf_get(lpfc_wcqe_c_status, wcqe); 2413 result = wcqe->parameter; 2414 2415 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2416 if (ctxp->flag & LPFC_NVMET_ABORT_OP) 2417 atomic_inc(&tgtp->xmt_fcp_abort_cmpl); 2418 2419 ctxp->state = LPFC_NVMET_STE_DONE; 2420 2421 /* Check if we already received a free context call 2422 * and we have completed processing an abort situation. 2423 */ 2424 spin_lock_irqsave(&ctxp->ctxlock, flags); 2425 if ((ctxp->flag & LPFC_NVMET_CTX_RLS) && 2426 !(ctxp->flag & LPFC_NVMET_XBUSY)) { 2427 list_del(&ctxp->list); 2428 released = true; 2429 } 2430 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 2431 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 2432 atomic_inc(&tgtp->xmt_abort_rsp); 2433 2434 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2435 "6165 ABORT cmpl: xri x%x flg x%x (%d) " 2436 "WCQE: %08x %08x %08x %08x\n", 2437 ctxp->oxid, ctxp->flag, released, 2438 wcqe->word0, wcqe->total_data_placed, 2439 result, wcqe->word3); 2440 2441 cmdwqe->context2 = NULL; 2442 cmdwqe->context3 = NULL; 2443 /* 2444 * if transport has released ctx, then can reuse it. Otherwise, 2445 * will be recycled by transport release call. 2446 */ 2447 if (released) 2448 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); 2449 2450 /* This is the iocbq for the abort, not the command */ 2451 lpfc_sli_release_iocbq(phba, cmdwqe); 2452 2453 /* Since iaab/iaar are NOT set, there is no work left. 2454 * For LPFC_NVMET_XBUSY, lpfc_sli4_nvmet_xri_aborted 2455 * should have been called already. 2456 */ 2457 } 2458 2459 /** 2460 * lpfc_nvmet_unsol_fcp_abort_cmp - Completion handler for ABTS 2461 * @phba: Pointer to HBA context object. 2462 * @cmdwqe: Pointer to driver command WQE object. 2463 * @wcqe: Pointer to driver response CQE object. 2464 * 2465 * The function is called from SLI ring event handler with no 2466 * lock held. This function is the completion handler for NVME ABTS for FCP cmds 2467 * The function frees memory resources used for the NVME commands. 2468 **/ 2469 static void 2470 lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 2471 struct lpfc_wcqe_complete *wcqe) 2472 { 2473 struct lpfc_nvmet_rcv_ctx *ctxp; 2474 struct lpfc_nvmet_tgtport *tgtp; 2475 unsigned long flags; 2476 uint32_t status, result; 2477 bool released = false; 2478 2479 ctxp = cmdwqe->context2; 2480 status = bf_get(lpfc_wcqe_c_status, wcqe); 2481 result = wcqe->parameter; 2482 2483 if (!ctxp) { 2484 /* if context is clear, related io alrady complete */ 2485 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2486 "6070 ABTS cmpl: WCQE: %08x %08x %08x %08x\n", 2487 wcqe->word0, wcqe->total_data_placed, 2488 result, wcqe->word3); 2489 return; 2490 } 2491 2492 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2493 if (ctxp->flag & LPFC_NVMET_ABORT_OP) 2494 atomic_inc(&tgtp->xmt_fcp_abort_cmpl); 2495 2496 /* Sanity check */ 2497 if (ctxp->state != LPFC_NVMET_STE_ABORT) { 2498 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 2499 "6112 ABTS Wrong state:%d oxid x%x\n", 2500 ctxp->state, ctxp->oxid); 2501 } 2502 2503 /* Check if we already received a free context call 2504 * and we have completed processing an abort situation. 2505 */ 2506 ctxp->state = LPFC_NVMET_STE_DONE; 2507 spin_lock_irqsave(&ctxp->ctxlock, flags); 2508 if ((ctxp->flag & LPFC_NVMET_CTX_RLS) && 2509 !(ctxp->flag & LPFC_NVMET_XBUSY)) { 2510 list_del(&ctxp->list); 2511 released = true; 2512 } 2513 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 2514 spin_unlock_irqrestore(&ctxp->ctxlock, flags); 2515 atomic_inc(&tgtp->xmt_abort_rsp); 2516 2517 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2518 "6316 ABTS cmpl xri x%x flg x%x (%x) " 2519 "WCQE: %08x %08x %08x %08x\n", 2520 ctxp->oxid, ctxp->flag, released, 2521 wcqe->word0, wcqe->total_data_placed, 2522 result, wcqe->word3); 2523 2524 cmdwqe->context2 = NULL; 2525 cmdwqe->context3 = NULL; 2526 /* 2527 * if transport has released ctx, then can reuse it. Otherwise, 2528 * will be recycled by transport release call. 2529 */ 2530 if (released) 2531 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); 2532 2533 /* Since iaab/iaar are NOT set, there is no work left. 2534 * For LPFC_NVMET_XBUSY, lpfc_sli4_nvmet_xri_aborted 2535 * should have been called already. 2536 */ 2537 } 2538 2539 /** 2540 * lpfc_nvmet_xmt_ls_abort_cmp - Completion handler for ABTS 2541 * @phba: Pointer to HBA context object. 2542 * @cmdwqe: Pointer to driver command WQE object. 2543 * @wcqe: Pointer to driver response CQE object. 2544 * 2545 * The function is called from SLI ring event handler with no 2546 * lock held. This function is the completion handler for NVME ABTS for LS cmds 2547 * The function frees memory resources used for the NVME commands. 2548 **/ 2549 static void 2550 lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 2551 struct lpfc_wcqe_complete *wcqe) 2552 { 2553 struct lpfc_nvmet_rcv_ctx *ctxp; 2554 struct lpfc_nvmet_tgtport *tgtp; 2555 uint32_t status, result; 2556 2557 ctxp = cmdwqe->context2; 2558 status = bf_get(lpfc_wcqe_c_status, wcqe); 2559 result = wcqe->parameter; 2560 2561 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2562 atomic_inc(&tgtp->xmt_ls_abort_cmpl); 2563 2564 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2565 "6083 Abort cmpl: ctx %p WCQE:%08x %08x %08x %08x\n", 2566 ctxp, wcqe->word0, wcqe->total_data_placed, 2567 result, wcqe->word3); 2568 2569 if (!ctxp) { 2570 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 2571 "6415 NVMET LS Abort No ctx: WCQE: " 2572 "%08x %08x %08x %08x\n", 2573 wcqe->word0, wcqe->total_data_placed, 2574 result, wcqe->word3); 2575 2576 lpfc_sli_release_iocbq(phba, cmdwqe); 2577 return; 2578 } 2579 2580 if (ctxp->state != LPFC_NVMET_STE_LS_ABORT) { 2581 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2582 "6416 NVMET LS abort cmpl state mismatch: " 2583 "oxid x%x: %d %d\n", 2584 ctxp->oxid, ctxp->state, ctxp->entry_cnt); 2585 } 2586 2587 cmdwqe->context2 = NULL; 2588 cmdwqe->context3 = NULL; 2589 lpfc_sli_release_iocbq(phba, cmdwqe); 2590 kfree(ctxp); 2591 } 2592 2593 static int 2594 lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba, 2595 struct lpfc_nvmet_rcv_ctx *ctxp, 2596 uint32_t sid, uint16_t xri) 2597 { 2598 struct lpfc_nvmet_tgtport *tgtp; 2599 struct lpfc_iocbq *abts_wqeq; 2600 union lpfc_wqe *wqe_abts; 2601 struct lpfc_nodelist *ndlp; 2602 2603 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2604 "6067 ABTS: sid %x xri x%x/x%x\n", 2605 sid, xri, ctxp->wqeq->sli4_xritag); 2606 2607 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2608 2609 ndlp = lpfc_findnode_did(phba->pport, sid); 2610 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || 2611 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && 2612 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { 2613 atomic_inc(&tgtp->xmt_abort_rsp_error); 2614 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 2615 "6134 Drop ABTS - wrong NDLP state x%x.\n", 2616 (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE); 2617 2618 /* No failure to an ABTS request. */ 2619 return 0; 2620 } 2621 2622 abts_wqeq = ctxp->wqeq; 2623 wqe_abts = &abts_wqeq->wqe; 2624 2625 /* 2626 * Since we zero the whole WQE, we need to ensure we set the WQE fields 2627 * that were initialized in lpfc_sli4_nvmet_alloc. 2628 */ 2629 memset(wqe_abts, 0, sizeof(union lpfc_wqe)); 2630 2631 /* Word 5 */ 2632 bf_set(wqe_dfctl, &wqe_abts->xmit_sequence.wge_ctl, 0); 2633 bf_set(wqe_ls, &wqe_abts->xmit_sequence.wge_ctl, 1); 2634 bf_set(wqe_la, &wqe_abts->xmit_sequence.wge_ctl, 0); 2635 bf_set(wqe_rctl, &wqe_abts->xmit_sequence.wge_ctl, FC_RCTL_BA_ABTS); 2636 bf_set(wqe_type, &wqe_abts->xmit_sequence.wge_ctl, FC_TYPE_BLS); 2637 2638 /* Word 6 */ 2639 bf_set(wqe_ctxt_tag, &wqe_abts->xmit_sequence.wqe_com, 2640 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 2641 bf_set(wqe_xri_tag, &wqe_abts->xmit_sequence.wqe_com, 2642 abts_wqeq->sli4_xritag); 2643 2644 /* Word 7 */ 2645 bf_set(wqe_cmnd, &wqe_abts->xmit_sequence.wqe_com, 2646 CMD_XMIT_SEQUENCE64_WQE); 2647 bf_set(wqe_ct, &wqe_abts->xmit_sequence.wqe_com, SLI4_CT_RPI); 2648 bf_set(wqe_class, &wqe_abts->xmit_sequence.wqe_com, CLASS3); 2649 bf_set(wqe_pu, &wqe_abts->xmit_sequence.wqe_com, 0); 2650 2651 /* Word 8 */ 2652 wqe_abts->xmit_sequence.wqe_com.abort_tag = abts_wqeq->iotag; 2653 2654 /* Word 9 */ 2655 bf_set(wqe_reqtag, &wqe_abts->xmit_sequence.wqe_com, abts_wqeq->iotag); 2656 /* Needs to be set by caller */ 2657 bf_set(wqe_rcvoxid, &wqe_abts->xmit_sequence.wqe_com, xri); 2658 2659 /* Word 10 */ 2660 bf_set(wqe_dbde, &wqe_abts->xmit_sequence.wqe_com, 1); 2661 bf_set(wqe_iod, &wqe_abts->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE); 2662 bf_set(wqe_lenloc, &wqe_abts->xmit_sequence.wqe_com, 2663 LPFC_WQE_LENLOC_WORD12); 2664 bf_set(wqe_ebde_cnt, &wqe_abts->xmit_sequence.wqe_com, 0); 2665 bf_set(wqe_qosd, &wqe_abts->xmit_sequence.wqe_com, 0); 2666 2667 /* Word 11 */ 2668 bf_set(wqe_cqid, &wqe_abts->xmit_sequence.wqe_com, 2669 LPFC_WQE_CQ_ID_DEFAULT); 2670 bf_set(wqe_cmd_type, &wqe_abts->xmit_sequence.wqe_com, 2671 OTHER_COMMAND); 2672 2673 abts_wqeq->vport = phba->pport; 2674 abts_wqeq->context1 = ndlp; 2675 abts_wqeq->context2 = ctxp; 2676 abts_wqeq->context3 = NULL; 2677 abts_wqeq->rsvd2 = 0; 2678 /* hba_wqidx should already be setup from command we are aborting */ 2679 abts_wqeq->iocb.ulpCommand = CMD_XMIT_SEQUENCE64_CR; 2680 abts_wqeq->iocb.ulpLe = 1; 2681 2682 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2683 "6069 Issue ABTS to xri x%x reqtag x%x\n", 2684 xri, abts_wqeq->iotag); 2685 return 1; 2686 } 2687 2688 static int 2689 lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, 2690 struct lpfc_nvmet_rcv_ctx *ctxp, 2691 uint32_t sid, uint16_t xri) 2692 { 2693 struct lpfc_nvmet_tgtport *tgtp; 2694 struct lpfc_iocbq *abts_wqeq; 2695 union lpfc_wqe *abts_wqe; 2696 struct lpfc_nodelist *ndlp; 2697 unsigned long flags; 2698 int rc; 2699 2700 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2701 if (!ctxp->wqeq) { 2702 ctxp->wqeq = ctxp->ctxbuf->iocbq; 2703 ctxp->wqeq->hba_wqidx = 0; 2704 } 2705 2706 ndlp = lpfc_findnode_did(phba->pport, sid); 2707 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || 2708 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && 2709 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { 2710 atomic_inc(&tgtp->xmt_abort_rsp_error); 2711 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 2712 "6160 Drop ABORT - wrong NDLP state x%x.\n", 2713 (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE); 2714 2715 /* No failure to an ABTS request. */ 2716 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 2717 return 0; 2718 } 2719 2720 /* Issue ABTS for this WQE based on iotag */ 2721 ctxp->abort_wqeq = lpfc_sli_get_iocbq(phba); 2722 if (!ctxp->abort_wqeq) { 2723 atomic_inc(&tgtp->xmt_abort_rsp_error); 2724 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 2725 "6161 ABORT failed: No wqeqs: " 2726 "xri: x%x\n", ctxp->oxid); 2727 /* No failure to an ABTS request. */ 2728 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 2729 return 0; 2730 } 2731 abts_wqeq = ctxp->abort_wqeq; 2732 abts_wqe = &abts_wqeq->wqe; 2733 ctxp->state = LPFC_NVMET_STE_ABORT; 2734 2735 /* Announce entry to new IO submit field. */ 2736 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2737 "6162 ABORT Request to rport DID x%06x " 2738 "for xri x%x x%x\n", 2739 ctxp->sid, ctxp->oxid, ctxp->wqeq->sli4_xritag); 2740 2741 /* If the hba is getting reset, this flag is set. It is 2742 * cleared when the reset is complete and rings reestablished. 2743 */ 2744 spin_lock_irqsave(&phba->hbalock, flags); 2745 /* driver queued commands are in process of being flushed */ 2746 if (phba->hba_flag & HBA_NVME_IOQ_FLUSH) { 2747 spin_unlock_irqrestore(&phba->hbalock, flags); 2748 atomic_inc(&tgtp->xmt_abort_rsp_error); 2749 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 2750 "6163 Driver in reset cleanup - flushing " 2751 "NVME Req now. hba_flag x%x oxid x%x\n", 2752 phba->hba_flag, ctxp->oxid); 2753 lpfc_sli_release_iocbq(phba, abts_wqeq); 2754 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 2755 return 0; 2756 } 2757 2758 /* Outstanding abort is in progress */ 2759 if (abts_wqeq->iocb_flag & LPFC_DRIVER_ABORTED) { 2760 spin_unlock_irqrestore(&phba->hbalock, flags); 2761 atomic_inc(&tgtp->xmt_abort_rsp_error); 2762 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 2763 "6164 Outstanding NVME I/O Abort Request " 2764 "still pending on oxid x%x\n", 2765 ctxp->oxid); 2766 lpfc_sli_release_iocbq(phba, abts_wqeq); 2767 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 2768 return 0; 2769 } 2770 2771 /* Ready - mark outstanding as aborted by driver. */ 2772 abts_wqeq->iocb_flag |= LPFC_DRIVER_ABORTED; 2773 2774 /* WQEs are reused. Clear stale data and set key fields to 2775 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag. 2776 */ 2777 memset(abts_wqe, 0, sizeof(union lpfc_wqe)); 2778 2779 /* word 3 */ 2780 bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG); 2781 2782 /* word 7 */ 2783 bf_set(wqe_ct, &abts_wqe->abort_cmd.wqe_com, 0); 2784 bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX); 2785 2786 /* word 8 - tell the FW to abort the IO associated with this 2787 * outstanding exchange ID. 2788 */ 2789 abts_wqe->abort_cmd.wqe_com.abort_tag = ctxp->wqeq->sli4_xritag; 2790 2791 /* word 9 - this is the iotag for the abts_wqe completion. */ 2792 bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com, 2793 abts_wqeq->iotag); 2794 2795 /* word 10 */ 2796 bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1); 2797 bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE); 2798 2799 /* word 11 */ 2800 bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND); 2801 bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1); 2802 bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 2803 2804 /* ABTS WQE must go to the same WQ as the WQE to be aborted */ 2805 abts_wqeq->hba_wqidx = ctxp->wqeq->hba_wqidx; 2806 abts_wqeq->wqe_cmpl = lpfc_nvmet_sol_fcp_abort_cmp; 2807 abts_wqeq->iocb_cmpl = 0; 2808 abts_wqeq->iocb_flag |= LPFC_IO_NVME; 2809 abts_wqeq->context2 = ctxp; 2810 abts_wqeq->vport = phba->pport; 2811 rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_wqeq); 2812 spin_unlock_irqrestore(&phba->hbalock, flags); 2813 if (rc == WQE_SUCCESS) { 2814 atomic_inc(&tgtp->xmt_abort_sol); 2815 return 0; 2816 } 2817 2818 atomic_inc(&tgtp->xmt_abort_rsp_error); 2819 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 2820 lpfc_sli_release_iocbq(phba, abts_wqeq); 2821 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 2822 "6166 Failed ABORT issue_wqe with status x%x " 2823 "for oxid x%x.\n", 2824 rc, ctxp->oxid); 2825 return 1; 2826 } 2827 2828 2829 static int 2830 lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba, 2831 struct lpfc_nvmet_rcv_ctx *ctxp, 2832 uint32_t sid, uint16_t xri) 2833 { 2834 struct lpfc_nvmet_tgtport *tgtp; 2835 struct lpfc_iocbq *abts_wqeq; 2836 unsigned long flags; 2837 int rc; 2838 2839 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2840 if (!ctxp->wqeq) { 2841 ctxp->wqeq = ctxp->ctxbuf->iocbq; 2842 ctxp->wqeq->hba_wqidx = 0; 2843 } 2844 2845 if (ctxp->state == LPFC_NVMET_STE_FREE) { 2846 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2847 "6417 NVMET ABORT ctx freed %d %d oxid x%x\n", 2848 ctxp->state, ctxp->entry_cnt, ctxp->oxid); 2849 rc = WQE_BUSY; 2850 goto aerr; 2851 } 2852 ctxp->state = LPFC_NVMET_STE_ABORT; 2853 ctxp->entry_cnt++; 2854 rc = lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri); 2855 if (rc == 0) 2856 goto aerr; 2857 2858 spin_lock_irqsave(&phba->hbalock, flags); 2859 abts_wqeq = ctxp->wqeq; 2860 abts_wqeq->wqe_cmpl = lpfc_nvmet_unsol_fcp_abort_cmp; 2861 abts_wqeq->iocb_cmpl = NULL; 2862 abts_wqeq->iocb_flag |= LPFC_IO_NVMET; 2863 rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_wqeq); 2864 spin_unlock_irqrestore(&phba->hbalock, flags); 2865 if (rc == WQE_SUCCESS) { 2866 return 0; 2867 } 2868 2869 aerr: 2870 ctxp->flag &= ~LPFC_NVMET_ABORT_OP; 2871 atomic_inc(&tgtp->xmt_abort_rsp_error); 2872 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 2873 "6135 Failed to Issue ABTS for oxid x%x. Status x%x\n", 2874 ctxp->oxid, rc); 2875 return 1; 2876 } 2877 2878 static int 2879 lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *phba, 2880 struct lpfc_nvmet_rcv_ctx *ctxp, 2881 uint32_t sid, uint16_t xri) 2882 { 2883 struct lpfc_nvmet_tgtport *tgtp; 2884 struct lpfc_iocbq *abts_wqeq; 2885 union lpfc_wqe *wqe_abts; 2886 unsigned long flags; 2887 int rc; 2888 2889 if ((ctxp->state == LPFC_NVMET_STE_LS_RCV && ctxp->entry_cnt == 1) || 2890 (ctxp->state == LPFC_NVMET_STE_LS_RSP && ctxp->entry_cnt == 2)) { 2891 ctxp->state = LPFC_NVMET_STE_LS_ABORT; 2892 ctxp->entry_cnt++; 2893 } else { 2894 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2895 "6418 NVMET LS abort state mismatch " 2896 "IO x%x: %d %d\n", 2897 ctxp->oxid, ctxp->state, ctxp->entry_cnt); 2898 ctxp->state = LPFC_NVMET_STE_LS_ABORT; 2899 } 2900 2901 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; 2902 if (!ctxp->wqeq) { 2903 /* Issue ABTS for this WQE based on iotag */ 2904 ctxp->wqeq = lpfc_sli_get_iocbq(phba); 2905 if (!ctxp->wqeq) { 2906 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 2907 "6068 Abort failed: No wqeqs: " 2908 "xri: x%x\n", xri); 2909 /* No failure to an ABTS request. */ 2910 kfree(ctxp); 2911 return 0; 2912 } 2913 } 2914 abts_wqeq = ctxp->wqeq; 2915 wqe_abts = &abts_wqeq->wqe; 2916 2917 if (lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri) == 0) { 2918 rc = WQE_BUSY; 2919 goto out; 2920 } 2921 2922 spin_lock_irqsave(&phba->hbalock, flags); 2923 abts_wqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_abort_cmp; 2924 abts_wqeq->iocb_cmpl = 0; 2925 abts_wqeq->iocb_flag |= LPFC_IO_NVME_LS; 2926 rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, abts_wqeq); 2927 spin_unlock_irqrestore(&phba->hbalock, flags); 2928 if (rc == WQE_SUCCESS) { 2929 atomic_inc(&tgtp->xmt_abort_unsol); 2930 return 0; 2931 } 2932 out: 2933 atomic_inc(&tgtp->xmt_abort_rsp_error); 2934 abts_wqeq->context2 = NULL; 2935 abts_wqeq->context3 = NULL; 2936 lpfc_sli_release_iocbq(phba, abts_wqeq); 2937 kfree(ctxp); 2938 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, 2939 "6056 Failed to Issue ABTS. Status x%x\n", rc); 2940 return 0; 2941 } 2942