1895427bdSJames Smart /******************************************************************* 2895427bdSJames Smart * This file is part of the Emulex Linux Device Driver for * 3895427bdSJames Smart * Fibre Channel Host Bus Adapters. * 4*f45775bfSJames Smart * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term * 53e21d1cbSJames Smart * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * 6895427bdSJames Smart * Copyright (C) 2004-2016 Emulex. All rights reserved. * 7895427bdSJames Smart * EMULEX and SLI are trademarks of Emulex. * 8d080abe0SJames Smart * www.broadcom.com * 9895427bdSJames Smart * Portions Copyright (C) 2004-2005 Christoph Hellwig * 10895427bdSJames Smart * * 11895427bdSJames Smart * This program is free software; you can redistribute it and/or * 12895427bdSJames Smart * modify it under the terms of version 2 of the GNU General * 13895427bdSJames Smart * Public License as published by the Free Software Foundation. * 14895427bdSJames Smart * This program is distributed in the hope that it will be useful. * 15895427bdSJames Smart * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 16895427bdSJames Smart * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 17895427bdSJames Smart * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 18895427bdSJames Smart * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 19895427bdSJames Smart * TO BE LEGALLY INVALID. See the GNU General Public License for * 20895427bdSJames Smart * more details, a copy of which can be found in the file COPYING * 21895427bdSJames Smart * included with this package. * 22895427bdSJames Smart ********************************************************************/ 23895427bdSJames Smart 242a1160a0SJames Smart #include <linux/nvme.h> 252a1160a0SJames Smart #include <linux/nvme-fc-driver.h> 262a1160a0SJames Smart #include <linux/nvme-fc.h> 272a1160a0SJames Smart 28a44e4e8bSJames Smart #define LPFC_NVME_DEFAULT_SEGS (64 + 1) /* 256K IOs */ 29895427bdSJames Smart 30895427bdSJames Smart #define LPFC_NVME_ERSP_LEN 0x20 31895427bdSJames Smart 32add9d6beSJames Smart #define LPFC_NVME_WAIT_TMO 10 33cf1a1d3eSJames Smart #define LPFC_NVME_EXPEDITE_XRICNT 8 340709263aSJames Smart #define LPFC_NVME_FB_SHIFT 9 350709263aSJames Smart #define LPFC_NVME_MAX_FB (1 << 20) /* 1M */ 36add9d6beSJames Smart 3701466024SJames Smart #define lpfc_ndlp_get_nrport(ndlp) \ 3806145683SJames Smart ((!ndlp->nrport || (ndlp->fc4_xpt_flags & NVME_XPT_UNREG_WAIT))\ 3901466024SJames Smart ? NULL : ndlp->nrport) 4001466024SJames Smart 4101649561SJames Smart struct lpfc_nvme_qhandle { 4201649561SJames Smart uint32_t index; /* WQ index to use */ 4301649561SJames Smart uint32_t qidx; /* queue index passed to create */ 4401649561SJames Smart uint32_t cpu_id; /* current cpu id at time of create */ 4501649561SJames Smart }; 4601649561SJames Smart 47895427bdSJames Smart /* Declare nvme-based local and remote port definitions. */ 48895427bdSJames Smart struct lpfc_nvme_lport { 49895427bdSJames Smart struct lpfc_vport *vport; 507961cba6SEwan D. Milne struct completion *lport_unreg_cmp; 514b056682SJames Smart /* Add stats counters here */ 5266a210ffSJames Smart atomic_t fc4NvmeLsRequests; 5366a210ffSJames Smart atomic_t fc4NvmeLsCmpls; 544b056682SJames Smart atomic_t xmt_fcp_noxri; 554b056682SJames Smart atomic_t xmt_fcp_bad_ndlp; 564b056682SJames Smart atomic_t xmt_fcp_qdepth; 574b056682SJames Smart atomic_t xmt_fcp_wqerr; 5844c2757bSJames Smart atomic_t xmt_fcp_err; 594b056682SJames Smart atomic_t xmt_fcp_abort; 604b056682SJames Smart atomic_t xmt_ls_abort; 614b056682SJames Smart atomic_t xmt_ls_err; 624b056682SJames Smart atomic_t cmpl_fcp_xb; 634b056682SJames Smart atomic_t cmpl_fcp_err; 644b056682SJames Smart atomic_t cmpl_ls_xb; 654b056682SJames Smart atomic_t cmpl_ls_err; 66895427bdSJames Smart }; 67895427bdSJames Smart 68895427bdSJames Smart struct lpfc_nvme_rport { 69895427bdSJames Smart struct lpfc_nvme_lport *lport; 70895427bdSJames Smart struct nvme_fc_remote_port *remoteport; 71895427bdSJames Smart struct lpfc_nodelist *ndlp; 72895427bdSJames Smart struct completion rport_unreg_done; 73895427bdSJames Smart }; 74895427bdSJames Smart 75bbe3012bSJames Smart struct lpfc_nvme_fcpreq_priv { 76c490850aSJames Smart struct lpfc_io_buf *nvme_buf; 77bbe3012bSJames Smart }; 782a1160a0SJames Smart 796514b25dSJames Smart /* 806514b25dSJames Smart * set NVME LS request timeouts to 30s. It is larger than the 2*R_A_TOV 816514b25dSJames Smart * set by the spec, which appears to have issues with some devices. 826514b25dSJames Smart */ 836514b25dSJames Smart #define LPFC_NVME_LS_TIMEOUT 30 846514b25dSJames Smart 852a1160a0SJames Smart 862a1160a0SJames Smart #define LPFC_NVMET_DEFAULT_SEGS (64 + 1) /* 256K IOs */ 872a1160a0SJames Smart #define LPFC_NVMET_RQE_MIN_POST 128 882a1160a0SJames Smart #define LPFC_NVMET_RQE_DEF_POST 512 892a1160a0SJames Smart #define LPFC_NVMET_RQE_DEF_COUNT 2048 902a1160a0SJames Smart #define LPFC_NVMET_SUCCESS_LEN 12 912a1160a0SJames Smart 922a1160a0SJames Smart #define LPFC_NVMET_MRQ_AUTO 0 932a1160a0SJames Smart #define LPFC_NVMET_MRQ_MAX 16 942a1160a0SJames Smart 952a1160a0SJames Smart #define LPFC_NVMET_WAIT_TMO (5 * MSEC_PER_SEC) 962a1160a0SJames Smart 972a1160a0SJames Smart /* Used for NVME Target */ 984c2805aaSJames Smart #define LPFC_NVMET_INV_HOST_ACTIVE 1 994c2805aaSJames Smart 1002a1160a0SJames Smart struct lpfc_nvmet_tgtport { 1012a1160a0SJames Smart struct lpfc_hba *phba; 1022a1160a0SJames Smart struct completion *tport_unreg_cmp; 1034c2805aaSJames Smart atomic_t state; /* tracks nvmet hosthandle invalidation */ 1042a1160a0SJames Smart 1052a1160a0SJames Smart /* Stats counters - lpfc_nvmet_unsol_ls_buffer */ 1062a1160a0SJames Smart atomic_t rcv_ls_req_in; 1072a1160a0SJames Smart atomic_t rcv_ls_req_out; 1082a1160a0SJames Smart atomic_t rcv_ls_req_drop; 1092a1160a0SJames Smart atomic_t xmt_ls_abort; 1102a1160a0SJames Smart atomic_t xmt_ls_abort_cmpl; 1112a1160a0SJames Smart 1122a1160a0SJames Smart /* Stats counters - lpfc_nvmet_xmt_ls_rsp */ 1132a1160a0SJames Smart atomic_t xmt_ls_rsp; 1142a1160a0SJames Smart atomic_t xmt_ls_drop; 1152a1160a0SJames Smart 1162a1160a0SJames Smart /* Stats counters - lpfc_nvmet_xmt_ls_rsp_cmp */ 1172a1160a0SJames Smart atomic_t xmt_ls_rsp_error; 1182a1160a0SJames Smart atomic_t xmt_ls_rsp_aborted; 1192a1160a0SJames Smart atomic_t xmt_ls_rsp_xb_set; 1202a1160a0SJames Smart atomic_t xmt_ls_rsp_cmpl; 1212a1160a0SJames Smart 1222a1160a0SJames Smart /* Stats counters - lpfc_nvmet_unsol_fcp_buffer */ 1232a1160a0SJames Smart atomic_t rcv_fcp_cmd_in; 1242a1160a0SJames Smart atomic_t rcv_fcp_cmd_out; 1252a1160a0SJames Smart atomic_t rcv_fcp_cmd_drop; 1262a1160a0SJames Smart atomic_t rcv_fcp_cmd_defer; 1272a1160a0SJames Smart atomic_t xmt_fcp_release; 1282a1160a0SJames Smart 1292a1160a0SJames Smart /* Stats counters - lpfc_nvmet_xmt_fcp_op */ 1302a1160a0SJames Smart atomic_t xmt_fcp_drop; 1312a1160a0SJames Smart atomic_t xmt_fcp_read_rsp; 1322a1160a0SJames Smart atomic_t xmt_fcp_read; 1332a1160a0SJames Smart atomic_t xmt_fcp_write; 1342a1160a0SJames Smart atomic_t xmt_fcp_rsp; 1352a1160a0SJames Smart 1362a1160a0SJames Smart /* Stats counters - lpfc_nvmet_xmt_fcp_op_cmp */ 1372a1160a0SJames Smart atomic_t xmt_fcp_rsp_xb_set; 1382a1160a0SJames Smart atomic_t xmt_fcp_rsp_cmpl; 1392a1160a0SJames Smart atomic_t xmt_fcp_rsp_error; 1402a1160a0SJames Smart atomic_t xmt_fcp_rsp_aborted; 1412a1160a0SJames Smart atomic_t xmt_fcp_rsp_drop; 1422a1160a0SJames Smart 1432a1160a0SJames Smart /* Stats counters - lpfc_nvmet_xmt_fcp_abort */ 1442a1160a0SJames Smart atomic_t xmt_fcp_xri_abort_cqe; 1452a1160a0SJames Smart atomic_t xmt_fcp_abort; 1462a1160a0SJames Smart atomic_t xmt_fcp_abort_cmpl; 1472a1160a0SJames Smart atomic_t xmt_abort_sol; 1482a1160a0SJames Smart atomic_t xmt_abort_unsol; 1492a1160a0SJames Smart atomic_t xmt_abort_rsp; 1502a1160a0SJames Smart atomic_t xmt_abort_rsp_error; 1512a1160a0SJames Smart 1522a1160a0SJames Smart /* Stats counters - defer IO */ 1532a1160a0SJames Smart atomic_t defer_ctx; 1542a1160a0SJames Smart atomic_t defer_fod; 1552a1160a0SJames Smart atomic_t defer_wqfull; 1562a1160a0SJames Smart }; 1572a1160a0SJames Smart 1582a1160a0SJames Smart struct lpfc_nvmet_ctx_info { 1592a1160a0SJames Smart struct list_head nvmet_ctx_list; 1602a1160a0SJames Smart spinlock_t nvmet_ctx_list_lock; /* lock per CPU */ 1612a1160a0SJames Smart struct lpfc_nvmet_ctx_info *nvmet_ctx_next_cpu; 1622a1160a0SJames Smart struct lpfc_nvmet_ctx_info *nvmet_ctx_start_cpu; 1632a1160a0SJames Smart uint16_t nvmet_ctx_list_cnt; 1642a1160a0SJames Smart char pad[16]; /* pad to a cache-line */ 1652a1160a0SJames Smart }; 1662a1160a0SJames Smart 1672a1160a0SJames Smart /* This retrieves the context info associated with the specified cpu / mrq */ 1682a1160a0SJames Smart #define lpfc_get_ctx_list(phba, cpu, mrq) \ 1692a1160a0SJames Smart (phba->sli4_hba.nvmet_ctx_info + ((cpu * phba->cfg_nvmet_mrq) + mrq)) 1702a1160a0SJames Smart 1717b7f551bSJames Smart /* Values for state field of struct lpfc_async_xchg_ctx */ 1727b7f551bSJames Smart #define LPFC_NVME_STE_LS_RCV 1 1737b7f551bSJames Smart #define LPFC_NVME_STE_LS_ABORT 2 1747b7f551bSJames Smart #define LPFC_NVME_STE_LS_RSP 3 1757b7f551bSJames Smart #define LPFC_NVME_STE_RCV 4 1767b7f551bSJames Smart #define LPFC_NVME_STE_DATA 5 1777b7f551bSJames Smart #define LPFC_NVME_STE_ABORT 6 1787b7f551bSJames Smart #define LPFC_NVME_STE_DONE 7 1797b7f551bSJames Smart #define LPFC_NVME_STE_FREE 0xff 1807b7f551bSJames Smart 1817b7f551bSJames Smart /* Values for flag field of struct lpfc_async_xchg_ctx */ 1827b7f551bSJames Smart #define LPFC_NVME_IO_INP 0x1 /* IO is in progress on exchange */ 1837b7f551bSJames Smart #define LPFC_NVME_ABORT_OP 0x2 /* Abort WQE issued on exchange */ 1847b7f551bSJames Smart #define LPFC_NVME_XBUSY 0x4 /* XB bit set on IO cmpl */ 1857b7f551bSJames Smart #define LPFC_NVME_CTX_RLS 0x8 /* ctx free requested */ 1867b7f551bSJames Smart #define LPFC_NVME_ABTS_RCV 0x10 /* ABTS received on exchange */ 1877b7f551bSJames Smart #define LPFC_NVME_CTX_REUSE_WQ 0x20 /* ctx reused via WQ */ 1887b7f551bSJames Smart #define LPFC_NVME_DEFER_WQFULL 0x40 /* Waiting on a free WQE */ 1897b7f551bSJames Smart #define LPFC_NVME_TNOTIFY 0x80 /* notify transport of abts */ 1907b7f551bSJames Smart 1917cacae2aSJames Smart struct lpfc_async_xchg_ctx { 1922a1160a0SJames Smart union { 1932a1160a0SJames Smart struct nvmefc_tgt_fcp_req fcp_req; 1947cacae2aSJames Smart } hdlrctx; 1952a1160a0SJames Smart struct list_head list; 1962a1160a0SJames Smart struct lpfc_hba *phba; 1973a8070c5SJames Smart struct lpfc_nodelist *ndlp; 1987cacae2aSJames Smart struct nvmefc_ls_req *ls_req; 1997cacae2aSJames Smart struct nvmefc_ls_rsp ls_rsp; 2002a1160a0SJames Smart struct lpfc_iocbq *wqeq; 2012a1160a0SJames Smart struct lpfc_iocbq *abort_wqeq; 2022a1160a0SJames Smart spinlock_t ctxlock; /* protect flag access */ 2032a1160a0SJames Smart uint32_t sid; 2042a1160a0SJames Smart uint32_t offset; 2052a1160a0SJames Smart uint16_t oxid; 2062a1160a0SJames Smart uint16_t size; 2072a1160a0SJames Smart uint16_t entry_cnt; 2082a1160a0SJames Smart uint16_t cpu; 2092a1160a0SJames Smart uint16_t idx; 2102a1160a0SJames Smart uint16_t state; 2112a1160a0SJames Smart uint16_t flag; 2123a8070c5SJames Smart void *payload; 2132a1160a0SJames Smart struct rqb_dmabuf *rqb_buffer; 2142a1160a0SJames Smart struct lpfc_nvmet_ctxbuf *ctxbuf; 2152a1160a0SJames Smart struct lpfc_sli4_hdw_queue *hdwq; 2162a1160a0SJames Smart 2172a1160a0SJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 2182a1160a0SJames Smart uint64_t ts_isr_cmd; 2192a1160a0SJames Smart uint64_t ts_cmd_nvme; 2202a1160a0SJames Smart uint64_t ts_nvme_data; 2212a1160a0SJames Smart uint64_t ts_data_wqput; 2222a1160a0SJames Smart uint64_t ts_isr_data; 2232a1160a0SJames Smart uint64_t ts_data_nvme; 2242a1160a0SJames Smart uint64_t ts_nvme_status; 2252a1160a0SJames Smart uint64_t ts_status_wqput; 2262a1160a0SJames Smart uint64_t ts_isr_status; 2272a1160a0SJames Smart uint64_t ts_status_nvme; 2282a1160a0SJames Smart #endif 2292a1160a0SJames Smart }; 2302a1160a0SJames Smart 2312a1160a0SJames Smart 2322a1160a0SJames Smart /* routines found in lpfc_nvme.c */ 2336514b25dSJames Smart int __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, 2346514b25dSJames Smart struct nvmefc_ls_req *pnvme_lsreq, 2356514b25dSJames Smart void (*gen_req_cmp)(struct lpfc_hba *phba, 2366514b25dSJames Smart struct lpfc_iocbq *cmdwqe, 237a680a929SJames Smart struct lpfc_iocbq *rspwqe)); 2386514b25dSJames Smart void __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport, 2396514b25dSJames Smart struct lpfc_iocbq *cmdwqe, struct lpfc_wcqe_complete *wcqe); 240e96a22b0SJames Smart int __lpfc_nvme_ls_abort(struct lpfc_vport *vport, 241e96a22b0SJames Smart struct lpfc_nodelist *ndlp, struct nvmefc_ls_req *pnvme_lsreq); 2422a1160a0SJames Smart 2432a1160a0SJames Smart /* routines found in lpfc_nvmet.c */ 2443a8070c5SJames Smart int lpfc_nvme_unsol_ls_issue_abort(struct lpfc_hba *phba, 2453a8070c5SJames Smart struct lpfc_async_xchg_ctx *ctxp, uint32_t sid, 2463a8070c5SJames Smart uint16_t xri); 247fe1bedecSJames Smart int __lpfc_nvme_xmt_ls_rsp(struct lpfc_async_xchg_ctx *axchg, 248fe1bedecSJames Smart struct nvmefc_ls_rsp *ls_rsp, 249fe1bedecSJames Smart void (*xmt_ls_rsp_cmp)(struct lpfc_hba *phba, 250fe1bedecSJames Smart struct lpfc_iocbq *cmdwqe, 251a680a929SJames Smart struct lpfc_iocbq *rspwqe)); 252fe1bedecSJames Smart void __lpfc_nvme_xmt_ls_rsp_cmp(struct lpfc_hba *phba, 253a680a929SJames Smart struct lpfc_iocbq *cmdwqe, struct lpfc_iocbq *rspwqe); 254