1942b7654SJitendra Bhivare /* 2942b7654SJitendra Bhivare * Copyright 2017 Broadcom. All Rights Reserved. 3942b7654SJitendra Bhivare * The term "Broadcom" refers to Broadcom Limited and/or its subsidiaries. 46733b39aSJayamohan Kallickal * 56733b39aSJayamohan Kallickal * This program is free software; you can redistribute it and/or 66733b39aSJayamohan Kallickal * modify it under the terms of the GNU General Public License version 2 76733b39aSJayamohan Kallickal * as published by the Free Software Foundation. The full GNU General 86733b39aSJayamohan Kallickal * Public License is included in this distribution in the file called COPYING. 96733b39aSJayamohan Kallickal * 106733b39aSJayamohan Kallickal * Contact Information: 1160f36e04SJitendra Bhivare * linux-drivers@broadcom.com 126733b39aSJayamohan Kallickal * 136733b39aSJayamohan Kallickal */ 14255fa9a3SJayamohan Kallickal 156733b39aSJayamohan Kallickal #include <linux/reboot.h> 166733b39aSJayamohan Kallickal #include <linux/delay.h> 175a0e3ad6STejun Heo #include <linux/slab.h> 186733b39aSJayamohan Kallickal #include <linux/interrupt.h> 196733b39aSJayamohan Kallickal #include <linux/blkdev.h> 206733b39aSJayamohan Kallickal #include <linux/pci.h> 216733b39aSJayamohan Kallickal #include <linux/string.h> 226733b39aSJayamohan Kallickal #include <linux/kernel.h> 236733b39aSJayamohan Kallickal #include <linux/semaphore.h> 24c7acc5b8SJayamohan Kallickal #include <linux/iscsi_boot_sysfs.h> 25acf3368fSPaul Gortmaker #include <linux/module.h> 26ffce3e2eSJayamohan Kallickal #include <linux/bsg-lib.h> 271094cf68SJitendra Bhivare #include <linux/irq_poll.h> 286733b39aSJayamohan Kallickal 296733b39aSJayamohan Kallickal #include <scsi/libiscsi.h> 30ffce3e2eSJayamohan Kallickal #include <scsi/scsi_bsg_iscsi.h> 31ffce3e2eSJayamohan Kallickal #include <scsi/scsi_netlink.h> 326733b39aSJayamohan Kallickal #include <scsi/scsi_transport_iscsi.h> 336733b39aSJayamohan Kallickal #include <scsi/scsi_transport.h> 346733b39aSJayamohan Kallickal #include <scsi/scsi_cmnd.h> 356733b39aSJayamohan Kallickal #include <scsi/scsi_device.h> 366733b39aSJayamohan Kallickal #include <scsi/scsi_host.h> 376733b39aSJayamohan Kallickal #include <scsi/scsi.h> 386733b39aSJayamohan Kallickal #include "be_main.h" 396733b39aSJayamohan Kallickal #include "be_iscsi.h" 406733b39aSJayamohan Kallickal #include "be_mgmt.h" 410a513dd8SJohn Soni Jose #include "be_cmds.h" 426733b39aSJayamohan Kallickal 436733b39aSJayamohan Kallickal static unsigned int be_iopoll_budget = 10; 446733b39aSJayamohan Kallickal static unsigned int be_max_phys_size = 64; 45bfead3b2SJayamohan Kallickal static unsigned int enable_msix = 1; 466733b39aSJayamohan Kallickal 476733b39aSJayamohan Kallickal MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR); 4876d15dbdSJayamohan Kallickal MODULE_VERSION(BUILD_STR); 49c4f39bdaSKetan Mukadam MODULE_AUTHOR("Emulex Corporation"); 506733b39aSJayamohan Kallickal MODULE_LICENSE("GPL"); 516733b39aSJayamohan Kallickal module_param(be_iopoll_budget, int, 0); 526733b39aSJayamohan Kallickal module_param(enable_msix, int, 0); 536733b39aSJayamohan Kallickal module_param(be_max_phys_size, uint, S_IRUGO); 5499bc5d55SJohn Soni Jose MODULE_PARM_DESC(be_max_phys_size, 5599bc5d55SJohn Soni Jose "Maximum Size (In Kilobytes) of physically contiguous " 5699bc5d55SJohn Soni Jose "memory that can be allocated. Range is 16 - 128"); 5799bc5d55SJohn Soni Jose 5899bc5d55SJohn Soni Jose #define beiscsi_disp_param(_name)\ 590825b8eeSBaoyou Xie static ssize_t \ 6099bc5d55SJohn Soni Jose beiscsi_##_name##_disp(struct device *dev,\ 6199bc5d55SJohn Soni Jose struct device_attribute *attrib, char *buf) \ 6299bc5d55SJohn Soni Jose { \ 6399bc5d55SJohn Soni Jose struct Scsi_Host *shost = class_to_shost(dev);\ 6499bc5d55SJohn Soni Jose struct beiscsi_hba *phba = iscsi_host_priv(shost); \ 6599bc5d55SJohn Soni Jose return snprintf(buf, PAGE_SIZE, "%d\n",\ 6699bc5d55SJohn Soni Jose phba->attr_##_name);\ 6799bc5d55SJohn Soni Jose } 6899bc5d55SJohn Soni Jose 6999bc5d55SJohn Soni Jose #define beiscsi_change_param(_name, _minval, _maxval, _defaval)\ 700825b8eeSBaoyou Xie static int \ 7199bc5d55SJohn Soni Jose beiscsi_##_name##_change(struct beiscsi_hba *phba, uint32_t val)\ 7299bc5d55SJohn Soni Jose {\ 7399bc5d55SJohn Soni Jose if (val >= _minval && val <= _maxval) {\ 7499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\ 7599bc5d55SJohn Soni Jose "BA_%d : beiscsi_"#_name" updated "\ 7699bc5d55SJohn Soni Jose "from 0x%x ==> 0x%x\n",\ 7799bc5d55SJohn Soni Jose phba->attr_##_name, val); \ 7899bc5d55SJohn Soni Jose phba->attr_##_name = val;\ 7999bc5d55SJohn Soni Jose return 0;\ 8099bc5d55SJohn Soni Jose } \ 8199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, \ 8299bc5d55SJohn Soni Jose "BA_%d beiscsi_"#_name" attribute "\ 8399bc5d55SJohn Soni Jose "cannot be updated to 0x%x, "\ 8499bc5d55SJohn Soni Jose "range allowed is ["#_minval" - "#_maxval"]\n", val);\ 8599bc5d55SJohn Soni Jose return -EINVAL;\ 8699bc5d55SJohn Soni Jose } 8799bc5d55SJohn Soni Jose 8899bc5d55SJohn Soni Jose #define beiscsi_store_param(_name) \ 890825b8eeSBaoyou Xie static ssize_t \ 9099bc5d55SJohn Soni Jose beiscsi_##_name##_store(struct device *dev,\ 9199bc5d55SJohn Soni Jose struct device_attribute *attr, const char *buf,\ 9299bc5d55SJohn Soni Jose size_t count) \ 9399bc5d55SJohn Soni Jose { \ 9499bc5d55SJohn Soni Jose struct Scsi_Host *shost = class_to_shost(dev);\ 9599bc5d55SJohn Soni Jose struct beiscsi_hba *phba = iscsi_host_priv(shost);\ 9699bc5d55SJohn Soni Jose uint32_t param_val = 0;\ 9799bc5d55SJohn Soni Jose if (!isdigit(buf[0]))\ 9899bc5d55SJohn Soni Jose return -EINVAL;\ 9999bc5d55SJohn Soni Jose if (sscanf(buf, "%i", ¶m_val) != 1)\ 10099bc5d55SJohn Soni Jose return -EINVAL;\ 10199bc5d55SJohn Soni Jose if (beiscsi_##_name##_change(phba, param_val) == 0) \ 10299bc5d55SJohn Soni Jose return strlen(buf);\ 10399bc5d55SJohn Soni Jose else \ 10499bc5d55SJohn Soni Jose return -EINVAL;\ 10599bc5d55SJohn Soni Jose } 10699bc5d55SJohn Soni Jose 10799bc5d55SJohn Soni Jose #define beiscsi_init_param(_name, _minval, _maxval, _defval) \ 1080825b8eeSBaoyou Xie static int \ 10999bc5d55SJohn Soni Jose beiscsi_##_name##_init(struct beiscsi_hba *phba, uint32_t val) \ 11099bc5d55SJohn Soni Jose { \ 11199bc5d55SJohn Soni Jose if (val >= _minval && val <= _maxval) {\ 11299bc5d55SJohn Soni Jose phba->attr_##_name = val;\ 11399bc5d55SJohn Soni Jose return 0;\ 11499bc5d55SJohn Soni Jose } \ 11599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\ 11699bc5d55SJohn Soni Jose "BA_%d beiscsi_"#_name" attribute " \ 11799bc5d55SJohn Soni Jose "cannot be updated to 0x%x, "\ 11899bc5d55SJohn Soni Jose "range allowed is ["#_minval" - "#_maxval"]\n", val);\ 11999bc5d55SJohn Soni Jose phba->attr_##_name = _defval;\ 12099bc5d55SJohn Soni Jose return -EINVAL;\ 12199bc5d55SJohn Soni Jose } 12299bc5d55SJohn Soni Jose 12399bc5d55SJohn Soni Jose #define BEISCSI_RW_ATTR(_name, _minval, _maxval, _defval, _descp) \ 12499bc5d55SJohn Soni Jose static uint beiscsi_##_name = _defval;\ 12599bc5d55SJohn Soni Jose module_param(beiscsi_##_name, uint, S_IRUGO);\ 12699bc5d55SJohn Soni Jose MODULE_PARM_DESC(beiscsi_##_name, _descp);\ 12799bc5d55SJohn Soni Jose beiscsi_disp_param(_name)\ 12899bc5d55SJohn Soni Jose beiscsi_change_param(_name, _minval, _maxval, _defval)\ 12999bc5d55SJohn Soni Jose beiscsi_store_param(_name)\ 13099bc5d55SJohn Soni Jose beiscsi_init_param(_name, _minval, _maxval, _defval)\ 13199bc5d55SJohn Soni Jose DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\ 13299bc5d55SJohn Soni Jose beiscsi_##_name##_disp, beiscsi_##_name##_store) 13399bc5d55SJohn Soni Jose 13499bc5d55SJohn Soni Jose /* 13599bc5d55SJohn Soni Jose * When new log level added update the 13699bc5d55SJohn Soni Jose * the MAX allowed value for log_enable 13799bc5d55SJohn Soni Jose */ 13899bc5d55SJohn Soni Jose BEISCSI_RW_ATTR(log_enable, 0x00, 13999bc5d55SJohn Soni Jose 0xFF, 0x00, "Enable logging Bit Mask\n" 14099bc5d55SJohn Soni Jose "\t\t\t\tInitialization Events : 0x01\n" 14199bc5d55SJohn Soni Jose "\t\t\t\tMailbox Events : 0x02\n" 14299bc5d55SJohn Soni Jose "\t\t\t\tMiscellaneous Events : 0x04\n" 14399bc5d55SJohn Soni Jose "\t\t\t\tError Handling : 0x08\n" 14499bc5d55SJohn Soni Jose "\t\t\t\tIO Path Events : 0x10\n" 145afb96058SJayamohan Kallickal "\t\t\t\tConfiguration Path : 0x20\n" 146afb96058SJayamohan Kallickal "\t\t\t\tiSCSI Protocol : 0x40\n"); 14799bc5d55SJohn Soni Jose 1485cac7596SJohn Soni Jose DEVICE_ATTR(beiscsi_drvr_ver, S_IRUGO, beiscsi_drvr_ver_disp, NULL); 14926000db7SJohn Soni Jose DEVICE_ATTR(beiscsi_adapter_family, S_IRUGO, beiscsi_adap_family_disp, NULL); 15022661e25SJayamohan Kallickal DEVICE_ATTR(beiscsi_fw_ver, S_IRUGO, beiscsi_fw_ver_disp, NULL); 151d3fea9afSJayamohan Kallickal DEVICE_ATTR(beiscsi_phys_port, S_IRUGO, beiscsi_phys_port_disp, NULL); 1526103c1f7SJayamohan Kallickal DEVICE_ATTR(beiscsi_active_session_count, S_IRUGO, 1536103c1f7SJayamohan Kallickal beiscsi_active_session_disp, NULL); 1546103c1f7SJayamohan Kallickal DEVICE_ATTR(beiscsi_free_session_count, S_IRUGO, 1556103c1f7SJayamohan Kallickal beiscsi_free_session_disp, NULL); 15699bc5d55SJohn Soni Jose struct device_attribute *beiscsi_attrs[] = { 15799bc5d55SJohn Soni Jose &dev_attr_beiscsi_log_enable, 1585cac7596SJohn Soni Jose &dev_attr_beiscsi_drvr_ver, 15926000db7SJohn Soni Jose &dev_attr_beiscsi_adapter_family, 16022661e25SJayamohan Kallickal &dev_attr_beiscsi_fw_ver, 1616103c1f7SJayamohan Kallickal &dev_attr_beiscsi_active_session_count, 1626103c1f7SJayamohan Kallickal &dev_attr_beiscsi_free_session_count, 163d3fea9afSJayamohan Kallickal &dev_attr_beiscsi_phys_port, 16499bc5d55SJohn Soni Jose NULL, 16599bc5d55SJohn Soni Jose }; 1666733b39aSJayamohan Kallickal 1676763daaeSJohn Soni Jose static char const *cqe_desc[] = { 1686763daaeSJohn Soni Jose "RESERVED_DESC", 1696763daaeSJohn Soni Jose "SOL_CMD_COMPLETE", 1706763daaeSJohn Soni Jose "SOL_CMD_KILLED_DATA_DIGEST_ERR", 1716763daaeSJohn Soni Jose "CXN_KILLED_PDU_SIZE_EXCEEDS_DSL", 1726763daaeSJohn Soni Jose "CXN_KILLED_BURST_LEN_MISMATCH", 1736763daaeSJohn Soni Jose "CXN_KILLED_AHS_RCVD", 1746763daaeSJohn Soni Jose "CXN_KILLED_HDR_DIGEST_ERR", 1756763daaeSJohn Soni Jose "CXN_KILLED_UNKNOWN_HDR", 1766763daaeSJohn Soni Jose "CXN_KILLED_STALE_ITT_TTT_RCVD", 1776763daaeSJohn Soni Jose "CXN_KILLED_INVALID_ITT_TTT_RCVD", 1786763daaeSJohn Soni Jose "CXN_KILLED_RST_RCVD", 1796763daaeSJohn Soni Jose "CXN_KILLED_TIMED_OUT", 1806763daaeSJohn Soni Jose "CXN_KILLED_RST_SENT", 1816763daaeSJohn Soni Jose "CXN_KILLED_FIN_RCVD", 1826763daaeSJohn Soni Jose "CXN_KILLED_BAD_UNSOL_PDU_RCVD", 1836763daaeSJohn Soni Jose "CXN_KILLED_BAD_WRB_INDEX_ERROR", 1846763daaeSJohn Soni Jose "CXN_KILLED_OVER_RUN_RESIDUAL", 1856763daaeSJohn Soni Jose "CXN_KILLED_UNDER_RUN_RESIDUAL", 1866763daaeSJohn Soni Jose "CMD_KILLED_INVALID_STATSN_RCVD", 1876763daaeSJohn Soni Jose "CMD_KILLED_INVALID_R2T_RCVD", 1886763daaeSJohn Soni Jose "CMD_CXN_KILLED_LUN_INVALID", 1896763daaeSJohn Soni Jose "CMD_CXN_KILLED_ICD_INVALID", 1906763daaeSJohn Soni Jose "CMD_CXN_KILLED_ITT_INVALID", 1916763daaeSJohn Soni Jose "CMD_CXN_KILLED_SEQ_OUTOFORDER", 1926763daaeSJohn Soni Jose "CMD_CXN_KILLED_INVALID_DATASN_RCVD", 1936763daaeSJohn Soni Jose "CXN_INVALIDATE_NOTIFY", 1946763daaeSJohn Soni Jose "CXN_INVALIDATE_INDEX_NOTIFY", 1956763daaeSJohn Soni Jose "CMD_INVALIDATED_NOTIFY", 1966763daaeSJohn Soni Jose "UNSOL_HDR_NOTIFY", 1976763daaeSJohn Soni Jose "UNSOL_DATA_NOTIFY", 1986763daaeSJohn Soni Jose "UNSOL_DATA_DIGEST_ERROR_NOTIFY", 1996763daaeSJohn Soni Jose "DRIVERMSG_NOTIFY", 2006763daaeSJohn Soni Jose "CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN", 2016763daaeSJohn Soni Jose "SOL_CMD_KILLED_DIF_ERR", 2026763daaeSJohn Soni Jose "CXN_KILLED_SYN_RCVD", 2036763daaeSJohn Soni Jose "CXN_KILLED_IMM_DATA_RCVD" 2046763daaeSJohn Soni Jose }; 2056763daaeSJohn Soni Jose 2066733b39aSJayamohan Kallickal static int beiscsi_slave_configure(struct scsi_device *sdev) 2076733b39aSJayamohan Kallickal { 2086733b39aSJayamohan Kallickal blk_queue_max_segment_size(sdev->request_queue, 65536); 2096733b39aSJayamohan Kallickal return 0; 2106733b39aSJayamohan Kallickal } 2116733b39aSJayamohan Kallickal 2124183122dSJayamohan Kallickal static int beiscsi_eh_abort(struct scsi_cmnd *sc) 2134183122dSJayamohan Kallickal { 214faa0a22dSJitendra Bhivare struct iscsi_task *abrt_task = (struct iscsi_task *)sc->SCp.ptr; 2154183122dSJayamohan Kallickal struct iscsi_cls_session *cls_session; 216faa0a22dSJitendra Bhivare struct beiscsi_io_task *abrt_io_task; 2174183122dSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn; 2184183122dSJayamohan Kallickal struct iscsi_session *session; 219f3505013SJitendra Bhivare struct invldt_cmd_tbl inv_tbl; 220faa0a22dSJitendra Bhivare struct beiscsi_hba *phba; 221faa0a22dSJitendra Bhivare struct iscsi_conn *conn; 2221957aa7fSJayamohan Kallickal int rc; 2234183122dSJayamohan Kallickal 2244183122dSJayamohan Kallickal cls_session = starget_to_session(scsi_target(sc->device)); 2254183122dSJayamohan Kallickal session = cls_session->dd_data; 2264183122dSJayamohan Kallickal 227faa0a22dSJitendra Bhivare /* check if we raced, task just got cleaned up under us */ 228faa0a22dSJitendra Bhivare spin_lock_bh(&session->back_lock); 229faa0a22dSJitendra Bhivare if (!abrt_task || !abrt_task->sc) { 230faa0a22dSJitendra Bhivare spin_unlock_bh(&session->back_lock); 2314183122dSJayamohan Kallickal return SUCCESS; 2324183122dSJayamohan Kallickal } 233faa0a22dSJitendra Bhivare /* get a task ref till FW processes the req for the ICD used */ 234faa0a22dSJitendra Bhivare __iscsi_get_task(abrt_task); 235faa0a22dSJitendra Bhivare abrt_io_task = abrt_task->dd_data; 236faa0a22dSJitendra Bhivare conn = abrt_task->conn; 2374183122dSJayamohan Kallickal beiscsi_conn = conn->dd_data; 2384183122dSJayamohan Kallickal phba = beiscsi_conn->phba; 239faa0a22dSJitendra Bhivare /* mark WRB invalid which have been not processed by FW yet */ 240392b7d2fSJitendra Bhivare if (is_chip_be2_be3r(phba)) { 241f3505013SJitendra Bhivare AMAP_SET_BITS(struct amap_iscsi_wrb, invld, 242faa0a22dSJitendra Bhivare abrt_io_task->pwrb_handle->pwrb, 1); 243392b7d2fSJitendra Bhivare } else { 244392b7d2fSJitendra Bhivare AMAP_SET_BITS(struct amap_iscsi_wrb_v2, invld, 245392b7d2fSJitendra Bhivare abrt_io_task->pwrb_handle->pwrb, 1); 246392b7d2fSJitendra Bhivare } 247faa0a22dSJitendra Bhivare inv_tbl.cid = beiscsi_conn->beiscsi_conn_cid; 248faa0a22dSJitendra Bhivare inv_tbl.icd = abrt_io_task->psgl_handle->sgl_index; 249faa0a22dSJitendra Bhivare spin_unlock_bh(&session->back_lock); 250faa0a22dSJitendra Bhivare 25198713216SJitendra Bhivare rc = beiscsi_mgmt_invalidate_icds(phba, &inv_tbl, 1); 252faa0a22dSJitendra Bhivare iscsi_put_task(abrt_task); 25398713216SJitendra Bhivare if (rc) { 25499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH, 25598713216SJitendra Bhivare "BM_%d : sc %p invalidation failed %d\n", 25698713216SJitendra Bhivare sc, rc); 2574183122dSJayamohan Kallickal return FAILED; 2584183122dSJayamohan Kallickal } 259e175defeSJohn Soni Jose 2604183122dSJayamohan Kallickal return iscsi_eh_abort(sc); 2614183122dSJayamohan Kallickal } 2624183122dSJayamohan Kallickal 2634183122dSJayamohan Kallickal static int beiscsi_eh_device_reset(struct scsi_cmnd *sc) 2644183122dSJayamohan Kallickal { 26598713216SJitendra Bhivare struct beiscsi_invldt_cmd_tbl { 26698713216SJitendra Bhivare struct invldt_cmd_tbl tbl[BE_INVLDT_CMD_TBL_SZ]; 26798713216SJitendra Bhivare struct iscsi_task *task[BE_INVLDT_CMD_TBL_SZ]; 26898713216SJitendra Bhivare } *inv_tbl; 2694183122dSJayamohan Kallickal struct iscsi_cls_session *cls_session; 27098713216SJitendra Bhivare struct beiscsi_conn *beiscsi_conn; 27198713216SJitendra Bhivare struct beiscsi_io_task *io_task; 27298713216SJitendra Bhivare struct iscsi_session *session; 27398713216SJitendra Bhivare struct beiscsi_hba *phba; 27498713216SJitendra Bhivare struct iscsi_conn *conn; 27598713216SJitendra Bhivare struct iscsi_task *task; 27698713216SJitendra Bhivare unsigned int i, nents; 277f3505013SJitendra Bhivare int rc, more = 0; 2784183122dSJayamohan Kallickal 2794183122dSJayamohan Kallickal cls_session = starget_to_session(scsi_target(sc->device)); 2804183122dSJayamohan Kallickal session = cls_session->dd_data; 28198713216SJitendra Bhivare 282659743b0SShlomo Pongratz spin_lock_bh(&session->frwd_lock); 283db7f7709SJayamohan Kallickal if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) { 284659743b0SShlomo Pongratz spin_unlock_bh(&session->frwd_lock); 285db7f7709SJayamohan Kallickal return FAILED; 286db7f7709SJayamohan Kallickal } 28798713216SJitendra Bhivare 2884183122dSJayamohan Kallickal conn = session->leadconn; 2894183122dSJayamohan Kallickal beiscsi_conn = conn->dd_data; 2904183122dSJayamohan Kallickal phba = beiscsi_conn->phba; 291f3505013SJitendra Bhivare 292f2534736SWei Yongjun inv_tbl = kzalloc(sizeof(*inv_tbl), GFP_ATOMIC); 293f3505013SJitendra Bhivare if (!inv_tbl) { 294f3505013SJitendra Bhivare spin_unlock_bh(&session->frwd_lock); 295f3505013SJitendra Bhivare beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH, 296f3505013SJitendra Bhivare "BM_%d : invldt_cmd_tbl alloc failed\n"); 297f3505013SJitendra Bhivare return FAILED; 298f3505013SJitendra Bhivare } 299f3505013SJitendra Bhivare nents = 0; 30098713216SJitendra Bhivare /* take back_lock to prevent task from getting cleaned up under us */ 30198713216SJitendra Bhivare spin_lock(&session->back_lock); 3024183122dSJayamohan Kallickal for (i = 0; i < conn->session->cmds_max; i++) { 30398713216SJitendra Bhivare task = conn->session->cmds[i]; 30498713216SJitendra Bhivare if (!task->sc) 3054183122dSJayamohan Kallickal continue; 3064183122dSJayamohan Kallickal 30798713216SJitendra Bhivare if (sc->device->lun != task->sc->device->lun) 3084183122dSJayamohan Kallickal continue; 309f3505013SJitendra Bhivare /** 310f3505013SJitendra Bhivare * Can't fit in more cmds? Normally this won't happen b'coz 311f3505013SJitendra Bhivare * BEISCSI_CMD_PER_LUN is same as BE_INVLDT_CMD_TBL_SZ. 312f3505013SJitendra Bhivare */ 313f3505013SJitendra Bhivare if (nents == BE_INVLDT_CMD_TBL_SZ) { 314f3505013SJitendra Bhivare more = 1; 315f3505013SJitendra Bhivare break; 316f3505013SJitendra Bhivare } 3174183122dSJayamohan Kallickal 31898713216SJitendra Bhivare /* get a task ref till FW processes the req for the ICD used */ 31998713216SJitendra Bhivare __iscsi_get_task(task); 32098713216SJitendra Bhivare io_task = task->dd_data; 32198713216SJitendra Bhivare /* mark WRB invalid which have been not processed by FW yet */ 322392b7d2fSJitendra Bhivare if (is_chip_be2_be3r(phba)) { 3237626c06bSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, invld, 324392b7d2fSJitendra Bhivare io_task->pwrb_handle->pwrb, 1); 325392b7d2fSJitendra Bhivare } else { 326392b7d2fSJitendra Bhivare AMAP_SET_BITS(struct amap_iscsi_wrb_v2, invld, 327392b7d2fSJitendra Bhivare io_task->pwrb_handle->pwrb, 1); 328392b7d2fSJitendra Bhivare } 3297626c06bSJayamohan Kallickal 33098713216SJitendra Bhivare inv_tbl->tbl[nents].cid = beiscsi_conn->beiscsi_conn_cid; 33198713216SJitendra Bhivare inv_tbl->tbl[nents].icd = io_task->psgl_handle->sgl_index; 33298713216SJitendra Bhivare inv_tbl->task[nents] = task; 333f3505013SJitendra Bhivare nents++; 3344183122dSJayamohan Kallickal } 335d1e1d63bSJitendra Bhivare spin_unlock(&session->back_lock); 336659743b0SShlomo Pongratz spin_unlock_bh(&session->frwd_lock); 3374183122dSJayamohan Kallickal 33898713216SJitendra Bhivare rc = SUCCESS; 33998713216SJitendra Bhivare if (!nents) 34098713216SJitendra Bhivare goto end_reset; 34198713216SJitendra Bhivare 342f3505013SJitendra Bhivare if (more) { 343f3505013SJitendra Bhivare beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH, 344f3505013SJitendra Bhivare "BM_%d : number of cmds exceeds size of invalidation table\n"); 34598713216SJitendra Bhivare rc = FAILED; 34698713216SJitendra Bhivare goto end_reset; 347f3505013SJitendra Bhivare } 348f3505013SJitendra Bhivare 34998713216SJitendra Bhivare if (beiscsi_mgmt_invalidate_icds(phba, &inv_tbl->tbl[0], nents)) { 35099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH, 35198713216SJitendra Bhivare "BM_%d : cid %u scmds invalidation failed\n", 35298713216SJitendra Bhivare beiscsi_conn->beiscsi_conn_cid); 35398713216SJitendra Bhivare rc = FAILED; 3544183122dSJayamohan Kallickal } 355e175defeSJohn Soni Jose 35698713216SJitendra Bhivare end_reset: 35798713216SJitendra Bhivare for (i = 0; i < nents; i++) 35898713216SJitendra Bhivare iscsi_put_task(inv_tbl->task[i]); 35998713216SJitendra Bhivare kfree(inv_tbl); 36098713216SJitendra Bhivare 36198713216SJitendra Bhivare if (rc == SUCCESS) 36298713216SJitendra Bhivare rc = iscsi_eh_device_reset(sc); 36398713216SJitendra Bhivare return rc; 3644183122dSJayamohan Kallickal } 3654183122dSJayamohan Kallickal 366bfead3b2SJayamohan Kallickal /*------------------- PCI Driver operations and data ----------------- */ 3679baa3c34SBenoit Taine static const struct pci_device_id beiscsi_pci_id_table[] = { 368bfead3b2SJayamohan Kallickal { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) }, 369f98c96b0SJayamohan Kallickal { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) }, 370bfead3b2SJayamohan Kallickal { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) }, 371bfead3b2SJayamohan Kallickal { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) }, 372bfead3b2SJayamohan Kallickal { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) }, 373139a1b1eSJohn Soni Jose { PCI_DEVICE(ELX_VENDOR_ID, OC_SKH_ID1) }, 374bfead3b2SJayamohan Kallickal { 0 } 375bfead3b2SJayamohan Kallickal }; 376bfead3b2SJayamohan Kallickal MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table); 377bfead3b2SJayamohan Kallickal 37899bc5d55SJohn Soni Jose 3796733b39aSJayamohan Kallickal static struct scsi_host_template beiscsi_sht = { 3806733b39aSJayamohan Kallickal .module = THIS_MODULE, 381c4f39bdaSKetan Mukadam .name = "Emulex 10Gbe open-iscsi Initiator Driver", 3826733b39aSJayamohan Kallickal .proc_name = DRV_NAME, 3836733b39aSJayamohan Kallickal .queuecommand = iscsi_queuecommand, 384db5ed4dfSChristoph Hellwig .change_queue_depth = scsi_change_queue_depth, 3856733b39aSJayamohan Kallickal .slave_configure = beiscsi_slave_configure, 3866733b39aSJayamohan Kallickal .target_alloc = iscsi_target_alloc, 387b6a05c82SChristoph Hellwig .eh_timed_out = iscsi_eh_cmd_timed_out, 3884183122dSJayamohan Kallickal .eh_abort_handler = beiscsi_eh_abort, 3894183122dSJayamohan Kallickal .eh_device_reset_handler = beiscsi_eh_device_reset, 390309ce156SJayamohan Kallickal .eh_target_reset_handler = iscsi_eh_session_reset, 39199bc5d55SJohn Soni Jose .shost_attrs = beiscsi_attrs, 3926733b39aSJayamohan Kallickal .sg_tablesize = BEISCSI_SGLIST_ELEMENTS, 3936733b39aSJayamohan Kallickal .can_queue = BE2_IO_DEPTH, 3946733b39aSJayamohan Kallickal .this_id = -1, 3956733b39aSJayamohan Kallickal .max_sectors = BEISCSI_MAX_SECTORS, 3966733b39aSJayamohan Kallickal .cmd_per_lun = BEISCSI_CMD_PER_LUN, 3976733b39aSJayamohan Kallickal .use_clustering = ENABLE_CLUSTERING, 398ffce3e2eSJayamohan Kallickal .vendor_id = SCSI_NL_VID_TYPE_PCI | BE_VENDOR_ID, 399c40ecc12SChristoph Hellwig .track_queue_depth = 1, 4006733b39aSJayamohan Kallickal }; 4016733b39aSJayamohan Kallickal 402bfead3b2SJayamohan Kallickal static struct scsi_transport_template *beiscsi_scsi_transport; 4036733b39aSJayamohan Kallickal 4046733b39aSJayamohan Kallickal static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev) 4056733b39aSJayamohan Kallickal { 4066733b39aSJayamohan Kallickal struct beiscsi_hba *phba; 4076733b39aSJayamohan Kallickal struct Scsi_Host *shost; 4086733b39aSJayamohan Kallickal 4096733b39aSJayamohan Kallickal shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0); 4106733b39aSJayamohan Kallickal if (!shost) { 41199bc5d55SJohn Soni Jose dev_err(&pcidev->dev, 41299bc5d55SJohn Soni Jose "beiscsi_hba_alloc - iscsi_host_alloc failed\n"); 4136733b39aSJayamohan Kallickal return NULL; 4146733b39aSJayamohan Kallickal } 4156733b39aSJayamohan Kallickal shost->max_id = BE2_MAX_SESSIONS; 4166733b39aSJayamohan Kallickal shost->max_channel = 0; 4176733b39aSJayamohan Kallickal shost->max_cmd_len = BEISCSI_MAX_CMD_LEN; 4186733b39aSJayamohan Kallickal shost->max_lun = BEISCSI_NUM_MAX_LUN; 4196733b39aSJayamohan Kallickal shost->transportt = beiscsi_scsi_transport; 4206733b39aSJayamohan Kallickal phba = iscsi_host_priv(shost); 4216733b39aSJayamohan Kallickal memset(phba, 0, sizeof(*phba)); 4226733b39aSJayamohan Kallickal phba->shost = shost; 4236733b39aSJayamohan Kallickal phba->pcidev = pci_dev_get(pcidev); 4242807afb7SJayamohan Kallickal pci_set_drvdata(pcidev, phba); 4250e43895eSMike Christie phba->interface_handle = 0xFFFFFFFF; 4266733b39aSJayamohan Kallickal 4276733b39aSJayamohan Kallickal return phba; 4286733b39aSJayamohan Kallickal } 4296733b39aSJayamohan Kallickal 4306733b39aSJayamohan Kallickal static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba) 4316733b39aSJayamohan Kallickal { 4326733b39aSJayamohan Kallickal if (phba->csr_va) { 4336733b39aSJayamohan Kallickal iounmap(phba->csr_va); 4346733b39aSJayamohan Kallickal phba->csr_va = NULL; 4356733b39aSJayamohan Kallickal } 4366733b39aSJayamohan Kallickal if (phba->db_va) { 4376733b39aSJayamohan Kallickal iounmap(phba->db_va); 4386733b39aSJayamohan Kallickal phba->db_va = NULL; 4396733b39aSJayamohan Kallickal } 4406733b39aSJayamohan Kallickal if (phba->pci_va) { 4416733b39aSJayamohan Kallickal iounmap(phba->pci_va); 4426733b39aSJayamohan Kallickal phba->pci_va = NULL; 4436733b39aSJayamohan Kallickal } 4446733b39aSJayamohan Kallickal } 4456733b39aSJayamohan Kallickal 4466733b39aSJayamohan Kallickal static int beiscsi_map_pci_bars(struct beiscsi_hba *phba, 4476733b39aSJayamohan Kallickal struct pci_dev *pcidev) 4486733b39aSJayamohan Kallickal { 4496733b39aSJayamohan Kallickal u8 __iomem *addr; 450f98c96b0SJayamohan Kallickal int pcicfg_reg; 4516733b39aSJayamohan Kallickal 4526733b39aSJayamohan Kallickal addr = ioremap_nocache(pci_resource_start(pcidev, 2), 4536733b39aSJayamohan Kallickal pci_resource_len(pcidev, 2)); 4546733b39aSJayamohan Kallickal if (addr == NULL) 4556733b39aSJayamohan Kallickal return -ENOMEM; 4566733b39aSJayamohan Kallickal phba->ctrl.csr = addr; 4576733b39aSJayamohan Kallickal phba->csr_va = addr; 4586733b39aSJayamohan Kallickal phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2); 4596733b39aSJayamohan Kallickal 4606733b39aSJayamohan Kallickal addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024); 4616733b39aSJayamohan Kallickal if (addr == NULL) 4626733b39aSJayamohan Kallickal goto pci_map_err; 4636733b39aSJayamohan Kallickal phba->ctrl.db = addr; 4646733b39aSJayamohan Kallickal phba->db_va = addr; 4656733b39aSJayamohan Kallickal phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4); 4666733b39aSJayamohan Kallickal 467f98c96b0SJayamohan Kallickal if (phba->generation == BE_GEN2) 468f98c96b0SJayamohan Kallickal pcicfg_reg = 1; 469f98c96b0SJayamohan Kallickal else 470f98c96b0SJayamohan Kallickal pcicfg_reg = 0; 471f98c96b0SJayamohan Kallickal 472f98c96b0SJayamohan Kallickal addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg), 473f98c96b0SJayamohan Kallickal pci_resource_len(pcidev, pcicfg_reg)); 474f98c96b0SJayamohan Kallickal 4756733b39aSJayamohan Kallickal if (addr == NULL) 4766733b39aSJayamohan Kallickal goto pci_map_err; 4776733b39aSJayamohan Kallickal phba->ctrl.pcicfg = addr; 4786733b39aSJayamohan Kallickal phba->pci_va = addr; 479f98c96b0SJayamohan Kallickal phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg); 4806733b39aSJayamohan Kallickal return 0; 4816733b39aSJayamohan Kallickal 4826733b39aSJayamohan Kallickal pci_map_err: 4836733b39aSJayamohan Kallickal beiscsi_unmap_pci_function(phba); 4846733b39aSJayamohan Kallickal return -ENOMEM; 4856733b39aSJayamohan Kallickal } 4866733b39aSJayamohan Kallickal 4876733b39aSJayamohan Kallickal static int beiscsi_enable_pci(struct pci_dev *pcidev) 4886733b39aSJayamohan Kallickal { 4896733b39aSJayamohan Kallickal int ret; 4906733b39aSJayamohan Kallickal 4916733b39aSJayamohan Kallickal ret = pci_enable_device(pcidev); 4926733b39aSJayamohan Kallickal if (ret) { 49399bc5d55SJohn Soni Jose dev_err(&pcidev->dev, 49499bc5d55SJohn Soni Jose "beiscsi_enable_pci - enable device failed\n"); 4956733b39aSJayamohan Kallickal return ret; 4966733b39aSJayamohan Kallickal } 4976733b39aSJayamohan Kallickal 498e307f3acSJohn Soni Jose ret = pci_request_regions(pcidev, DRV_NAME); 499e307f3acSJohn Soni Jose if (ret) { 500e307f3acSJohn Soni Jose dev_err(&pcidev->dev, 501e307f3acSJohn Soni Jose "beiscsi_enable_pci - request region failed\n"); 502e307f3acSJohn Soni Jose goto pci_dev_disable; 503e307f3acSJohn Soni Jose } 504e307f3acSJohn Soni Jose 505bfead3b2SJayamohan Kallickal pci_set_master(pcidev); 5066c57625bSJayamohan Kallickal ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(64)); 5076c57625bSJayamohan Kallickal if (ret) { 5086c57625bSJayamohan Kallickal ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(32)); 5096c57625bSJayamohan Kallickal if (ret) { 5106c57625bSJayamohan Kallickal dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n"); 511e307f3acSJohn Soni Jose goto pci_region_release; 5126c57625bSJayamohan Kallickal } else { 5136c57625bSJayamohan Kallickal ret = pci_set_consistent_dma_mask(pcidev, 5146c57625bSJayamohan Kallickal DMA_BIT_MASK(32)); 5156c57625bSJayamohan Kallickal } 5166c57625bSJayamohan Kallickal } else { 5176c57625bSJayamohan Kallickal ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64)); 5186733b39aSJayamohan Kallickal if (ret) { 5196733b39aSJayamohan Kallickal dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n"); 520e307f3acSJohn Soni Jose goto pci_region_release; 5216733b39aSJayamohan Kallickal } 5226733b39aSJayamohan Kallickal } 5236733b39aSJayamohan Kallickal return 0; 524e307f3acSJohn Soni Jose 525e307f3acSJohn Soni Jose pci_region_release: 526e307f3acSJohn Soni Jose pci_release_regions(pcidev); 527e307f3acSJohn Soni Jose pci_dev_disable: 528e307f3acSJohn Soni Jose pci_disable_device(pcidev); 529e307f3acSJohn Soni Jose 530e307f3acSJohn Soni Jose return ret; 5316733b39aSJayamohan Kallickal } 5326733b39aSJayamohan Kallickal 5336733b39aSJayamohan Kallickal static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev) 5346733b39aSJayamohan Kallickal { 5356733b39aSJayamohan Kallickal struct be_ctrl_info *ctrl = &phba->ctrl; 5366733b39aSJayamohan Kallickal struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced; 5376733b39aSJayamohan Kallickal struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem; 5386733b39aSJayamohan Kallickal int status = 0; 5396733b39aSJayamohan Kallickal 5406733b39aSJayamohan Kallickal ctrl->pdev = pdev; 5416733b39aSJayamohan Kallickal status = beiscsi_map_pci_bars(phba, pdev); 5426733b39aSJayamohan Kallickal if (status) 5436733b39aSJayamohan Kallickal return status; 5446733b39aSJayamohan Kallickal mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16; 5456733b39aSJayamohan Kallickal mbox_mem_alloc->va = pci_alloc_consistent(pdev, 5466733b39aSJayamohan Kallickal mbox_mem_alloc->size, 5476733b39aSJayamohan Kallickal &mbox_mem_alloc->dma); 5486733b39aSJayamohan Kallickal if (!mbox_mem_alloc->va) { 5496733b39aSJayamohan Kallickal beiscsi_unmap_pci_function(phba); 550a49e06d5SJayamohan Kallickal return -ENOMEM; 5516733b39aSJayamohan Kallickal } 5526733b39aSJayamohan Kallickal 5536733b39aSJayamohan Kallickal mbox_mem_align->size = sizeof(struct be_mcc_mailbox); 5546733b39aSJayamohan Kallickal mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16); 5556733b39aSJayamohan Kallickal mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16); 5566733b39aSJayamohan Kallickal memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox)); 557c03a50f7SJitendra Bhivare mutex_init(&ctrl->mbox_lock); 558bfead3b2SJayamohan Kallickal spin_lock_init(&phba->ctrl.mcc_lock); 559bfead3b2SJayamohan Kallickal 5606733b39aSJayamohan Kallickal return status; 5616733b39aSJayamohan Kallickal } 5626733b39aSJayamohan Kallickal 563843ae752SJayamohan Kallickal /** 564843ae752SJayamohan Kallickal * beiscsi_get_params()- Set the config paramters 565843ae752SJayamohan Kallickal * @phba: ptr device priv structure 566843ae752SJayamohan Kallickal **/ 5676733b39aSJayamohan Kallickal static void beiscsi_get_params(struct beiscsi_hba *phba) 5686733b39aSJayamohan Kallickal { 569843ae752SJayamohan Kallickal uint32_t total_cid_count = 0; 570843ae752SJayamohan Kallickal uint32_t total_icd_count = 0; 571843ae752SJayamohan Kallickal uint8_t ulp_num = 0; 572843ae752SJayamohan Kallickal 573843ae752SJayamohan Kallickal total_cid_count = BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP0) + 574843ae752SJayamohan Kallickal BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP1); 575843ae752SJayamohan Kallickal 576cf987b79SJayamohan Kallickal for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 577cf987b79SJayamohan Kallickal uint32_t align_mask = 0; 578cf987b79SJayamohan Kallickal uint32_t icd_post_per_page = 0; 579cf987b79SJayamohan Kallickal uint32_t icd_count_unavailable = 0; 580cf987b79SJayamohan Kallickal uint32_t icd_start = 0, icd_count = 0; 581cf987b79SJayamohan Kallickal uint32_t icd_start_align = 0, icd_count_align = 0; 582cf987b79SJayamohan Kallickal 583843ae752SJayamohan Kallickal if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 584cf987b79SJayamohan Kallickal icd_start = phba->fw_config.iscsi_icd_start[ulp_num]; 585cf987b79SJayamohan Kallickal icd_count = phba->fw_config.iscsi_icd_count[ulp_num]; 586cf987b79SJayamohan Kallickal 587cf987b79SJayamohan Kallickal /* Get ICD count that can be posted on each page */ 588cf987b79SJayamohan Kallickal icd_post_per_page = (PAGE_SIZE / (BE2_SGE * 589cf987b79SJayamohan Kallickal sizeof(struct iscsi_sge))); 590cf987b79SJayamohan Kallickal align_mask = (icd_post_per_page - 1); 591cf987b79SJayamohan Kallickal 592cf987b79SJayamohan Kallickal /* Check if icd_start is aligned ICD per page posting */ 593cf987b79SJayamohan Kallickal if (icd_start % icd_post_per_page) { 594cf987b79SJayamohan Kallickal icd_start_align = ((icd_start + 595cf987b79SJayamohan Kallickal icd_post_per_page) & 596cf987b79SJayamohan Kallickal ~(align_mask)); 597cf987b79SJayamohan Kallickal phba->fw_config. 598cf987b79SJayamohan Kallickal iscsi_icd_start[ulp_num] = 599cf987b79SJayamohan Kallickal icd_start_align; 600843ae752SJayamohan Kallickal } 601843ae752SJayamohan Kallickal 602cf987b79SJayamohan Kallickal icd_count_align = (icd_count & ~align_mask); 603cf987b79SJayamohan Kallickal 604cf987b79SJayamohan Kallickal /* ICD discarded in the process of alignment */ 605cf987b79SJayamohan Kallickal if (icd_start_align) 606cf987b79SJayamohan Kallickal icd_count_unavailable = ((icd_start_align - 607cf987b79SJayamohan Kallickal icd_start) + 608cf987b79SJayamohan Kallickal (icd_count - 609cf987b79SJayamohan Kallickal icd_count_align)); 610cf987b79SJayamohan Kallickal 611cf987b79SJayamohan Kallickal /* Updated ICD count available */ 612cf987b79SJayamohan Kallickal phba->fw_config.iscsi_icd_count[ulp_num] = (icd_count - 613cf987b79SJayamohan Kallickal icd_count_unavailable); 614cf987b79SJayamohan Kallickal 615cf987b79SJayamohan Kallickal beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 616cf987b79SJayamohan Kallickal "BM_%d : Aligned ICD values\n" 617cf987b79SJayamohan Kallickal "\t ICD Start : %d\n" 618cf987b79SJayamohan Kallickal "\t ICD Count : %d\n" 619cf987b79SJayamohan Kallickal "\t ICD Discarded : %d\n", 620cf987b79SJayamohan Kallickal phba->fw_config. 621cf987b79SJayamohan Kallickal iscsi_icd_start[ulp_num], 622cf987b79SJayamohan Kallickal phba->fw_config. 623cf987b79SJayamohan Kallickal iscsi_icd_count[ulp_num], 624cf987b79SJayamohan Kallickal icd_count_unavailable); 625cf987b79SJayamohan Kallickal break; 626cf987b79SJayamohan Kallickal } 627cf987b79SJayamohan Kallickal } 628cf987b79SJayamohan Kallickal 629cf987b79SJayamohan Kallickal total_icd_count = phba->fw_config.iscsi_icd_count[ulp_num]; 630843ae752SJayamohan Kallickal phba->params.ios_per_ctrl = (total_icd_count - 631843ae752SJayamohan Kallickal (total_cid_count + 632843ae752SJayamohan Kallickal BE2_TMFS + BE2_NOPOUT_REQ)); 633843ae752SJayamohan Kallickal phba->params.cxns_per_ctrl = total_cid_count; 634843ae752SJayamohan Kallickal phba->params.icds_per_ctrl = total_icd_count; 6356733b39aSJayamohan Kallickal phba->params.num_sge_per_io = BE2_SGE; 6366733b39aSJayamohan Kallickal phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ; 6376733b39aSJayamohan Kallickal phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ; 638843ae752SJayamohan Kallickal phba->params.num_eq_entries = 1024; 639843ae752SJayamohan Kallickal phba->params.num_cq_entries = 1024; 6406733b39aSJayamohan Kallickal phba->params.wrbs_per_cxn = 256; 6416733b39aSJayamohan Kallickal } 6426733b39aSJayamohan Kallickal 6436733b39aSJayamohan Kallickal static void hwi_ring_eq_db(struct beiscsi_hba *phba, 6446733b39aSJayamohan Kallickal unsigned int id, unsigned int clr_interrupt, 6456733b39aSJayamohan Kallickal unsigned int num_processed, 6466733b39aSJayamohan Kallickal unsigned char rearm, unsigned char event) 6476733b39aSJayamohan Kallickal { 6486733b39aSJayamohan Kallickal u32 val = 0; 649e08b3c8bSJayamohan Kallickal 6506733b39aSJayamohan Kallickal if (rearm) 6516733b39aSJayamohan Kallickal val |= 1 << DB_EQ_REARM_SHIFT; 6526733b39aSJayamohan Kallickal if (clr_interrupt) 6536733b39aSJayamohan Kallickal val |= 1 << DB_EQ_CLR_SHIFT; 6546733b39aSJayamohan Kallickal if (event) 6556733b39aSJayamohan Kallickal val |= 1 << DB_EQ_EVNT_SHIFT; 656e08b3c8bSJayamohan Kallickal 6576733b39aSJayamohan Kallickal val |= num_processed << DB_EQ_NUM_POPPED_SHIFT; 658e08b3c8bSJayamohan Kallickal /* Setting lower order EQ_ID Bits */ 659e08b3c8bSJayamohan Kallickal val |= (id & DB_EQ_RING_ID_LOW_MASK); 660e08b3c8bSJayamohan Kallickal 661e08b3c8bSJayamohan Kallickal /* Setting Higher order EQ_ID Bits */ 662e08b3c8bSJayamohan Kallickal val |= (((id >> DB_EQ_HIGH_FEILD_SHIFT) & 663e08b3c8bSJayamohan Kallickal DB_EQ_RING_ID_HIGH_MASK) 664e08b3c8bSJayamohan Kallickal << DB_EQ_HIGH_SET_SHIFT); 665e08b3c8bSJayamohan Kallickal 6666733b39aSJayamohan Kallickal iowrite32(val, phba->db_va + DB_EQ_OFFSET); 6676733b39aSJayamohan Kallickal } 6686733b39aSJayamohan Kallickal 6696733b39aSJayamohan Kallickal /** 670bfead3b2SJayamohan Kallickal * be_isr_mcc - The isr routine of the driver. 671bfead3b2SJayamohan Kallickal * @irq: Not used 672bfead3b2SJayamohan Kallickal * @dev_id: Pointer to host adapter structure 673bfead3b2SJayamohan Kallickal */ 674bfead3b2SJayamohan Kallickal static irqreturn_t be_isr_mcc(int irq, void *dev_id) 675bfead3b2SJayamohan Kallickal { 676bfead3b2SJayamohan Kallickal struct beiscsi_hba *phba; 677a3095016SJitendra Bhivare struct be_eq_entry *eqe; 678bfead3b2SJayamohan Kallickal struct be_queue_info *eq; 679bfead3b2SJayamohan Kallickal struct be_queue_info *mcc; 680a3095016SJitendra Bhivare unsigned int mcc_events; 681bfead3b2SJayamohan Kallickal struct be_eq_obj *pbe_eq; 682bfead3b2SJayamohan Kallickal 683bfead3b2SJayamohan Kallickal pbe_eq = dev_id; 684bfead3b2SJayamohan Kallickal eq = &pbe_eq->q; 685bfead3b2SJayamohan Kallickal phba = pbe_eq->phba; 686bfead3b2SJayamohan Kallickal mcc = &phba->ctrl.mcc_obj.cq; 687bfead3b2SJayamohan Kallickal eqe = queue_tail_node(eq); 688bfead3b2SJayamohan Kallickal 689a3095016SJitendra Bhivare mcc_events = 0; 690bfead3b2SJayamohan Kallickal while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 691bfead3b2SJayamohan Kallickal & EQE_VALID_MASK) { 692bfead3b2SJayamohan Kallickal if (((eqe->dw[offsetof(struct amap_eq_entry, 693bfead3b2SJayamohan Kallickal resource_id) / 32] & 694bfead3b2SJayamohan Kallickal EQE_RESID_MASK) >> 16) == mcc->id) { 695a3095016SJitendra Bhivare mcc_events++; 696bfead3b2SJayamohan Kallickal } 697bfead3b2SJayamohan Kallickal AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 698bfead3b2SJayamohan Kallickal queue_tail_inc(eq); 699bfead3b2SJayamohan Kallickal eqe = queue_tail_node(eq); 700bfead3b2SJayamohan Kallickal } 701bfead3b2SJayamohan Kallickal 702a3095016SJitendra Bhivare if (mcc_events) { 703a3095016SJitendra Bhivare queue_work(phba->wq, &pbe_eq->mcc_work); 704a3095016SJitendra Bhivare hwi_ring_eq_db(phba, eq->id, 1, mcc_events, 1, 1); 705a3095016SJitendra Bhivare } 706bfead3b2SJayamohan Kallickal return IRQ_HANDLED; 707bfead3b2SJayamohan Kallickal } 708bfead3b2SJayamohan Kallickal 709bfead3b2SJayamohan Kallickal /** 710bfead3b2SJayamohan Kallickal * be_isr_msix - The isr routine of the driver. 711bfead3b2SJayamohan Kallickal * @irq: Not used 712bfead3b2SJayamohan Kallickal * @dev_id: Pointer to host adapter structure 713bfead3b2SJayamohan Kallickal */ 714bfead3b2SJayamohan Kallickal static irqreturn_t be_isr_msix(int irq, void *dev_id) 715bfead3b2SJayamohan Kallickal { 716bfead3b2SJayamohan Kallickal struct beiscsi_hba *phba; 717bfead3b2SJayamohan Kallickal struct be_queue_info *eq; 718bfead3b2SJayamohan Kallickal struct be_eq_obj *pbe_eq; 719bfead3b2SJayamohan Kallickal 720bfead3b2SJayamohan Kallickal pbe_eq = dev_id; 721bfead3b2SJayamohan Kallickal eq = &pbe_eq->q; 722bfead3b2SJayamohan Kallickal 723bfead3b2SJayamohan Kallickal phba = pbe_eq->phba; 7241094cf68SJitendra Bhivare /* disable interrupt till iopoll completes */ 7251094cf68SJitendra Bhivare hwi_ring_eq_db(phba, eq->id, 1, 0, 0, 1); 726511cbce2SChristoph Hellwig irq_poll_sched(&pbe_eq->iopoll); 727bfead3b2SJayamohan Kallickal 728bfead3b2SJayamohan Kallickal return IRQ_HANDLED; 729bfead3b2SJayamohan Kallickal } 730bfead3b2SJayamohan Kallickal 731bfead3b2SJayamohan Kallickal /** 7326733b39aSJayamohan Kallickal * be_isr - The isr routine of the driver. 7336733b39aSJayamohan Kallickal * @irq: Not used 7346733b39aSJayamohan Kallickal * @dev_id: Pointer to host adapter structure 7356733b39aSJayamohan Kallickal */ 7366733b39aSJayamohan Kallickal static irqreturn_t be_isr(int irq, void *dev_id) 7376733b39aSJayamohan Kallickal { 7386733b39aSJayamohan Kallickal struct beiscsi_hba *phba; 7396733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 7406733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context; 741a3095016SJitendra Bhivare struct be_eq_entry *eqe; 7426733b39aSJayamohan Kallickal struct be_queue_info *eq; 743bfead3b2SJayamohan Kallickal struct be_queue_info *mcc; 744a3095016SJitendra Bhivare unsigned int mcc_events, io_events; 7456733b39aSJayamohan Kallickal struct be_ctrl_info *ctrl; 746bfead3b2SJayamohan Kallickal struct be_eq_obj *pbe_eq; 747a3095016SJitendra Bhivare int isr, rearm; 7486733b39aSJayamohan Kallickal 7496733b39aSJayamohan Kallickal phba = dev_id; 7506eab04a8SJustin P. Mattock ctrl = &phba->ctrl; 7516733b39aSJayamohan Kallickal isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET + 7526733b39aSJayamohan Kallickal (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE)); 7536733b39aSJayamohan Kallickal if (!isr) 7546733b39aSJayamohan Kallickal return IRQ_NONE; 7556733b39aSJayamohan Kallickal 7566733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 7576733b39aSJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 758bfead3b2SJayamohan Kallickal pbe_eq = &phwi_context->be_eq[0]; 759bfead3b2SJayamohan Kallickal 760bfead3b2SJayamohan Kallickal eq = &phwi_context->be_eq[0].q; 761bfead3b2SJayamohan Kallickal mcc = &phba->ctrl.mcc_obj.cq; 7626733b39aSJayamohan Kallickal eqe = queue_tail_node(eq); 7636733b39aSJayamohan Kallickal 764a3095016SJitendra Bhivare io_events = 0; 765a3095016SJitendra Bhivare mcc_events = 0; 7666733b39aSJayamohan Kallickal while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 7676733b39aSJayamohan Kallickal & EQE_VALID_MASK) { 768bfead3b2SJayamohan Kallickal if (((eqe->dw[offsetof(struct amap_eq_entry, 769a3095016SJitendra Bhivare resource_id) / 32] & EQE_RESID_MASK) >> 16) == mcc->id) 770a3095016SJitendra Bhivare mcc_events++; 771a3095016SJitendra Bhivare else 772a3095016SJitendra Bhivare io_events++; 7736733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 7746733b39aSJayamohan Kallickal queue_tail_inc(eq); 7756733b39aSJayamohan Kallickal eqe = queue_tail_node(eq); 7766733b39aSJayamohan Kallickal } 777a3095016SJitendra Bhivare if (!io_events && !mcc_events) 7786733b39aSJayamohan Kallickal return IRQ_NONE; 779a3095016SJitendra Bhivare 780a3095016SJitendra Bhivare /* no need to rearm if interrupt is only for IOs */ 781a3095016SJitendra Bhivare rearm = 0; 782a3095016SJitendra Bhivare if (mcc_events) { 783a3095016SJitendra Bhivare queue_work(phba->wq, &pbe_eq->mcc_work); 784a3095016SJitendra Bhivare /* rearm for MCCQ */ 785a3095016SJitendra Bhivare rearm = 1; 786a3095016SJitendra Bhivare } 787a3095016SJitendra Bhivare if (io_events) 788a3095016SJitendra Bhivare irq_poll_sched(&pbe_eq->iopoll); 789a3095016SJitendra Bhivare hwi_ring_eq_db(phba, eq->id, 0, (io_events + mcc_events), rearm, 1); 790a3095016SJitendra Bhivare return IRQ_HANDLED; 7916733b39aSJayamohan Kallickal } 7926733b39aSJayamohan Kallickal 79345371aa3SJitendra Bhivare static void beiscsi_free_irqs(struct beiscsi_hba *phba) 79445371aa3SJitendra Bhivare { 79545371aa3SJitendra Bhivare struct hwi_context_memory *phwi_context; 79645371aa3SJitendra Bhivare int i; 79745371aa3SJitendra Bhivare 79845371aa3SJitendra Bhivare if (!phba->pcidev->msix_enabled) { 79945371aa3SJitendra Bhivare if (phba->pcidev->irq) 80045371aa3SJitendra Bhivare free_irq(phba->pcidev->irq, phba); 80145371aa3SJitendra Bhivare return; 80245371aa3SJitendra Bhivare } 80345371aa3SJitendra Bhivare 80445371aa3SJitendra Bhivare phwi_context = phba->phwi_ctrlr->phwi_ctxt; 80545371aa3SJitendra Bhivare for (i = 0; i <= phba->num_cpus; i++) { 80645371aa3SJitendra Bhivare free_irq(pci_irq_vector(phba->pcidev, i), 80745371aa3SJitendra Bhivare &phwi_context->be_eq[i]); 80845371aa3SJitendra Bhivare kfree(phba->msi_name[i]); 80945371aa3SJitendra Bhivare } 81045371aa3SJitendra Bhivare } 8111094cf68SJitendra Bhivare 8126733b39aSJayamohan Kallickal static int beiscsi_init_irqs(struct beiscsi_hba *phba) 8136733b39aSJayamohan Kallickal { 8146733b39aSJayamohan Kallickal struct pci_dev *pcidev = phba->pcidev; 815bfead3b2SJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 816bfead3b2SJayamohan Kallickal struct hwi_context_memory *phwi_context; 81783148866SChristoph Hellwig int ret, i, j; 8186733b39aSJayamohan Kallickal 819bfead3b2SJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 820bfead3b2SJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 821bfead3b2SJayamohan Kallickal 82283148866SChristoph Hellwig if (pcidev->msix_enabled) { 823bfead3b2SJayamohan Kallickal for (i = 0; i < phba->num_cpus; i++) { 8248fcfb210SJayamohan Kallickal phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, 8258fcfb210SJayamohan Kallickal GFP_KERNEL); 8268fcfb210SJayamohan Kallickal if (!phba->msi_name[i]) { 8278fcfb210SJayamohan Kallickal ret = -ENOMEM; 8288fcfb210SJayamohan Kallickal goto free_msix_irqs; 8298fcfb210SJayamohan Kallickal } 8308fcfb210SJayamohan Kallickal 8318fcfb210SJayamohan Kallickal sprintf(phba->msi_name[i], "beiscsi_%02x_%02x", 8328fcfb210SJayamohan Kallickal phba->shost->host_no, i); 83383148866SChristoph Hellwig ret = request_irq(pci_irq_vector(pcidev, i), 83483148866SChristoph Hellwig be_isr_msix, 0, phba->msi_name[i], 835bfead3b2SJayamohan Kallickal &phwi_context->be_eq[i]); 8364f5af07eSJayamohan Kallickal if (ret) { 83799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 83899bc5d55SJohn Soni Jose "BM_%d : beiscsi_init_irqs-Failed to" 83999bc5d55SJohn Soni Jose "register msix for i = %d\n", 84099bc5d55SJohn Soni Jose i); 8418fcfb210SJayamohan Kallickal kfree(phba->msi_name[i]); 8424f5af07eSJayamohan Kallickal goto free_msix_irqs; 8434f5af07eSJayamohan Kallickal } 844bfead3b2SJayamohan Kallickal } 8458fcfb210SJayamohan Kallickal phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL); 8468fcfb210SJayamohan Kallickal if (!phba->msi_name[i]) { 8478fcfb210SJayamohan Kallickal ret = -ENOMEM; 8488fcfb210SJayamohan Kallickal goto free_msix_irqs; 8498fcfb210SJayamohan Kallickal } 8508fcfb210SJayamohan Kallickal sprintf(phba->msi_name[i], "beiscsi_mcc_%02x", 8518fcfb210SJayamohan Kallickal phba->shost->host_no); 85283148866SChristoph Hellwig ret = request_irq(pci_irq_vector(pcidev, i), be_isr_mcc, 0, 85383148866SChristoph Hellwig phba->msi_name[i], &phwi_context->be_eq[i]); 8544f5af07eSJayamohan Kallickal if (ret) { 85599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT , 85699bc5d55SJohn Soni Jose "BM_%d : beiscsi_init_irqs-" 8574f5af07eSJayamohan Kallickal "Failed to register beiscsi_msix_mcc\n"); 8588fcfb210SJayamohan Kallickal kfree(phba->msi_name[i]); 8594f5af07eSJayamohan Kallickal goto free_msix_irqs; 8604f5af07eSJayamohan Kallickal } 8614f5af07eSJayamohan Kallickal 862bfead3b2SJayamohan Kallickal } else { 863bfead3b2SJayamohan Kallickal ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED, 864bfead3b2SJayamohan Kallickal "beiscsi", phba); 8656733b39aSJayamohan Kallickal if (ret) { 86699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 86799bc5d55SJohn Soni Jose "BM_%d : beiscsi_init_irqs-" 8686733b39aSJayamohan Kallickal "Failed to register irq\\n"); 8696733b39aSJayamohan Kallickal return ret; 8706733b39aSJayamohan Kallickal } 871bfead3b2SJayamohan Kallickal } 8726733b39aSJayamohan Kallickal return 0; 8734f5af07eSJayamohan Kallickal free_msix_irqs: 8748fcfb210SJayamohan Kallickal for (j = i - 1; j >= 0; j--) { 87583148866SChristoph Hellwig free_irq(pci_irq_vector(pcidev, i), &phwi_context->be_eq[j]); 8768fcfb210SJayamohan Kallickal kfree(phba->msi_name[j]); 8778fcfb210SJayamohan Kallickal } 8784f5af07eSJayamohan Kallickal return ret; 8796733b39aSJayamohan Kallickal } 8806733b39aSJayamohan Kallickal 881e08b3c8bSJayamohan Kallickal void hwi_ring_cq_db(struct beiscsi_hba *phba, 8826733b39aSJayamohan Kallickal unsigned int id, unsigned int num_processed, 8831094cf68SJitendra Bhivare unsigned char rearm) 8846733b39aSJayamohan Kallickal { 8856733b39aSJayamohan Kallickal u32 val = 0; 886e08b3c8bSJayamohan Kallickal 8876733b39aSJayamohan Kallickal if (rearm) 8886733b39aSJayamohan Kallickal val |= 1 << DB_CQ_REARM_SHIFT; 889e08b3c8bSJayamohan Kallickal 8906733b39aSJayamohan Kallickal val |= num_processed << DB_CQ_NUM_POPPED_SHIFT; 891e08b3c8bSJayamohan Kallickal 892e08b3c8bSJayamohan Kallickal /* Setting lower order CQ_ID Bits */ 893e08b3c8bSJayamohan Kallickal val |= (id & DB_CQ_RING_ID_LOW_MASK); 894e08b3c8bSJayamohan Kallickal 895e08b3c8bSJayamohan Kallickal /* Setting Higher order CQ_ID Bits */ 896e08b3c8bSJayamohan Kallickal val |= (((id >> DB_CQ_HIGH_FEILD_SHIFT) & 897e08b3c8bSJayamohan Kallickal DB_CQ_RING_ID_HIGH_MASK) 898e08b3c8bSJayamohan Kallickal << DB_CQ_HIGH_SET_SHIFT); 899e08b3c8bSJayamohan Kallickal 9006733b39aSJayamohan Kallickal iowrite32(val, phba->db_va + DB_CQ_OFFSET); 9016733b39aSJayamohan Kallickal } 9026733b39aSJayamohan Kallickal 9036733b39aSJayamohan Kallickal static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba) 9046733b39aSJayamohan Kallickal { 9056733b39aSJayamohan Kallickal struct sgl_handle *psgl_handle; 9067d2c0d64SJitendra Bhivare unsigned long flags; 9076733b39aSJayamohan Kallickal 9087d2c0d64SJitendra Bhivare spin_lock_irqsave(&phba->io_sgl_lock, flags); 9096733b39aSJayamohan Kallickal if (phba->io_sgl_hndl_avbl) { 91099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO, 91199bc5d55SJohn Soni Jose "BM_%d : In alloc_io_sgl_handle," 91299bc5d55SJohn Soni Jose " io_sgl_alloc_index=%d\n", 9136733b39aSJayamohan Kallickal phba->io_sgl_alloc_index); 91499bc5d55SJohn Soni Jose 9156733b39aSJayamohan Kallickal psgl_handle = phba->io_sgl_hndl_base[phba-> 9166733b39aSJayamohan Kallickal io_sgl_alloc_index]; 9176733b39aSJayamohan Kallickal phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL; 9186733b39aSJayamohan Kallickal phba->io_sgl_hndl_avbl--; 919bfead3b2SJayamohan Kallickal if (phba->io_sgl_alloc_index == (phba->params. 920bfead3b2SJayamohan Kallickal ios_per_ctrl - 1)) 9216733b39aSJayamohan Kallickal phba->io_sgl_alloc_index = 0; 9226733b39aSJayamohan Kallickal else 9236733b39aSJayamohan Kallickal phba->io_sgl_alloc_index++; 9246733b39aSJayamohan Kallickal } else 9256733b39aSJayamohan Kallickal psgl_handle = NULL; 9267d2c0d64SJitendra Bhivare spin_unlock_irqrestore(&phba->io_sgl_lock, flags); 9276733b39aSJayamohan Kallickal return psgl_handle; 9286733b39aSJayamohan Kallickal } 9296733b39aSJayamohan Kallickal 9306733b39aSJayamohan Kallickal static void 9316733b39aSJayamohan Kallickal free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) 9326733b39aSJayamohan Kallickal { 9337d2c0d64SJitendra Bhivare unsigned long flags; 9347d2c0d64SJitendra Bhivare 9357d2c0d64SJitendra Bhivare spin_lock_irqsave(&phba->io_sgl_lock, flags); 93699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO, 93799bc5d55SJohn Soni Jose "BM_%d : In free_,io_sgl_free_index=%d\n", 9386733b39aSJayamohan Kallickal phba->io_sgl_free_index); 93999bc5d55SJohn Soni Jose 9406733b39aSJayamohan Kallickal if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) { 9416733b39aSJayamohan Kallickal /* 9426733b39aSJayamohan Kallickal * this can happen if clean_task is called on a task that 9436733b39aSJayamohan Kallickal * failed in xmit_task or alloc_pdu. 9446733b39aSJayamohan Kallickal */ 94599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO, 94699bc5d55SJohn Soni Jose "BM_%d : Double Free in IO SGL io_sgl_free_index=%d," 9476733b39aSJayamohan Kallickal "value there=%p\n", phba->io_sgl_free_index, 94899bc5d55SJohn Soni Jose phba->io_sgl_hndl_base 94999bc5d55SJohn Soni Jose [phba->io_sgl_free_index]); 9507d2c0d64SJitendra Bhivare spin_unlock_irqrestore(&phba->io_sgl_lock, flags); 9516733b39aSJayamohan Kallickal return; 9526733b39aSJayamohan Kallickal } 9536733b39aSJayamohan Kallickal phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle; 9546733b39aSJayamohan Kallickal phba->io_sgl_hndl_avbl++; 9556733b39aSJayamohan Kallickal if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1)) 9566733b39aSJayamohan Kallickal phba->io_sgl_free_index = 0; 9576733b39aSJayamohan Kallickal else 9586733b39aSJayamohan Kallickal phba->io_sgl_free_index++; 9597d2c0d64SJitendra Bhivare spin_unlock_irqrestore(&phba->io_sgl_lock, flags); 9606733b39aSJayamohan Kallickal } 9616733b39aSJayamohan Kallickal 962cb564c6bSJitendra Bhivare static inline struct wrb_handle * 963cb564c6bSJitendra Bhivare beiscsi_get_wrb_handle(struct hwi_wrb_context *pwrb_context, 964cb564c6bSJitendra Bhivare unsigned int wrbs_per_cxn) 965cb564c6bSJitendra Bhivare { 966cb564c6bSJitendra Bhivare struct wrb_handle *pwrb_handle; 9677d2c0d64SJitendra Bhivare unsigned long flags; 968cb564c6bSJitendra Bhivare 9697d2c0d64SJitendra Bhivare spin_lock_irqsave(&pwrb_context->wrb_lock, flags); 9703f7f62eeSJitendra Bhivare if (!pwrb_context->wrb_handles_available) { 9713f7f62eeSJitendra Bhivare spin_unlock_irqrestore(&pwrb_context->wrb_lock, flags); 9723f7f62eeSJitendra Bhivare return NULL; 9733f7f62eeSJitendra Bhivare } 974cb564c6bSJitendra Bhivare pwrb_handle = pwrb_context->pwrb_handle_base[pwrb_context->alloc_index]; 975cb564c6bSJitendra Bhivare pwrb_context->wrb_handles_available--; 976cb564c6bSJitendra Bhivare if (pwrb_context->alloc_index == (wrbs_per_cxn - 1)) 977cb564c6bSJitendra Bhivare pwrb_context->alloc_index = 0; 978cb564c6bSJitendra Bhivare else 979cb564c6bSJitendra Bhivare pwrb_context->alloc_index++; 9807d2c0d64SJitendra Bhivare spin_unlock_irqrestore(&pwrb_context->wrb_lock, flags); 981bf9b7554SJitendra Bhivare 982bf9b7554SJitendra Bhivare if (pwrb_handle) 983e1f9d31eSJitendra Bhivare memset(pwrb_handle->pwrb, 0, sizeof(*pwrb_handle->pwrb)); 984cb564c6bSJitendra Bhivare 985cb564c6bSJitendra Bhivare return pwrb_handle; 986cb564c6bSJitendra Bhivare } 987cb564c6bSJitendra Bhivare 9886733b39aSJayamohan Kallickal /** 9896733b39aSJayamohan Kallickal * alloc_wrb_handle - To allocate a wrb handle 9906733b39aSJayamohan Kallickal * @phba: The hba pointer 9916733b39aSJayamohan Kallickal * @cid: The cid to use for allocation 992340c99e9SJohn Soni Jose * @pwrb_context: ptr to ptr to wrb context 9936733b39aSJayamohan Kallickal * 9946733b39aSJayamohan Kallickal * This happens under session_lock until submission to chip 9956733b39aSJayamohan Kallickal */ 996340c99e9SJohn Soni Jose struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid, 997340c99e9SJohn Soni Jose struct hwi_wrb_context **pcontext) 9986733b39aSJayamohan Kallickal { 9996733b39aSJayamohan Kallickal struct hwi_wrb_context *pwrb_context; 10006733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 1001a7909b39SJayamohan Kallickal uint16_t cri_index = BE_GET_CRI_FROM_CID(cid); 10026733b39aSJayamohan Kallickal 10036733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 1004a7909b39SJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[cri_index]; 1005cb564c6bSJitendra Bhivare /* return the context address */ 1006340c99e9SJohn Soni Jose *pcontext = pwrb_context; 1007cb564c6bSJitendra Bhivare return beiscsi_get_wrb_handle(pwrb_context, phba->params.wrbs_per_cxn); 1008cb564c6bSJitendra Bhivare } 1009cb564c6bSJitendra Bhivare 1010cb564c6bSJitendra Bhivare static inline void 1011cb564c6bSJitendra Bhivare beiscsi_put_wrb_handle(struct hwi_wrb_context *pwrb_context, 1012cb564c6bSJitendra Bhivare struct wrb_handle *pwrb_handle, 1013cb564c6bSJitendra Bhivare unsigned int wrbs_per_cxn) 1014cb564c6bSJitendra Bhivare { 10157d2c0d64SJitendra Bhivare unsigned long flags; 10167d2c0d64SJitendra Bhivare 10177d2c0d64SJitendra Bhivare spin_lock_irqsave(&pwrb_context->wrb_lock, flags); 1018cb564c6bSJitendra Bhivare pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle; 1019cb564c6bSJitendra Bhivare pwrb_context->wrb_handles_available++; 1020cb564c6bSJitendra Bhivare if (pwrb_context->free_index == (wrbs_per_cxn - 1)) 1021cb564c6bSJitendra Bhivare pwrb_context->free_index = 0; 1022cb564c6bSJitendra Bhivare else 1023cb564c6bSJitendra Bhivare pwrb_context->free_index++; 10243f7f62eeSJitendra Bhivare pwrb_handle->pio_handle = NULL; 10257d2c0d64SJitendra Bhivare spin_unlock_irqrestore(&pwrb_context->wrb_lock, flags); 10266733b39aSJayamohan Kallickal } 10276733b39aSJayamohan Kallickal 10286733b39aSJayamohan Kallickal /** 10296733b39aSJayamohan Kallickal * free_wrb_handle - To free the wrb handle back to pool 10306733b39aSJayamohan Kallickal * @phba: The hba pointer 10316733b39aSJayamohan Kallickal * @pwrb_context: The context to free from 10326733b39aSJayamohan Kallickal * @pwrb_handle: The wrb_handle to free 10336733b39aSJayamohan Kallickal * 10346733b39aSJayamohan Kallickal * This happens under session_lock until submission to chip 10356733b39aSJayamohan Kallickal */ 10366733b39aSJayamohan Kallickal static void 10376733b39aSJayamohan Kallickal free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context, 10386733b39aSJayamohan Kallickal struct wrb_handle *pwrb_handle) 10396733b39aSJayamohan Kallickal { 1040cb564c6bSJitendra Bhivare beiscsi_put_wrb_handle(pwrb_context, 1041cb564c6bSJitendra Bhivare pwrb_handle, 1042cb564c6bSJitendra Bhivare phba->params.wrbs_per_cxn); 104399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, 104499bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 104599bc5d55SJohn Soni Jose "BM_%d : FREE WRB: pwrb_handle=%p free_index=0x%x" 10466733b39aSJayamohan Kallickal "wrb_handles_available=%d\n", 10476733b39aSJayamohan Kallickal pwrb_handle, pwrb_context->free_index, 1048bfead3b2SJayamohan Kallickal pwrb_context->wrb_handles_available); 10496733b39aSJayamohan Kallickal } 10506733b39aSJayamohan Kallickal 10516733b39aSJayamohan Kallickal static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba) 10526733b39aSJayamohan Kallickal { 10536733b39aSJayamohan Kallickal struct sgl_handle *psgl_handle; 10547d2c0d64SJitendra Bhivare unsigned long flags; 10556733b39aSJayamohan Kallickal 10567d2c0d64SJitendra Bhivare spin_lock_irqsave(&phba->mgmt_sgl_lock, flags); 10576733b39aSJayamohan Kallickal if (phba->eh_sgl_hndl_avbl) { 10586733b39aSJayamohan Kallickal psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index]; 10596733b39aSJayamohan Kallickal phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL; 106099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG, 106199bc5d55SJohn Soni Jose "BM_%d : mgmt_sgl_alloc_index=%d=0x%x\n", 106299bc5d55SJohn Soni Jose phba->eh_sgl_alloc_index, 106399bc5d55SJohn Soni Jose phba->eh_sgl_alloc_index); 106499bc5d55SJohn Soni Jose 10656733b39aSJayamohan Kallickal phba->eh_sgl_hndl_avbl--; 10666733b39aSJayamohan Kallickal if (phba->eh_sgl_alloc_index == 10676733b39aSJayamohan Kallickal (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 10686733b39aSJayamohan Kallickal 1)) 10696733b39aSJayamohan Kallickal phba->eh_sgl_alloc_index = 0; 10706733b39aSJayamohan Kallickal else 10716733b39aSJayamohan Kallickal phba->eh_sgl_alloc_index++; 10726733b39aSJayamohan Kallickal } else 10736733b39aSJayamohan Kallickal psgl_handle = NULL; 10747d2c0d64SJitendra Bhivare spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags); 10756733b39aSJayamohan Kallickal return psgl_handle; 10766733b39aSJayamohan Kallickal } 10776733b39aSJayamohan Kallickal 10786733b39aSJayamohan Kallickal void 10796733b39aSJayamohan Kallickal free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) 10806733b39aSJayamohan Kallickal { 10817d2c0d64SJitendra Bhivare unsigned long flags; 10827d2c0d64SJitendra Bhivare 10837d2c0d64SJitendra Bhivare spin_lock_irqsave(&phba->mgmt_sgl_lock, flags); 108499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG, 108599bc5d55SJohn Soni Jose "BM_%d : In free_mgmt_sgl_handle," 108699bc5d55SJohn Soni Jose "eh_sgl_free_index=%d\n", 1087bfead3b2SJayamohan Kallickal phba->eh_sgl_free_index); 108899bc5d55SJohn Soni Jose 10896733b39aSJayamohan Kallickal if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) { 10906733b39aSJayamohan Kallickal /* 10916733b39aSJayamohan Kallickal * this can happen if clean_task is called on a task that 10926733b39aSJayamohan Kallickal * failed in xmit_task or alloc_pdu. 10936733b39aSJayamohan Kallickal */ 109499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG, 109599bc5d55SJohn Soni Jose "BM_%d : Double Free in eh SGL ," 109699bc5d55SJohn Soni Jose "eh_sgl_free_index=%d\n", 10976733b39aSJayamohan Kallickal phba->eh_sgl_free_index); 10987d2c0d64SJitendra Bhivare spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags); 10996733b39aSJayamohan Kallickal return; 11006733b39aSJayamohan Kallickal } 11016733b39aSJayamohan Kallickal phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle; 11026733b39aSJayamohan Kallickal phba->eh_sgl_hndl_avbl++; 11036733b39aSJayamohan Kallickal if (phba->eh_sgl_free_index == 11046733b39aSJayamohan Kallickal (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1)) 11056733b39aSJayamohan Kallickal phba->eh_sgl_free_index = 0; 11066733b39aSJayamohan Kallickal else 11076733b39aSJayamohan Kallickal phba->eh_sgl_free_index++; 11087d2c0d64SJitendra Bhivare spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags); 11096733b39aSJayamohan Kallickal } 11106733b39aSJayamohan Kallickal 11116733b39aSJayamohan Kallickal static void 11126733b39aSJayamohan Kallickal be_complete_io(struct beiscsi_conn *beiscsi_conn, 111373133261SJohn Soni Jose struct iscsi_task *task, 111473133261SJohn Soni Jose struct common_sol_cqe *csol_cqe) 11156733b39aSJayamohan Kallickal { 11166733b39aSJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 11176733b39aSJayamohan Kallickal struct be_status_bhs *sts_bhs = 11186733b39aSJayamohan Kallickal (struct be_status_bhs *)io_task->cmd_bhs; 11196733b39aSJayamohan Kallickal struct iscsi_conn *conn = beiscsi_conn->conn; 11206733b39aSJayamohan Kallickal unsigned char *sense; 11216733b39aSJayamohan Kallickal u32 resid = 0, exp_cmdsn, max_cmdsn; 11226733b39aSJayamohan Kallickal u8 rsp, status, flags; 11236733b39aSJayamohan Kallickal 112473133261SJohn Soni Jose exp_cmdsn = csol_cqe->exp_cmdsn; 112573133261SJohn Soni Jose max_cmdsn = (csol_cqe->exp_cmdsn + 112673133261SJohn Soni Jose csol_cqe->cmd_wnd - 1); 112773133261SJohn Soni Jose rsp = csol_cqe->i_resp; 112873133261SJohn Soni Jose status = csol_cqe->i_sts; 112973133261SJohn Soni Jose flags = csol_cqe->i_flags; 113073133261SJohn Soni Jose resid = csol_cqe->res_cnt; 113173133261SJohn Soni Jose 1132bd535451SJayamohan Kallickal if (!task->sc) { 1133da334977SJayamohan Kallickal if (io_task->scsi_cmnd) { 1134bd535451SJayamohan Kallickal scsi_dma_unmap(io_task->scsi_cmnd); 1135da334977SJayamohan Kallickal io_task->scsi_cmnd = NULL; 1136da334977SJayamohan Kallickal } 11376733b39aSJayamohan Kallickal 1138bd535451SJayamohan Kallickal return; 1139bd535451SJayamohan Kallickal } 11406733b39aSJayamohan Kallickal task->sc->result = (DID_OK << 16) | status; 11416733b39aSJayamohan Kallickal if (rsp != ISCSI_STATUS_CMD_COMPLETED) { 11426733b39aSJayamohan Kallickal task->sc->result = DID_ERROR << 16; 11436733b39aSJayamohan Kallickal goto unmap; 11446733b39aSJayamohan Kallickal } 11456733b39aSJayamohan Kallickal 11466733b39aSJayamohan Kallickal /* bidi not initially supported */ 11476733b39aSJayamohan Kallickal if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) { 11486733b39aSJayamohan Kallickal if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW)) 11496733b39aSJayamohan Kallickal task->sc->result = DID_ERROR << 16; 11506733b39aSJayamohan Kallickal 11516733b39aSJayamohan Kallickal if (flags & ISCSI_FLAG_CMD_UNDERFLOW) { 11526733b39aSJayamohan Kallickal scsi_set_resid(task->sc, resid); 11536733b39aSJayamohan Kallickal if (!status && (scsi_bufflen(task->sc) - resid < 11546733b39aSJayamohan Kallickal task->sc->underflow)) 11556733b39aSJayamohan Kallickal task->sc->result = DID_ERROR << 16; 11566733b39aSJayamohan Kallickal } 11576733b39aSJayamohan Kallickal } 11586733b39aSJayamohan Kallickal 11596733b39aSJayamohan Kallickal if (status == SAM_STAT_CHECK_CONDITION) { 11604053a4beSDan Carpenter u16 sense_len; 1161bfead3b2SJayamohan Kallickal unsigned short *slen = (unsigned short *)sts_bhs->sense_info; 11624053a4beSDan Carpenter 11636733b39aSJayamohan Kallickal sense = sts_bhs->sense_info + sizeof(unsigned short); 11644053a4beSDan Carpenter sense_len = be16_to_cpu(*slen); 11656733b39aSJayamohan Kallickal memcpy(task->sc->sense_buffer, sense, 11666733b39aSJayamohan Kallickal min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE)); 11676733b39aSJayamohan Kallickal } 1168756d29c8SJayamohan Kallickal 116973133261SJohn Soni Jose if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) 117073133261SJohn Soni Jose conn->rxdata_octets += resid; 11716733b39aSJayamohan Kallickal unmap: 1172eb1c4692SJohn Soni Jose if (io_task->scsi_cmnd) { 11736733b39aSJayamohan Kallickal scsi_dma_unmap(io_task->scsi_cmnd); 1174da334977SJayamohan Kallickal io_task->scsi_cmnd = NULL; 1175eb1c4692SJohn Soni Jose } 11766733b39aSJayamohan Kallickal iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn); 11776733b39aSJayamohan Kallickal } 11786733b39aSJayamohan Kallickal 11796733b39aSJayamohan Kallickal static void 11806733b39aSJayamohan Kallickal be_complete_logout(struct beiscsi_conn *beiscsi_conn, 118173133261SJohn Soni Jose struct iscsi_task *task, 118273133261SJohn Soni Jose struct common_sol_cqe *csol_cqe) 11836733b39aSJayamohan Kallickal { 11846733b39aSJayamohan Kallickal struct iscsi_logout_rsp *hdr; 1185bfead3b2SJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 11866733b39aSJayamohan Kallickal struct iscsi_conn *conn = beiscsi_conn->conn; 11876733b39aSJayamohan Kallickal 11886733b39aSJayamohan Kallickal hdr = (struct iscsi_logout_rsp *)task->hdr; 11897bd6e25cSJayamohan Kallickal hdr->opcode = ISCSI_OP_LOGOUT_RSP; 11906733b39aSJayamohan Kallickal hdr->t2wait = 5; 11916733b39aSJayamohan Kallickal hdr->t2retain = 0; 119273133261SJohn Soni Jose hdr->flags = csol_cqe->i_flags; 119373133261SJohn Soni Jose hdr->response = csol_cqe->i_resp; 1194702dc5e8SJayamohan Kallickal hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn); 1195702dc5e8SJayamohan Kallickal hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn + 1196702dc5e8SJayamohan Kallickal csol_cqe->cmd_wnd - 1); 119773133261SJohn Soni Jose 11987bd6e25cSJayamohan Kallickal hdr->dlength[0] = 0; 11997bd6e25cSJayamohan Kallickal hdr->dlength[1] = 0; 12007bd6e25cSJayamohan Kallickal hdr->dlength[2] = 0; 12016733b39aSJayamohan Kallickal hdr->hlength = 0; 1202bfead3b2SJayamohan Kallickal hdr->itt = io_task->libiscsi_itt; 12036733b39aSJayamohan Kallickal __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 12046733b39aSJayamohan Kallickal } 12056733b39aSJayamohan Kallickal 12066733b39aSJayamohan Kallickal static void 12076733b39aSJayamohan Kallickal be_complete_tmf(struct beiscsi_conn *beiscsi_conn, 120873133261SJohn Soni Jose struct iscsi_task *task, 120973133261SJohn Soni Jose struct common_sol_cqe *csol_cqe) 12106733b39aSJayamohan Kallickal { 12116733b39aSJayamohan Kallickal struct iscsi_tm_rsp *hdr; 12126733b39aSJayamohan Kallickal struct iscsi_conn *conn = beiscsi_conn->conn; 1213bfead3b2SJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 12146733b39aSJayamohan Kallickal 12156733b39aSJayamohan Kallickal hdr = (struct iscsi_tm_rsp *)task->hdr; 12167bd6e25cSJayamohan Kallickal hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP; 121773133261SJohn Soni Jose hdr->flags = csol_cqe->i_flags; 121873133261SJohn Soni Jose hdr->response = csol_cqe->i_resp; 1219702dc5e8SJayamohan Kallickal hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn); 1220702dc5e8SJayamohan Kallickal hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn + 122173133261SJohn Soni Jose csol_cqe->cmd_wnd - 1); 122273133261SJohn Soni Jose 1223bfead3b2SJayamohan Kallickal hdr->itt = io_task->libiscsi_itt; 12246733b39aSJayamohan Kallickal __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 12256733b39aSJayamohan Kallickal } 12266733b39aSJayamohan Kallickal 12276733b39aSJayamohan Kallickal static void 12286733b39aSJayamohan Kallickal hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn, 12296733b39aSJayamohan Kallickal struct beiscsi_hba *phba, struct sol_cqe *psol) 12306733b39aSJayamohan Kallickal { 12316733b39aSJayamohan Kallickal struct hwi_wrb_context *pwrb_context; 1232a7909b39SJayamohan Kallickal uint16_t wrb_index, cid, cri_index; 1233e1f9d31eSJitendra Bhivare struct hwi_controller *phwi_ctrlr; 1234e1f9d31eSJitendra Bhivare struct wrb_handle *pwrb_handle; 12353f7f62eeSJitendra Bhivare struct iscsi_session *session; 1236e1f9d31eSJitendra Bhivare struct iscsi_task *task; 12376733b39aSJayamohan Kallickal 12386733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 12392c9dfd36SJayamohan Kallickal if (is_chip_be2_be3r(phba)) { 124073133261SJohn Soni Jose wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe, 124173133261SJohn Soni Jose wrb_idx, psol); 124273133261SJohn Soni Jose cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe, 124373133261SJohn Soni Jose cid, psol); 12442c9dfd36SJayamohan Kallickal } else { 12452c9dfd36SJayamohan Kallickal wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2, 12462c9dfd36SJayamohan Kallickal wrb_idx, psol); 12472c9dfd36SJayamohan Kallickal cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2, 12482c9dfd36SJayamohan Kallickal cid, psol); 124973133261SJohn Soni Jose } 125073133261SJohn Soni Jose 1251a7909b39SJayamohan Kallickal cri_index = BE_GET_CRI_FROM_CID(cid); 1252a7909b39SJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[cri_index]; 125373133261SJohn Soni Jose pwrb_handle = pwrb_context->pwrb_handle_basestd[wrb_index]; 12543f7f62eeSJitendra Bhivare session = beiscsi_conn->conn->session; 12553f7f62eeSJitendra Bhivare spin_lock_bh(&session->back_lock); 1256bfead3b2SJayamohan Kallickal task = pwrb_handle->pio_handle; 12573f7f62eeSJitendra Bhivare if (task) 12583f7f62eeSJitendra Bhivare __iscsi_put_task(task); 12593f7f62eeSJitendra Bhivare spin_unlock_bh(&session->back_lock); 12606733b39aSJayamohan Kallickal } 12616733b39aSJayamohan Kallickal 12626733b39aSJayamohan Kallickal static void 12636733b39aSJayamohan Kallickal be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn, 126473133261SJohn Soni Jose struct iscsi_task *task, 126573133261SJohn Soni Jose struct common_sol_cqe *csol_cqe) 12666733b39aSJayamohan Kallickal { 12676733b39aSJayamohan Kallickal struct iscsi_nopin *hdr; 12686733b39aSJayamohan Kallickal struct iscsi_conn *conn = beiscsi_conn->conn; 1269bfead3b2SJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 12706733b39aSJayamohan Kallickal 12716733b39aSJayamohan Kallickal hdr = (struct iscsi_nopin *)task->hdr; 127273133261SJohn Soni Jose hdr->flags = csol_cqe->i_flags; 127373133261SJohn Soni Jose hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn); 1274702dc5e8SJayamohan Kallickal hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn + 127573133261SJohn Soni Jose csol_cqe->cmd_wnd - 1); 127673133261SJohn Soni Jose 12776733b39aSJayamohan Kallickal hdr->opcode = ISCSI_OP_NOOP_IN; 1278bfead3b2SJayamohan Kallickal hdr->itt = io_task->libiscsi_itt; 12796733b39aSJayamohan Kallickal __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 12806733b39aSJayamohan Kallickal } 12816733b39aSJayamohan Kallickal 128273133261SJohn Soni Jose static void adapter_get_sol_cqe(struct beiscsi_hba *phba, 128373133261SJohn Soni Jose struct sol_cqe *psol, 128473133261SJohn Soni Jose struct common_sol_cqe *csol_cqe) 128573133261SJohn Soni Jose { 12862c9dfd36SJayamohan Kallickal if (is_chip_be2_be3r(phba)) { 12872c9dfd36SJayamohan Kallickal csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe, 12882c9dfd36SJayamohan Kallickal i_exp_cmd_sn, psol); 12892c9dfd36SJayamohan Kallickal csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe, 12902c9dfd36SJayamohan Kallickal i_res_cnt, psol); 12912c9dfd36SJayamohan Kallickal csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe, 12922c9dfd36SJayamohan Kallickal i_cmd_wnd, psol); 12932c9dfd36SJayamohan Kallickal csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe, 12942c9dfd36SJayamohan Kallickal wrb_index, psol); 12952c9dfd36SJayamohan Kallickal csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe, 12962c9dfd36SJayamohan Kallickal cid, psol); 12972c9dfd36SJayamohan Kallickal csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe, 12982c9dfd36SJayamohan Kallickal hw_sts, psol); 12992c9dfd36SJayamohan Kallickal csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe, 13002c9dfd36SJayamohan Kallickal i_resp, psol); 13012c9dfd36SJayamohan Kallickal csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe, 13022c9dfd36SJayamohan Kallickal i_sts, psol); 13032c9dfd36SJayamohan Kallickal csol_cqe->i_flags = AMAP_GET_BITS(struct amap_sol_cqe, 13042c9dfd36SJayamohan Kallickal i_flags, psol); 13052c9dfd36SJayamohan Kallickal } else { 130673133261SJohn Soni Jose csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe_v2, 130773133261SJohn Soni Jose i_exp_cmd_sn, psol); 130873133261SJohn Soni Jose csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe_v2, 130973133261SJohn Soni Jose i_res_cnt, psol); 131073133261SJohn Soni Jose csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe_v2, 131173133261SJohn Soni Jose wrb_index, psol); 131273133261SJohn Soni Jose csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe_v2, 131373133261SJohn Soni Jose cid, psol); 131473133261SJohn Soni Jose csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2, 131573133261SJohn Soni Jose hw_sts, psol); 1316702dc5e8SJayamohan Kallickal csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe_v2, 131773133261SJohn Soni Jose i_cmd_wnd, psol); 131873133261SJohn Soni Jose if (AMAP_GET_BITS(struct amap_sol_cqe_v2, 131973133261SJohn Soni Jose cmd_cmpl, psol)) 132073133261SJohn Soni Jose csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2, 132173133261SJohn Soni Jose i_sts, psol); 132273133261SJohn Soni Jose else 132373133261SJohn Soni Jose csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe_v2, 132473133261SJohn Soni Jose i_sts, psol); 132573133261SJohn Soni Jose if (AMAP_GET_BITS(struct amap_sol_cqe_v2, 132673133261SJohn Soni Jose u, psol)) 132773133261SJohn Soni Jose csol_cqe->i_flags = ISCSI_FLAG_CMD_UNDERFLOW; 132873133261SJohn Soni Jose 132973133261SJohn Soni Jose if (AMAP_GET_BITS(struct amap_sol_cqe_v2, 133073133261SJohn Soni Jose o, psol)) 133173133261SJohn Soni Jose csol_cqe->i_flags |= ISCSI_FLAG_CMD_OVERFLOW; 133273133261SJohn Soni Jose } 133373133261SJohn Soni Jose } 133473133261SJohn Soni Jose 133573133261SJohn Soni Jose 13366733b39aSJayamohan Kallickal static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn, 13376733b39aSJayamohan Kallickal struct beiscsi_hba *phba, struct sol_cqe *psol) 13386733b39aSJayamohan Kallickal { 13396733b39aSJayamohan Kallickal struct iscsi_conn *conn = beiscsi_conn->conn; 13406733b39aSJayamohan Kallickal struct iscsi_session *session = conn->session; 134173133261SJohn Soni Jose struct common_sol_cqe csol_cqe = {0}; 13423f7f62eeSJitendra Bhivare struct hwi_wrb_context *pwrb_context; 13433f7f62eeSJitendra Bhivare struct hwi_controller *phwi_ctrlr; 13443f7f62eeSJitendra Bhivare struct wrb_handle *pwrb_handle; 13453f7f62eeSJitendra Bhivare struct iscsi_task *task; 1346a7909b39SJayamohan Kallickal uint16_t cri_index = 0; 13473f7f62eeSJitendra Bhivare uint8_t type; 13486733b39aSJayamohan Kallickal 13496733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 135073133261SJohn Soni Jose 135173133261SJohn Soni Jose /* Copy the elements to a common structure */ 135273133261SJohn Soni Jose adapter_get_sol_cqe(phba, psol, &csol_cqe); 135373133261SJohn Soni Jose 1354a7909b39SJayamohan Kallickal cri_index = BE_GET_CRI_FROM_CID(csol_cqe.cid); 1355a7909b39SJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[cri_index]; 135673133261SJohn Soni Jose 135773133261SJohn Soni Jose pwrb_handle = pwrb_context->pwrb_handle_basestd[ 135873133261SJohn Soni Jose csol_cqe.wrb_index]; 135973133261SJohn Soni Jose 13603f7f62eeSJitendra Bhivare spin_lock_bh(&session->back_lock); 13616733b39aSJayamohan Kallickal task = pwrb_handle->pio_handle; 13623f7f62eeSJitendra Bhivare if (!task) { 13633f7f62eeSJitendra Bhivare spin_unlock_bh(&session->back_lock); 13643f7f62eeSJitendra Bhivare return; 13653f7f62eeSJitendra Bhivare } 136673133261SJohn Soni Jose type = ((struct beiscsi_io_task *)task->dd_data)->wrb_type; 136732951dd8SJayamohan Kallickal 1368bfead3b2SJayamohan Kallickal switch (type) { 13696733b39aSJayamohan Kallickal case HWH_TYPE_IO: 13706733b39aSJayamohan Kallickal case HWH_TYPE_IO_RD: 13716733b39aSJayamohan Kallickal if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == 1372dafab8e0SJayamohan Kallickal ISCSI_OP_NOOP_OUT) 137373133261SJohn Soni Jose be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe); 1374dafab8e0SJayamohan Kallickal else 137573133261SJohn Soni Jose be_complete_io(beiscsi_conn, task, &csol_cqe); 13766733b39aSJayamohan Kallickal break; 13776733b39aSJayamohan Kallickal 13786733b39aSJayamohan Kallickal case HWH_TYPE_LOGOUT: 1379dafab8e0SJayamohan Kallickal if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT) 138073133261SJohn Soni Jose be_complete_logout(beiscsi_conn, task, &csol_cqe); 1381dafab8e0SJayamohan Kallickal else 138273133261SJohn Soni Jose be_complete_tmf(beiscsi_conn, task, &csol_cqe); 13836733b39aSJayamohan Kallickal break; 13846733b39aSJayamohan Kallickal 13856733b39aSJayamohan Kallickal case HWH_TYPE_LOGIN: 138699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 138799bc5d55SJohn Soni Jose BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 138899bc5d55SJohn Soni Jose "BM_%d :\t\t No HWH_TYPE_LOGIN Expected in" 138999bc5d55SJohn Soni Jose " hwi_complete_cmd- Solicited path\n"); 13906733b39aSJayamohan Kallickal break; 13916733b39aSJayamohan Kallickal 13926733b39aSJayamohan Kallickal case HWH_TYPE_NOP: 139373133261SJohn Soni Jose be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe); 13946733b39aSJayamohan Kallickal break; 13956733b39aSJayamohan Kallickal 13966733b39aSJayamohan Kallickal default: 139799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, 139899bc5d55SJohn Soni Jose BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 139999bc5d55SJohn Soni Jose "BM_%d : In hwi_complete_cmd, unknown type = %d" 1400bfead3b2SJayamohan Kallickal "wrb_index 0x%x CID 0x%x\n", type, 140173133261SJohn Soni Jose csol_cqe.wrb_index, 140273133261SJohn Soni Jose csol_cqe.cid); 14036733b39aSJayamohan Kallickal break; 14046733b39aSJayamohan Kallickal } 140535e66019SJayamohan Kallickal 1406659743b0SShlomo Pongratz spin_unlock_bh(&session->back_lock); 14076733b39aSJayamohan Kallickal } 14086733b39aSJayamohan Kallickal 1409938f372cSJitendra Bhivare /** 1410938f372cSJitendra Bhivare * ASYNC PDUs include 1411938f372cSJitendra Bhivare * a. Unsolicited NOP-In (target initiated NOP-In) 1412938f372cSJitendra Bhivare * b. ASYNC Messages 1413938f372cSJitendra Bhivare * c. Reject PDU 1414938f372cSJitendra Bhivare * d. Login response 1415938f372cSJitendra Bhivare * These headers arrive unprocessed by the EP firmware. 1416938f372cSJitendra Bhivare * iSCSI layer processes them. 1417938f372cSJitendra Bhivare */ 14186733b39aSJayamohan Kallickal static unsigned int 1419938f372cSJitendra Bhivare beiscsi_complete_pdu(struct beiscsi_conn *beiscsi_conn, 1420938f372cSJitendra Bhivare struct pdu_base *phdr, void *pdata, unsigned int dlen) 14216733b39aSJayamohan Kallickal { 1422938f372cSJitendra Bhivare struct beiscsi_hba *phba = beiscsi_conn->phba; 1423938f372cSJitendra Bhivare struct iscsi_conn *conn = beiscsi_conn->conn; 1424938f372cSJitendra Bhivare struct beiscsi_io_task *io_task; 1425938f372cSJitendra Bhivare struct iscsi_hdr *login_hdr; 1426938f372cSJitendra Bhivare struct iscsi_task *task; 1427938f372cSJitendra Bhivare u8 code; 14286733b39aSJayamohan Kallickal 1429938f372cSJitendra Bhivare code = AMAP_GET_BITS(struct amap_pdu_base, opcode, phdr); 1430938f372cSJitendra Bhivare switch (code) { 1431938f372cSJitendra Bhivare case ISCSI_OP_NOOP_IN: 1432938f372cSJitendra Bhivare pdata = NULL; 1433938f372cSJitendra Bhivare dlen = 0; 1434938f372cSJitendra Bhivare break; 1435938f372cSJitendra Bhivare case ISCSI_OP_ASYNC_EVENT: 1436938f372cSJitendra Bhivare break; 1437938f372cSJitendra Bhivare case ISCSI_OP_REJECT: 1438938f372cSJitendra Bhivare WARN_ON(!pdata); 1439938f372cSJitendra Bhivare WARN_ON(!(dlen == 48)); 144099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 144199bc5d55SJohn Soni Jose BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 1442938f372cSJitendra Bhivare "BM_%d : In ISCSI_OP_REJECT\n"); 1443938f372cSJitendra Bhivare break; 1444938f372cSJitendra Bhivare case ISCSI_OP_LOGIN_RSP: 1445938f372cSJitendra Bhivare case ISCSI_OP_TEXT_RSP: 1446938f372cSJitendra Bhivare task = conn->login_task; 1447938f372cSJitendra Bhivare io_task = task->dd_data; 1448938f372cSJitendra Bhivare login_hdr = (struct iscsi_hdr *)phdr; 1449938f372cSJitendra Bhivare login_hdr->itt = io_task->libiscsi_itt; 1450938f372cSJitendra Bhivare break; 1451938f372cSJitendra Bhivare default: 1452938f372cSJitendra Bhivare beiscsi_log(phba, KERN_WARNING, 1453938f372cSJitendra Bhivare BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 1454938f372cSJitendra Bhivare "BM_%d : unrecognized async PDU opcode 0x%x\n", 1455938f372cSJitendra Bhivare code); 1456938f372cSJitendra Bhivare return 1; 14576733b39aSJayamohan Kallickal } 1458938f372cSJitendra Bhivare __iscsi_complete_pdu(conn, (struct iscsi_hdr *)phdr, pdata, dlen); 14596733b39aSJayamohan Kallickal return 0; 14606733b39aSJayamohan Kallickal } 14616733b39aSJayamohan Kallickal 1462938f372cSJitendra Bhivare static inline void 1463938f372cSJitendra Bhivare beiscsi_hdl_put_handle(struct hd_async_context *pasync_ctx, 1464938f372cSJitendra Bhivare struct hd_async_handle *pasync_handle) 14656733b39aSJayamohan Kallickal { 1466ba6983a7SJitendra Bhivare pasync_handle->is_final = 0; 1467ba6983a7SJitendra Bhivare pasync_handle->buffer_len = 0; 1468ba6983a7SJitendra Bhivare pasync_handle->in_use = 0; 1469ba6983a7SJitendra Bhivare list_del_init(&pasync_handle->link); 14706733b39aSJayamohan Kallickal } 1471ba6983a7SJitendra Bhivare 1472ba6983a7SJitendra Bhivare static void 1473ba6983a7SJitendra Bhivare beiscsi_hdl_purge_handles(struct beiscsi_hba *phba, 1474ba6983a7SJitendra Bhivare struct hd_async_context *pasync_ctx, 1475ba6983a7SJitendra Bhivare u16 cri) 1476ba6983a7SJitendra Bhivare { 1477ba6983a7SJitendra Bhivare struct hd_async_handle *pasync_handle, *tmp_handle; 1478ba6983a7SJitendra Bhivare struct list_head *plist; 1479ba6983a7SJitendra Bhivare 1480ba6983a7SJitendra Bhivare plist = &pasync_ctx->async_entry[cri].wq.list; 1481ba6983a7SJitendra Bhivare list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) 1482ba6983a7SJitendra Bhivare beiscsi_hdl_put_handle(pasync_ctx, pasync_handle); 1483ba6983a7SJitendra Bhivare 1484ba6983a7SJitendra Bhivare INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wq.list); 1485ba6983a7SJitendra Bhivare pasync_ctx->async_entry[cri].wq.hdr_len = 0; 1486ba6983a7SJitendra Bhivare pasync_ctx->async_entry[cri].wq.bytes_received = 0; 1487ba6983a7SJitendra Bhivare pasync_ctx->async_entry[cri].wq.bytes_needed = 0; 14886733b39aSJayamohan Kallickal } 14896733b39aSJayamohan Kallickal 1490938f372cSJitendra Bhivare static struct hd_async_handle * 1491938f372cSJitendra Bhivare beiscsi_hdl_get_handle(struct beiscsi_conn *beiscsi_conn, 1492938f372cSJitendra Bhivare struct hd_async_context *pasync_ctx, 14931e2931f1SJitendra Bhivare struct i_t_dpdu_cqe *pdpdu_cqe, 14941e2931f1SJitendra Bhivare u8 *header) 1495938f372cSJitendra Bhivare { 1496938f372cSJitendra Bhivare struct beiscsi_hba *phba = beiscsi_conn->phba; 1497938f372cSJitendra Bhivare struct hd_async_handle *pasync_handle; 1498938f372cSJitendra Bhivare struct be_bus_address phys_addr; 1499ba6983a7SJitendra Bhivare u16 cid, code, ci, cri; 1500938f372cSJitendra Bhivare u8 final, error = 0; 1501938f372cSJitendra Bhivare u32 dpl; 1502938f372cSJitendra Bhivare 1503938f372cSJitendra Bhivare cid = beiscsi_conn->beiscsi_conn_cid; 1504ba6983a7SJitendra Bhivare cri = BE_GET_ASYNC_CRI_FROM_CID(cid); 1505938f372cSJitendra Bhivare /** 1506938f372cSJitendra Bhivare * This function is invoked to get the right async_handle structure 1507938f372cSJitendra Bhivare * from a given DEF PDU CQ entry. 1508938f372cSJitendra Bhivare * 1509938f372cSJitendra Bhivare * - index in CQ entry gives the vertical index 1510938f372cSJitendra Bhivare * - address in CQ entry is the offset where the DMA last ended 1511938f372cSJitendra Bhivare * - final - no more notifications for this PDU 1512938f372cSJitendra Bhivare */ 1513938f372cSJitendra Bhivare if (is_chip_be2_be3r(phba)) { 1514938f372cSJitendra Bhivare dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe, 1515938f372cSJitendra Bhivare dpl, pdpdu_cqe); 1516938f372cSJitendra Bhivare ci = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe, 1517938f372cSJitendra Bhivare index, pdpdu_cqe); 1518938f372cSJitendra Bhivare final = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe, 1519938f372cSJitendra Bhivare final, pdpdu_cqe); 1520938f372cSJitendra Bhivare } else { 1521938f372cSJitendra Bhivare dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2, 1522938f372cSJitendra Bhivare dpl, pdpdu_cqe); 1523938f372cSJitendra Bhivare ci = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2, 1524938f372cSJitendra Bhivare index, pdpdu_cqe); 1525938f372cSJitendra Bhivare final = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2, 1526938f372cSJitendra Bhivare final, pdpdu_cqe); 15276733b39aSJayamohan Kallickal } 15286733b39aSJayamohan Kallickal 1529938f372cSJitendra Bhivare /** 1530938f372cSJitendra Bhivare * DB addr Hi/Lo is same for BE and SKH. 1531938f372cSJitendra Bhivare * Subtract the dataplacementlength to get to the base. 1532938f372cSJitendra Bhivare */ 1533938f372cSJitendra Bhivare phys_addr.u.a32.address_lo = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe, 1534938f372cSJitendra Bhivare db_addr_lo, pdpdu_cqe); 1535938f372cSJitendra Bhivare phys_addr.u.a32.address_lo -= dpl; 1536938f372cSJitendra Bhivare phys_addr.u.a32.address_hi = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe, 1537938f372cSJitendra Bhivare db_addr_hi, pdpdu_cqe); 15386733b39aSJayamohan Kallickal 1539938f372cSJitendra Bhivare code = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe, code, pdpdu_cqe); 1540938f372cSJitendra Bhivare switch (code) { 1541938f372cSJitendra Bhivare case UNSOL_HDR_NOTIFY: 1542938f372cSJitendra Bhivare pasync_handle = pasync_ctx->async_entry[ci].header; 15431e2931f1SJitendra Bhivare *header = 1; 1544938f372cSJitendra Bhivare break; 1545938f372cSJitendra Bhivare case UNSOL_DATA_DIGEST_ERROR_NOTIFY: 1546938f372cSJitendra Bhivare error = 1; 1547938f372cSJitendra Bhivare case UNSOL_DATA_NOTIFY: 1548938f372cSJitendra Bhivare pasync_handle = pasync_ctx->async_entry[ci].data; 1549938f372cSJitendra Bhivare break; 1550938f372cSJitendra Bhivare /* called only for above codes */ 1551938f372cSJitendra Bhivare default: 1552ba6983a7SJitendra Bhivare return NULL; 1553938f372cSJitendra Bhivare } 1554938f372cSJitendra Bhivare 1555938f372cSJitendra Bhivare if (pasync_handle->pa.u.a64.address != phys_addr.u.a64.address || 1556938f372cSJitendra Bhivare pasync_handle->index != ci) { 1557938f372cSJitendra Bhivare /* driver bug - if ci does not match async handle index */ 1558938f372cSJitendra Bhivare error = 1; 1559938f372cSJitendra Bhivare beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI, 1560938f372cSJitendra Bhivare "BM_%d : cid %u async PDU handle mismatch - addr in %cQE %llx at %u:addr in CQE %llx ci %u\n", 1561938f372cSJitendra Bhivare cid, pasync_handle->is_header ? 'H' : 'D', 1562938f372cSJitendra Bhivare pasync_handle->pa.u.a64.address, 1563938f372cSJitendra Bhivare pasync_handle->index, 1564938f372cSJitendra Bhivare phys_addr.u.a64.address, ci); 1565938f372cSJitendra Bhivare /* FW has stale address - attempt continuing by dropping */ 1566938f372cSJitendra Bhivare } 1567938f372cSJitendra Bhivare 1568938f372cSJitendra Bhivare /** 1569938f372cSJitendra Bhivare * DEF PDU header and data buffers with errors should be simply 1570938f372cSJitendra Bhivare * dropped as there are no consumers for it. 1571938f372cSJitendra Bhivare */ 1572938f372cSJitendra Bhivare if (error) { 1573938f372cSJitendra Bhivare beiscsi_hdl_put_handle(pasync_ctx, pasync_handle); 1574ba6983a7SJitendra Bhivare return NULL; 1575938f372cSJitendra Bhivare } 1576ba6983a7SJitendra Bhivare 1577ba6983a7SJitendra Bhivare if (pasync_handle->in_use || !list_empty(&pasync_handle->link)) { 1578ba6983a7SJitendra Bhivare beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI, 1579ba6983a7SJitendra Bhivare "BM_%d : cid %d async PDU handle in use - code %d ci %d addr %llx\n", 1580ba6983a7SJitendra Bhivare cid, code, ci, phys_addr.u.a64.address); 1581ba6983a7SJitendra Bhivare beiscsi_hdl_purge_handles(phba, pasync_ctx, cri); 1582ba6983a7SJitendra Bhivare } 1583ba6983a7SJitendra Bhivare 1584ba6983a7SJitendra Bhivare list_del_init(&pasync_handle->link); 1585ba6983a7SJitendra Bhivare /** 1586ba6983a7SJitendra Bhivare * Each CID is associated with unique CRI. 1587ba6983a7SJitendra Bhivare * ASYNC_CRI_FROM_CID mapping and CRI_FROM_CID are totaly different. 1588ba6983a7SJitendra Bhivare **/ 1589ba6983a7SJitendra Bhivare pasync_handle->cri = cri; 1590ba6983a7SJitendra Bhivare pasync_handle->is_final = final; 1591ba6983a7SJitendra Bhivare pasync_handle->buffer_len = dpl; 1592ba6983a7SJitendra Bhivare pasync_handle->in_use = 1; 1593ba6983a7SJitendra Bhivare 1594938f372cSJitendra Bhivare return pasync_handle; 15956733b39aSJayamohan Kallickal } 15966733b39aSJayamohan Kallickal 1597938f372cSJitendra Bhivare static unsigned int 1598938f372cSJitendra Bhivare beiscsi_hdl_fwd_pdu(struct beiscsi_conn *beiscsi_conn, 1599938f372cSJitendra Bhivare struct hd_async_context *pasync_ctx, 1600938f372cSJitendra Bhivare u16 cri) 1601938f372cSJitendra Bhivare { 1602938f372cSJitendra Bhivare struct iscsi_session *session = beiscsi_conn->conn->session; 1603938f372cSJitendra Bhivare struct hd_async_handle *pasync_handle, *plast_handle; 1604938f372cSJitendra Bhivare struct beiscsi_hba *phba = beiscsi_conn->phba; 1605938f372cSJitendra Bhivare void *phdr = NULL, *pdata = NULL; 1606938f372cSJitendra Bhivare u32 dlen = 0, status = 0; 1607938f372cSJitendra Bhivare struct list_head *plist; 1608938f372cSJitendra Bhivare 1609938f372cSJitendra Bhivare plist = &pasync_ctx->async_entry[cri].wq.list; 1610938f372cSJitendra Bhivare plast_handle = NULL; 1611938f372cSJitendra Bhivare list_for_each_entry(pasync_handle, plist, link) { 1612938f372cSJitendra Bhivare plast_handle = pasync_handle; 1613938f372cSJitendra Bhivare /* get the header, the first entry */ 1614938f372cSJitendra Bhivare if (!phdr) { 1615938f372cSJitendra Bhivare phdr = pasync_handle->pbuffer; 1616938f372cSJitendra Bhivare continue; 1617938f372cSJitendra Bhivare } 1618938f372cSJitendra Bhivare /* use first buffer to collect all the data */ 1619938f372cSJitendra Bhivare if (!pdata) { 1620938f372cSJitendra Bhivare pdata = pasync_handle->pbuffer; 1621938f372cSJitendra Bhivare dlen = pasync_handle->buffer_len; 1622938f372cSJitendra Bhivare continue; 1623938f372cSJitendra Bhivare } 16240ddee50eSJitendra Bhivare if (!pasync_handle->buffer_len || 16250ddee50eSJitendra Bhivare (dlen + pasync_handle->buffer_len) > 16260ddee50eSJitendra Bhivare pasync_ctx->async_data.buffer_size) 16270ddee50eSJitendra Bhivare break; 1628938f372cSJitendra Bhivare memcpy(pdata + dlen, pasync_handle->pbuffer, 1629938f372cSJitendra Bhivare pasync_handle->buffer_len); 1630938f372cSJitendra Bhivare dlen += pasync_handle->buffer_len; 1631938f372cSJitendra Bhivare } 1632938f372cSJitendra Bhivare 1633938f372cSJitendra Bhivare if (!plast_handle->is_final) { 1634938f372cSJitendra Bhivare /* last handle should have final PDU notification from FW */ 1635938f372cSJitendra Bhivare beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI, 16360ddee50eSJitendra Bhivare "BM_%d : cid %u %p fwd async PDU opcode %x with last handle missing - HL%u:DN%u:DR%u\n", 1637938f372cSJitendra Bhivare beiscsi_conn->beiscsi_conn_cid, plast_handle, 16380ddee50eSJitendra Bhivare AMAP_GET_BITS(struct amap_pdu_base, opcode, phdr), 1639938f372cSJitendra Bhivare pasync_ctx->async_entry[cri].wq.hdr_len, 1640938f372cSJitendra Bhivare pasync_ctx->async_entry[cri].wq.bytes_needed, 1641938f372cSJitendra Bhivare pasync_ctx->async_entry[cri].wq.bytes_received); 1642938f372cSJitendra Bhivare } 1643938f372cSJitendra Bhivare spin_lock_bh(&session->back_lock); 1644938f372cSJitendra Bhivare status = beiscsi_complete_pdu(beiscsi_conn, phdr, pdata, dlen); 1645938f372cSJitendra Bhivare spin_unlock_bh(&session->back_lock); 1646938f372cSJitendra Bhivare beiscsi_hdl_purge_handles(phba, pasync_ctx, cri); 1647938f372cSJitendra Bhivare return status; 1648938f372cSJitendra Bhivare } 1649938f372cSJitendra Bhivare 1650938f372cSJitendra Bhivare static unsigned int 1651938f372cSJitendra Bhivare beiscsi_hdl_gather_pdu(struct beiscsi_conn *beiscsi_conn, 1652938f372cSJitendra Bhivare struct hd_async_context *pasync_ctx, 1653938f372cSJitendra Bhivare struct hd_async_handle *pasync_handle) 1654938f372cSJitendra Bhivare { 1655938f372cSJitendra Bhivare unsigned int bytes_needed = 0, status = 0; 1656938f372cSJitendra Bhivare u16 cri = pasync_handle->cri; 1657938f372cSJitendra Bhivare struct cri_wait_queue *wq; 1658938f372cSJitendra Bhivare struct beiscsi_hba *phba; 1659938f372cSJitendra Bhivare struct pdu_base *ppdu; 1660938f372cSJitendra Bhivare char *err = ""; 1661938f372cSJitendra Bhivare 1662938f372cSJitendra Bhivare phba = beiscsi_conn->phba; 1663938f372cSJitendra Bhivare wq = &pasync_ctx->async_entry[cri].wq; 1664938f372cSJitendra Bhivare if (pasync_handle->is_header) { 1665938f372cSJitendra Bhivare /* check if PDU hdr is rcv'd when old hdr not completed */ 1666938f372cSJitendra Bhivare if (wq->hdr_len) { 1667938f372cSJitendra Bhivare err = "incomplete"; 1668938f372cSJitendra Bhivare goto drop_pdu; 1669938f372cSJitendra Bhivare } 1670938f372cSJitendra Bhivare ppdu = pasync_handle->pbuffer; 1671938f372cSJitendra Bhivare bytes_needed = AMAP_GET_BITS(struct amap_pdu_base, 1672938f372cSJitendra Bhivare data_len_hi, ppdu); 1673938f372cSJitendra Bhivare bytes_needed <<= 16; 1674938f372cSJitendra Bhivare bytes_needed |= be16_to_cpu(AMAP_GET_BITS(struct amap_pdu_base, 1675938f372cSJitendra Bhivare data_len_lo, ppdu)); 1676938f372cSJitendra Bhivare wq->hdr_len = pasync_handle->buffer_len; 1677938f372cSJitendra Bhivare wq->bytes_received = 0; 1678938f372cSJitendra Bhivare wq->bytes_needed = bytes_needed; 1679938f372cSJitendra Bhivare list_add_tail(&pasync_handle->link, &wq->list); 1680938f372cSJitendra Bhivare if (!bytes_needed) 1681938f372cSJitendra Bhivare status = beiscsi_hdl_fwd_pdu(beiscsi_conn, 1682938f372cSJitendra Bhivare pasync_ctx, cri); 1683938f372cSJitendra Bhivare } else { 1684938f372cSJitendra Bhivare /* check if data received has header and is needed */ 1685938f372cSJitendra Bhivare if (!wq->hdr_len || !wq->bytes_needed) { 1686938f372cSJitendra Bhivare err = "header less"; 1687938f372cSJitendra Bhivare goto drop_pdu; 1688938f372cSJitendra Bhivare } 1689938f372cSJitendra Bhivare wq->bytes_received += pasync_handle->buffer_len; 1690938f372cSJitendra Bhivare /* Something got overwritten? Better catch it here. */ 1691938f372cSJitendra Bhivare if (wq->bytes_received > wq->bytes_needed) { 1692938f372cSJitendra Bhivare err = "overflow"; 1693938f372cSJitendra Bhivare goto drop_pdu; 1694938f372cSJitendra Bhivare } 1695938f372cSJitendra Bhivare list_add_tail(&pasync_handle->link, &wq->list); 1696938f372cSJitendra Bhivare if (wq->bytes_received == wq->bytes_needed) 1697938f372cSJitendra Bhivare status = beiscsi_hdl_fwd_pdu(beiscsi_conn, 1698938f372cSJitendra Bhivare pasync_ctx, cri); 1699938f372cSJitendra Bhivare } 1700938f372cSJitendra Bhivare return status; 1701938f372cSJitendra Bhivare 1702938f372cSJitendra Bhivare drop_pdu: 1703938f372cSJitendra Bhivare beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI, 1704938f372cSJitendra Bhivare "BM_%d : cid %u async PDU %s - def-%c:HL%u:DN%u:DR%u\n", 1705938f372cSJitendra Bhivare beiscsi_conn->beiscsi_conn_cid, err, 1706938f372cSJitendra Bhivare pasync_handle->is_header ? 'H' : 'D', 1707938f372cSJitendra Bhivare wq->hdr_len, wq->bytes_needed, 1708938f372cSJitendra Bhivare pasync_handle->buffer_len); 1709938f372cSJitendra Bhivare /* discard this handle */ 1710938f372cSJitendra Bhivare beiscsi_hdl_put_handle(pasync_ctx, pasync_handle); 1711938f372cSJitendra Bhivare /* free all the other handles in cri_wait_queue */ 1712938f372cSJitendra Bhivare beiscsi_hdl_purge_handles(phba, pasync_ctx, cri); 1713938f372cSJitendra Bhivare /* try continuing */ 1714938f372cSJitendra Bhivare return status; 1715938f372cSJitendra Bhivare } 1716938f372cSJitendra Bhivare 1717938f372cSJitendra Bhivare static void 1718938f372cSJitendra Bhivare beiscsi_hdq_post_handles(struct beiscsi_hba *phba, 17191e2931f1SJitendra Bhivare u8 header, u8 ulp_num, u16 nbuf) 1720938f372cSJitendra Bhivare { 17211e2931f1SJitendra Bhivare struct hd_async_handle *pasync_handle; 1722938f372cSJitendra Bhivare struct hd_async_context *pasync_ctx; 17236733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 17246733b39aSJayamohan Kallickal struct phys_addr *pasync_sge; 1725938f372cSJitendra Bhivare u32 ring_id, doorbell = 0; 1726938f372cSJitendra Bhivare u32 doorbell_offset; 17271e2931f1SJitendra Bhivare u16 prod, pi; 17286733b39aSJayamohan Kallickal 17296733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 17308a86e833SJayamohan Kallickal pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, ulp_num); 1731938f372cSJitendra Bhivare if (header) { 17321e2931f1SJitendra Bhivare pasync_sge = pasync_ctx->async_header.ring_base; 17331e2931f1SJitendra Bhivare pi = pasync_ctx->async_header.pi; 17348a86e833SJayamohan Kallickal ring_id = phwi_ctrlr->default_pdu_hdr[ulp_num].id; 17358a86e833SJayamohan Kallickal doorbell_offset = phwi_ctrlr->default_pdu_hdr[ulp_num]. 17368a86e833SJayamohan Kallickal doorbell_offset; 17376733b39aSJayamohan Kallickal } else { 17381e2931f1SJitendra Bhivare pasync_sge = pasync_ctx->async_data.ring_base; 17391e2931f1SJitendra Bhivare pi = pasync_ctx->async_data.pi; 17408a86e833SJayamohan Kallickal ring_id = phwi_ctrlr->default_pdu_data[ulp_num].id; 17418a86e833SJayamohan Kallickal doorbell_offset = phwi_ctrlr->default_pdu_data[ulp_num]. 17428a86e833SJayamohan Kallickal doorbell_offset; 17436733b39aSJayamohan Kallickal } 17446733b39aSJayamohan Kallickal 17451e2931f1SJitendra Bhivare for (prod = 0; prod < nbuf; prod++) { 17461e2931f1SJitendra Bhivare if (header) 17471e2931f1SJitendra Bhivare pasync_handle = pasync_ctx->async_entry[pi].header; 17481e2931f1SJitendra Bhivare else 17491e2931f1SJitendra Bhivare pasync_handle = pasync_ctx->async_entry[pi].data; 1750938f372cSJitendra Bhivare WARN_ON(pasync_handle->is_header != header); 17511e2931f1SJitendra Bhivare WARN_ON(pasync_handle->index != pi); 17521e2931f1SJitendra Bhivare /* setup the ring only once */ 17531e2931f1SJitendra Bhivare if (nbuf == pasync_ctx->num_entries) { 17541e2931f1SJitendra Bhivare /* note hi is lo */ 17551e2931f1SJitendra Bhivare pasync_sge[pi].hi = pasync_handle->pa.u.a32.address_lo; 17561e2931f1SJitendra Bhivare pasync_sge[pi].lo = pasync_handle->pa.u.a32.address_hi; 1757938f372cSJitendra Bhivare } 17581e2931f1SJitendra Bhivare if (++pi == pasync_ctx->num_entries) 17591e2931f1SJitendra Bhivare pi = 0; 17601e2931f1SJitendra Bhivare } 17616733b39aSJayamohan Kallickal 1762938f372cSJitendra Bhivare if (header) 17631e2931f1SJitendra Bhivare pasync_ctx->async_header.pi = pi; 1764938f372cSJitendra Bhivare else 17651e2931f1SJitendra Bhivare pasync_ctx->async_data.pi = pi; 17666733b39aSJayamohan Kallickal 17676733b39aSJayamohan Kallickal doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK; 17686733b39aSJayamohan Kallickal doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT; 17696733b39aSJayamohan Kallickal doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT; 1770938f372cSJitendra Bhivare doorbell |= (prod & DB_DEF_PDU_CQPROC_MASK) << DB_DEF_PDU_CQPROC_SHIFT; 17718a86e833SJayamohan Kallickal iowrite32(doorbell, phba->db_va + doorbell_offset); 17726733b39aSJayamohan Kallickal } 17736733b39aSJayamohan Kallickal 1774938f372cSJitendra Bhivare static void 1775938f372cSJitendra Bhivare beiscsi_hdq_process_compl(struct beiscsi_conn *beiscsi_conn, 17766733b39aSJayamohan Kallickal struct i_t_dpdu_cqe *pdpdu_cqe) 17776733b39aSJayamohan Kallickal { 1778938f372cSJitendra Bhivare struct beiscsi_hba *phba = beiscsi_conn->phba; 1779938f372cSJitendra Bhivare struct hd_async_handle *pasync_handle = NULL; 1780938f372cSJitendra Bhivare struct hd_async_context *pasync_ctx; 17816733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 17821e2931f1SJitendra Bhivare u8 ulp_num, consumed, header = 0; 1783938f372cSJitendra Bhivare u16 cid_cri; 17846733b39aSJayamohan Kallickal 17856733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 1786938f372cSJitendra Bhivare cid_cri = BE_GET_CRI_FROM_CID(beiscsi_conn->beiscsi_conn_cid); 1787938f372cSJitendra Bhivare ulp_num = BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr, cid_cri); 1788938f372cSJitendra Bhivare pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, ulp_num); 1789938f372cSJitendra Bhivare pasync_handle = beiscsi_hdl_get_handle(beiscsi_conn, pasync_ctx, 17901e2931f1SJitendra Bhivare pdpdu_cqe, &header); 17911e2931f1SJitendra Bhivare if (is_chip_be2_be3r(phba)) 17921e2931f1SJitendra Bhivare consumed = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe, 17931e2931f1SJitendra Bhivare num_cons, pdpdu_cqe); 17941e2931f1SJitendra Bhivare else 17951e2931f1SJitendra Bhivare consumed = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2, 17961e2931f1SJitendra Bhivare num_cons, pdpdu_cqe); 17971e2931f1SJitendra Bhivare if (pasync_handle) 1798938f372cSJitendra Bhivare beiscsi_hdl_gather_pdu(beiscsi_conn, pasync_ctx, pasync_handle); 17991e2931f1SJitendra Bhivare /* num_cons indicates number of 8 RQEs consumed */ 18001e2931f1SJitendra Bhivare if (consumed) 18011e2931f1SJitendra Bhivare beiscsi_hdq_post_handles(phba, header, ulp_num, 8 * consumed); 18026733b39aSJayamohan Kallickal } 18036733b39aSJayamohan Kallickal 18042e4e8f65SJitendra Bhivare void beiscsi_process_mcc_cq(struct beiscsi_hba *phba) 1805756d29c8SJayamohan Kallickal { 1806756d29c8SJayamohan Kallickal struct be_queue_info *mcc_cq; 1807756d29c8SJayamohan Kallickal struct be_mcc_compl *mcc_compl; 1808756d29c8SJayamohan Kallickal unsigned int num_processed = 0; 1809756d29c8SJayamohan Kallickal 1810756d29c8SJayamohan Kallickal mcc_cq = &phba->ctrl.mcc_obj.cq; 1811756d29c8SJayamohan Kallickal mcc_compl = queue_tail_node(mcc_cq); 1812756d29c8SJayamohan Kallickal mcc_compl->flags = le32_to_cpu(mcc_compl->flags); 1813756d29c8SJayamohan Kallickal while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) { 18149122e991SJitendra Bhivare if (beiscsi_hba_in_error(phba)) 18159122e991SJitendra Bhivare return; 18169122e991SJitendra Bhivare 1817756d29c8SJayamohan Kallickal if (num_processed >= 32) { 1818756d29c8SJayamohan Kallickal hwi_ring_cq_db(phba, mcc_cq->id, 18191094cf68SJitendra Bhivare num_processed, 0); 1820756d29c8SJayamohan Kallickal num_processed = 0; 1821756d29c8SJayamohan Kallickal } 1822756d29c8SJayamohan Kallickal if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) { 182353aefe25SJitendra Bhivare beiscsi_process_async_event(phba, mcc_compl); 1824756d29c8SJayamohan Kallickal } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) { 18252e4e8f65SJitendra Bhivare beiscsi_process_mcc_compl(&phba->ctrl, mcc_compl); 1826756d29c8SJayamohan Kallickal } 1827756d29c8SJayamohan Kallickal 1828756d29c8SJayamohan Kallickal mcc_compl->flags = 0; 1829756d29c8SJayamohan Kallickal queue_tail_inc(mcc_cq); 1830756d29c8SJayamohan Kallickal mcc_compl = queue_tail_node(mcc_cq); 1831756d29c8SJayamohan Kallickal mcc_compl->flags = le32_to_cpu(mcc_compl->flags); 1832756d29c8SJayamohan Kallickal num_processed++; 1833756d29c8SJayamohan Kallickal } 1834756d29c8SJayamohan Kallickal 1835756d29c8SJayamohan Kallickal if (num_processed > 0) 18361094cf68SJitendra Bhivare hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1); 1837756d29c8SJayamohan Kallickal } 1838bfead3b2SJayamohan Kallickal 1839a3095016SJitendra Bhivare static void beiscsi_mcc_work(struct work_struct *work) 1840a3095016SJitendra Bhivare { 1841a3095016SJitendra Bhivare struct be_eq_obj *pbe_eq; 1842a3095016SJitendra Bhivare struct beiscsi_hba *phba; 1843a3095016SJitendra Bhivare 1844a3095016SJitendra Bhivare pbe_eq = container_of(work, struct be_eq_obj, mcc_work); 1845a3095016SJitendra Bhivare phba = pbe_eq->phba; 1846a3095016SJitendra Bhivare beiscsi_process_mcc_cq(phba); 1847a3095016SJitendra Bhivare /* rearm EQ for further interrupts */ 18489122e991SJitendra Bhivare if (!beiscsi_hba_in_error(phba)) 1849a3095016SJitendra Bhivare hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1); 1850a3095016SJitendra Bhivare } 1851a3095016SJitendra Bhivare 18526763daaeSJohn Soni Jose /** 18536763daaeSJohn Soni Jose * beiscsi_process_cq()- Process the Completion Queue 18546763daaeSJohn Soni Jose * @pbe_eq: Event Q on which the Completion has come 18551094cf68SJitendra Bhivare * @budget: Max number of events to processed 18566763daaeSJohn Soni Jose * 18576763daaeSJohn Soni Jose * return 18586763daaeSJohn Soni Jose * Number of Completion Entries processed. 18596763daaeSJohn Soni Jose **/ 18601094cf68SJitendra Bhivare unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq, int budget) 18616733b39aSJayamohan Kallickal { 18626733b39aSJayamohan Kallickal struct be_queue_info *cq; 18636733b39aSJayamohan Kallickal struct sol_cqe *sol; 18646733b39aSJayamohan Kallickal struct dmsg_cqe *dmsg; 18651094cf68SJitendra Bhivare unsigned int total = 0; 18666733b39aSJayamohan Kallickal unsigned int num_processed = 0; 18670a513dd8SJohn Soni Jose unsigned short code = 0, cid = 0; 1868a7909b39SJayamohan Kallickal uint16_t cri_index = 0; 18696733b39aSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn; 1870c2462288SJayamohan Kallickal struct beiscsi_endpoint *beiscsi_ep; 1871c2462288SJayamohan Kallickal struct iscsi_endpoint *ep; 1872bfead3b2SJayamohan Kallickal struct beiscsi_hba *phba; 18736733b39aSJayamohan Kallickal 1874bfead3b2SJayamohan Kallickal cq = pbe_eq->cq; 18756733b39aSJayamohan Kallickal sol = queue_tail_node(cq); 1876bfead3b2SJayamohan Kallickal phba = pbe_eq->phba; 18776733b39aSJayamohan Kallickal 18786733b39aSJayamohan Kallickal while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] & 18796733b39aSJayamohan Kallickal CQE_VALID_MASK) { 18809122e991SJitendra Bhivare if (beiscsi_hba_in_error(phba)) 18819122e991SJitendra Bhivare return 0; 18829122e991SJitendra Bhivare 18836733b39aSJayamohan Kallickal be_dws_le_to_cpu(sol, sizeof(struct sol_cqe)); 18846733b39aSJayamohan Kallickal 188573133261SJohn Soni Jose code = (sol->dw[offsetof(struct amap_sol_cqe, code) / 188673133261SJohn Soni Jose 32] & CQE_CODE_MASK); 188732951dd8SJayamohan Kallickal 188873133261SJohn Soni Jose /* Get the CID */ 18892c9dfd36SJayamohan Kallickal if (is_chip_be2_be3r(phba)) { 18902c9dfd36SJayamohan Kallickal cid = AMAP_GET_BITS(struct amap_sol_cqe, cid, sol); 18912c9dfd36SJayamohan Kallickal } else { 189273133261SJohn Soni Jose if ((code == DRIVERMSG_NOTIFY) || 189373133261SJohn Soni Jose (code == UNSOL_HDR_NOTIFY) || 189473133261SJohn Soni Jose (code == UNSOL_DATA_NOTIFY)) 189573133261SJohn Soni Jose cid = AMAP_GET_BITS( 189673133261SJohn Soni Jose struct amap_i_t_dpdu_cqe_v2, 189773133261SJohn Soni Jose cid, sol); 189873133261SJohn Soni Jose else 189973133261SJohn Soni Jose cid = AMAP_GET_BITS(struct amap_sol_cqe_v2, 190073133261SJohn Soni Jose cid, sol); 19012c9dfd36SJayamohan Kallickal } 190273133261SJohn Soni Jose 1903a7909b39SJayamohan Kallickal cri_index = BE_GET_CRI_FROM_CID(cid); 1904a7909b39SJayamohan Kallickal ep = phba->ep_array[cri_index]; 1905b7ab35b1SJayamohan Kallickal 1906b7ab35b1SJayamohan Kallickal if (ep == NULL) { 1907b7ab35b1SJayamohan Kallickal /* connection has already been freed 1908b7ab35b1SJayamohan Kallickal * just move on to next one 1909b7ab35b1SJayamohan Kallickal */ 1910b7ab35b1SJayamohan Kallickal beiscsi_log(phba, KERN_WARNING, 1911b7ab35b1SJayamohan Kallickal BEISCSI_LOG_INIT, 1912b7ab35b1SJayamohan Kallickal "BM_%d : proc cqe of disconn ep: cid %d\n", 1913b7ab35b1SJayamohan Kallickal cid); 1914b7ab35b1SJayamohan Kallickal goto proc_next_cqe; 1915b7ab35b1SJayamohan Kallickal } 1916b7ab35b1SJayamohan Kallickal 1917c2462288SJayamohan Kallickal beiscsi_ep = ep->dd_data; 1918c2462288SJayamohan Kallickal beiscsi_conn = beiscsi_ep->conn; 1919756d29c8SJayamohan Kallickal 19201094cf68SJitendra Bhivare /* replenish cq */ 19211094cf68SJitendra Bhivare if (num_processed == 32) { 19221094cf68SJitendra Bhivare hwi_ring_cq_db(phba, cq->id, 32, 0); 19236733b39aSJayamohan Kallickal num_processed = 0; 19246733b39aSJayamohan Kallickal } 19251094cf68SJitendra Bhivare total++; 19266733b39aSJayamohan Kallickal 19270a513dd8SJohn Soni Jose switch (code) { 19286733b39aSJayamohan Kallickal case SOL_CMD_COMPLETE: 19296733b39aSJayamohan Kallickal hwi_complete_cmd(beiscsi_conn, phba, sol); 19306733b39aSJayamohan Kallickal break; 19316733b39aSJayamohan Kallickal case DRIVERMSG_NOTIFY: 193299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, 193399bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 19346763daaeSJohn Soni Jose "BM_%d : Received %s[%d] on CID : %d\n", 19356763daaeSJohn Soni Jose cqe_desc[code], code, cid); 193699bc5d55SJohn Soni Jose 19376733b39aSJayamohan Kallickal dmsg = (struct dmsg_cqe *)sol; 19386733b39aSJayamohan Kallickal hwi_complete_drvr_msgs(beiscsi_conn, phba, sol); 19396733b39aSJayamohan Kallickal break; 19406733b39aSJayamohan Kallickal case UNSOL_HDR_NOTIFY: 194199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, 194299bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 19436763daaeSJohn Soni Jose "BM_%d : Received %s[%d] on CID : %d\n", 19446763daaeSJohn Soni Jose cqe_desc[code], code, cid); 194599bc5d55SJohn Soni Jose 19468f09a3b9SJayamohan Kallickal spin_lock_bh(&phba->async_pdu_lock); 1947938f372cSJitendra Bhivare beiscsi_hdq_process_compl(beiscsi_conn, 1948bfead3b2SJayamohan Kallickal (struct i_t_dpdu_cqe *)sol); 19498f09a3b9SJayamohan Kallickal spin_unlock_bh(&phba->async_pdu_lock); 1950bfead3b2SJayamohan Kallickal break; 19516733b39aSJayamohan Kallickal case UNSOL_DATA_NOTIFY: 195299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, 195399bc5d55SJohn Soni Jose BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 19546763daaeSJohn Soni Jose "BM_%d : Received %s[%d] on CID : %d\n", 19556763daaeSJohn Soni Jose cqe_desc[code], code, cid); 195699bc5d55SJohn Soni Jose 19578f09a3b9SJayamohan Kallickal spin_lock_bh(&phba->async_pdu_lock); 1958938f372cSJitendra Bhivare beiscsi_hdq_process_compl(beiscsi_conn, 19596733b39aSJayamohan Kallickal (struct i_t_dpdu_cqe *)sol); 19608f09a3b9SJayamohan Kallickal spin_unlock_bh(&phba->async_pdu_lock); 19616733b39aSJayamohan Kallickal break; 19626733b39aSJayamohan Kallickal case CXN_INVALIDATE_INDEX_NOTIFY: 19636733b39aSJayamohan Kallickal case CMD_INVALIDATED_NOTIFY: 19646733b39aSJayamohan Kallickal case CXN_INVALIDATE_NOTIFY: 196599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 196699bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 19676763daaeSJohn Soni Jose "BM_%d : Ignoring %s[%d] on CID : %d\n", 19686763daaeSJohn Soni Jose cqe_desc[code], code, cid); 19696733b39aSJayamohan Kallickal break; 19701094cf68SJitendra Bhivare case CXN_KILLED_HDR_DIGEST_ERR: 19716733b39aSJayamohan Kallickal case SOL_CMD_KILLED_DATA_DIGEST_ERR: 19721094cf68SJitendra Bhivare beiscsi_log(phba, KERN_ERR, 19731094cf68SJitendra Bhivare BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 19741094cf68SJitendra Bhivare "BM_%d : Cmd Notification %s[%d] on CID : %d\n", 19751094cf68SJitendra Bhivare cqe_desc[code], code, cid); 19761094cf68SJitendra Bhivare break; 19776733b39aSJayamohan Kallickal case CMD_KILLED_INVALID_STATSN_RCVD: 19786733b39aSJayamohan Kallickal case CMD_KILLED_INVALID_R2T_RCVD: 19796733b39aSJayamohan Kallickal case CMD_CXN_KILLED_LUN_INVALID: 19806733b39aSJayamohan Kallickal case CMD_CXN_KILLED_ICD_INVALID: 19816733b39aSJayamohan Kallickal case CMD_CXN_KILLED_ITT_INVALID: 19826733b39aSJayamohan Kallickal case CMD_CXN_KILLED_SEQ_OUTOFORDER: 19836733b39aSJayamohan Kallickal case CMD_CXN_KILLED_INVALID_DATASN_RCVD: 198499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 198599bc5d55SJohn Soni Jose BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 19866763daaeSJohn Soni Jose "BM_%d : Cmd Notification %s[%d] on CID : %d\n", 19876763daaeSJohn Soni Jose cqe_desc[code], code, cid); 19886733b39aSJayamohan Kallickal break; 19896733b39aSJayamohan Kallickal case UNSOL_DATA_DIGEST_ERROR_NOTIFY: 199099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 199199bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 19926763daaeSJohn Soni Jose "BM_%d : Dropping %s[%d] on DPDU ring on CID : %d\n", 19936763daaeSJohn Soni Jose cqe_desc[code], code, cid); 19948f09a3b9SJayamohan Kallickal spin_lock_bh(&phba->async_pdu_lock); 1995938f372cSJitendra Bhivare /* driver consumes the entry and drops the contents */ 1996938f372cSJitendra Bhivare beiscsi_hdq_process_compl(beiscsi_conn, 19976733b39aSJayamohan Kallickal (struct i_t_dpdu_cqe *)sol); 19988f09a3b9SJayamohan Kallickal spin_unlock_bh(&phba->async_pdu_lock); 19996733b39aSJayamohan Kallickal break; 20006733b39aSJayamohan Kallickal case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL: 20016733b39aSJayamohan Kallickal case CXN_KILLED_BURST_LEN_MISMATCH: 20026733b39aSJayamohan Kallickal case CXN_KILLED_AHS_RCVD: 20036733b39aSJayamohan Kallickal case CXN_KILLED_UNKNOWN_HDR: 20046733b39aSJayamohan Kallickal case CXN_KILLED_STALE_ITT_TTT_RCVD: 20056733b39aSJayamohan Kallickal case CXN_KILLED_INVALID_ITT_TTT_RCVD: 20066733b39aSJayamohan Kallickal case CXN_KILLED_TIMED_OUT: 20076733b39aSJayamohan Kallickal case CXN_KILLED_FIN_RCVD: 20086763daaeSJohn Soni Jose case CXN_KILLED_RST_SENT: 20096763daaeSJohn Soni Jose case CXN_KILLED_RST_RCVD: 20106733b39aSJayamohan Kallickal case CXN_KILLED_BAD_UNSOL_PDU_RCVD: 20116733b39aSJayamohan Kallickal case CXN_KILLED_BAD_WRB_INDEX_ERROR: 20126733b39aSJayamohan Kallickal case CXN_KILLED_OVER_RUN_RESIDUAL: 20136733b39aSJayamohan Kallickal case CXN_KILLED_UNDER_RUN_RESIDUAL: 20146733b39aSJayamohan Kallickal case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN: 201599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 201699bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 20176763daaeSJohn Soni Jose "BM_%d : Event %s[%d] received on CID : %d\n", 20186763daaeSJohn Soni Jose cqe_desc[code], code, cid); 20190a513dd8SJohn Soni Jose if (beiscsi_conn) 20206733b39aSJayamohan Kallickal iscsi_conn_failure(beiscsi_conn->conn, 20216733b39aSJayamohan Kallickal ISCSI_ERR_CONN_FAILED); 20226733b39aSJayamohan Kallickal break; 20236733b39aSJayamohan Kallickal default: 202499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 202599bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 20266763daaeSJohn Soni Jose "BM_%d : Invalid CQE Event Received Code : %d" 20276763daaeSJohn Soni Jose "CID 0x%x...\n", 20280a513dd8SJohn Soni Jose code, cid); 20296733b39aSJayamohan Kallickal break; 20306733b39aSJayamohan Kallickal } 20316733b39aSJayamohan Kallickal 2032b7ab35b1SJayamohan Kallickal proc_next_cqe: 20336733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0); 20346733b39aSJayamohan Kallickal queue_tail_inc(cq); 20356733b39aSJayamohan Kallickal sol = queue_tail_node(cq); 20366733b39aSJayamohan Kallickal num_processed++; 20371094cf68SJitendra Bhivare if (total == budget) 20381094cf68SJitendra Bhivare break; 20396733b39aSJayamohan Kallickal } 20406733b39aSJayamohan Kallickal 20411094cf68SJitendra Bhivare hwi_ring_cq_db(phba, cq->id, num_processed, 1); 20421094cf68SJitendra Bhivare return total; 20436733b39aSJayamohan Kallickal } 20446733b39aSJayamohan Kallickal 2045511cbce2SChristoph Hellwig static int be_iopoll(struct irq_poll *iop, int budget) 20466733b39aSJayamohan Kallickal { 2047a3095016SJitendra Bhivare unsigned int ret, io_events; 20486733b39aSJayamohan Kallickal struct beiscsi_hba *phba; 2049bfead3b2SJayamohan Kallickal struct be_eq_obj *pbe_eq; 20501094cf68SJitendra Bhivare struct be_eq_entry *eqe = NULL; 20511094cf68SJitendra Bhivare struct be_queue_info *eq; 20526733b39aSJayamohan Kallickal 2053bfead3b2SJayamohan Kallickal pbe_eq = container_of(iop, struct be_eq_obj, iopoll); 20541094cf68SJitendra Bhivare phba = pbe_eq->phba; 20559122e991SJitendra Bhivare if (beiscsi_hba_in_error(phba)) { 20569122e991SJitendra Bhivare irq_poll_complete(iop); 20579122e991SJitendra Bhivare return 0; 20589122e991SJitendra Bhivare } 20599122e991SJitendra Bhivare 20609122e991SJitendra Bhivare io_events = 0; 20611094cf68SJitendra Bhivare eq = &pbe_eq->q; 20621094cf68SJitendra Bhivare eqe = queue_tail_node(eq); 20631094cf68SJitendra Bhivare while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] & 20641094cf68SJitendra Bhivare EQE_VALID_MASK) { 20651094cf68SJitendra Bhivare AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 20661094cf68SJitendra Bhivare queue_tail_inc(eq); 20671094cf68SJitendra Bhivare eqe = queue_tail_node(eq); 2068a3095016SJitendra Bhivare io_events++; 20691094cf68SJitendra Bhivare } 2070a3095016SJitendra Bhivare hwi_ring_eq_db(phba, eq->id, 1, io_events, 0, 1); 20711094cf68SJitendra Bhivare 20721094cf68SJitendra Bhivare ret = beiscsi_process_cq(pbe_eq, budget); 207373af08e1SJayamohan Kallickal pbe_eq->cq_count += ret; 20746733b39aSJayamohan Kallickal if (ret < budget) { 2075511cbce2SChristoph Hellwig irq_poll_complete(iop); 207699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, 207799bc5d55SJohn Soni Jose BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 20781094cf68SJitendra Bhivare "BM_%d : rearm pbe_eq->q.id =%d ret %d\n", 20791094cf68SJitendra Bhivare pbe_eq->q.id, ret); 20809122e991SJitendra Bhivare if (!beiscsi_hba_in_error(phba)) 2081bfead3b2SJayamohan Kallickal hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1); 20826733b39aSJayamohan Kallickal } 20836733b39aSJayamohan Kallickal return ret; 20846733b39aSJayamohan Kallickal } 20856733b39aSJayamohan Kallickal 20866733b39aSJayamohan Kallickal static void 208709a1093aSJohn Soni Jose hwi_write_sgl_v2(struct iscsi_wrb *pwrb, struct scatterlist *sg, 208809a1093aSJohn Soni Jose unsigned int num_sg, struct beiscsi_io_task *io_task) 208909a1093aSJohn Soni Jose { 209009a1093aSJohn Soni Jose struct iscsi_sge *psgl; 209109a1093aSJohn Soni Jose unsigned int sg_len, index; 209209a1093aSJohn Soni Jose unsigned int sge_len = 0; 209309a1093aSJohn Soni Jose unsigned long long addr; 209409a1093aSJohn Soni Jose struct scatterlist *l_sg; 209509a1093aSJohn Soni Jose unsigned int offset; 209609a1093aSJohn Soni Jose 209709a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_lo, pwrb, 209809a1093aSJohn Soni Jose io_task->bhs_pa.u.a32.address_lo); 209909a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_hi, pwrb, 210009a1093aSJohn Soni Jose io_task->bhs_pa.u.a32.address_hi); 210109a1093aSJohn Soni Jose 210209a1093aSJohn Soni Jose l_sg = sg; 210309a1093aSJohn Soni Jose for (index = 0; (index < num_sg) && (index < 2); index++, 210409a1093aSJohn Soni Jose sg = sg_next(sg)) { 210509a1093aSJohn Soni Jose if (index == 0) { 210609a1093aSJohn Soni Jose sg_len = sg_dma_len(sg); 210709a1093aSJohn Soni Jose addr = (u64) sg_dma_address(sg); 210809a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 210909a1093aSJohn Soni Jose sge0_addr_lo, pwrb, 211009a1093aSJohn Soni Jose lower_32_bits(addr)); 211109a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 211209a1093aSJohn Soni Jose sge0_addr_hi, pwrb, 211309a1093aSJohn Soni Jose upper_32_bits(addr)); 211409a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 211509a1093aSJohn Soni Jose sge0_len, pwrb, 211609a1093aSJohn Soni Jose sg_len); 211709a1093aSJohn Soni Jose sge_len = sg_len; 211809a1093aSJohn Soni Jose } else { 211909a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_r2t_offset, 212009a1093aSJohn Soni Jose pwrb, sge_len); 212109a1093aSJohn Soni Jose sg_len = sg_dma_len(sg); 212209a1093aSJohn Soni Jose addr = (u64) sg_dma_address(sg); 212309a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 212409a1093aSJohn Soni Jose sge1_addr_lo, pwrb, 212509a1093aSJohn Soni Jose lower_32_bits(addr)); 212609a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 212709a1093aSJohn Soni Jose sge1_addr_hi, pwrb, 212809a1093aSJohn Soni Jose upper_32_bits(addr)); 212909a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 213009a1093aSJohn Soni Jose sge1_len, pwrb, 213109a1093aSJohn Soni Jose sg_len); 213209a1093aSJohn Soni Jose } 213309a1093aSJohn Soni Jose } 213409a1093aSJohn Soni Jose psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag; 213509a1093aSJohn Soni Jose memset(psgl, 0, sizeof(*psgl) * BE2_SGE); 213609a1093aSJohn Soni Jose 213709a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2); 213809a1093aSJohn Soni Jose 213909a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 214009a1093aSJohn Soni Jose io_task->bhs_pa.u.a32.address_hi); 214109a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 214209a1093aSJohn Soni Jose io_task->bhs_pa.u.a32.address_lo); 214309a1093aSJohn Soni Jose 214409a1093aSJohn Soni Jose if (num_sg == 1) { 214509a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb, 214609a1093aSJohn Soni Jose 1); 214709a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb, 214809a1093aSJohn Soni Jose 0); 214909a1093aSJohn Soni Jose } else if (num_sg == 2) { 215009a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb, 215109a1093aSJohn Soni Jose 0); 215209a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb, 215309a1093aSJohn Soni Jose 1); 215409a1093aSJohn Soni Jose } else { 215509a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb, 215609a1093aSJohn Soni Jose 0); 215709a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb, 215809a1093aSJohn Soni Jose 0); 215909a1093aSJohn Soni Jose } 216009a1093aSJohn Soni Jose 216109a1093aSJohn Soni Jose sg = l_sg; 216209a1093aSJohn Soni Jose psgl++; 216309a1093aSJohn Soni Jose psgl++; 216409a1093aSJohn Soni Jose offset = 0; 216509a1093aSJohn Soni Jose for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) { 216609a1093aSJohn Soni Jose sg_len = sg_dma_len(sg); 216709a1093aSJohn Soni Jose addr = (u64) sg_dma_address(sg); 216809a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 216909a1093aSJohn Soni Jose lower_32_bits(addr)); 217009a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 217109a1093aSJohn Soni Jose upper_32_bits(addr)); 217209a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len); 217309a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset); 217409a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0); 217509a1093aSJohn Soni Jose offset += sg_len; 217609a1093aSJohn Soni Jose } 217709a1093aSJohn Soni Jose psgl--; 217809a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1); 217909a1093aSJohn Soni Jose } 218009a1093aSJohn Soni Jose 218109a1093aSJohn Soni Jose static void 21826733b39aSJayamohan Kallickal hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg, 21836733b39aSJayamohan Kallickal unsigned int num_sg, struct beiscsi_io_task *io_task) 21846733b39aSJayamohan Kallickal { 21856733b39aSJayamohan Kallickal struct iscsi_sge *psgl; 218658ff4bd0SJayamohan Kallickal unsigned int sg_len, index; 21876733b39aSJayamohan Kallickal unsigned int sge_len = 0; 21886733b39aSJayamohan Kallickal unsigned long long addr; 21896733b39aSJayamohan Kallickal struct scatterlist *l_sg; 21906733b39aSJayamohan Kallickal unsigned int offset; 21916733b39aSJayamohan Kallickal 21926733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb, 21936733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_lo); 21946733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb, 21956733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_hi); 21966733b39aSJayamohan Kallickal 21976733b39aSJayamohan Kallickal l_sg = sg; 219848bd86cfSJayamohan Kallickal for (index = 0; (index < num_sg) && (index < 2); index++, 219948bd86cfSJayamohan Kallickal sg = sg_next(sg)) { 22006733b39aSJayamohan Kallickal if (index == 0) { 22016733b39aSJayamohan Kallickal sg_len = sg_dma_len(sg); 22026733b39aSJayamohan Kallickal addr = (u64) sg_dma_address(sg); 22036733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb, 2204457ff3b7SJayamohan Kallickal ((u32)(addr & 0xFFFFFFFF))); 22056733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb, 2206457ff3b7SJayamohan Kallickal ((u32)(addr >> 32))); 22076733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb, 22086733b39aSJayamohan Kallickal sg_len); 22096733b39aSJayamohan Kallickal sge_len = sg_len; 22106733b39aSJayamohan Kallickal } else { 22116733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset, 22126733b39aSJayamohan Kallickal pwrb, sge_len); 22136733b39aSJayamohan Kallickal sg_len = sg_dma_len(sg); 22146733b39aSJayamohan Kallickal addr = (u64) sg_dma_address(sg); 22156733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb, 2216457ff3b7SJayamohan Kallickal ((u32)(addr & 0xFFFFFFFF))); 22176733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb, 2218457ff3b7SJayamohan Kallickal ((u32)(addr >> 32))); 22196733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb, 22206733b39aSJayamohan Kallickal sg_len); 22216733b39aSJayamohan Kallickal } 22226733b39aSJayamohan Kallickal } 22236733b39aSJayamohan Kallickal psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag; 22246733b39aSJayamohan Kallickal memset(psgl, 0, sizeof(*psgl) * BE2_SGE); 22256733b39aSJayamohan Kallickal 22266733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2); 22276733b39aSJayamohan Kallickal 22286733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 22296733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_hi); 22306733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 22316733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_lo); 22326733b39aSJayamohan Kallickal 2233caf818f1SJayamohan Kallickal if (num_sg == 1) { 2234caf818f1SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 2235caf818f1SJayamohan Kallickal 1); 2236caf818f1SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 2237caf818f1SJayamohan Kallickal 0); 2238caf818f1SJayamohan Kallickal } else if (num_sg == 2) { 2239caf818f1SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 2240caf818f1SJayamohan Kallickal 0); 2241caf818f1SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 2242caf818f1SJayamohan Kallickal 1); 2243caf818f1SJayamohan Kallickal } else { 2244caf818f1SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 2245caf818f1SJayamohan Kallickal 0); 2246caf818f1SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 2247caf818f1SJayamohan Kallickal 0); 2248caf818f1SJayamohan Kallickal } 22496733b39aSJayamohan Kallickal sg = l_sg; 22506733b39aSJayamohan Kallickal psgl++; 22516733b39aSJayamohan Kallickal psgl++; 22526733b39aSJayamohan Kallickal offset = 0; 225348bd86cfSJayamohan Kallickal for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) { 22546733b39aSJayamohan Kallickal sg_len = sg_dma_len(sg); 22556733b39aSJayamohan Kallickal addr = (u64) sg_dma_address(sg); 22566733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 22576733b39aSJayamohan Kallickal (addr & 0xFFFFFFFF)); 22586733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 22596733b39aSJayamohan Kallickal (addr >> 32)); 22606733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len); 22616733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset); 22626733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0); 22636733b39aSJayamohan Kallickal offset += sg_len; 22646733b39aSJayamohan Kallickal } 22656733b39aSJayamohan Kallickal psgl--; 22666733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1); 22676733b39aSJayamohan Kallickal } 22686733b39aSJayamohan Kallickal 2269d629c471SJohn Soni Jose /** 2270d629c471SJohn Soni Jose * hwi_write_buffer()- Populate the WRB with task info 2271d629c471SJohn Soni Jose * @pwrb: ptr to the WRB entry 2272d629c471SJohn Soni Jose * @task: iscsi task which is to be executed 2273d629c471SJohn Soni Jose **/ 2274e0493627SAlexey Khoroshilov static int hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task) 22756733b39aSJayamohan Kallickal { 22766733b39aSJayamohan Kallickal struct iscsi_sge *psgl; 22776733b39aSJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 22786733b39aSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn = io_task->conn; 22796733b39aSJayamohan Kallickal struct beiscsi_hba *phba = beiscsi_conn->phba; 228009a1093aSJohn Soni Jose uint8_t dsp_value = 0; 22816733b39aSJayamohan Kallickal 22826733b39aSJayamohan Kallickal io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2; 22836733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb, 22846733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_lo); 22856733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb, 22866733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_hi); 22876733b39aSJayamohan Kallickal 22886733b39aSJayamohan Kallickal if (task->data) { 228909a1093aSJohn Soni Jose 229009a1093aSJohn Soni Jose /* Check for the data_count */ 229109a1093aSJohn Soni Jose dsp_value = (task->data_count) ? 1 : 0; 229209a1093aSJohn Soni Jose 22932c9dfd36SJayamohan Kallickal if (is_chip_be2_be3r(phba)) 22942c9dfd36SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, 229509a1093aSJohn Soni Jose pwrb, dsp_value); 229609a1093aSJohn Soni Jose else 22972c9dfd36SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, 229809a1093aSJohn Soni Jose pwrb, dsp_value); 229909a1093aSJohn Soni Jose 230009a1093aSJohn Soni Jose /* Map addr only if there is data_count */ 230109a1093aSJohn Soni Jose if (dsp_value) { 2302d629c471SJohn Soni Jose io_task->mtask_addr = pci_map_single(phba->pcidev, 23036733b39aSJayamohan Kallickal task->data, 2304d629c471SJohn Soni Jose task->data_count, 2305d629c471SJohn Soni Jose PCI_DMA_TODEVICE); 2306e0493627SAlexey Khoroshilov if (pci_dma_mapping_error(phba->pcidev, 2307e0493627SAlexey Khoroshilov io_task->mtask_addr)) 2308e0493627SAlexey Khoroshilov return -ENOMEM; 2309d629c471SJohn Soni Jose io_task->mtask_data_count = task->data_count; 231009a1093aSJohn Soni Jose } else 2311d629c471SJohn Soni Jose io_task->mtask_addr = 0; 231209a1093aSJohn Soni Jose 23136733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb, 2314d629c471SJohn Soni Jose lower_32_bits(io_task->mtask_addr)); 23156733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb, 2316d629c471SJohn Soni Jose upper_32_bits(io_task->mtask_addr)); 23176733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb, 23186733b39aSJayamohan Kallickal task->data_count); 23196733b39aSJayamohan Kallickal 23206733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1); 23216733b39aSJayamohan Kallickal } else { 23226733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); 2323d629c471SJohn Soni Jose io_task->mtask_addr = 0; 23246733b39aSJayamohan Kallickal } 23256733b39aSJayamohan Kallickal 23266733b39aSJayamohan Kallickal psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag; 23276733b39aSJayamohan Kallickal 23286733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len); 23296733b39aSJayamohan Kallickal 23306733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 23316733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_hi); 23326733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 23336733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_lo); 23346733b39aSJayamohan Kallickal if (task->data) { 23356733b39aSJayamohan Kallickal psgl++; 23366733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0); 23376733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0); 23386733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0); 23396733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0); 23406733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0); 23416733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0); 23426733b39aSJayamohan Kallickal 23436733b39aSJayamohan Kallickal psgl++; 23446733b39aSJayamohan Kallickal if (task->data) { 23456733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 2346d629c471SJohn Soni Jose lower_32_bits(io_task->mtask_addr)); 23476733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 2348d629c471SJohn Soni Jose upper_32_bits(io_task->mtask_addr)); 23496733b39aSJayamohan Kallickal } 23506733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106); 23516733b39aSJayamohan Kallickal } 23526733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1); 2353e0493627SAlexey Khoroshilov return 0; 23546733b39aSJayamohan Kallickal } 23556733b39aSJayamohan Kallickal 2356843ae752SJayamohan Kallickal /** 2357843ae752SJayamohan Kallickal * beiscsi_find_mem_req()- Find mem needed 2358843ae752SJayamohan Kallickal * @phba: ptr to HBA struct 2359843ae752SJayamohan Kallickal **/ 23606733b39aSJayamohan Kallickal static void beiscsi_find_mem_req(struct beiscsi_hba *phba) 23616733b39aSJayamohan Kallickal { 23628a86e833SJayamohan Kallickal uint8_t mem_descr_index, ulp_num; 2363fa1261c4SJitendra Bhivare unsigned int num_async_pdu_buf_pages; 23646733b39aSJayamohan Kallickal unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn; 23656733b39aSJayamohan Kallickal unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages; 23666733b39aSJayamohan Kallickal 23676733b39aSJayamohan Kallickal phba->params.hwi_ws_sz = sizeof(struct hwi_controller); 23686733b39aSJayamohan Kallickal 23696733b39aSJayamohan Kallickal phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 * 23706733b39aSJayamohan Kallickal BE_ISCSI_PDU_HEADER_SIZE; 23716733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_ADDN_CONTEXT] = 23726733b39aSJayamohan Kallickal sizeof(struct hwi_context_memory); 23736733b39aSJayamohan Kallickal 23746733b39aSJayamohan Kallickal 23756733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb) 23766733b39aSJayamohan Kallickal * (phba->params.wrbs_per_cxn) 23776733b39aSJayamohan Kallickal * phba->params.cxns_per_ctrl; 23786733b39aSJayamohan Kallickal wrb_sz_per_cxn = sizeof(struct wrb_handle) * 23796733b39aSJayamohan Kallickal (phba->params.wrbs_per_cxn); 23806733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) * 23816733b39aSJayamohan Kallickal phba->params.cxns_per_ctrl); 23826733b39aSJayamohan Kallickal 23836733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) * 23846733b39aSJayamohan Kallickal phba->params.icds_per_ctrl; 23856733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) * 23866733b39aSJayamohan Kallickal phba->params.num_sge_per_io * phba->params.icds_per_ctrl; 23878a86e833SJayamohan Kallickal for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 23888a86e833SJayamohan Kallickal if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 23896733b39aSJayamohan Kallickal 23908a86e833SJayamohan Kallickal num_async_pdu_buf_sgl_pages = 2391fecc3824SJitendra Bhivare PAGES_REQUIRED(BEISCSI_ASYNC_HDQ_SIZE( 23928a86e833SJayamohan Kallickal phba, ulp_num) * 23938a86e833SJayamohan Kallickal sizeof(struct phys_addr)); 23948a86e833SJayamohan Kallickal 23958a86e833SJayamohan Kallickal num_async_pdu_buf_pages = 2396fecc3824SJitendra Bhivare PAGES_REQUIRED(BEISCSI_ASYNC_HDQ_SIZE( 23978a86e833SJayamohan Kallickal phba, ulp_num) * 23988a86e833SJayamohan Kallickal phba->params.defpdu_hdr_sz); 23998a86e833SJayamohan Kallickal 24008a86e833SJayamohan Kallickal num_async_pdu_data_pages = 2401fecc3824SJitendra Bhivare PAGES_REQUIRED(BEISCSI_ASYNC_HDQ_SIZE( 24028a86e833SJayamohan Kallickal phba, ulp_num) * 24038a86e833SJayamohan Kallickal phba->params.defpdu_data_sz); 24048a86e833SJayamohan Kallickal 24058a86e833SJayamohan Kallickal num_async_pdu_data_sgl_pages = 2406fecc3824SJitendra Bhivare PAGES_REQUIRED(BEISCSI_ASYNC_HDQ_SIZE( 24078a86e833SJayamohan Kallickal phba, ulp_num) * 24088a86e833SJayamohan Kallickal sizeof(struct phys_addr)); 24098a86e833SJayamohan Kallickal 2410a129d92fSJayamohan Kallickal mem_descr_index = (HWI_MEM_TEMPLATE_HDR_ULP0 + 2411a129d92fSJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET)); 2412a129d92fSJayamohan Kallickal phba->mem_req[mem_descr_index] = 2413a129d92fSJayamohan Kallickal BEISCSI_GET_CID_COUNT(phba, ulp_num) * 2414a129d92fSJayamohan Kallickal BEISCSI_TEMPLATE_HDR_PER_CXN_SIZE; 2415a129d92fSJayamohan Kallickal 24168a86e833SJayamohan Kallickal mem_descr_index = (HWI_MEM_ASYNC_HEADER_BUF_ULP0 + 24178a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET)); 24188a86e833SJayamohan Kallickal phba->mem_req[mem_descr_index] = 24198a86e833SJayamohan Kallickal num_async_pdu_buf_pages * 24208a86e833SJayamohan Kallickal PAGE_SIZE; 24218a86e833SJayamohan Kallickal 24228a86e833SJayamohan Kallickal mem_descr_index = (HWI_MEM_ASYNC_DATA_BUF_ULP0 + 24238a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET)); 24248a86e833SJayamohan Kallickal phba->mem_req[mem_descr_index] = 24258a86e833SJayamohan Kallickal num_async_pdu_data_pages * 24268a86e833SJayamohan Kallickal PAGE_SIZE; 24278a86e833SJayamohan Kallickal 24288a86e833SJayamohan Kallickal mem_descr_index = (HWI_MEM_ASYNC_HEADER_RING_ULP0 + 24298a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET)); 24308a86e833SJayamohan Kallickal phba->mem_req[mem_descr_index] = 24318a86e833SJayamohan Kallickal num_async_pdu_buf_sgl_pages * 24328a86e833SJayamohan Kallickal PAGE_SIZE; 24338a86e833SJayamohan Kallickal 24348a86e833SJayamohan Kallickal mem_descr_index = (HWI_MEM_ASYNC_DATA_RING_ULP0 + 24358a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET)); 24368a86e833SJayamohan Kallickal phba->mem_req[mem_descr_index] = 24378a86e833SJayamohan Kallickal num_async_pdu_data_sgl_pages * 24388a86e833SJayamohan Kallickal PAGE_SIZE; 24398a86e833SJayamohan Kallickal 24408a86e833SJayamohan Kallickal mem_descr_index = (HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 + 24418a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET)); 24428a86e833SJayamohan Kallickal phba->mem_req[mem_descr_index] = 2443fecc3824SJitendra Bhivare BEISCSI_ASYNC_HDQ_SIZE(phba, ulp_num) * 2444938f372cSJitendra Bhivare sizeof(struct hd_async_handle); 24458a86e833SJayamohan Kallickal 24468a86e833SJayamohan Kallickal mem_descr_index = (HWI_MEM_ASYNC_DATA_HANDLE_ULP0 + 24478a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET)); 24488a86e833SJayamohan Kallickal phba->mem_req[mem_descr_index] = 2449fecc3824SJitendra Bhivare BEISCSI_ASYNC_HDQ_SIZE(phba, ulp_num) * 2450938f372cSJitendra Bhivare sizeof(struct hd_async_handle); 24518a86e833SJayamohan Kallickal 24528a86e833SJayamohan Kallickal mem_descr_index = (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 + 24538a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET)); 24548a86e833SJayamohan Kallickal phba->mem_req[mem_descr_index] = 2455938f372cSJitendra Bhivare sizeof(struct hd_async_context) + 2456fecc3824SJitendra Bhivare (BEISCSI_ASYNC_HDQ_SIZE(phba, ulp_num) * 2457938f372cSJitendra Bhivare sizeof(struct hd_async_entry)); 24588a86e833SJayamohan Kallickal } 24598a86e833SJayamohan Kallickal } 24606733b39aSJayamohan Kallickal } 24616733b39aSJayamohan Kallickal 24626733b39aSJayamohan Kallickal static int beiscsi_alloc_mem(struct beiscsi_hba *phba) 24636733b39aSJayamohan Kallickal { 24646733b39aSJayamohan Kallickal dma_addr_t bus_add; 2465a7909b39SJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 2466a7909b39SJayamohan Kallickal struct be_mem_descriptor *mem_descr; 24676733b39aSJayamohan Kallickal struct mem_array *mem_arr, *mem_arr_orig; 24686733b39aSJayamohan Kallickal unsigned int i, j, alloc_size, curr_alloc_size; 24696733b39aSJayamohan Kallickal 24703ec78271SJayamohan Kallickal phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL); 24716733b39aSJayamohan Kallickal if (!phba->phwi_ctrlr) 24726733b39aSJayamohan Kallickal return -ENOMEM; 24736733b39aSJayamohan Kallickal 2474a7909b39SJayamohan Kallickal /* Allocate memory for wrb_context */ 2475a7909b39SJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 2476a7909b39SJayamohan Kallickal phwi_ctrlr->wrb_context = kzalloc(sizeof(struct hwi_wrb_context) * 2477a7909b39SJayamohan Kallickal phba->params.cxns_per_ctrl, 2478a7909b39SJayamohan Kallickal GFP_KERNEL); 24790c88740dSMaurizio Lombardi if (!phwi_ctrlr->wrb_context) { 24800c88740dSMaurizio Lombardi kfree(phba->phwi_ctrlr); 2481a7909b39SJayamohan Kallickal return -ENOMEM; 24820c88740dSMaurizio Lombardi } 2483a7909b39SJayamohan Kallickal 24846733b39aSJayamohan Kallickal phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr), 24856733b39aSJayamohan Kallickal GFP_KERNEL); 24866733b39aSJayamohan Kallickal if (!phba->init_mem) { 2487a7909b39SJayamohan Kallickal kfree(phwi_ctrlr->wrb_context); 24886733b39aSJayamohan Kallickal kfree(phba->phwi_ctrlr); 24896733b39aSJayamohan Kallickal return -ENOMEM; 24906733b39aSJayamohan Kallickal } 24916733b39aSJayamohan Kallickal 24926733b39aSJayamohan Kallickal mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT, 24936733b39aSJayamohan Kallickal GFP_KERNEL); 24946733b39aSJayamohan Kallickal if (!mem_arr_orig) { 24956733b39aSJayamohan Kallickal kfree(phba->init_mem); 2496a7909b39SJayamohan Kallickal kfree(phwi_ctrlr->wrb_context); 24976733b39aSJayamohan Kallickal kfree(phba->phwi_ctrlr); 24986733b39aSJayamohan Kallickal return -ENOMEM; 24996733b39aSJayamohan Kallickal } 25006733b39aSJayamohan Kallickal 25016733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 25026733b39aSJayamohan Kallickal for (i = 0; i < SE_MEM_MAX; i++) { 25038a86e833SJayamohan Kallickal if (!phba->mem_req[i]) { 25048a86e833SJayamohan Kallickal mem_descr->mem_array = NULL; 25058a86e833SJayamohan Kallickal mem_descr++; 25068a86e833SJayamohan Kallickal continue; 25078a86e833SJayamohan Kallickal } 25088a86e833SJayamohan Kallickal 25096733b39aSJayamohan Kallickal j = 0; 25106733b39aSJayamohan Kallickal mem_arr = mem_arr_orig; 25116733b39aSJayamohan Kallickal alloc_size = phba->mem_req[i]; 25126733b39aSJayamohan Kallickal memset(mem_arr, 0, sizeof(struct mem_array) * 25136733b39aSJayamohan Kallickal BEISCSI_MAX_FRAGS_INIT); 25146733b39aSJayamohan Kallickal curr_alloc_size = min(be_max_phys_size * 1024, alloc_size); 25156733b39aSJayamohan Kallickal do { 25166733b39aSJayamohan Kallickal mem_arr->virtual_address = pci_alloc_consistent( 25176733b39aSJayamohan Kallickal phba->pcidev, 25186733b39aSJayamohan Kallickal curr_alloc_size, 25196733b39aSJayamohan Kallickal &bus_add); 25206733b39aSJayamohan Kallickal if (!mem_arr->virtual_address) { 25216733b39aSJayamohan Kallickal if (curr_alloc_size <= BE_MIN_MEM_SIZE) 25226733b39aSJayamohan Kallickal goto free_mem; 25236733b39aSJayamohan Kallickal if (curr_alloc_size - 25246733b39aSJayamohan Kallickal rounddown_pow_of_two(curr_alloc_size)) 25256733b39aSJayamohan Kallickal curr_alloc_size = rounddown_pow_of_two 25266733b39aSJayamohan Kallickal (curr_alloc_size); 25276733b39aSJayamohan Kallickal else 25286733b39aSJayamohan Kallickal curr_alloc_size = curr_alloc_size / 2; 25296733b39aSJayamohan Kallickal } else { 25306733b39aSJayamohan Kallickal mem_arr->bus_address.u. 25316733b39aSJayamohan Kallickal a64.address = (__u64) bus_add; 25326733b39aSJayamohan Kallickal mem_arr->size = curr_alloc_size; 25336733b39aSJayamohan Kallickal alloc_size -= curr_alloc_size; 25346733b39aSJayamohan Kallickal curr_alloc_size = min(be_max_phys_size * 25356733b39aSJayamohan Kallickal 1024, alloc_size); 25366733b39aSJayamohan Kallickal j++; 25376733b39aSJayamohan Kallickal mem_arr++; 25386733b39aSJayamohan Kallickal } 25396733b39aSJayamohan Kallickal } while (alloc_size); 25406733b39aSJayamohan Kallickal mem_descr->num_elements = j; 25416733b39aSJayamohan Kallickal mem_descr->size_in_bytes = phba->mem_req[i]; 25426733b39aSJayamohan Kallickal mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j, 25436733b39aSJayamohan Kallickal GFP_KERNEL); 25446733b39aSJayamohan Kallickal if (!mem_descr->mem_array) 25456733b39aSJayamohan Kallickal goto free_mem; 25466733b39aSJayamohan Kallickal 25476733b39aSJayamohan Kallickal memcpy(mem_descr->mem_array, mem_arr_orig, 25486733b39aSJayamohan Kallickal sizeof(struct mem_array) * j); 25496733b39aSJayamohan Kallickal mem_descr++; 25506733b39aSJayamohan Kallickal } 25516733b39aSJayamohan Kallickal kfree(mem_arr_orig); 25526733b39aSJayamohan Kallickal return 0; 25536733b39aSJayamohan Kallickal free_mem: 25546733b39aSJayamohan Kallickal mem_descr->num_elements = j; 25556733b39aSJayamohan Kallickal while ((i) || (j)) { 25566733b39aSJayamohan Kallickal for (j = mem_descr->num_elements; j > 0; j--) { 25576733b39aSJayamohan Kallickal pci_free_consistent(phba->pcidev, 25586733b39aSJayamohan Kallickal mem_descr->mem_array[j - 1].size, 25596733b39aSJayamohan Kallickal mem_descr->mem_array[j - 1]. 25606733b39aSJayamohan Kallickal virtual_address, 2561457ff3b7SJayamohan Kallickal (unsigned long)mem_descr-> 2562457ff3b7SJayamohan Kallickal mem_array[j - 1]. 25636733b39aSJayamohan Kallickal bus_address.u.a64.address); 25646733b39aSJayamohan Kallickal } 25656733b39aSJayamohan Kallickal if (i) { 25666733b39aSJayamohan Kallickal i--; 25676733b39aSJayamohan Kallickal kfree(mem_descr->mem_array); 25686733b39aSJayamohan Kallickal mem_descr--; 25696733b39aSJayamohan Kallickal } 25706733b39aSJayamohan Kallickal } 25716733b39aSJayamohan Kallickal kfree(mem_arr_orig); 25726733b39aSJayamohan Kallickal kfree(phba->init_mem); 2573a7909b39SJayamohan Kallickal kfree(phba->phwi_ctrlr->wrb_context); 25746733b39aSJayamohan Kallickal kfree(phba->phwi_ctrlr); 25756733b39aSJayamohan Kallickal return -ENOMEM; 25766733b39aSJayamohan Kallickal } 25776733b39aSJayamohan Kallickal 25786733b39aSJayamohan Kallickal static int beiscsi_get_memory(struct beiscsi_hba *phba) 25796733b39aSJayamohan Kallickal { 25806733b39aSJayamohan Kallickal beiscsi_find_mem_req(phba); 25816733b39aSJayamohan Kallickal return beiscsi_alloc_mem(phba); 25826733b39aSJayamohan Kallickal } 25836733b39aSJayamohan Kallickal 25846733b39aSJayamohan Kallickal static void iscsi_init_global_templates(struct beiscsi_hba *phba) 25856733b39aSJayamohan Kallickal { 25866733b39aSJayamohan Kallickal struct pdu_data_out *pdata_out; 25876733b39aSJayamohan Kallickal struct pdu_nop_out *pnop_out; 25886733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 25896733b39aSJayamohan Kallickal 25906733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 25916733b39aSJayamohan Kallickal mem_descr += ISCSI_MEM_GLOBAL_HEADER; 25926733b39aSJayamohan Kallickal pdata_out = 25936733b39aSJayamohan Kallickal (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address; 25946733b39aSJayamohan Kallickal memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE); 25956733b39aSJayamohan Kallickal 25966733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out, 25976733b39aSJayamohan Kallickal IIOC_SCSI_DATA); 25986733b39aSJayamohan Kallickal 25996733b39aSJayamohan Kallickal pnop_out = 26006733b39aSJayamohan Kallickal (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0]. 26016733b39aSJayamohan Kallickal virtual_address + BE_ISCSI_PDU_HEADER_SIZE); 26026733b39aSJayamohan Kallickal 26036733b39aSJayamohan Kallickal memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE); 26046733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF); 26056733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1); 26066733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0); 26076733b39aSJayamohan Kallickal } 26086733b39aSJayamohan Kallickal 26093ec78271SJayamohan Kallickal static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba) 26106733b39aSJayamohan Kallickal { 26116733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb; 2612a7909b39SJayamohan Kallickal struct hwi_context_memory *phwi_ctxt; 26133ec78271SJayamohan Kallickal struct wrb_handle *pwrb_handle = NULL; 26146733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 26156733b39aSJayamohan Kallickal struct hwi_wrb_context *pwrb_context; 26163ec78271SJayamohan Kallickal struct iscsi_wrb *pwrb = NULL; 26173ec78271SJayamohan Kallickal unsigned int num_cxn_wrbh = 0; 26183ec78271SJayamohan Kallickal unsigned int num_cxn_wrb = 0, j, idx = 0, index; 26196733b39aSJayamohan Kallickal 26206733b39aSJayamohan Kallickal mem_descr_wrbh = phba->init_mem; 26216733b39aSJayamohan Kallickal mem_descr_wrbh += HWI_MEM_WRBH; 26226733b39aSJayamohan Kallickal 26236733b39aSJayamohan Kallickal mem_descr_wrb = phba->init_mem; 26246733b39aSJayamohan Kallickal mem_descr_wrb += HWI_MEM_WRB; 26256733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 26266733b39aSJayamohan Kallickal 2627a7909b39SJayamohan Kallickal /* Allocate memory for WRBQ */ 2628a7909b39SJayamohan Kallickal phwi_ctxt = phwi_ctrlr->phwi_ctxt; 2629a7909b39SJayamohan Kallickal phwi_ctxt->be_wrbq = kzalloc(sizeof(struct be_queue_info) * 2630843ae752SJayamohan Kallickal phba->params.cxns_per_ctrl, 2631a7909b39SJayamohan Kallickal GFP_KERNEL); 2632a7909b39SJayamohan Kallickal if (!phwi_ctxt->be_wrbq) { 2633a7909b39SJayamohan Kallickal beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 2634a7909b39SJayamohan Kallickal "BM_%d : WRBQ Mem Alloc Failed\n"); 2635a7909b39SJayamohan Kallickal return -ENOMEM; 2636a7909b39SJayamohan Kallickal } 2637a7909b39SJayamohan Kallickal 2638a7909b39SJayamohan Kallickal for (index = 0; index < phba->params.cxns_per_ctrl; index++) { 26396733b39aSJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[index]; 26406733b39aSJayamohan Kallickal pwrb_context->pwrb_handle_base = 26416733b39aSJayamohan Kallickal kzalloc(sizeof(struct wrb_handle *) * 26426733b39aSJayamohan Kallickal phba->params.wrbs_per_cxn, GFP_KERNEL); 26433ec78271SJayamohan Kallickal if (!pwrb_context->pwrb_handle_base) { 264499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 264599bc5d55SJohn Soni Jose "BM_%d : Mem Alloc Failed. Failing to load\n"); 26463ec78271SJayamohan Kallickal goto init_wrb_hndl_failed; 26473ec78271SJayamohan Kallickal } 26486733b39aSJayamohan Kallickal pwrb_context->pwrb_handle_basestd = 26496733b39aSJayamohan Kallickal kzalloc(sizeof(struct wrb_handle *) * 26506733b39aSJayamohan Kallickal phba->params.wrbs_per_cxn, GFP_KERNEL); 26513ec78271SJayamohan Kallickal if (!pwrb_context->pwrb_handle_basestd) { 265299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 265399bc5d55SJohn Soni Jose "BM_%d : Mem Alloc Failed. Failing to load\n"); 26543ec78271SJayamohan Kallickal goto init_wrb_hndl_failed; 26556733b39aSJayamohan Kallickal } 26563ec78271SJayamohan Kallickal if (!num_cxn_wrbh) { 26576733b39aSJayamohan Kallickal pwrb_handle = 26586733b39aSJayamohan Kallickal mem_descr_wrbh->mem_array[idx].virtual_address; 26593ec78271SJayamohan Kallickal num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) / 26606733b39aSJayamohan Kallickal ((sizeof(struct wrb_handle)) * 26616733b39aSJayamohan Kallickal phba->params.wrbs_per_cxn)); 26623ec78271SJayamohan Kallickal idx++; 26633ec78271SJayamohan Kallickal } 26646733b39aSJayamohan Kallickal pwrb_context->alloc_index = 0; 26653ec78271SJayamohan Kallickal pwrb_context->wrb_handles_available = 0; 26663ec78271SJayamohan Kallickal pwrb_context->free_index = 0; 26673ec78271SJayamohan Kallickal 26683ec78271SJayamohan Kallickal if (num_cxn_wrbh) { 26696733b39aSJayamohan Kallickal for (j = 0; j < phba->params.wrbs_per_cxn; j++) { 26706733b39aSJayamohan Kallickal pwrb_context->pwrb_handle_base[j] = pwrb_handle; 26716733b39aSJayamohan Kallickal pwrb_context->pwrb_handle_basestd[j] = 26726733b39aSJayamohan Kallickal pwrb_handle; 26736733b39aSJayamohan Kallickal pwrb_context->wrb_handles_available++; 2674bfead3b2SJayamohan Kallickal pwrb_handle->wrb_index = j; 26756733b39aSJayamohan Kallickal pwrb_handle++; 26766733b39aSJayamohan Kallickal } 26776733b39aSJayamohan Kallickal num_cxn_wrbh--; 26786733b39aSJayamohan Kallickal } 2679f64d92e6SJitendra Bhivare spin_lock_init(&pwrb_context->wrb_lock); 26806733b39aSJayamohan Kallickal } 26816733b39aSJayamohan Kallickal idx = 0; 2682a7909b39SJayamohan Kallickal for (index = 0; index < phba->params.cxns_per_ctrl; index++) { 26833ec78271SJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[index]; 26843ec78271SJayamohan Kallickal if (!num_cxn_wrb) { 26856733b39aSJayamohan Kallickal pwrb = mem_descr_wrb->mem_array[idx].virtual_address; 26867c56533cSJayamohan Kallickal num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) / 26877c56533cSJayamohan Kallickal ((sizeof(struct iscsi_wrb) * 26887c56533cSJayamohan Kallickal phba->params.wrbs_per_cxn)); 26893ec78271SJayamohan Kallickal idx++; 26903ec78271SJayamohan Kallickal } 26913ec78271SJayamohan Kallickal 26926733b39aSJayamohan Kallickal if (num_cxn_wrb) { 26936733b39aSJayamohan Kallickal for (j = 0; j < phba->params.wrbs_per_cxn; j++) { 26946733b39aSJayamohan Kallickal pwrb_handle = pwrb_context->pwrb_handle_base[j]; 26956733b39aSJayamohan Kallickal pwrb_handle->pwrb = pwrb; 26966733b39aSJayamohan Kallickal pwrb++; 26976733b39aSJayamohan Kallickal } 26986733b39aSJayamohan Kallickal num_cxn_wrb--; 26996733b39aSJayamohan Kallickal } 27006733b39aSJayamohan Kallickal } 27013ec78271SJayamohan Kallickal return 0; 27023ec78271SJayamohan Kallickal init_wrb_hndl_failed: 27033ec78271SJayamohan Kallickal for (j = index; j > 0; j--) { 27043ec78271SJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[j]; 27053ec78271SJayamohan Kallickal kfree(pwrb_context->pwrb_handle_base); 27063ec78271SJayamohan Kallickal kfree(pwrb_context->pwrb_handle_basestd); 27073ec78271SJayamohan Kallickal } 27083ec78271SJayamohan Kallickal return -ENOMEM; 27096733b39aSJayamohan Kallickal } 27106733b39aSJayamohan Kallickal 2711a7909b39SJayamohan Kallickal static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba) 27126733b39aSJayamohan Kallickal { 27138a86e833SJayamohan Kallickal uint8_t ulp_num; 27146733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 27156733b39aSJayamohan Kallickal struct hba_parameters *p = &phba->params; 2716938f372cSJitendra Bhivare struct hd_async_context *pasync_ctx; 2717938f372cSJitendra Bhivare struct hd_async_handle *pasync_header_h, *pasync_data_h; 2718dc63aac6SJayamohan Kallickal unsigned int index, idx, num_per_mem, num_async_data; 27196733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 27206733b39aSJayamohan Kallickal 27218a86e833SJayamohan Kallickal for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 27228a86e833SJayamohan Kallickal if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 2723938f372cSJitendra Bhivare /* get async_ctx for each ULP */ 27246733b39aSJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 27258a86e833SJayamohan Kallickal mem_descr += (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 + 27268a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET)); 27276733b39aSJayamohan Kallickal 27286733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 27298a86e833SJayamohan Kallickal phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num] = 2730938f372cSJitendra Bhivare (struct hd_async_context *) 27316733b39aSJayamohan Kallickal mem_descr->mem_array[0].virtual_address; 27328a86e833SJayamohan Kallickal 27338a86e833SJayamohan Kallickal pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num]; 27346733b39aSJayamohan Kallickal memset(pasync_ctx, 0, sizeof(*pasync_ctx)); 27356733b39aSJayamohan Kallickal 27368a86e833SJayamohan Kallickal pasync_ctx->async_entry = 2737938f372cSJitendra Bhivare (struct hd_async_entry *) 27388a86e833SJayamohan Kallickal ((long unsigned int)pasync_ctx + 2739938f372cSJitendra Bhivare sizeof(struct hd_async_context)); 2740a7909b39SJayamohan Kallickal 2741fecc3824SJitendra Bhivare pasync_ctx->num_entries = BEISCSI_ASYNC_HDQ_SIZE(phba, 27428a86e833SJayamohan Kallickal ulp_num); 2743938f372cSJitendra Bhivare /* setup header buffers */ 27446733b39aSJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 27458a86e833SJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_HEADER_BUF_ULP0 + 27468a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET); 27476733b39aSJayamohan Kallickal if (mem_descr->mem_array[0].virtual_address) { 274899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 274999bc5d55SJohn Soni Jose "BM_%d : hwi_init_async_pdu_ctx" 27508a86e833SJayamohan Kallickal " HWI_MEM_ASYNC_HEADER_BUF_ULP%d va=%p\n", 27518a86e833SJayamohan Kallickal ulp_num, 27528a86e833SJayamohan Kallickal mem_descr->mem_array[0]. 27538a86e833SJayamohan Kallickal virtual_address); 27546733b39aSJayamohan Kallickal } else 27558a86e833SJayamohan Kallickal beiscsi_log(phba, KERN_WARNING, 27568a86e833SJayamohan Kallickal BEISCSI_LOG_INIT, 27578a86e833SJayamohan Kallickal "BM_%d : No Virtual address for ULP : %d\n", 27588a86e833SJayamohan Kallickal ulp_num); 27596733b39aSJayamohan Kallickal 27601e2931f1SJitendra Bhivare pasync_ctx->async_header.pi = 0; 2761938f372cSJitendra Bhivare pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz; 27626733b39aSJayamohan Kallickal pasync_ctx->async_header.va_base = 27636733b39aSJayamohan Kallickal mem_descr->mem_array[0].virtual_address; 27646733b39aSJayamohan Kallickal 27656733b39aSJayamohan Kallickal pasync_ctx->async_header.pa_base.u.a64.address = 27668a86e833SJayamohan Kallickal mem_descr->mem_array[0]. 27678a86e833SJayamohan Kallickal bus_address.u.a64.address; 27686733b39aSJayamohan Kallickal 2769938f372cSJitendra Bhivare /* setup header buffer sgls */ 27706733b39aSJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 27718a86e833SJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 + 27728a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET); 27736733b39aSJayamohan Kallickal if (mem_descr->mem_array[0].virtual_address) { 277499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 277599bc5d55SJohn Soni Jose "BM_%d : hwi_init_async_pdu_ctx" 27768a86e833SJayamohan Kallickal " HWI_MEM_ASYNC_HEADER_RING_ULP%d va=%p\n", 27778a86e833SJayamohan Kallickal ulp_num, 27788a86e833SJayamohan Kallickal mem_descr->mem_array[0]. 27798a86e833SJayamohan Kallickal virtual_address); 27806733b39aSJayamohan Kallickal } else 27818a86e833SJayamohan Kallickal beiscsi_log(phba, KERN_WARNING, 27828a86e833SJayamohan Kallickal BEISCSI_LOG_INIT, 27838a86e833SJayamohan Kallickal "BM_%d : No Virtual address for ULP : %d\n", 27848a86e833SJayamohan Kallickal ulp_num); 278599bc5d55SJohn Soni Jose 27866733b39aSJayamohan Kallickal pasync_ctx->async_header.ring_base = 27876733b39aSJayamohan Kallickal mem_descr->mem_array[0].virtual_address; 27886733b39aSJayamohan Kallickal 2789938f372cSJitendra Bhivare /* setup header buffer handles */ 27906733b39aSJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 27918a86e833SJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 + 27928a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET); 27936733b39aSJayamohan Kallickal if (mem_descr->mem_array[0].virtual_address) { 279499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 279599bc5d55SJohn Soni Jose "BM_%d : hwi_init_async_pdu_ctx" 27968a86e833SJayamohan Kallickal " HWI_MEM_ASYNC_HEADER_HANDLE_ULP%d va=%p\n", 27978a86e833SJayamohan Kallickal ulp_num, 27988a86e833SJayamohan Kallickal mem_descr->mem_array[0]. 27998a86e833SJayamohan Kallickal virtual_address); 28006733b39aSJayamohan Kallickal } else 28018a86e833SJayamohan Kallickal beiscsi_log(phba, KERN_WARNING, 28028a86e833SJayamohan Kallickal BEISCSI_LOG_INIT, 28038a86e833SJayamohan Kallickal "BM_%d : No Virtual address for ULP : %d\n", 28048a86e833SJayamohan Kallickal ulp_num); 28056733b39aSJayamohan Kallickal 28066733b39aSJayamohan Kallickal pasync_ctx->async_header.handle_base = 28076733b39aSJayamohan Kallickal mem_descr->mem_array[0].virtual_address; 28086733b39aSJayamohan Kallickal 2809938f372cSJitendra Bhivare /* setup data buffer sgls */ 28106733b39aSJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 28118a86e833SJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 + 28128a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET); 28136733b39aSJayamohan Kallickal if (mem_descr->mem_array[0].virtual_address) { 281499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 281599bc5d55SJohn Soni Jose "BM_%d : hwi_init_async_pdu_ctx" 28168a86e833SJayamohan Kallickal " HWI_MEM_ASYNC_DATA_RING_ULP%d va=%p\n", 28178a86e833SJayamohan Kallickal ulp_num, 28188a86e833SJayamohan Kallickal mem_descr->mem_array[0]. 28198a86e833SJayamohan Kallickal virtual_address); 28206733b39aSJayamohan Kallickal } else 28218a86e833SJayamohan Kallickal beiscsi_log(phba, KERN_WARNING, 28228a86e833SJayamohan Kallickal BEISCSI_LOG_INIT, 28238a86e833SJayamohan Kallickal "BM_%d : No Virtual address for ULP : %d\n", 28248a86e833SJayamohan Kallickal ulp_num); 28256733b39aSJayamohan Kallickal 28266733b39aSJayamohan Kallickal pasync_ctx->async_data.ring_base = 28276733b39aSJayamohan Kallickal mem_descr->mem_array[0].virtual_address; 28286733b39aSJayamohan Kallickal 2829938f372cSJitendra Bhivare /* setup data buffer handles */ 28306733b39aSJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 28318a86e833SJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_DATA_HANDLE_ULP0 + 28328a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET); 28336733b39aSJayamohan Kallickal if (!mem_descr->mem_array[0].virtual_address) 28348a86e833SJayamohan Kallickal beiscsi_log(phba, KERN_WARNING, 28358a86e833SJayamohan Kallickal BEISCSI_LOG_INIT, 28368a86e833SJayamohan Kallickal "BM_%d : No Virtual address for ULP : %d\n", 28378a86e833SJayamohan Kallickal ulp_num); 28386733b39aSJayamohan Kallickal 28396733b39aSJayamohan Kallickal pasync_ctx->async_data.handle_base = 28406733b39aSJayamohan Kallickal mem_descr->mem_array[0].virtual_address; 28416733b39aSJayamohan Kallickal 28426733b39aSJayamohan Kallickal pasync_header_h = 2843938f372cSJitendra Bhivare (struct hd_async_handle *) 28448a86e833SJayamohan Kallickal pasync_ctx->async_header.handle_base; 28456733b39aSJayamohan Kallickal pasync_data_h = 2846938f372cSJitendra Bhivare (struct hd_async_handle *) 28478a86e833SJayamohan Kallickal pasync_ctx->async_data.handle_base; 28486733b39aSJayamohan Kallickal 2849938f372cSJitendra Bhivare /* setup data buffers */ 2850dc63aac6SJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 28518a86e833SJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_DATA_BUF_ULP0 + 28528a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET); 2853dc63aac6SJayamohan Kallickal if (mem_descr->mem_array[0].virtual_address) { 285499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 285599bc5d55SJohn Soni Jose "BM_%d : hwi_init_async_pdu_ctx" 28568a86e833SJayamohan Kallickal " HWI_MEM_ASYNC_DATA_BUF_ULP%d va=%p\n", 28578a86e833SJayamohan Kallickal ulp_num, 28588a86e833SJayamohan Kallickal mem_descr->mem_array[0]. 28598a86e833SJayamohan Kallickal virtual_address); 2860dc63aac6SJayamohan Kallickal } else 28618a86e833SJayamohan Kallickal beiscsi_log(phba, KERN_WARNING, 28628a86e833SJayamohan Kallickal BEISCSI_LOG_INIT, 28638a86e833SJayamohan Kallickal "BM_%d : No Virtual address for ULP : %d\n", 28648a86e833SJayamohan Kallickal ulp_num); 286599bc5d55SJohn Soni Jose 2866dc63aac6SJayamohan Kallickal idx = 0; 28671e2931f1SJitendra Bhivare pasync_ctx->async_data.pi = 0; 2868938f372cSJitendra Bhivare pasync_ctx->async_data.buffer_size = p->defpdu_data_sz; 2869dc63aac6SJayamohan Kallickal pasync_ctx->async_data.va_base = 2870dc63aac6SJayamohan Kallickal mem_descr->mem_array[idx].virtual_address; 2871dc63aac6SJayamohan Kallickal pasync_ctx->async_data.pa_base.u.a64.address = 28728a86e833SJayamohan Kallickal mem_descr->mem_array[idx]. 28738a86e833SJayamohan Kallickal bus_address.u.a64.address; 2874dc63aac6SJayamohan Kallickal 2875dc63aac6SJayamohan Kallickal num_async_data = ((mem_descr->mem_array[idx].size) / 2876dc63aac6SJayamohan Kallickal phba->params.defpdu_data_sz); 2877dc63aac6SJayamohan Kallickal num_per_mem = 0; 2878dc63aac6SJayamohan Kallickal 2879fecc3824SJitendra Bhivare for (index = 0; index < BEISCSI_ASYNC_HDQ_SIZE 28808a86e833SJayamohan Kallickal (phba, ulp_num); index++) { 28816733b39aSJayamohan Kallickal pasync_header_h->cri = -1; 2882938f372cSJitendra Bhivare pasync_header_h->is_header = 1; 2883938f372cSJitendra Bhivare pasync_header_h->index = index; 28846733b39aSJayamohan Kallickal INIT_LIST_HEAD(&pasync_header_h->link); 28856733b39aSJayamohan Kallickal pasync_header_h->pbuffer = 28866733b39aSJayamohan Kallickal (void *)((unsigned long) 28878a86e833SJayamohan Kallickal (pasync_ctx-> 28888a86e833SJayamohan Kallickal async_header.va_base) + 28896733b39aSJayamohan Kallickal (p->defpdu_hdr_sz * index)); 28906733b39aSJayamohan Kallickal 28916733b39aSJayamohan Kallickal pasync_header_h->pa.u.a64.address = 28928a86e833SJayamohan Kallickal pasync_ctx->async_header.pa_base.u.a64. 28938a86e833SJayamohan Kallickal address + (p->defpdu_hdr_sz * index); 28946733b39aSJayamohan Kallickal 28951e2931f1SJitendra Bhivare pasync_ctx->async_entry[index].header = 28961e2931f1SJitendra Bhivare pasync_header_h; 28976733b39aSJayamohan Kallickal pasync_header_h++; 2898938f372cSJitendra Bhivare INIT_LIST_HEAD(&pasync_ctx->async_entry[index]. 2899938f372cSJitendra Bhivare wq.list); 29006733b39aSJayamohan Kallickal 29016733b39aSJayamohan Kallickal pasync_data_h->cri = -1; 2902938f372cSJitendra Bhivare pasync_data_h->is_header = 0; 2903938f372cSJitendra Bhivare pasync_data_h->index = index; 29046733b39aSJayamohan Kallickal INIT_LIST_HEAD(&pasync_data_h->link); 2905dc63aac6SJayamohan Kallickal 2906dc63aac6SJayamohan Kallickal if (!num_async_data) { 2907dc63aac6SJayamohan Kallickal num_per_mem = 0; 2908dc63aac6SJayamohan Kallickal idx++; 2909dc63aac6SJayamohan Kallickal pasync_ctx->async_data.va_base = 29108a86e833SJayamohan Kallickal mem_descr->mem_array[idx]. 29118a86e833SJayamohan Kallickal virtual_address; 29128a86e833SJayamohan Kallickal pasync_ctx->async_data.pa_base.u. 29138a86e833SJayamohan Kallickal a64.address = 2914dc63aac6SJayamohan Kallickal mem_descr->mem_array[idx]. 2915dc63aac6SJayamohan Kallickal bus_address.u.a64.address; 29168a86e833SJayamohan Kallickal num_async_data = 29178a86e833SJayamohan Kallickal ((mem_descr->mem_array[idx]. 29188a86e833SJayamohan Kallickal size) / 2919dc63aac6SJayamohan Kallickal phba->params.defpdu_data_sz); 2920dc63aac6SJayamohan Kallickal } 29216733b39aSJayamohan Kallickal pasync_data_h->pbuffer = 29226733b39aSJayamohan Kallickal (void *)((unsigned long) 29236733b39aSJayamohan Kallickal (pasync_ctx->async_data.va_base) + 2924dc63aac6SJayamohan Kallickal (p->defpdu_data_sz * num_per_mem)); 29256733b39aSJayamohan Kallickal 29266733b39aSJayamohan Kallickal pasync_data_h->pa.u.a64.address = 29278a86e833SJayamohan Kallickal pasync_ctx->async_data.pa_base.u.a64. 29288a86e833SJayamohan Kallickal address + (p->defpdu_data_sz * 29298a86e833SJayamohan Kallickal num_per_mem); 2930dc63aac6SJayamohan Kallickal num_per_mem++; 2931dc63aac6SJayamohan Kallickal num_async_data--; 29326733b39aSJayamohan Kallickal 29331e2931f1SJitendra Bhivare pasync_ctx->async_entry[index].data = 29341e2931f1SJitendra Bhivare pasync_data_h; 29356733b39aSJayamohan Kallickal pasync_data_h++; 29366733b39aSJayamohan Kallickal } 29378a86e833SJayamohan Kallickal } 29388a86e833SJayamohan Kallickal } 2939a7909b39SJayamohan Kallickal 2940a7909b39SJayamohan Kallickal return 0; 29416733b39aSJayamohan Kallickal } 29426733b39aSJayamohan Kallickal 29436733b39aSJayamohan Kallickal static int 29446733b39aSJayamohan Kallickal be_sgl_create_contiguous(void *virtual_address, 29456733b39aSJayamohan Kallickal u64 physical_address, u32 length, 29466733b39aSJayamohan Kallickal struct be_dma_mem *sgl) 29476733b39aSJayamohan Kallickal { 29486733b39aSJayamohan Kallickal WARN_ON(!virtual_address); 29496733b39aSJayamohan Kallickal WARN_ON(!physical_address); 2950dd29dae0STim Gardner WARN_ON(!length); 29516733b39aSJayamohan Kallickal WARN_ON(!sgl); 29526733b39aSJayamohan Kallickal 29536733b39aSJayamohan Kallickal sgl->va = virtual_address; 2954457ff3b7SJayamohan Kallickal sgl->dma = (unsigned long)physical_address; 29556733b39aSJayamohan Kallickal sgl->size = length; 29566733b39aSJayamohan Kallickal 29576733b39aSJayamohan Kallickal return 0; 29586733b39aSJayamohan Kallickal } 29596733b39aSJayamohan Kallickal 29606733b39aSJayamohan Kallickal static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl) 29616733b39aSJayamohan Kallickal { 29626733b39aSJayamohan Kallickal memset(sgl, 0, sizeof(*sgl)); 29636733b39aSJayamohan Kallickal } 29646733b39aSJayamohan Kallickal 29656733b39aSJayamohan Kallickal static void 29666733b39aSJayamohan Kallickal hwi_build_be_sgl_arr(struct beiscsi_hba *phba, 29676733b39aSJayamohan Kallickal struct mem_array *pmem, struct be_dma_mem *sgl) 29686733b39aSJayamohan Kallickal { 29696733b39aSJayamohan Kallickal if (sgl->va) 29706733b39aSJayamohan Kallickal be_sgl_destroy_contiguous(sgl); 29716733b39aSJayamohan Kallickal 29726733b39aSJayamohan Kallickal be_sgl_create_contiguous(pmem->virtual_address, 29736733b39aSJayamohan Kallickal pmem->bus_address.u.a64.address, 29746733b39aSJayamohan Kallickal pmem->size, sgl); 29756733b39aSJayamohan Kallickal } 29766733b39aSJayamohan Kallickal 29776733b39aSJayamohan Kallickal static void 29786733b39aSJayamohan Kallickal hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba, 29796733b39aSJayamohan Kallickal struct mem_array *pmem, struct be_dma_mem *sgl) 29806733b39aSJayamohan Kallickal { 29816733b39aSJayamohan Kallickal if (sgl->va) 29826733b39aSJayamohan Kallickal be_sgl_destroy_contiguous(sgl); 29836733b39aSJayamohan Kallickal 29846733b39aSJayamohan Kallickal be_sgl_create_contiguous((unsigned char *)pmem->virtual_address, 29856733b39aSJayamohan Kallickal pmem->bus_address.u.a64.address, 29866733b39aSJayamohan Kallickal pmem->size, sgl); 29876733b39aSJayamohan Kallickal } 29886733b39aSJayamohan Kallickal 29896733b39aSJayamohan Kallickal static int be_fill_queue(struct be_queue_info *q, 29906733b39aSJayamohan Kallickal u16 len, u16 entry_size, void *vaddress) 29916733b39aSJayamohan Kallickal { 29926733b39aSJayamohan Kallickal struct be_dma_mem *mem = &q->dma_mem; 29936733b39aSJayamohan Kallickal 29946733b39aSJayamohan Kallickal memset(q, 0, sizeof(*q)); 29956733b39aSJayamohan Kallickal q->len = len; 29966733b39aSJayamohan Kallickal q->entry_size = entry_size; 29976733b39aSJayamohan Kallickal mem->size = len * entry_size; 29986733b39aSJayamohan Kallickal mem->va = vaddress; 29996733b39aSJayamohan Kallickal if (!mem->va) 30006733b39aSJayamohan Kallickal return -ENOMEM; 30016733b39aSJayamohan Kallickal memset(mem->va, 0, mem->size); 30026733b39aSJayamohan Kallickal return 0; 30036733b39aSJayamohan Kallickal } 30046733b39aSJayamohan Kallickal 3005bfead3b2SJayamohan Kallickal static int beiscsi_create_eqs(struct beiscsi_hba *phba, 30066733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context) 30076733b39aSJayamohan Kallickal { 3008deeea8edSChristophe JAILLET int ret = -ENOMEM, eq_for_mcc; 3009bfead3b2SJayamohan Kallickal unsigned int i, num_eq_pages; 30106733b39aSJayamohan Kallickal struct be_queue_info *eq; 30116733b39aSJayamohan Kallickal struct be_dma_mem *mem; 30126733b39aSJayamohan Kallickal void *eq_vaddress; 3013bfead3b2SJayamohan Kallickal dma_addr_t paddr; 30146733b39aSJayamohan Kallickal 3015bfead3b2SJayamohan Kallickal num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \ 3016bfead3b2SJayamohan Kallickal sizeof(struct be_eq_entry)); 3017bfead3b2SJayamohan Kallickal 301883148866SChristoph Hellwig if (phba->pcidev->msix_enabled) 3019bfead3b2SJayamohan Kallickal eq_for_mcc = 1; 3020bfead3b2SJayamohan Kallickal else 3021bfead3b2SJayamohan Kallickal eq_for_mcc = 0; 3022bfead3b2SJayamohan Kallickal for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) { 3023bfead3b2SJayamohan Kallickal eq = &phwi_context->be_eq[i].q; 30246733b39aSJayamohan Kallickal mem = &eq->dma_mem; 3025bfead3b2SJayamohan Kallickal phwi_context->be_eq[i].phba = phba; 3026bfead3b2SJayamohan Kallickal eq_vaddress = pci_alloc_consistent(phba->pcidev, 3027bfead3b2SJayamohan Kallickal num_eq_pages * PAGE_SIZE, 3028bfead3b2SJayamohan Kallickal &paddr); 302984a261ffSPan Bian if (!eq_vaddress) { 303084a261ffSPan Bian ret = -ENOMEM; 3031bfead3b2SJayamohan Kallickal goto create_eq_error; 303284a261ffSPan Bian } 30336733b39aSJayamohan Kallickal 3034bfead3b2SJayamohan Kallickal mem->va = eq_vaddress; 30356733b39aSJayamohan Kallickal ret = be_fill_queue(eq, phba->params.num_eq_entries, 30366733b39aSJayamohan Kallickal sizeof(struct be_eq_entry), eq_vaddress); 30376733b39aSJayamohan Kallickal if (ret) { 303899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 303999bc5d55SJohn Soni Jose "BM_%d : be_fill_queue Failed for EQ\n"); 3040bfead3b2SJayamohan Kallickal goto create_eq_error; 30416733b39aSJayamohan Kallickal } 30426733b39aSJayamohan Kallickal 3043bfead3b2SJayamohan Kallickal mem->dma = paddr; 30446733b39aSJayamohan Kallickal ret = beiscsi_cmd_eq_create(&phba->ctrl, eq, 3045bfead3b2SJayamohan Kallickal phwi_context->cur_eqd); 30466733b39aSJayamohan Kallickal if (ret) { 304799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 304899bc5d55SJohn Soni Jose "BM_%d : beiscsi_cmd_eq_create" 30496733b39aSJayamohan Kallickal "Failed for EQ\n"); 3050bfead3b2SJayamohan Kallickal goto create_eq_error; 3051bfead3b2SJayamohan Kallickal } 305299bc5d55SJohn Soni Jose 305399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 305499bc5d55SJohn Soni Jose "BM_%d : eqid = %d\n", 305599bc5d55SJohn Soni Jose phwi_context->be_eq[i].q.id); 3056bfead3b2SJayamohan Kallickal } 3057bfead3b2SJayamohan Kallickal return 0; 3058deeea8edSChristophe JAILLET 3059bfead3b2SJayamohan Kallickal create_eq_error: 3060107dfcbaSJohn Soni Jose for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) { 3061bfead3b2SJayamohan Kallickal eq = &phwi_context->be_eq[i].q; 3062bfead3b2SJayamohan Kallickal mem = &eq->dma_mem; 3063bfead3b2SJayamohan Kallickal if (mem->va) 3064bfead3b2SJayamohan Kallickal pci_free_consistent(phba->pcidev, num_eq_pages 3065bfead3b2SJayamohan Kallickal * PAGE_SIZE, 3066bfead3b2SJayamohan Kallickal mem->va, mem->dma); 3067bfead3b2SJayamohan Kallickal } 30686733b39aSJayamohan Kallickal return ret; 30696733b39aSJayamohan Kallickal } 30706733b39aSJayamohan Kallickal 3071bfead3b2SJayamohan Kallickal static int beiscsi_create_cqs(struct beiscsi_hba *phba, 30726733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context) 30736733b39aSJayamohan Kallickal { 3074bfead3b2SJayamohan Kallickal unsigned int i, num_cq_pages; 30756733b39aSJayamohan Kallickal struct be_queue_info *cq, *eq; 30766733b39aSJayamohan Kallickal struct be_dma_mem *mem; 3077bfead3b2SJayamohan Kallickal struct be_eq_obj *pbe_eq; 30786733b39aSJayamohan Kallickal void *cq_vaddress; 3079deeea8edSChristophe JAILLET int ret = -ENOMEM; 3080bfead3b2SJayamohan Kallickal dma_addr_t paddr; 30816733b39aSJayamohan Kallickal 3082bfead3b2SJayamohan Kallickal num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \ 3083bfead3b2SJayamohan Kallickal sizeof(struct sol_cqe)); 3084bfead3b2SJayamohan Kallickal 3085bfead3b2SJayamohan Kallickal for (i = 0; i < phba->num_cpus; i++) { 3086bfead3b2SJayamohan Kallickal cq = &phwi_context->be_cq[i]; 3087bfead3b2SJayamohan Kallickal eq = &phwi_context->be_eq[i].q; 3088bfead3b2SJayamohan Kallickal pbe_eq = &phwi_context->be_eq[i]; 3089bfead3b2SJayamohan Kallickal pbe_eq->cq = cq; 3090bfead3b2SJayamohan Kallickal pbe_eq->phba = phba; 30916733b39aSJayamohan Kallickal mem = &cq->dma_mem; 3092bfead3b2SJayamohan Kallickal cq_vaddress = pci_alloc_consistent(phba->pcidev, 3093bfead3b2SJayamohan Kallickal num_cq_pages * PAGE_SIZE, 3094bfead3b2SJayamohan Kallickal &paddr); 309529b33252SPan Bian if (!cq_vaddress) { 309629b33252SPan Bian ret = -ENOMEM; 3097bfead3b2SJayamohan Kallickal goto create_cq_error; 309829b33252SPan Bian } 3099deeea8edSChristophe JAILLET 31007da50879SJayamohan Kallickal ret = be_fill_queue(cq, phba->params.num_cq_entries, 31016733b39aSJayamohan Kallickal sizeof(struct sol_cqe), cq_vaddress); 31026733b39aSJayamohan Kallickal if (ret) { 310399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 310499bc5d55SJohn Soni Jose "BM_%d : be_fill_queue Failed " 310599bc5d55SJohn Soni Jose "for ISCSI CQ\n"); 3106bfead3b2SJayamohan Kallickal goto create_cq_error; 31076733b39aSJayamohan Kallickal } 31086733b39aSJayamohan Kallickal 3109bfead3b2SJayamohan Kallickal mem->dma = paddr; 3110bfead3b2SJayamohan Kallickal ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false, 3111bfead3b2SJayamohan Kallickal false, 0); 31126733b39aSJayamohan Kallickal if (ret) { 311399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 311499bc5d55SJohn Soni Jose "BM_%d : beiscsi_cmd_eq_create" 3115bfead3b2SJayamohan Kallickal "Failed for ISCSI CQ\n"); 3116bfead3b2SJayamohan Kallickal goto create_cq_error; 31176733b39aSJayamohan Kallickal } 311899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 311999bc5d55SJohn Soni Jose "BM_%d : iscsi cq_id is %d for eq_id %d\n" 312099bc5d55SJohn Soni Jose "iSCSI CQ CREATED\n", cq->id, eq->id); 3121bfead3b2SJayamohan Kallickal } 31226733b39aSJayamohan Kallickal return 0; 3123bfead3b2SJayamohan Kallickal 3124bfead3b2SJayamohan Kallickal create_cq_error: 3125bfead3b2SJayamohan Kallickal for (i = 0; i < phba->num_cpus; i++) { 3126bfead3b2SJayamohan Kallickal cq = &phwi_context->be_cq[i]; 3127bfead3b2SJayamohan Kallickal mem = &cq->dma_mem; 3128bfead3b2SJayamohan Kallickal if (mem->va) 3129bfead3b2SJayamohan Kallickal pci_free_consistent(phba->pcidev, num_cq_pages 3130bfead3b2SJayamohan Kallickal * PAGE_SIZE, 3131bfead3b2SJayamohan Kallickal mem->va, mem->dma); 3132bfead3b2SJayamohan Kallickal } 3133bfead3b2SJayamohan Kallickal return ret; 31346733b39aSJayamohan Kallickal } 31356733b39aSJayamohan Kallickal 31366733b39aSJayamohan Kallickal static int 31376733b39aSJayamohan Kallickal beiscsi_create_def_hdr(struct beiscsi_hba *phba, 31386733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context, 31396733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr, 31408a86e833SJayamohan Kallickal unsigned int def_pdu_ring_sz, uint8_t ulp_num) 31416733b39aSJayamohan Kallickal { 31426733b39aSJayamohan Kallickal unsigned int idx; 31436733b39aSJayamohan Kallickal int ret; 31446733b39aSJayamohan Kallickal struct be_queue_info *dq, *cq; 31456733b39aSJayamohan Kallickal struct be_dma_mem *mem; 31466733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 31476733b39aSJayamohan Kallickal void *dq_vaddress; 31486733b39aSJayamohan Kallickal 31496733b39aSJayamohan Kallickal idx = 0; 31508a86e833SJayamohan Kallickal dq = &phwi_context->be_def_hdrq[ulp_num]; 3151bfead3b2SJayamohan Kallickal cq = &phwi_context->be_cq[0]; 31526733b39aSJayamohan Kallickal mem = &dq->dma_mem; 31536733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 31548a86e833SJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 + 31558a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET); 31566733b39aSJayamohan Kallickal dq_vaddress = mem_descr->mem_array[idx].virtual_address; 31576733b39aSJayamohan Kallickal ret = be_fill_queue(dq, mem_descr->mem_array[0].size / 31586733b39aSJayamohan Kallickal sizeof(struct phys_addr), 31596733b39aSJayamohan Kallickal sizeof(struct phys_addr), dq_vaddress); 31606733b39aSJayamohan Kallickal if (ret) { 316199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 31628a86e833SJayamohan Kallickal "BM_%d : be_fill_queue Failed for DEF PDU HDR on ULP : %d\n", 31638a86e833SJayamohan Kallickal ulp_num); 31648a86e833SJayamohan Kallickal 31656733b39aSJayamohan Kallickal return ret; 31666733b39aSJayamohan Kallickal } 3167457ff3b7SJayamohan Kallickal mem->dma = (unsigned long)mem_descr->mem_array[idx]. 3168457ff3b7SJayamohan Kallickal bus_address.u.a64.address; 31696733b39aSJayamohan Kallickal ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq, 31706733b39aSJayamohan Kallickal def_pdu_ring_sz, 31718a86e833SJayamohan Kallickal phba->params.defpdu_hdr_sz, 31728a86e833SJayamohan Kallickal BEISCSI_DEFQ_HDR, ulp_num); 31736733b39aSJayamohan Kallickal if (ret) { 317499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 31758a86e833SJayamohan Kallickal "BM_%d : be_cmd_create_default_pdu_queue Failed DEFHDR on ULP : %d\n", 31768a86e833SJayamohan Kallickal ulp_num); 31778a86e833SJayamohan Kallickal 31786733b39aSJayamohan Kallickal return ret; 31796733b39aSJayamohan Kallickal } 318099bc5d55SJohn Soni Jose 31818a86e833SJayamohan Kallickal beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 31828a86e833SJayamohan Kallickal "BM_%d : iscsi hdr def pdu id for ULP : %d is %d\n", 31838a86e833SJayamohan Kallickal ulp_num, 31848a86e833SJayamohan Kallickal phwi_context->be_def_hdrq[ulp_num].id); 31856733b39aSJayamohan Kallickal return 0; 31866733b39aSJayamohan Kallickal } 31876733b39aSJayamohan Kallickal 31886733b39aSJayamohan Kallickal static int 31896733b39aSJayamohan Kallickal beiscsi_create_def_data(struct beiscsi_hba *phba, 31906733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context, 31916733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr, 31928a86e833SJayamohan Kallickal unsigned int def_pdu_ring_sz, uint8_t ulp_num) 31936733b39aSJayamohan Kallickal { 31946733b39aSJayamohan Kallickal unsigned int idx; 31956733b39aSJayamohan Kallickal int ret; 31966733b39aSJayamohan Kallickal struct be_queue_info *dataq, *cq; 31976733b39aSJayamohan Kallickal struct be_dma_mem *mem; 31986733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 31996733b39aSJayamohan Kallickal void *dq_vaddress; 32006733b39aSJayamohan Kallickal 32016733b39aSJayamohan Kallickal idx = 0; 32028a86e833SJayamohan Kallickal dataq = &phwi_context->be_def_dataq[ulp_num]; 3203bfead3b2SJayamohan Kallickal cq = &phwi_context->be_cq[0]; 32046733b39aSJayamohan Kallickal mem = &dataq->dma_mem; 32056733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 32068a86e833SJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 + 32078a86e833SJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET); 32086733b39aSJayamohan Kallickal dq_vaddress = mem_descr->mem_array[idx].virtual_address; 32096733b39aSJayamohan Kallickal ret = be_fill_queue(dataq, mem_descr->mem_array[0].size / 32106733b39aSJayamohan Kallickal sizeof(struct phys_addr), 32116733b39aSJayamohan Kallickal sizeof(struct phys_addr), dq_vaddress); 32126733b39aSJayamohan Kallickal if (ret) { 321399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 32148a86e833SJayamohan Kallickal "BM_%d : be_fill_queue Failed for DEF PDU " 32158a86e833SJayamohan Kallickal "DATA on ULP : %d\n", 32168a86e833SJayamohan Kallickal ulp_num); 32178a86e833SJayamohan Kallickal 32186733b39aSJayamohan Kallickal return ret; 32196733b39aSJayamohan Kallickal } 3220457ff3b7SJayamohan Kallickal mem->dma = (unsigned long)mem_descr->mem_array[idx]. 3221457ff3b7SJayamohan Kallickal bus_address.u.a64.address; 32226733b39aSJayamohan Kallickal ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq, 32236733b39aSJayamohan Kallickal def_pdu_ring_sz, 32248a86e833SJayamohan Kallickal phba->params.defpdu_data_sz, 32258a86e833SJayamohan Kallickal BEISCSI_DEFQ_DATA, ulp_num); 32266733b39aSJayamohan Kallickal if (ret) { 322799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 322899bc5d55SJohn Soni Jose "BM_%d be_cmd_create_default_pdu_queue" 32298a86e833SJayamohan Kallickal " Failed for DEF PDU DATA on ULP : %d\n", 32308a86e833SJayamohan Kallickal ulp_num); 32316733b39aSJayamohan Kallickal return ret; 32326733b39aSJayamohan Kallickal } 323399bc5d55SJohn Soni Jose 323499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 32358a86e833SJayamohan Kallickal "BM_%d : iscsi def data id on ULP : %d is %d\n", 32368a86e833SJayamohan Kallickal ulp_num, 32378a86e833SJayamohan Kallickal phwi_context->be_def_dataq[ulp_num].id); 32388a86e833SJayamohan Kallickal 32398a86e833SJayamohan Kallickal beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 32408a86e833SJayamohan Kallickal "BM_%d : DEFAULT PDU DATA RING CREATED" 32418a86e833SJayamohan Kallickal "on ULP : %d\n", ulp_num); 32426733b39aSJayamohan Kallickal return 0; 32436733b39aSJayamohan Kallickal } 32446733b39aSJayamohan Kallickal 324515a90fe0SJayamohan Kallickal 324615a90fe0SJayamohan Kallickal static int 324715a90fe0SJayamohan Kallickal beiscsi_post_template_hdr(struct beiscsi_hba *phba) 324815a90fe0SJayamohan Kallickal { 324915a90fe0SJayamohan Kallickal struct be_mem_descriptor *mem_descr; 325015a90fe0SJayamohan Kallickal struct mem_array *pm_arr; 325115a90fe0SJayamohan Kallickal struct be_dma_mem sgl; 3252a129d92fSJayamohan Kallickal int status, ulp_num; 325315a90fe0SJayamohan Kallickal 3254a129d92fSJayamohan Kallickal for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 3255a129d92fSJayamohan Kallickal if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 3256a129d92fSJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 3257a129d92fSJayamohan Kallickal mem_descr += HWI_MEM_TEMPLATE_HDR_ULP0 + 3258a129d92fSJayamohan Kallickal (ulp_num * MEM_DESCR_OFFSET); 325915a90fe0SJayamohan Kallickal pm_arr = mem_descr->mem_array; 326015a90fe0SJayamohan Kallickal 326115a90fe0SJayamohan Kallickal hwi_build_be_sgl_arr(phba, pm_arr, &sgl); 3262a129d92fSJayamohan Kallickal status = be_cmd_iscsi_post_template_hdr( 3263a129d92fSJayamohan Kallickal &phba->ctrl, &sgl); 326415a90fe0SJayamohan Kallickal 326515a90fe0SJayamohan Kallickal if (status != 0) { 326615a90fe0SJayamohan Kallickal beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3267a129d92fSJayamohan Kallickal "BM_%d : Post Template HDR Failed for" 3268a129d92fSJayamohan Kallickal "ULP_%d\n", ulp_num); 326915a90fe0SJayamohan Kallickal return status; 327015a90fe0SJayamohan Kallickal } 327115a90fe0SJayamohan Kallickal 327215a90fe0SJayamohan Kallickal beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 3273a129d92fSJayamohan Kallickal "BM_%d : Template HDR Pages Posted for" 3274a129d92fSJayamohan Kallickal "ULP_%d\n", ulp_num); 3275a129d92fSJayamohan Kallickal } 3276a129d92fSJayamohan Kallickal } 327715a90fe0SJayamohan Kallickal return 0; 327815a90fe0SJayamohan Kallickal } 327915a90fe0SJayamohan Kallickal 32806733b39aSJayamohan Kallickal static int 32816733b39aSJayamohan Kallickal beiscsi_post_pages(struct beiscsi_hba *phba) 32826733b39aSJayamohan Kallickal { 32836733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 32846733b39aSJayamohan Kallickal struct mem_array *pm_arr; 32856733b39aSJayamohan Kallickal unsigned int page_offset, i; 32866733b39aSJayamohan Kallickal struct be_dma_mem sgl; 3287843ae752SJayamohan Kallickal int status, ulp_num = 0; 32886733b39aSJayamohan Kallickal 32896733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 32906733b39aSJayamohan Kallickal mem_descr += HWI_MEM_SGE; 32916733b39aSJayamohan Kallickal pm_arr = mem_descr->mem_array; 32926733b39aSJayamohan Kallickal 329390622db3SJayamohan Kallickal for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) 329490622db3SJayamohan Kallickal if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) 329590622db3SJayamohan Kallickal break; 329690622db3SJayamohan Kallickal 32976733b39aSJayamohan Kallickal page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io * 3298843ae752SJayamohan Kallickal phba->fw_config.iscsi_icd_start[ulp_num]) / PAGE_SIZE; 32996733b39aSJayamohan Kallickal for (i = 0; i < mem_descr->num_elements; i++) { 33006733b39aSJayamohan Kallickal hwi_build_be_sgl_arr(phba, pm_arr, &sgl); 33016733b39aSJayamohan Kallickal status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl, 33026733b39aSJayamohan Kallickal page_offset, 33036733b39aSJayamohan Kallickal (pm_arr->size / PAGE_SIZE)); 33046733b39aSJayamohan Kallickal page_offset += pm_arr->size / PAGE_SIZE; 33056733b39aSJayamohan Kallickal if (status != 0) { 330699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 330799bc5d55SJohn Soni Jose "BM_%d : post sgl failed.\n"); 33086733b39aSJayamohan Kallickal return status; 33096733b39aSJayamohan Kallickal } 33106733b39aSJayamohan Kallickal pm_arr++; 33116733b39aSJayamohan Kallickal } 331299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 331399bc5d55SJohn Soni Jose "BM_%d : POSTED PAGES\n"); 33146733b39aSJayamohan Kallickal return 0; 33156733b39aSJayamohan Kallickal } 33166733b39aSJayamohan Kallickal 3317bfead3b2SJayamohan Kallickal static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q) 3318bfead3b2SJayamohan Kallickal { 3319bfead3b2SJayamohan Kallickal struct be_dma_mem *mem = &q->dma_mem; 3320c8b25598SJayamohan Kallickal if (mem->va) { 3321bfead3b2SJayamohan Kallickal pci_free_consistent(phba->pcidev, mem->size, 3322bfead3b2SJayamohan Kallickal mem->va, mem->dma); 3323c8b25598SJayamohan Kallickal mem->va = NULL; 3324c8b25598SJayamohan Kallickal } 3325bfead3b2SJayamohan Kallickal } 3326bfead3b2SJayamohan Kallickal 3327bfead3b2SJayamohan Kallickal static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q, 3328bfead3b2SJayamohan Kallickal u16 len, u16 entry_size) 3329bfead3b2SJayamohan Kallickal { 3330bfead3b2SJayamohan Kallickal struct be_dma_mem *mem = &q->dma_mem; 3331bfead3b2SJayamohan Kallickal 3332bfead3b2SJayamohan Kallickal memset(q, 0, sizeof(*q)); 3333bfead3b2SJayamohan Kallickal q->len = len; 3334bfead3b2SJayamohan Kallickal q->entry_size = entry_size; 3335bfead3b2SJayamohan Kallickal mem->size = len * entry_size; 33367c845eb5SJoe Perches mem->va = pci_zalloc_consistent(phba->pcidev, mem->size, &mem->dma); 3337bfead3b2SJayamohan Kallickal if (!mem->va) 3338d3ad2bb3SJayamohan Kallickal return -ENOMEM; 3339bfead3b2SJayamohan Kallickal return 0; 3340bfead3b2SJayamohan Kallickal } 3341bfead3b2SJayamohan Kallickal 33426733b39aSJayamohan Kallickal static int 33436733b39aSJayamohan Kallickal beiscsi_create_wrb_rings(struct beiscsi_hba *phba, 33446733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context, 33456733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr) 33466733b39aSJayamohan Kallickal { 3347fa1261c4SJitendra Bhivare unsigned int num_wrb_rings; 33486733b39aSJayamohan Kallickal u64 pa_addr_lo; 33494eea99d5SJayamohan Kallickal unsigned int idx, num, i, ulp_num; 33506733b39aSJayamohan Kallickal struct mem_array *pwrb_arr; 33516733b39aSJayamohan Kallickal void *wrb_vaddr; 33526733b39aSJayamohan Kallickal struct be_dma_mem sgl; 33536733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 3354a7909b39SJayamohan Kallickal struct hwi_wrb_context *pwrb_context; 33556733b39aSJayamohan Kallickal int status; 33564eea99d5SJayamohan Kallickal uint8_t ulp_count = 0, ulp_base_num = 0; 33574eea99d5SJayamohan Kallickal uint16_t cid_count_ulp[BEISCSI_ULP_COUNT] = { 0 }; 33586733b39aSJayamohan Kallickal 33596733b39aSJayamohan Kallickal idx = 0; 33606733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 33616733b39aSJayamohan Kallickal mem_descr += HWI_MEM_WRB; 33626733b39aSJayamohan Kallickal pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl, 33636733b39aSJayamohan Kallickal GFP_KERNEL); 33646733b39aSJayamohan Kallickal if (!pwrb_arr) { 336599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 336699bc5d55SJohn Soni Jose "BM_%d : Memory alloc failed in create wrb ring.\n"); 33676733b39aSJayamohan Kallickal return -ENOMEM; 33686733b39aSJayamohan Kallickal } 33696733b39aSJayamohan Kallickal wrb_vaddr = mem_descr->mem_array[idx].virtual_address; 33706733b39aSJayamohan Kallickal pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address; 33716733b39aSJayamohan Kallickal num_wrb_rings = mem_descr->mem_array[idx].size / 33726733b39aSJayamohan Kallickal (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb)); 33736733b39aSJayamohan Kallickal 33746733b39aSJayamohan Kallickal for (num = 0; num < phba->params.cxns_per_ctrl; num++) { 33756733b39aSJayamohan Kallickal if (num_wrb_rings) { 33766733b39aSJayamohan Kallickal pwrb_arr[num].virtual_address = wrb_vaddr; 33776733b39aSJayamohan Kallickal pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo; 33786733b39aSJayamohan Kallickal pwrb_arr[num].size = phba->params.wrbs_per_cxn * 33796733b39aSJayamohan Kallickal sizeof(struct iscsi_wrb); 33806733b39aSJayamohan Kallickal wrb_vaddr += pwrb_arr[num].size; 33816733b39aSJayamohan Kallickal pa_addr_lo += pwrb_arr[num].size; 33826733b39aSJayamohan Kallickal num_wrb_rings--; 33836733b39aSJayamohan Kallickal } else { 33846733b39aSJayamohan Kallickal idx++; 33856733b39aSJayamohan Kallickal wrb_vaddr = mem_descr->mem_array[idx].virtual_address; 33866733b39aSJayamohan Kallickal pa_addr_lo = mem_descr->mem_array[idx].\ 33876733b39aSJayamohan Kallickal bus_address.u.a64.address; 33886733b39aSJayamohan Kallickal num_wrb_rings = mem_descr->mem_array[idx].size / 33896733b39aSJayamohan Kallickal (phba->params.wrbs_per_cxn * 33906733b39aSJayamohan Kallickal sizeof(struct iscsi_wrb)); 33916733b39aSJayamohan Kallickal pwrb_arr[num].virtual_address = wrb_vaddr; 33926733b39aSJayamohan Kallickal pwrb_arr[num].bus_address.u.a64.address\ 33936733b39aSJayamohan Kallickal = pa_addr_lo; 33946733b39aSJayamohan Kallickal pwrb_arr[num].size = phba->params.wrbs_per_cxn * 33956733b39aSJayamohan Kallickal sizeof(struct iscsi_wrb); 33966733b39aSJayamohan Kallickal wrb_vaddr += pwrb_arr[num].size; 33976733b39aSJayamohan Kallickal pa_addr_lo += pwrb_arr[num].size; 33986733b39aSJayamohan Kallickal num_wrb_rings--; 33996733b39aSJayamohan Kallickal } 34006733b39aSJayamohan Kallickal } 34014eea99d5SJayamohan Kallickal 34024eea99d5SJayamohan Kallickal /* Get the ULP Count */ 34034eea99d5SJayamohan Kallickal for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) 34044eea99d5SJayamohan Kallickal if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 34054eea99d5SJayamohan Kallickal ulp_count++; 34064eea99d5SJayamohan Kallickal ulp_base_num = ulp_num; 34074eea99d5SJayamohan Kallickal cid_count_ulp[ulp_num] = 34084eea99d5SJayamohan Kallickal BEISCSI_GET_CID_COUNT(phba, ulp_num); 34094eea99d5SJayamohan Kallickal } 34104eea99d5SJayamohan Kallickal 34116733b39aSJayamohan Kallickal for (i = 0; i < phba->params.cxns_per_ctrl; i++) { 34124eea99d5SJayamohan Kallickal if (ulp_count > 1) { 34134eea99d5SJayamohan Kallickal ulp_base_num = (ulp_base_num + 1) % BEISCSI_ULP_COUNT; 34144eea99d5SJayamohan Kallickal 34154eea99d5SJayamohan Kallickal if (!cid_count_ulp[ulp_base_num]) 34164eea99d5SJayamohan Kallickal ulp_base_num = (ulp_base_num + 1) % 34174eea99d5SJayamohan Kallickal BEISCSI_ULP_COUNT; 34184eea99d5SJayamohan Kallickal 34194eea99d5SJayamohan Kallickal cid_count_ulp[ulp_base_num]--; 34204eea99d5SJayamohan Kallickal } 34214eea99d5SJayamohan Kallickal 34224eea99d5SJayamohan Kallickal 34236733b39aSJayamohan Kallickal hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl); 34246733b39aSJayamohan Kallickal status = be_cmd_wrbq_create(&phba->ctrl, &sgl, 34254eea99d5SJayamohan Kallickal &phwi_context->be_wrbq[i], 34264eea99d5SJayamohan Kallickal &phwi_ctrlr->wrb_context[i], 34274eea99d5SJayamohan Kallickal ulp_base_num); 34286733b39aSJayamohan Kallickal if (status != 0) { 342999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 343099bc5d55SJohn Soni Jose "BM_%d : wrbq create failed."); 34311462b8ffSDan Carpenter kfree(pwrb_arr); 34326733b39aSJayamohan Kallickal return status; 34336733b39aSJayamohan Kallickal } 3434a7909b39SJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[i]; 3435a7909b39SJayamohan Kallickal BE_SET_CID_TO_CRI(i, pwrb_context->cid); 34366733b39aSJayamohan Kallickal } 34376733b39aSJayamohan Kallickal kfree(pwrb_arr); 34386733b39aSJayamohan Kallickal return 0; 34396733b39aSJayamohan Kallickal } 34406733b39aSJayamohan Kallickal 34416733b39aSJayamohan Kallickal static void free_wrb_handles(struct beiscsi_hba *phba) 34426733b39aSJayamohan Kallickal { 34436733b39aSJayamohan Kallickal unsigned int index; 34446733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 34456733b39aSJayamohan Kallickal struct hwi_wrb_context *pwrb_context; 34466733b39aSJayamohan Kallickal 34476733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 3448a7909b39SJayamohan Kallickal for (index = 0; index < phba->params.cxns_per_ctrl; index++) { 34496733b39aSJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[index]; 34506733b39aSJayamohan Kallickal kfree(pwrb_context->pwrb_handle_base); 34516733b39aSJayamohan Kallickal kfree(pwrb_context->pwrb_handle_basestd); 34526733b39aSJayamohan Kallickal } 34536733b39aSJayamohan Kallickal } 34546733b39aSJayamohan Kallickal 3455bfead3b2SJayamohan Kallickal static void be_mcc_queues_destroy(struct beiscsi_hba *phba) 3456bfead3b2SJayamohan Kallickal { 3457bfead3b2SJayamohan Kallickal struct be_ctrl_info *ctrl = &phba->ctrl; 3458d1d5ca88SJitendra Bhivare struct be_dma_mem *ptag_mem; 3459d1d5ca88SJitendra Bhivare struct be_queue_info *q; 3460d1d5ca88SJitendra Bhivare int i, tag; 3461bfead3b2SJayamohan Kallickal 3462bfead3b2SJayamohan Kallickal q = &phba->ctrl.mcc_obj.q; 3463d1d5ca88SJitendra Bhivare for (i = 0; i < MAX_MCC_CMD; i++) { 3464d1d5ca88SJitendra Bhivare tag = i + 1; 3465d1d5ca88SJitendra Bhivare if (!test_bit(MCC_TAG_STATE_RUNNING, 3466d1d5ca88SJitendra Bhivare &ctrl->ptag_state[tag].tag_state)) 3467d1d5ca88SJitendra Bhivare continue; 3468d1d5ca88SJitendra Bhivare 3469d1d5ca88SJitendra Bhivare if (test_bit(MCC_TAG_STATE_TIMEOUT, 3470d1d5ca88SJitendra Bhivare &ctrl->ptag_state[tag].tag_state)) { 3471d1d5ca88SJitendra Bhivare ptag_mem = &ctrl->ptag_state[tag].tag_mem_state; 3472d1d5ca88SJitendra Bhivare if (ptag_mem->size) { 3473d1d5ca88SJitendra Bhivare pci_free_consistent(ctrl->pdev, 3474d1d5ca88SJitendra Bhivare ptag_mem->size, 3475d1d5ca88SJitendra Bhivare ptag_mem->va, 3476d1d5ca88SJitendra Bhivare ptag_mem->dma); 3477d1d5ca88SJitendra Bhivare ptag_mem->size = 0; 3478d1d5ca88SJitendra Bhivare } 3479d1d5ca88SJitendra Bhivare continue; 3480d1d5ca88SJitendra Bhivare } 3481d1d5ca88SJitendra Bhivare /** 3482d1d5ca88SJitendra Bhivare * If MCC is still active and waiting then wake up the process. 3483d1d5ca88SJitendra Bhivare * We are here only because port is going offline. The process 3484d1d5ca88SJitendra Bhivare * sees that (BEISCSI_HBA_ONLINE is cleared) and EIO error is 3485d1d5ca88SJitendra Bhivare * returned for the operation and allocated memory cleaned up. 3486d1d5ca88SJitendra Bhivare */ 3487d1d5ca88SJitendra Bhivare if (waitqueue_active(&ctrl->mcc_wait[tag])) { 3488d1d5ca88SJitendra Bhivare ctrl->mcc_tag_status[tag] = MCC_STATUS_FAILED; 3489d1d5ca88SJitendra Bhivare ctrl->mcc_tag_status[tag] |= CQE_VALID_MASK; 3490d1d5ca88SJitendra Bhivare wake_up_interruptible(&ctrl->mcc_wait[tag]); 3491d1d5ca88SJitendra Bhivare /* 3492d1d5ca88SJitendra Bhivare * Control tag info gets reinitialized in enable 3493d1d5ca88SJitendra Bhivare * so wait for the process to clear running state. 3494d1d5ca88SJitendra Bhivare */ 3495d1d5ca88SJitendra Bhivare while (test_bit(MCC_TAG_STATE_RUNNING, 3496d1d5ca88SJitendra Bhivare &ctrl->ptag_state[tag].tag_state)) 3497d1d5ca88SJitendra Bhivare schedule_timeout_uninterruptible(HZ); 3498d1d5ca88SJitendra Bhivare } 3499d1d5ca88SJitendra Bhivare /** 3500d1d5ca88SJitendra Bhivare * For MCC with tag_states MCC_TAG_STATE_ASYNC and 3501d1d5ca88SJitendra Bhivare * MCC_TAG_STATE_IGNORE nothing needs to done. 3502d1d5ca88SJitendra Bhivare */ 3503d1d5ca88SJitendra Bhivare } 35044e2bdf7aSJohn Soni Jose if (q->created) { 3505bfead3b2SJayamohan Kallickal beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ); 3506bfead3b2SJayamohan Kallickal be_queue_free(phba, q); 35074e2bdf7aSJohn Soni Jose } 3508bfead3b2SJayamohan Kallickal 3509bfead3b2SJayamohan Kallickal q = &phba->ctrl.mcc_obj.cq; 35104e2bdf7aSJohn Soni Jose if (q->created) { 3511bfead3b2SJayamohan Kallickal beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ); 3512bfead3b2SJayamohan Kallickal be_queue_free(phba, q); 3513bfead3b2SJayamohan Kallickal } 35144e2bdf7aSJohn Soni Jose } 3515bfead3b2SJayamohan Kallickal 3516bfead3b2SJayamohan Kallickal static int be_mcc_queues_create(struct beiscsi_hba *phba, 3517bfead3b2SJayamohan Kallickal struct hwi_context_memory *phwi_context) 3518bfead3b2SJayamohan Kallickal { 3519bfead3b2SJayamohan Kallickal struct be_queue_info *q, *cq; 3520bfead3b2SJayamohan Kallickal struct be_ctrl_info *ctrl = &phba->ctrl; 3521bfead3b2SJayamohan Kallickal 3522bfead3b2SJayamohan Kallickal /* Alloc MCC compl queue */ 3523bfead3b2SJayamohan Kallickal cq = &phba->ctrl.mcc_obj.cq; 3524bfead3b2SJayamohan Kallickal if (be_queue_alloc(phba, cq, MCC_CQ_LEN, 3525bfead3b2SJayamohan Kallickal sizeof(struct be_mcc_compl))) 3526bfead3b2SJayamohan Kallickal goto err; 3527bfead3b2SJayamohan Kallickal /* Ask BE to create MCC compl queue; */ 352883148866SChristoph Hellwig if (phba->pcidev->msix_enabled) { 3529bfead3b2SJayamohan Kallickal if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq 3530bfead3b2SJayamohan Kallickal [phba->num_cpus].q, false, true, 0)) 3531bfead3b2SJayamohan Kallickal goto mcc_cq_free; 3532bfead3b2SJayamohan Kallickal } else { 3533bfead3b2SJayamohan Kallickal if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q, 3534bfead3b2SJayamohan Kallickal false, true, 0)) 3535bfead3b2SJayamohan Kallickal goto mcc_cq_free; 3536bfead3b2SJayamohan Kallickal } 3537bfead3b2SJayamohan Kallickal 3538bfead3b2SJayamohan Kallickal /* Alloc MCC queue */ 3539bfead3b2SJayamohan Kallickal q = &phba->ctrl.mcc_obj.q; 3540bfead3b2SJayamohan Kallickal if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb))) 3541bfead3b2SJayamohan Kallickal goto mcc_cq_destroy; 3542bfead3b2SJayamohan Kallickal 3543bfead3b2SJayamohan Kallickal /* Ask BE to create MCC queue */ 354435e66019SJayamohan Kallickal if (beiscsi_cmd_mccq_create(phba, q, cq)) 3545bfead3b2SJayamohan Kallickal goto mcc_q_free; 3546bfead3b2SJayamohan Kallickal 3547bfead3b2SJayamohan Kallickal return 0; 3548bfead3b2SJayamohan Kallickal 3549bfead3b2SJayamohan Kallickal mcc_q_free: 3550bfead3b2SJayamohan Kallickal be_queue_free(phba, q); 3551bfead3b2SJayamohan Kallickal mcc_cq_destroy: 3552bfead3b2SJayamohan Kallickal beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ); 3553bfead3b2SJayamohan Kallickal mcc_cq_free: 3554bfead3b2SJayamohan Kallickal be_queue_free(phba, cq); 3555bfead3b2SJayamohan Kallickal err: 3556d3ad2bb3SJayamohan Kallickal return -ENOMEM; 3557bfead3b2SJayamohan Kallickal } 3558bfead3b2SJayamohan Kallickal 355983148866SChristoph Hellwig static void be2iscsi_enable_msix(struct beiscsi_hba *phba) 3560bfead3b2SJayamohan Kallickal { 356183148866SChristoph Hellwig int nvec = 1; 3562bfead3b2SJayamohan Kallickal 356322abeef0SJohn Soni Jose switch (phba->generation) { 356422abeef0SJohn Soni Jose case BE_GEN2: 356522abeef0SJohn Soni Jose case BE_GEN3: 356683148866SChristoph Hellwig nvec = BEISCSI_MAX_NUM_CPUS + 1; 356722abeef0SJohn Soni Jose break; 356822abeef0SJohn Soni Jose case BE_GEN4: 356983148866SChristoph Hellwig nvec = phba->fw_config.eqid_count; 357022abeef0SJohn Soni Jose break; 357122abeef0SJohn Soni Jose default: 357283148866SChristoph Hellwig nvec = 2; 357383148866SChristoph Hellwig break; 357422abeef0SJohn Soni Jose } 357583148866SChristoph Hellwig 357683148866SChristoph Hellwig /* if eqid_count == 1 fall back to INTX */ 357783148866SChristoph Hellwig if (enable_msix && nvec > 1) { 357883148866SChristoph Hellwig const struct irq_affinity desc = { .post_vectors = 1 }; 357983148866SChristoph Hellwig 358083148866SChristoph Hellwig if (pci_alloc_irq_vectors_affinity(phba->pcidev, 2, nvec, 358183148866SChristoph Hellwig PCI_IRQ_MSIX | PCI_IRQ_AFFINITY, &desc) < 0) { 358283148866SChristoph Hellwig phba->num_cpus = nvec - 1; 358383148866SChristoph Hellwig return; 358483148866SChristoph Hellwig } 358583148866SChristoph Hellwig } 358683148866SChristoph Hellwig 358783148866SChristoph Hellwig phba->num_cpus = 1; 3588bfead3b2SJayamohan Kallickal } 35896733b39aSJayamohan Kallickal 3590d1d5ca88SJitendra Bhivare static void hwi_purge_eq(struct beiscsi_hba *phba) 3591d1d5ca88SJitendra Bhivare { 3592d1d5ca88SJitendra Bhivare struct hwi_controller *phwi_ctrlr; 3593d1d5ca88SJitendra Bhivare struct hwi_context_memory *phwi_context; 3594d1d5ca88SJitendra Bhivare struct be_queue_info *eq; 3595d1d5ca88SJitendra Bhivare struct be_eq_entry *eqe = NULL; 3596d1d5ca88SJitendra Bhivare int i, eq_msix; 3597d1d5ca88SJitendra Bhivare unsigned int num_processed; 3598d1d5ca88SJitendra Bhivare 3599d1d5ca88SJitendra Bhivare if (beiscsi_hba_in_error(phba)) 3600d1d5ca88SJitendra Bhivare return; 3601d1d5ca88SJitendra Bhivare 3602d1d5ca88SJitendra Bhivare phwi_ctrlr = phba->phwi_ctrlr; 3603d1d5ca88SJitendra Bhivare phwi_context = phwi_ctrlr->phwi_ctxt; 360483148866SChristoph Hellwig if (phba->pcidev->msix_enabled) 3605d1d5ca88SJitendra Bhivare eq_msix = 1; 3606d1d5ca88SJitendra Bhivare else 3607d1d5ca88SJitendra Bhivare eq_msix = 0; 3608d1d5ca88SJitendra Bhivare 3609d1d5ca88SJitendra Bhivare for (i = 0; i < (phba->num_cpus + eq_msix); i++) { 3610d1d5ca88SJitendra Bhivare eq = &phwi_context->be_eq[i].q; 3611d1d5ca88SJitendra Bhivare eqe = queue_tail_node(eq); 3612d1d5ca88SJitendra Bhivare num_processed = 0; 3613d1d5ca88SJitendra Bhivare while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 3614d1d5ca88SJitendra Bhivare & EQE_VALID_MASK) { 3615d1d5ca88SJitendra Bhivare AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 3616d1d5ca88SJitendra Bhivare queue_tail_inc(eq); 3617d1d5ca88SJitendra Bhivare eqe = queue_tail_node(eq); 3618d1d5ca88SJitendra Bhivare num_processed++; 3619d1d5ca88SJitendra Bhivare } 3620d1d5ca88SJitendra Bhivare 3621d1d5ca88SJitendra Bhivare if (num_processed) 3622d1d5ca88SJitendra Bhivare hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1); 3623d1d5ca88SJitendra Bhivare } 3624d1d5ca88SJitendra Bhivare } 3625d1d5ca88SJitendra Bhivare 3626d1d5ca88SJitendra Bhivare static void hwi_cleanup_port(struct beiscsi_hba *phba) 3627d1d5ca88SJitendra Bhivare { 3628d1d5ca88SJitendra Bhivare struct be_queue_info *q; 3629d1d5ca88SJitendra Bhivare struct be_ctrl_info *ctrl = &phba->ctrl; 3630d1d5ca88SJitendra Bhivare struct hwi_controller *phwi_ctrlr; 3631d1d5ca88SJitendra Bhivare struct hwi_context_memory *phwi_context; 3632d1d5ca88SJitendra Bhivare int i, eq_for_mcc, ulp_num; 3633d1d5ca88SJitendra Bhivare 3634d1d5ca88SJitendra Bhivare for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) 3635d1d5ca88SJitendra Bhivare if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) 3636d1d5ca88SJitendra Bhivare beiscsi_cmd_iscsi_cleanup(phba, ulp_num); 3637d1d5ca88SJitendra Bhivare 3638d1d5ca88SJitendra Bhivare /** 3639d1d5ca88SJitendra Bhivare * Purge all EQ entries that may have been left out. This is to 3640d1d5ca88SJitendra Bhivare * workaround a problem we've seen occasionally where driver gets an 3641d1d5ca88SJitendra Bhivare * interrupt with EQ entry bit set after stopping the controller. 3642d1d5ca88SJitendra Bhivare */ 3643d1d5ca88SJitendra Bhivare hwi_purge_eq(phba); 3644d1d5ca88SJitendra Bhivare 3645d1d5ca88SJitendra Bhivare phwi_ctrlr = phba->phwi_ctrlr; 3646d1d5ca88SJitendra Bhivare phwi_context = phwi_ctrlr->phwi_ctxt; 3647d1d5ca88SJitendra Bhivare 3648d1d5ca88SJitendra Bhivare be_cmd_iscsi_remove_template_hdr(ctrl); 3649d1d5ca88SJitendra Bhivare 3650d1d5ca88SJitendra Bhivare for (i = 0; i < phba->params.cxns_per_ctrl; i++) { 3651d1d5ca88SJitendra Bhivare q = &phwi_context->be_wrbq[i]; 3652d1d5ca88SJitendra Bhivare if (q->created) 3653d1d5ca88SJitendra Bhivare beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ); 3654d1d5ca88SJitendra Bhivare } 3655d1d5ca88SJitendra Bhivare kfree(phwi_context->be_wrbq); 3656d1d5ca88SJitendra Bhivare free_wrb_handles(phba); 3657d1d5ca88SJitendra Bhivare 3658d1d5ca88SJitendra Bhivare for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 3659d1d5ca88SJitendra Bhivare if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 3660d1d5ca88SJitendra Bhivare 3661d1d5ca88SJitendra Bhivare q = &phwi_context->be_def_hdrq[ulp_num]; 3662d1d5ca88SJitendra Bhivare if (q->created) 3663d1d5ca88SJitendra Bhivare beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ); 3664d1d5ca88SJitendra Bhivare 3665d1d5ca88SJitendra Bhivare q = &phwi_context->be_def_dataq[ulp_num]; 3666d1d5ca88SJitendra Bhivare if (q->created) 3667d1d5ca88SJitendra Bhivare beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ); 3668d1d5ca88SJitendra Bhivare } 3669d1d5ca88SJitendra Bhivare } 3670d1d5ca88SJitendra Bhivare 3671d1d5ca88SJitendra Bhivare beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL); 3672d1d5ca88SJitendra Bhivare 3673d1d5ca88SJitendra Bhivare for (i = 0; i < (phba->num_cpus); i++) { 3674d1d5ca88SJitendra Bhivare q = &phwi_context->be_cq[i]; 3675d1d5ca88SJitendra Bhivare if (q->created) { 3676d1d5ca88SJitendra Bhivare be_queue_free(phba, q); 3677d1d5ca88SJitendra Bhivare beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ); 3678d1d5ca88SJitendra Bhivare } 3679d1d5ca88SJitendra Bhivare } 3680d1d5ca88SJitendra Bhivare 3681d1d5ca88SJitendra Bhivare be_mcc_queues_destroy(phba); 368283148866SChristoph Hellwig if (phba->pcidev->msix_enabled) 3683d1d5ca88SJitendra Bhivare eq_for_mcc = 1; 3684d1d5ca88SJitendra Bhivare else 3685d1d5ca88SJitendra Bhivare eq_for_mcc = 0; 3686d1d5ca88SJitendra Bhivare for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) { 3687d1d5ca88SJitendra Bhivare q = &phwi_context->be_eq[i].q; 3688d1d5ca88SJitendra Bhivare if (q->created) { 3689d1d5ca88SJitendra Bhivare be_queue_free(phba, q); 3690d1d5ca88SJitendra Bhivare beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ); 3691d1d5ca88SJitendra Bhivare } 3692d1d5ca88SJitendra Bhivare } 36934ee1ec42SJitendra Bhivare /* this ensures complete FW cleanup */ 36944ee1ec42SJitendra Bhivare beiscsi_cmd_function_reset(phba); 3695d1d5ca88SJitendra Bhivare /* last communication, indicate driver is unloading */ 3696d1d5ca88SJitendra Bhivare beiscsi_cmd_special_wrb(&phba->ctrl, 0); 3697d1d5ca88SJitendra Bhivare } 36984ee1ec42SJitendra Bhivare 36996733b39aSJayamohan Kallickal static int hwi_init_port(struct beiscsi_hba *phba) 37006733b39aSJayamohan Kallickal { 37016733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 37026733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context; 37036733b39aSJayamohan Kallickal unsigned int def_pdu_ring_sz; 37046733b39aSJayamohan Kallickal struct be_ctrl_info *ctrl = &phba->ctrl; 37058a86e833SJayamohan Kallickal int status, ulp_num; 37061e2931f1SJitendra Bhivare u16 nbufs; 37076733b39aSJayamohan Kallickal 37086733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 37096733b39aSJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 371073af08e1SJayamohan Kallickal phwi_context->max_eqd = 128; 3711bfead3b2SJayamohan Kallickal phwi_context->min_eqd = 0; 37121b7a7ddcSJitendra Bhivare phwi_context->cur_eqd = 32; 37134d2ee1e6SJitendra Bhivare /* set port optic state to unknown */ 371453aefe25SJitendra Bhivare phba->optic_state = 0xff; 3715bfead3b2SJayamohan Kallickal 3716bfead3b2SJayamohan Kallickal status = beiscsi_create_eqs(phba, phwi_context); 37176733b39aSJayamohan Kallickal if (status != 0) { 371899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 371999bc5d55SJohn Soni Jose "BM_%d : EQ not created\n"); 37206733b39aSJayamohan Kallickal goto error; 37216733b39aSJayamohan Kallickal } 37226733b39aSJayamohan Kallickal 3723bfead3b2SJayamohan Kallickal status = be_mcc_queues_create(phba, phwi_context); 3724bfead3b2SJayamohan Kallickal if (status != 0) 3725bfead3b2SJayamohan Kallickal goto error; 3726bfead3b2SJayamohan Kallickal 3727480195c2SJitendra Bhivare status = beiscsi_check_supported_fw(ctrl, phba); 37286733b39aSJayamohan Kallickal if (status != 0) { 372999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 373099bc5d55SJohn Soni Jose "BM_%d : Unsupported fw version\n"); 37316733b39aSJayamohan Kallickal goto error; 37326733b39aSJayamohan Kallickal } 37336733b39aSJayamohan Kallickal 3734bfead3b2SJayamohan Kallickal status = beiscsi_create_cqs(phba, phwi_context); 37356733b39aSJayamohan Kallickal if (status != 0) { 373699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 373799bc5d55SJohn Soni Jose "BM_%d : CQ not created\n"); 37386733b39aSJayamohan Kallickal goto error; 37396733b39aSJayamohan Kallickal } 37406733b39aSJayamohan Kallickal 37418a86e833SJayamohan Kallickal for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 37428a86e833SJayamohan Kallickal if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 37431e2931f1SJitendra Bhivare nbufs = phwi_context->pasync_ctx[ulp_num]->num_entries; 37441e2931f1SJitendra Bhivare def_pdu_ring_sz = nbufs * sizeof(struct phys_addr); 37458a86e833SJayamohan Kallickal 37468a86e833SJayamohan Kallickal status = beiscsi_create_def_hdr(phba, phwi_context, 37478a86e833SJayamohan Kallickal phwi_ctrlr, 37488a86e833SJayamohan Kallickal def_pdu_ring_sz, 37498a86e833SJayamohan Kallickal ulp_num); 37506733b39aSJayamohan Kallickal if (status != 0) { 375199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 37528a86e833SJayamohan Kallickal "BM_%d : Default Header not created for ULP : %d\n", 37538a86e833SJayamohan Kallickal ulp_num); 37546733b39aSJayamohan Kallickal goto error; 37556733b39aSJayamohan Kallickal } 37566733b39aSJayamohan Kallickal 37576733b39aSJayamohan Kallickal status = beiscsi_create_def_data(phba, phwi_context, 37588a86e833SJayamohan Kallickal phwi_ctrlr, 37598a86e833SJayamohan Kallickal def_pdu_ring_sz, 37608a86e833SJayamohan Kallickal ulp_num); 37616733b39aSJayamohan Kallickal if (status != 0) { 376299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 37638a86e833SJayamohan Kallickal "BM_%d : Default Data not created for ULP : %d\n", 37648a86e833SJayamohan Kallickal ulp_num); 37656733b39aSJayamohan Kallickal goto error; 37666733b39aSJayamohan Kallickal } 3767f79929deSJitendra Bhivare /** 3768f79929deSJitendra Bhivare * Now that the default PDU rings have been created, 3769f79929deSJitendra Bhivare * let EP know about it. 3770f79929deSJitendra Bhivare */ 3771938f372cSJitendra Bhivare beiscsi_hdq_post_handles(phba, BEISCSI_DEFQ_HDR, 37721e2931f1SJitendra Bhivare ulp_num, nbufs); 3773938f372cSJitendra Bhivare beiscsi_hdq_post_handles(phba, BEISCSI_DEFQ_DATA, 37741e2931f1SJitendra Bhivare ulp_num, nbufs); 37758a86e833SJayamohan Kallickal } 37768a86e833SJayamohan Kallickal } 37776733b39aSJayamohan Kallickal 37786733b39aSJayamohan Kallickal status = beiscsi_post_pages(phba); 37796733b39aSJayamohan Kallickal if (status != 0) { 378099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 378199bc5d55SJohn Soni Jose "BM_%d : Post SGL Pages Failed\n"); 37826733b39aSJayamohan Kallickal goto error; 37836733b39aSJayamohan Kallickal } 37846733b39aSJayamohan Kallickal 378515a90fe0SJayamohan Kallickal status = beiscsi_post_template_hdr(phba); 378615a90fe0SJayamohan Kallickal if (status != 0) { 378715a90fe0SJayamohan Kallickal beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 378815a90fe0SJayamohan Kallickal "BM_%d : Template HDR Posting for CXN Failed\n"); 378915a90fe0SJayamohan Kallickal } 379015a90fe0SJayamohan Kallickal 37916733b39aSJayamohan Kallickal status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr); 37926733b39aSJayamohan Kallickal if (status != 0) { 379399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 379499bc5d55SJohn Soni Jose "BM_%d : WRB Rings not created\n"); 37956733b39aSJayamohan Kallickal goto error; 37966733b39aSJayamohan Kallickal } 37976733b39aSJayamohan Kallickal 37988a86e833SJayamohan Kallickal for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 37998a86e833SJayamohan Kallickal uint16_t async_arr_idx = 0; 38008a86e833SJayamohan Kallickal 38018a86e833SJayamohan Kallickal if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 38028a86e833SJayamohan Kallickal uint16_t cri = 0; 3803938f372cSJitendra Bhivare struct hd_async_context *pasync_ctx; 38048a86e833SJayamohan Kallickal 38058a86e833SJayamohan Kallickal pasync_ctx = HWI_GET_ASYNC_PDU_CTX( 38068a86e833SJayamohan Kallickal phwi_ctrlr, ulp_num); 38078a86e833SJayamohan Kallickal for (cri = 0; cri < 38088a86e833SJayamohan Kallickal phba->params.cxns_per_ctrl; cri++) { 38098a86e833SJayamohan Kallickal if (ulp_num == BEISCSI_GET_ULP_FROM_CRI 38108a86e833SJayamohan Kallickal (phwi_ctrlr, cri)) 38118a86e833SJayamohan Kallickal pasync_ctx->cid_to_async_cri_map[ 38128a86e833SJayamohan Kallickal phwi_ctrlr->wrb_context[cri].cid] = 38138a86e833SJayamohan Kallickal async_arr_idx++; 38148a86e833SJayamohan Kallickal } 38158a86e833SJayamohan Kallickal } 38168a86e833SJayamohan Kallickal } 38178a86e833SJayamohan Kallickal 381899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 381999bc5d55SJohn Soni Jose "BM_%d : hwi_init_port success\n"); 38206733b39aSJayamohan Kallickal return 0; 38216733b39aSJayamohan Kallickal 38226733b39aSJayamohan Kallickal error: 382399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 382499bc5d55SJohn Soni Jose "BM_%d : hwi_init_port failed"); 38254d2ee1e6SJitendra Bhivare hwi_cleanup_port(phba); 3826a49e06d5SJayamohan Kallickal return status; 38276733b39aSJayamohan Kallickal } 38286733b39aSJayamohan Kallickal 38296733b39aSJayamohan Kallickal static int hwi_init_controller(struct beiscsi_hba *phba) 38306733b39aSJayamohan Kallickal { 38316733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 38326733b39aSJayamohan Kallickal 38336733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 38346733b39aSJayamohan Kallickal if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) { 38356733b39aSJayamohan Kallickal phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba-> 38366733b39aSJayamohan Kallickal init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address; 383799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 383899bc5d55SJohn Soni Jose "BM_%d : phwi_ctrlr->phwi_ctxt=%p\n", 38396733b39aSJayamohan Kallickal phwi_ctrlr->phwi_ctxt); 38406733b39aSJayamohan Kallickal } else { 384199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 384299bc5d55SJohn Soni Jose "BM_%d : HWI_MEM_ADDN_CONTEXT is more " 384399bc5d55SJohn Soni Jose "than one element.Failing to load\n"); 38446733b39aSJayamohan Kallickal return -ENOMEM; 38456733b39aSJayamohan Kallickal } 38466733b39aSJayamohan Kallickal 38476733b39aSJayamohan Kallickal iscsi_init_global_templates(phba); 38483ec78271SJayamohan Kallickal if (beiscsi_init_wrb_handle(phba)) 38493ec78271SJayamohan Kallickal return -ENOMEM; 38503ec78271SJayamohan Kallickal 3851a7909b39SJayamohan Kallickal if (hwi_init_async_pdu_ctx(phba)) { 3852a7909b39SJayamohan Kallickal beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3853a7909b39SJayamohan Kallickal "BM_%d : hwi_init_async_pdu_ctx failed\n"); 3854a7909b39SJayamohan Kallickal return -ENOMEM; 3855a7909b39SJayamohan Kallickal } 3856a7909b39SJayamohan Kallickal 38576733b39aSJayamohan Kallickal if (hwi_init_port(phba) != 0) { 385899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 385999bc5d55SJohn Soni Jose "BM_%d : hwi_init_controller failed\n"); 386099bc5d55SJohn Soni Jose 38616733b39aSJayamohan Kallickal return -ENOMEM; 38626733b39aSJayamohan Kallickal } 38636733b39aSJayamohan Kallickal return 0; 38646733b39aSJayamohan Kallickal } 38656733b39aSJayamohan Kallickal 38666733b39aSJayamohan Kallickal static void beiscsi_free_mem(struct beiscsi_hba *phba) 38676733b39aSJayamohan Kallickal { 38686733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 38696733b39aSJayamohan Kallickal int i, j; 38706733b39aSJayamohan Kallickal 38716733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 38726733b39aSJayamohan Kallickal i = 0; 38736733b39aSJayamohan Kallickal j = 0; 38746733b39aSJayamohan Kallickal for (i = 0; i < SE_MEM_MAX; i++) { 38756733b39aSJayamohan Kallickal for (j = mem_descr->num_elements; j > 0; j--) { 38766733b39aSJayamohan Kallickal pci_free_consistent(phba->pcidev, 38776733b39aSJayamohan Kallickal mem_descr->mem_array[j - 1].size, 38786733b39aSJayamohan Kallickal mem_descr->mem_array[j - 1].virtual_address, 3879457ff3b7SJayamohan Kallickal (unsigned long)mem_descr->mem_array[j - 1]. 3880457ff3b7SJayamohan Kallickal bus_address.u.a64.address); 38816733b39aSJayamohan Kallickal } 38828a86e833SJayamohan Kallickal 38836733b39aSJayamohan Kallickal kfree(mem_descr->mem_array); 38846733b39aSJayamohan Kallickal mem_descr++; 38856733b39aSJayamohan Kallickal } 38866733b39aSJayamohan Kallickal kfree(phba->init_mem); 3887a7909b39SJayamohan Kallickal kfree(phba->phwi_ctrlr->wrb_context); 38886733b39aSJayamohan Kallickal kfree(phba->phwi_ctrlr); 38896733b39aSJayamohan Kallickal } 38906733b39aSJayamohan Kallickal 38916733b39aSJayamohan Kallickal static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba) 38926733b39aSJayamohan Kallickal { 38936733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg; 38946733b39aSJayamohan Kallickal struct sgl_handle *psgl_handle; 38956733b39aSJayamohan Kallickal struct iscsi_sge *pfrag; 389690622db3SJayamohan Kallickal unsigned int arr_index, i, idx; 389790622db3SJayamohan Kallickal unsigned int ulp_icd_start, ulp_num = 0; 38986733b39aSJayamohan Kallickal 38996733b39aSJayamohan Kallickal phba->io_sgl_hndl_avbl = 0; 39006733b39aSJayamohan Kallickal phba->eh_sgl_hndl_avbl = 0; 3901bfead3b2SJayamohan Kallickal 39026733b39aSJayamohan Kallickal mem_descr_sglh = phba->init_mem; 39036733b39aSJayamohan Kallickal mem_descr_sglh += HWI_MEM_SGLH; 39046733b39aSJayamohan Kallickal if (1 == mem_descr_sglh->num_elements) { 39056733b39aSJayamohan Kallickal phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) * 39066733b39aSJayamohan Kallickal phba->params.ios_per_ctrl, 39076733b39aSJayamohan Kallickal GFP_KERNEL); 39086733b39aSJayamohan Kallickal if (!phba->io_sgl_hndl_base) { 390999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 391099bc5d55SJohn Soni Jose "BM_%d : Mem Alloc Failed. Failing to load\n"); 39116733b39aSJayamohan Kallickal return -ENOMEM; 39126733b39aSJayamohan Kallickal } 39136733b39aSJayamohan Kallickal phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) * 39146733b39aSJayamohan Kallickal (phba->params.icds_per_ctrl - 39156733b39aSJayamohan Kallickal phba->params.ios_per_ctrl), 39166733b39aSJayamohan Kallickal GFP_KERNEL); 39176733b39aSJayamohan Kallickal if (!phba->eh_sgl_hndl_base) { 39186733b39aSJayamohan Kallickal kfree(phba->io_sgl_hndl_base); 391999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 392099bc5d55SJohn Soni Jose "BM_%d : Mem Alloc Failed. Failing to load\n"); 39216733b39aSJayamohan Kallickal return -ENOMEM; 39226733b39aSJayamohan Kallickal } 39236733b39aSJayamohan Kallickal } else { 392499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 392599bc5d55SJohn Soni Jose "BM_%d : HWI_MEM_SGLH is more than one element." 39266733b39aSJayamohan Kallickal "Failing to load\n"); 39276733b39aSJayamohan Kallickal return -ENOMEM; 39286733b39aSJayamohan Kallickal } 39296733b39aSJayamohan Kallickal 39306733b39aSJayamohan Kallickal arr_index = 0; 39316733b39aSJayamohan Kallickal idx = 0; 39326733b39aSJayamohan Kallickal while (idx < mem_descr_sglh->num_elements) { 39336733b39aSJayamohan Kallickal psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address; 39346733b39aSJayamohan Kallickal 39356733b39aSJayamohan Kallickal for (i = 0; i < (mem_descr_sglh->mem_array[idx].size / 39366733b39aSJayamohan Kallickal sizeof(struct sgl_handle)); i++) { 39376733b39aSJayamohan Kallickal if (arr_index < phba->params.ios_per_ctrl) { 39386733b39aSJayamohan Kallickal phba->io_sgl_hndl_base[arr_index] = psgl_handle; 39396733b39aSJayamohan Kallickal phba->io_sgl_hndl_avbl++; 39406733b39aSJayamohan Kallickal arr_index++; 39416733b39aSJayamohan Kallickal } else { 39426733b39aSJayamohan Kallickal phba->eh_sgl_hndl_base[arr_index - 39436733b39aSJayamohan Kallickal phba->params.ios_per_ctrl] = 39446733b39aSJayamohan Kallickal psgl_handle; 39456733b39aSJayamohan Kallickal arr_index++; 39466733b39aSJayamohan Kallickal phba->eh_sgl_hndl_avbl++; 39476733b39aSJayamohan Kallickal } 39486733b39aSJayamohan Kallickal psgl_handle++; 39496733b39aSJayamohan Kallickal } 39506733b39aSJayamohan Kallickal idx++; 39516733b39aSJayamohan Kallickal } 395299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 395399bc5d55SJohn Soni Jose "BM_%d : phba->io_sgl_hndl_avbl=%d" 39546733b39aSJayamohan Kallickal "phba->eh_sgl_hndl_avbl=%d\n", 39556733b39aSJayamohan Kallickal phba->io_sgl_hndl_avbl, 39566733b39aSJayamohan Kallickal phba->eh_sgl_hndl_avbl); 395799bc5d55SJohn Soni Jose 39586733b39aSJayamohan Kallickal mem_descr_sg = phba->init_mem; 39596733b39aSJayamohan Kallickal mem_descr_sg += HWI_MEM_SGE; 396099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 396199bc5d55SJohn Soni Jose "\n BM_%d : mem_descr_sg->num_elements=%d\n", 39626733b39aSJayamohan Kallickal mem_descr_sg->num_elements); 396399bc5d55SJohn Soni Jose 396490622db3SJayamohan Kallickal for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) 396590622db3SJayamohan Kallickal if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) 396690622db3SJayamohan Kallickal break; 396790622db3SJayamohan Kallickal 396890622db3SJayamohan Kallickal ulp_icd_start = phba->fw_config.iscsi_icd_start[ulp_num]; 396990622db3SJayamohan Kallickal 39706733b39aSJayamohan Kallickal arr_index = 0; 39716733b39aSJayamohan Kallickal idx = 0; 39726733b39aSJayamohan Kallickal while (idx < mem_descr_sg->num_elements) { 39736733b39aSJayamohan Kallickal pfrag = mem_descr_sg->mem_array[idx].virtual_address; 39746733b39aSJayamohan Kallickal 39756733b39aSJayamohan Kallickal for (i = 0; 39766733b39aSJayamohan Kallickal i < (mem_descr_sg->mem_array[idx].size) / 39776733b39aSJayamohan Kallickal (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io); 39786733b39aSJayamohan Kallickal i++) { 39796733b39aSJayamohan Kallickal if (arr_index < phba->params.ios_per_ctrl) 39806733b39aSJayamohan Kallickal psgl_handle = phba->io_sgl_hndl_base[arr_index]; 39816733b39aSJayamohan Kallickal else 39826733b39aSJayamohan Kallickal psgl_handle = phba->eh_sgl_hndl_base[arr_index - 39836733b39aSJayamohan Kallickal phba->params.ios_per_ctrl]; 39846733b39aSJayamohan Kallickal psgl_handle->pfrag = pfrag; 39856733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0); 39866733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0); 39876733b39aSJayamohan Kallickal pfrag += phba->params.num_sge_per_io; 398890622db3SJayamohan Kallickal psgl_handle->sgl_index = ulp_icd_start + arr_index++; 39896733b39aSJayamohan Kallickal } 39906733b39aSJayamohan Kallickal idx++; 39916733b39aSJayamohan Kallickal } 39926733b39aSJayamohan Kallickal phba->io_sgl_free_index = 0; 39936733b39aSJayamohan Kallickal phba->io_sgl_alloc_index = 0; 39946733b39aSJayamohan Kallickal phba->eh_sgl_free_index = 0; 39956733b39aSJayamohan Kallickal phba->eh_sgl_alloc_index = 0; 39966733b39aSJayamohan Kallickal return 0; 39976733b39aSJayamohan Kallickal } 39986733b39aSJayamohan Kallickal 39996733b39aSJayamohan Kallickal static int hba_setup_cid_tbls(struct beiscsi_hba *phba) 40006733b39aSJayamohan Kallickal { 40010a3db7c0SJayamohan Kallickal int ret; 40020a3db7c0SJayamohan Kallickal uint16_t i, ulp_num; 40030a3db7c0SJayamohan Kallickal struct ulp_cid_info *ptr_cid_info = NULL; 40046733b39aSJayamohan Kallickal 40050a3db7c0SJayamohan Kallickal for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 40060a3db7c0SJayamohan Kallickal if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) { 40070a3db7c0SJayamohan Kallickal ptr_cid_info = kzalloc(sizeof(struct ulp_cid_info), 40086733b39aSJayamohan Kallickal GFP_KERNEL); 40090a3db7c0SJayamohan Kallickal 40100a3db7c0SJayamohan Kallickal if (!ptr_cid_info) { 401199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 40120a3db7c0SJayamohan Kallickal "BM_%d : Failed to allocate memory" 40130a3db7c0SJayamohan Kallickal "for ULP_CID_INFO for ULP : %d\n", 40140a3db7c0SJayamohan Kallickal ulp_num); 40150a3db7c0SJayamohan Kallickal ret = -ENOMEM; 40160a3db7c0SJayamohan Kallickal goto free_memory; 40170a3db7c0SJayamohan Kallickal 40180a3db7c0SJayamohan Kallickal } 40190a3db7c0SJayamohan Kallickal 40200a3db7c0SJayamohan Kallickal /* Allocate memory for CID array */ 4021413f3656SJitendra Bhivare ptr_cid_info->cid_array = 4022413f3656SJitendra Bhivare kcalloc(BEISCSI_GET_CID_COUNT(phba, ulp_num), 4023413f3656SJitendra Bhivare sizeof(*ptr_cid_info->cid_array), 4024413f3656SJitendra Bhivare GFP_KERNEL); 40250a3db7c0SJayamohan Kallickal if (!ptr_cid_info->cid_array) { 40260a3db7c0SJayamohan Kallickal beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 40270a3db7c0SJayamohan Kallickal "BM_%d : Failed to allocate memory" 40280a3db7c0SJayamohan Kallickal "for CID_ARRAY for ULP : %d\n", 40290a3db7c0SJayamohan Kallickal ulp_num); 40300a3db7c0SJayamohan Kallickal kfree(ptr_cid_info); 40310a3db7c0SJayamohan Kallickal ptr_cid_info = NULL; 40320a3db7c0SJayamohan Kallickal ret = -ENOMEM; 40330a3db7c0SJayamohan Kallickal 40340a3db7c0SJayamohan Kallickal goto free_memory; 40350a3db7c0SJayamohan Kallickal } 40360a3db7c0SJayamohan Kallickal ptr_cid_info->avlbl_cids = BEISCSI_GET_CID_COUNT( 40370a3db7c0SJayamohan Kallickal phba, ulp_num); 40380a3db7c0SJayamohan Kallickal 40390a3db7c0SJayamohan Kallickal /* Save the cid_info_array ptr */ 40400a3db7c0SJayamohan Kallickal phba->cid_array_info[ulp_num] = ptr_cid_info; 40410a3db7c0SJayamohan Kallickal } 40426733b39aSJayamohan Kallickal } 4043c2462288SJayamohan Kallickal phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) * 4044a7909b39SJayamohan Kallickal phba->params.cxns_per_ctrl, GFP_KERNEL); 40456733b39aSJayamohan Kallickal if (!phba->ep_array) { 404699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 404799bc5d55SJohn Soni Jose "BM_%d : Failed to allocate memory in " 40486733b39aSJayamohan Kallickal "hba_setup_cid_tbls\n"); 40490a3db7c0SJayamohan Kallickal ret = -ENOMEM; 40500a3db7c0SJayamohan Kallickal 40510a3db7c0SJayamohan Kallickal goto free_memory; 40526733b39aSJayamohan Kallickal } 4053a7909b39SJayamohan Kallickal 4054a7909b39SJayamohan Kallickal phba->conn_table = kzalloc(sizeof(struct beiscsi_conn *) * 4055a7909b39SJayamohan Kallickal phba->params.cxns_per_ctrl, GFP_KERNEL); 4056a7909b39SJayamohan Kallickal if (!phba->conn_table) { 4057a7909b39SJayamohan Kallickal beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4058a7909b39SJayamohan Kallickal "BM_%d : Failed to allocate memory in" 4059a7909b39SJayamohan Kallickal "hba_setup_cid_tbls\n"); 4060a7909b39SJayamohan Kallickal 4061a7909b39SJayamohan Kallickal kfree(phba->ep_array); 4062a7909b39SJayamohan Kallickal phba->ep_array = NULL; 40630a3db7c0SJayamohan Kallickal ret = -ENOMEM; 40645f2d25efSTomas Henzl 40655f2d25efSTomas Henzl goto free_memory; 40666733b39aSJayamohan Kallickal } 4067a7909b39SJayamohan Kallickal 40680a3db7c0SJayamohan Kallickal for (i = 0; i < phba->params.cxns_per_ctrl; i++) { 40690a3db7c0SJayamohan Kallickal ulp_num = phba->phwi_ctrlr->wrb_context[i].ulp_num; 4070a7909b39SJayamohan Kallickal 40710a3db7c0SJayamohan Kallickal ptr_cid_info = phba->cid_array_info[ulp_num]; 40720a3db7c0SJayamohan Kallickal ptr_cid_info->cid_array[ptr_cid_info->cid_alloc++] = 40730a3db7c0SJayamohan Kallickal phba->phwi_ctrlr->wrb_context[i].cid; 40740a3db7c0SJayamohan Kallickal 40750a3db7c0SJayamohan Kallickal } 40760a3db7c0SJayamohan Kallickal 40770a3db7c0SJayamohan Kallickal for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 40780a3db7c0SJayamohan Kallickal if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) { 40790a3db7c0SJayamohan Kallickal ptr_cid_info = phba->cid_array_info[ulp_num]; 40800a3db7c0SJayamohan Kallickal 40810a3db7c0SJayamohan Kallickal ptr_cid_info->cid_alloc = 0; 40820a3db7c0SJayamohan Kallickal ptr_cid_info->cid_free = 0; 40830a3db7c0SJayamohan Kallickal } 40840a3db7c0SJayamohan Kallickal } 40856733b39aSJayamohan Kallickal return 0; 40860a3db7c0SJayamohan Kallickal 40870a3db7c0SJayamohan Kallickal free_memory: 40880a3db7c0SJayamohan Kallickal for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 40890a3db7c0SJayamohan Kallickal if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) { 40900a3db7c0SJayamohan Kallickal ptr_cid_info = phba->cid_array_info[ulp_num]; 40910a3db7c0SJayamohan Kallickal 40920a3db7c0SJayamohan Kallickal if (ptr_cid_info) { 40930a3db7c0SJayamohan Kallickal kfree(ptr_cid_info->cid_array); 40940a3db7c0SJayamohan Kallickal kfree(ptr_cid_info); 40950a3db7c0SJayamohan Kallickal phba->cid_array_info[ulp_num] = NULL; 40960a3db7c0SJayamohan Kallickal } 40970a3db7c0SJayamohan Kallickal } 40980a3db7c0SJayamohan Kallickal } 40990a3db7c0SJayamohan Kallickal 41000a3db7c0SJayamohan Kallickal return ret; 41016733b39aSJayamohan Kallickal } 41026733b39aSJayamohan Kallickal 4103238f6b72SJayamohan Kallickal static void hwi_enable_intr(struct beiscsi_hba *phba) 41046733b39aSJayamohan Kallickal { 41056733b39aSJayamohan Kallickal struct be_ctrl_info *ctrl = &phba->ctrl; 41066733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 41076733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context; 41086733b39aSJayamohan Kallickal struct be_queue_info *eq; 41096733b39aSJayamohan Kallickal u8 __iomem *addr; 4110bfead3b2SJayamohan Kallickal u32 reg, i; 41116733b39aSJayamohan Kallickal u32 enabled; 41126733b39aSJayamohan Kallickal 41136733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 41146733b39aSJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 41156733b39aSJayamohan Kallickal 41166733b39aSJayamohan Kallickal addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg + 41176733b39aSJayamohan Kallickal PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET); 41186733b39aSJayamohan Kallickal reg = ioread32(addr); 41196733b39aSJayamohan Kallickal 41206733b39aSJayamohan Kallickal enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 41216733b39aSJayamohan Kallickal if (!enabled) { 41226733b39aSJayamohan Kallickal reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 412399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 412499bc5d55SJohn Soni Jose "BM_%d : reg =x%08x addr=%p\n", reg, addr); 41256733b39aSJayamohan Kallickal iowrite32(reg, addr); 4126665d6d94SJayamohan Kallickal } 4127665d6d94SJayamohan Kallickal 412883148866SChristoph Hellwig if (!phba->pcidev->msix_enabled) { 4129c03af1aeSJayamohan Kallickal eq = &phwi_context->be_eq[0].q; 413099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 413199bc5d55SJohn Soni Jose "BM_%d : eq->id=%d\n", eq->id); 413299bc5d55SJohn Soni Jose 4133c03af1aeSJayamohan Kallickal hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1); 4134c03af1aeSJayamohan Kallickal } else { 4135bfead3b2SJayamohan Kallickal for (i = 0; i <= phba->num_cpus; i++) { 4136bfead3b2SJayamohan Kallickal eq = &phwi_context->be_eq[i].q; 413799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 413899bc5d55SJohn Soni Jose "BM_%d : eq->id=%d\n", eq->id); 41396733b39aSJayamohan Kallickal hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1); 4140bfead3b2SJayamohan Kallickal } 4141c03af1aeSJayamohan Kallickal } 4142c03af1aeSJayamohan Kallickal } 41436733b39aSJayamohan Kallickal 41446733b39aSJayamohan Kallickal static void hwi_disable_intr(struct beiscsi_hba *phba) 41456733b39aSJayamohan Kallickal { 41466733b39aSJayamohan Kallickal struct be_ctrl_info *ctrl = &phba->ctrl; 41476733b39aSJayamohan Kallickal 41486733b39aSJayamohan Kallickal u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET; 41496733b39aSJayamohan Kallickal u32 reg = ioread32(addr); 41506733b39aSJayamohan Kallickal 41516733b39aSJayamohan Kallickal u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 41526733b39aSJayamohan Kallickal if (enabled) { 41536733b39aSJayamohan Kallickal reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 41546733b39aSJayamohan Kallickal iowrite32(reg, addr); 41556733b39aSJayamohan Kallickal } else 415699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 415799bc5d55SJohn Soni Jose "BM_%d : In hwi_disable_intr, Already Disabled\n"); 41586733b39aSJayamohan Kallickal } 41596733b39aSJayamohan Kallickal 41606733b39aSJayamohan Kallickal static int beiscsi_init_port(struct beiscsi_hba *phba) 41616733b39aSJayamohan Kallickal { 41626733b39aSJayamohan Kallickal int ret; 41636733b39aSJayamohan Kallickal 4164dd940972SJitendra Bhivare ret = hwi_init_controller(phba); 41656733b39aSJayamohan Kallickal if (ret < 0) { 416699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4167dd940972SJitendra Bhivare "BM_%d : init controller failed\n"); 41686733b39aSJayamohan Kallickal return ret; 41696733b39aSJayamohan Kallickal } 41706733b39aSJayamohan Kallickal ret = beiscsi_init_sgl_handle(phba); 41716733b39aSJayamohan Kallickal if (ret < 0) { 417299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4173dd940972SJitendra Bhivare "BM_%d : init sgl handles failed\n"); 4174dd940972SJitendra Bhivare goto cleanup_port; 41756733b39aSJayamohan Kallickal } 41766733b39aSJayamohan Kallickal 4177deeea8edSChristophe JAILLET ret = hba_setup_cid_tbls(phba); 4178deeea8edSChristophe JAILLET if (ret < 0) { 417999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4180dd940972SJitendra Bhivare "BM_%d : setup CID table failed\n"); 41816733b39aSJayamohan Kallickal kfree(phba->io_sgl_hndl_base); 41826733b39aSJayamohan Kallickal kfree(phba->eh_sgl_hndl_base); 4183dd940972SJitendra Bhivare goto cleanup_port; 41846733b39aSJayamohan Kallickal } 41856733b39aSJayamohan Kallickal return ret; 41866733b39aSJayamohan Kallickal 4187dd940972SJitendra Bhivare cleanup_port: 41884d2ee1e6SJitendra Bhivare hwi_cleanup_port(phba); 41896733b39aSJayamohan Kallickal return ret; 41906733b39aSJayamohan Kallickal } 41916733b39aSJayamohan Kallickal 41924d2ee1e6SJitendra Bhivare static void beiscsi_cleanup_port(struct beiscsi_hba *phba) 41936733b39aSJayamohan Kallickal { 41940a3db7c0SJayamohan Kallickal struct ulp_cid_info *ptr_cid_info = NULL; 4195f79929deSJitendra Bhivare int ulp_num; 41966733b39aSJayamohan Kallickal 41976733b39aSJayamohan Kallickal kfree(phba->io_sgl_hndl_base); 41986733b39aSJayamohan Kallickal kfree(phba->eh_sgl_hndl_base); 41996733b39aSJayamohan Kallickal kfree(phba->ep_array); 4200a7909b39SJayamohan Kallickal kfree(phba->conn_table); 42010a3db7c0SJayamohan Kallickal 42020a3db7c0SJayamohan Kallickal for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 42030a3db7c0SJayamohan Kallickal if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) { 42040a3db7c0SJayamohan Kallickal ptr_cid_info = phba->cid_array_info[ulp_num]; 42050a3db7c0SJayamohan Kallickal 42060a3db7c0SJayamohan Kallickal if (ptr_cid_info) { 42070a3db7c0SJayamohan Kallickal kfree(ptr_cid_info->cid_array); 42080a3db7c0SJayamohan Kallickal kfree(ptr_cid_info); 42090a3db7c0SJayamohan Kallickal phba->cid_array_info[ulp_num] = NULL; 42100a3db7c0SJayamohan Kallickal } 42110a3db7c0SJayamohan Kallickal } 42120a3db7c0SJayamohan Kallickal } 42136733b39aSJayamohan Kallickal } 42146733b39aSJayamohan Kallickal 4215d629c471SJohn Soni Jose /** 421643f388b0SJayamohan Kallickal * beiscsi_free_mgmt_task_handles()- Free driver CXN resources 421743f388b0SJayamohan Kallickal * @beiscsi_conn: ptr to the conn to be cleaned up 42184a4a11b9SJayamohan Kallickal * @task: ptr to iscsi_task resource to be freed. 421943f388b0SJayamohan Kallickal * 422043f388b0SJayamohan Kallickal * Free driver mgmt resources binded to CXN. 422143f388b0SJayamohan Kallickal **/ 422243f388b0SJayamohan Kallickal void 42234a4a11b9SJayamohan Kallickal beiscsi_free_mgmt_task_handles(struct beiscsi_conn *beiscsi_conn, 42244a4a11b9SJayamohan Kallickal struct iscsi_task *task) 422543f388b0SJayamohan Kallickal { 422643f388b0SJayamohan Kallickal struct beiscsi_io_task *io_task; 422743f388b0SJayamohan Kallickal struct beiscsi_hba *phba = beiscsi_conn->phba; 422843f388b0SJayamohan Kallickal struct hwi_wrb_context *pwrb_context; 422943f388b0SJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 4230a7909b39SJayamohan Kallickal uint16_t cri_index = BE_GET_CRI_FROM_CID( 4231a7909b39SJayamohan Kallickal beiscsi_conn->beiscsi_conn_cid); 423243f388b0SJayamohan Kallickal 423343f388b0SJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 4234a7909b39SJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[cri_index]; 4235a7909b39SJayamohan Kallickal 42364a4a11b9SJayamohan Kallickal io_task = task->dd_data; 423743f388b0SJayamohan Kallickal 423843f388b0SJayamohan Kallickal if (io_task->pwrb_handle) { 4239e1f9d31eSJitendra Bhivare free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle); 424043f388b0SJayamohan Kallickal io_task->pwrb_handle = NULL; 424143f388b0SJayamohan Kallickal } 424243f388b0SJayamohan Kallickal 424343f388b0SJayamohan Kallickal if (io_task->psgl_handle) { 4244e1f9d31eSJitendra Bhivare free_mgmt_sgl_handle(phba, io_task->psgl_handle); 424543f388b0SJayamohan Kallickal io_task->psgl_handle = NULL; 424643f388b0SJayamohan Kallickal } 424743f388b0SJayamohan Kallickal 4248eb1c4692SJohn Soni Jose if (io_task->mtask_addr) { 424943f388b0SJayamohan Kallickal pci_unmap_single(phba->pcidev, 425043f388b0SJayamohan Kallickal io_task->mtask_addr, 425143f388b0SJayamohan Kallickal io_task->mtask_data_count, 425243f388b0SJayamohan Kallickal PCI_DMA_TODEVICE); 4253eb1c4692SJohn Soni Jose io_task->mtask_addr = 0; 4254eb1c4692SJohn Soni Jose } 425543f388b0SJayamohan Kallickal } 425643f388b0SJayamohan Kallickal 425743f388b0SJayamohan Kallickal /** 4258d629c471SJohn Soni Jose * beiscsi_cleanup_task()- Free driver resources of the task 4259d629c471SJohn Soni Jose * @task: ptr to the iscsi task 4260d629c471SJohn Soni Jose * 4261d629c471SJohn Soni Jose **/ 42621282ab76SMike Christie static void beiscsi_cleanup_task(struct iscsi_task *task) 42631282ab76SMike Christie { 42641282ab76SMike Christie struct beiscsi_io_task *io_task = task->dd_data; 42651282ab76SMike Christie struct iscsi_conn *conn = task->conn; 42661282ab76SMike Christie struct beiscsi_conn *beiscsi_conn = conn->dd_data; 42671282ab76SMike Christie struct beiscsi_hba *phba = beiscsi_conn->phba; 42681282ab76SMike Christie struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess; 42691282ab76SMike Christie struct hwi_wrb_context *pwrb_context; 42701282ab76SMike Christie struct hwi_controller *phwi_ctrlr; 4271a7909b39SJayamohan Kallickal uint16_t cri_index = BE_GET_CRI_FROM_CID( 4272a7909b39SJayamohan Kallickal beiscsi_conn->beiscsi_conn_cid); 42731282ab76SMike Christie 42741282ab76SMike Christie phwi_ctrlr = phba->phwi_ctrlr; 4275a7909b39SJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[cri_index]; 42761282ab76SMike Christie 42771282ab76SMike Christie if (io_task->cmd_bhs) { 4278af007b02SRomain Perier dma_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs, 42791282ab76SMike Christie io_task->bhs_pa.u.a64.address); 42801282ab76SMike Christie io_task->cmd_bhs = NULL; 4281e1f9d31eSJitendra Bhivare task->hdr = NULL; 42821282ab76SMike Christie } 42831282ab76SMike Christie 42841282ab76SMike Christie if (task->sc) { 42851282ab76SMike Christie if (io_task->pwrb_handle) { 42861282ab76SMike Christie free_wrb_handle(phba, pwrb_context, 42871282ab76SMike Christie io_task->pwrb_handle); 42881282ab76SMike Christie io_task->pwrb_handle = NULL; 42891282ab76SMike Christie } 42901282ab76SMike Christie 42911282ab76SMike Christie if (io_task->psgl_handle) { 42921282ab76SMike Christie free_io_sgl_handle(phba, io_task->psgl_handle); 42931282ab76SMike Christie io_task->psgl_handle = NULL; 42941282ab76SMike Christie } 4295da334977SJayamohan Kallickal 4296da334977SJayamohan Kallickal if (io_task->scsi_cmnd) { 42979122e991SJitendra Bhivare if (io_task->num_sg) 4298da334977SJayamohan Kallickal scsi_dma_unmap(io_task->scsi_cmnd); 4299da334977SJayamohan Kallickal io_task->scsi_cmnd = NULL; 4300da334977SJayamohan Kallickal } 43011282ab76SMike Christie } else { 430243f388b0SJayamohan Kallickal if (!beiscsi_conn->login_in_progress) 43034a4a11b9SJayamohan Kallickal beiscsi_free_mgmt_task_handles(beiscsi_conn, task); 43041282ab76SMike Christie } 43051282ab76SMike Christie } 43061282ab76SMike Christie 43076733b39aSJayamohan Kallickal void 43086733b39aSJayamohan Kallickal beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn, 43096733b39aSJayamohan Kallickal struct beiscsi_offload_params *params) 43106733b39aSJayamohan Kallickal { 43116733b39aSJayamohan Kallickal struct wrb_handle *pwrb_handle; 4312340c99e9SJohn Soni Jose struct hwi_wrb_context *pwrb_context = NULL; 43136733b39aSJayamohan Kallickal struct beiscsi_hba *phba = beiscsi_conn->phba; 43141282ab76SMike Christie struct iscsi_task *task = beiscsi_conn->task; 43151282ab76SMike Christie struct iscsi_session *session = task->conn->session; 43166733b39aSJayamohan Kallickal u32 doorbell = 0; 43176733b39aSJayamohan Kallickal 43186733b39aSJayamohan Kallickal /* 43196733b39aSJayamohan Kallickal * We can always use 0 here because it is reserved by libiscsi for 43206733b39aSJayamohan Kallickal * login/startup related tasks. 43216733b39aSJayamohan Kallickal */ 43221282ab76SMike Christie beiscsi_conn->login_in_progress = 0; 4323659743b0SShlomo Pongratz spin_lock_bh(&session->back_lock); 43241282ab76SMike Christie beiscsi_cleanup_task(task); 4325659743b0SShlomo Pongratz spin_unlock_bh(&session->back_lock); 43261282ab76SMike Christie 4327340c99e9SJohn Soni Jose pwrb_handle = alloc_wrb_handle(phba, beiscsi_conn->beiscsi_conn_cid, 4328340c99e9SJohn Soni Jose &pwrb_context); 43296733b39aSJayamohan Kallickal 4330acb9693cSJohn Soni Jose /* Check for the adapter family */ 43312c9dfd36SJayamohan Kallickal if (is_chip_be2_be3r(phba)) 4332acb9693cSJohn Soni Jose beiscsi_offload_cxn_v0(params, pwrb_handle, 4333340c99e9SJohn Soni Jose phba->init_mem, 4334340c99e9SJohn Soni Jose pwrb_context); 43352c9dfd36SJayamohan Kallickal else 4336340c99e9SJohn Soni Jose beiscsi_offload_cxn_v2(params, pwrb_handle, 4337340c99e9SJohn Soni Jose pwrb_context); 43386733b39aSJayamohan Kallickal 4339acb9693cSJohn Soni Jose be_dws_le_to_cpu(pwrb_handle->pwrb, 4340acb9693cSJohn Soni Jose sizeof(struct iscsi_target_context_update_wrb)); 43416733b39aSJayamohan Kallickal 43426733b39aSJayamohan Kallickal doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK; 4343bfead3b2SJayamohan Kallickal doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK) 4344bfead3b2SJayamohan Kallickal << DB_DEF_PDU_WRB_INDEX_SHIFT; 43456733b39aSJayamohan Kallickal doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 43461e4be6ffSJayamohan Kallickal iowrite32(doorbell, phba->db_va + 43471e4be6ffSJayamohan Kallickal beiscsi_conn->doorbell_offset); 4348cb564c6bSJitendra Bhivare 4349cb564c6bSJitendra Bhivare /* 4350cb564c6bSJitendra Bhivare * There is no completion for CONTEXT_UPDATE. The completion of next 4351cb564c6bSJitendra Bhivare * WRB posted guarantees FW's processing and DMA'ing of it. 4352cb564c6bSJitendra Bhivare * Use beiscsi_put_wrb_handle to put it back in the pool which makes 4353cb564c6bSJitendra Bhivare * sure zero'ing or reuse of the WRB only after wrbs_per_cxn. 4354cb564c6bSJitendra Bhivare */ 4355cb564c6bSJitendra Bhivare beiscsi_put_wrb_handle(pwrb_context, pwrb_handle, 4356cb564c6bSJitendra Bhivare phba->params.wrbs_per_cxn); 4357cb564c6bSJitendra Bhivare beiscsi_log(phba, KERN_INFO, 4358cb564c6bSJitendra Bhivare BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 4359cb564c6bSJitendra Bhivare "BM_%d : put CONTEXT_UPDATE pwrb_handle=%p free_index=0x%x wrb_handles_available=%d\n", 4360cb564c6bSJitendra Bhivare pwrb_handle, pwrb_context->free_index, 4361cb564c6bSJitendra Bhivare pwrb_context->wrb_handles_available); 43626733b39aSJayamohan Kallickal } 43636733b39aSJayamohan Kallickal 43646733b39aSJayamohan Kallickal static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt, 43656733b39aSJayamohan Kallickal int *index, int *age) 43666733b39aSJayamohan Kallickal { 4367bfead3b2SJayamohan Kallickal *index = (int)itt; 43686733b39aSJayamohan Kallickal if (age) 43696733b39aSJayamohan Kallickal *age = conn->session->age; 43706733b39aSJayamohan Kallickal } 43716733b39aSJayamohan Kallickal 43726733b39aSJayamohan Kallickal /** 43736733b39aSJayamohan Kallickal * beiscsi_alloc_pdu - allocates pdu and related resources 43746733b39aSJayamohan Kallickal * @task: libiscsi task 43756733b39aSJayamohan Kallickal * @opcode: opcode of pdu for task 43766733b39aSJayamohan Kallickal * 43776733b39aSJayamohan Kallickal * This is called with the session lock held. It will allocate 43786733b39aSJayamohan Kallickal * the wrb and sgl if needed for the command. And it will prep 43796733b39aSJayamohan Kallickal * the pdu's itt. beiscsi_parse_pdu will later translate 43806733b39aSJayamohan Kallickal * the pdu itt to the libiscsi task itt. 43816733b39aSJayamohan Kallickal */ 43826733b39aSJayamohan Kallickal static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode) 43836733b39aSJayamohan Kallickal { 43846733b39aSJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 43856733b39aSJayamohan Kallickal struct iscsi_conn *conn = task->conn; 43866733b39aSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn = conn->dd_data; 43876733b39aSJayamohan Kallickal struct beiscsi_hba *phba = beiscsi_conn->phba; 43886733b39aSJayamohan Kallickal struct hwi_wrb_context *pwrb_context; 43896733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 43906733b39aSJayamohan Kallickal itt_t itt; 4391a7909b39SJayamohan Kallickal uint16_t cri_index = 0; 43922afc95bfSJayamohan Kallickal struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess; 43932afc95bfSJayamohan Kallickal dma_addr_t paddr; 43946733b39aSJayamohan Kallickal 4395af007b02SRomain Perier io_task->cmd_bhs = dma_pool_alloc(beiscsi_sess->bhs_pool, 4396bc7accecSMike Christie GFP_ATOMIC, &paddr); 43972afc95bfSJayamohan Kallickal if (!io_task->cmd_bhs) 43982afc95bfSJayamohan Kallickal return -ENOMEM; 43992afc95bfSJayamohan Kallickal io_task->bhs_pa.u.a64.address = paddr; 4400bfead3b2SJayamohan Kallickal io_task->libiscsi_itt = (itt_t)task->itt; 44016733b39aSJayamohan Kallickal io_task->conn = beiscsi_conn; 44026733b39aSJayamohan Kallickal 44036733b39aSJayamohan Kallickal task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr; 44046733b39aSJayamohan Kallickal task->hdr_max = sizeof(struct be_cmd_bhs); 4405d2cecf0dSJayamohan Kallickal io_task->psgl_handle = NULL; 44063ec78271SJayamohan Kallickal io_task->pwrb_handle = NULL; 44076733b39aSJayamohan Kallickal 44086733b39aSJayamohan Kallickal if (task->sc) { 44096733b39aSJayamohan Kallickal io_task->psgl_handle = alloc_io_sgl_handle(phba); 44108359c79bSJohn Soni Jose if (!io_task->psgl_handle) { 44118359c79bSJohn Soni Jose beiscsi_log(phba, KERN_ERR, 44128359c79bSJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 44138359c79bSJohn Soni Jose "BM_%d : Alloc of IO_SGL_ICD Failed" 44148359c79bSJohn Soni Jose "for the CID : %d\n", 44158359c79bSJohn Soni Jose beiscsi_conn->beiscsi_conn_cid); 44162afc95bfSJayamohan Kallickal goto free_hndls; 44178359c79bSJohn Soni Jose } 4418d2cecf0dSJayamohan Kallickal io_task->pwrb_handle = alloc_wrb_handle(phba, 4419340c99e9SJohn Soni Jose beiscsi_conn->beiscsi_conn_cid, 4420340c99e9SJohn Soni Jose &io_task->pwrb_context); 44218359c79bSJohn Soni Jose if (!io_task->pwrb_handle) { 44228359c79bSJohn Soni Jose beiscsi_log(phba, KERN_ERR, 44238359c79bSJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 44248359c79bSJohn Soni Jose "BM_%d : Alloc of WRB_HANDLE Failed" 44258359c79bSJohn Soni Jose "for the CID : %d\n", 44268359c79bSJohn Soni Jose beiscsi_conn->beiscsi_conn_cid); 4427d2cecf0dSJayamohan Kallickal goto free_io_hndls; 44288359c79bSJohn Soni Jose } 44296733b39aSJayamohan Kallickal } else { 44306733b39aSJayamohan Kallickal io_task->scsi_cmnd = NULL; 4431d7aea67bSJayamohan Kallickal if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) { 443243f388b0SJayamohan Kallickal beiscsi_conn->task = task; 44336733b39aSJayamohan Kallickal if (!beiscsi_conn->login_in_progress) { 44346733b39aSJayamohan Kallickal io_task->psgl_handle = (struct sgl_handle *) 44356733b39aSJayamohan Kallickal alloc_mgmt_sgl_handle(phba); 44368359c79bSJohn Soni Jose if (!io_task->psgl_handle) { 44378359c79bSJohn Soni Jose beiscsi_log(phba, KERN_ERR, 44388359c79bSJohn Soni Jose BEISCSI_LOG_IO | 44398359c79bSJohn Soni Jose BEISCSI_LOG_CONFIG, 44408359c79bSJohn Soni Jose "BM_%d : Alloc of MGMT_SGL_ICD Failed" 44418359c79bSJohn Soni Jose "for the CID : %d\n", 44428359c79bSJohn Soni Jose beiscsi_conn-> 44438359c79bSJohn Soni Jose beiscsi_conn_cid); 44442afc95bfSJayamohan Kallickal goto free_hndls; 44458359c79bSJohn Soni Jose } 44462afc95bfSJayamohan Kallickal 44476733b39aSJayamohan Kallickal beiscsi_conn->login_in_progress = 1; 44486733b39aSJayamohan Kallickal beiscsi_conn->plogin_sgl_handle = 44496733b39aSJayamohan Kallickal io_task->psgl_handle; 4450d2cecf0dSJayamohan Kallickal io_task->pwrb_handle = 4451d2cecf0dSJayamohan Kallickal alloc_wrb_handle(phba, 4452340c99e9SJohn Soni Jose beiscsi_conn->beiscsi_conn_cid, 4453340c99e9SJohn Soni Jose &io_task->pwrb_context); 44548359c79bSJohn Soni Jose if (!io_task->pwrb_handle) { 44558359c79bSJohn Soni Jose beiscsi_log(phba, KERN_ERR, 44568359c79bSJohn Soni Jose BEISCSI_LOG_IO | 44578359c79bSJohn Soni Jose BEISCSI_LOG_CONFIG, 44588359c79bSJohn Soni Jose "BM_%d : Alloc of WRB_HANDLE Failed" 44598359c79bSJohn Soni Jose "for the CID : %d\n", 44608359c79bSJohn Soni Jose beiscsi_conn-> 44618359c79bSJohn Soni Jose beiscsi_conn_cid); 44628359c79bSJohn Soni Jose goto free_mgmt_hndls; 44638359c79bSJohn Soni Jose } 4464d2cecf0dSJayamohan Kallickal beiscsi_conn->plogin_wrb_handle = 4465d2cecf0dSJayamohan Kallickal io_task->pwrb_handle; 4466d2cecf0dSJayamohan Kallickal 44676733b39aSJayamohan Kallickal } else { 44686733b39aSJayamohan Kallickal io_task->psgl_handle = 44696733b39aSJayamohan Kallickal beiscsi_conn->plogin_sgl_handle; 4470d2cecf0dSJayamohan Kallickal io_task->pwrb_handle = 4471d2cecf0dSJayamohan Kallickal beiscsi_conn->plogin_wrb_handle; 44726733b39aSJayamohan Kallickal } 44736733b39aSJayamohan Kallickal } else { 44746733b39aSJayamohan Kallickal io_task->psgl_handle = alloc_mgmt_sgl_handle(phba); 44758359c79bSJohn Soni Jose if (!io_task->psgl_handle) { 44768359c79bSJohn Soni Jose beiscsi_log(phba, KERN_ERR, 44778359c79bSJohn Soni Jose BEISCSI_LOG_IO | 44788359c79bSJohn Soni Jose BEISCSI_LOG_CONFIG, 44798359c79bSJohn Soni Jose "BM_%d : Alloc of MGMT_SGL_ICD Failed" 44808359c79bSJohn Soni Jose "for the CID : %d\n", 44818359c79bSJohn Soni Jose beiscsi_conn-> 44828359c79bSJohn Soni Jose beiscsi_conn_cid); 44832afc95bfSJayamohan Kallickal goto free_hndls; 44848359c79bSJohn Soni Jose } 4485d2cecf0dSJayamohan Kallickal io_task->pwrb_handle = 4486d2cecf0dSJayamohan Kallickal alloc_wrb_handle(phba, 4487340c99e9SJohn Soni Jose beiscsi_conn->beiscsi_conn_cid, 4488340c99e9SJohn Soni Jose &io_task->pwrb_context); 44898359c79bSJohn Soni Jose if (!io_task->pwrb_handle) { 44908359c79bSJohn Soni Jose beiscsi_log(phba, KERN_ERR, 44918359c79bSJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 44928359c79bSJohn Soni Jose "BM_%d : Alloc of WRB_HANDLE Failed" 44938359c79bSJohn Soni Jose "for the CID : %d\n", 44948359c79bSJohn Soni Jose beiscsi_conn->beiscsi_conn_cid); 4495d2cecf0dSJayamohan Kallickal goto free_mgmt_hndls; 44968359c79bSJohn Soni Jose } 4497d2cecf0dSJayamohan Kallickal 44986733b39aSJayamohan Kallickal } 44996733b39aSJayamohan Kallickal } 4500bfead3b2SJayamohan Kallickal itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle-> 4501bfead3b2SJayamohan Kallickal wrb_index << 16) | (unsigned int) 4502bfead3b2SJayamohan Kallickal (io_task->psgl_handle->sgl_index)); 4503bfead3b2SJayamohan Kallickal io_task->pwrb_handle->pio_handle = task; 4504bfead3b2SJayamohan Kallickal 45056733b39aSJayamohan Kallickal io_task->cmd_bhs->iscsi_hdr.itt = itt; 45066733b39aSJayamohan Kallickal return 0; 45072afc95bfSJayamohan Kallickal 4508d2cecf0dSJayamohan Kallickal free_io_hndls: 4509d2cecf0dSJayamohan Kallickal free_io_sgl_handle(phba, io_task->psgl_handle); 4510d2cecf0dSJayamohan Kallickal goto free_hndls; 4511d2cecf0dSJayamohan Kallickal free_mgmt_hndls: 4512d2cecf0dSJayamohan Kallickal free_mgmt_sgl_handle(phba, io_task->psgl_handle); 4513a7909b39SJayamohan Kallickal io_task->psgl_handle = NULL; 45142afc95bfSJayamohan Kallickal free_hndls: 45152afc95bfSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 4516a7909b39SJayamohan Kallickal cri_index = BE_GET_CRI_FROM_CID( 4517a7909b39SJayamohan Kallickal beiscsi_conn->beiscsi_conn_cid); 4518a7909b39SJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[cri_index]; 4519d2cecf0dSJayamohan Kallickal if (io_task->pwrb_handle) 45202afc95bfSJayamohan Kallickal free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle); 45212afc95bfSJayamohan Kallickal io_task->pwrb_handle = NULL; 4522af007b02SRomain Perier dma_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs, 45232afc95bfSJayamohan Kallickal io_task->bhs_pa.u.a64.address); 45241282ab76SMike Christie io_task->cmd_bhs = NULL; 45252afc95bfSJayamohan Kallickal return -ENOMEM; 45266733b39aSJayamohan Kallickal } 45270825b8eeSBaoyou Xie static int beiscsi_iotask_v2(struct iscsi_task *task, struct scatterlist *sg, 452809a1093aSJohn Soni Jose unsigned int num_sg, unsigned int xferlen, 452909a1093aSJohn Soni Jose unsigned int writedir) 453009a1093aSJohn Soni Jose { 453109a1093aSJohn Soni Jose 453209a1093aSJohn Soni Jose struct beiscsi_io_task *io_task = task->dd_data; 453309a1093aSJohn Soni Jose struct iscsi_conn *conn = task->conn; 453409a1093aSJohn Soni Jose struct beiscsi_conn *beiscsi_conn = conn->dd_data; 453509a1093aSJohn Soni Jose struct beiscsi_hba *phba = beiscsi_conn->phba; 453609a1093aSJohn Soni Jose struct iscsi_wrb *pwrb = NULL; 453709a1093aSJohn Soni Jose unsigned int doorbell = 0; 453809a1093aSJohn Soni Jose 453909a1093aSJohn Soni Jose pwrb = io_task->pwrb_handle->pwrb; 454009a1093aSJohn Soni Jose 454109a1093aSJohn Soni Jose io_task->bhs_len = sizeof(struct be_cmd_bhs); 454209a1093aSJohn Soni Jose 454309a1093aSJohn Soni Jose if (writedir) { 454409a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb, 454509a1093aSJohn Soni Jose INI_WR_CMD); 454609a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 1); 454709a1093aSJohn Soni Jose } else { 454809a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb, 454909a1093aSJohn Soni Jose INI_RD_CMD); 455009a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 0); 455109a1093aSJohn Soni Jose } 455209a1093aSJohn Soni Jose 455309a1093aSJohn Soni Jose io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb_v2, 455409a1093aSJohn Soni Jose type, pwrb); 455509a1093aSJohn Soni Jose 455609a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, lun, pwrb, 455709a1093aSJohn Soni Jose cpu_to_be16(*(unsigned short *) 455809a1093aSJohn Soni Jose &io_task->cmd_bhs->iscsi_hdr.lun)); 455909a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb, xferlen); 456009a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb, 456109a1093aSJohn Soni Jose io_task->pwrb_handle->wrb_index); 456209a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb, 456309a1093aSJohn Soni Jose be32_to_cpu(task->cmdsn)); 456409a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb, 456509a1093aSJohn Soni Jose io_task->psgl_handle->sgl_index); 456609a1093aSJohn Soni Jose 456709a1093aSJohn Soni Jose hwi_write_sgl_v2(pwrb, sg, num_sg, io_task); 456809a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb, 4569340c99e9SJohn Soni Jose io_task->pwrb_handle->wrb_index); 4570340c99e9SJohn Soni Jose if (io_task->pwrb_context->plast_wrb) 4571340c99e9SJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, 4572340c99e9SJohn Soni Jose io_task->pwrb_context->plast_wrb, 4573340c99e9SJohn Soni Jose io_task->pwrb_handle->wrb_index); 4574340c99e9SJohn Soni Jose io_task->pwrb_context->plast_wrb = pwrb; 457509a1093aSJohn Soni Jose 457609a1093aSJohn Soni Jose be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb)); 457709a1093aSJohn Soni Jose 457809a1093aSJohn Soni Jose doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK; 457909a1093aSJohn Soni Jose doorbell |= (io_task->pwrb_handle->wrb_index & 458009a1093aSJohn Soni Jose DB_DEF_PDU_WRB_INDEX_MASK) << 458109a1093aSJohn Soni Jose DB_DEF_PDU_WRB_INDEX_SHIFT; 458209a1093aSJohn Soni Jose doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 45831e4be6ffSJayamohan Kallickal iowrite32(doorbell, phba->db_va + 45841e4be6ffSJayamohan Kallickal beiscsi_conn->doorbell_offset); 458509a1093aSJohn Soni Jose return 0; 458609a1093aSJohn Soni Jose } 45876733b39aSJayamohan Kallickal 45886733b39aSJayamohan Kallickal static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg, 45896733b39aSJayamohan Kallickal unsigned int num_sg, unsigned int xferlen, 45906733b39aSJayamohan Kallickal unsigned int writedir) 45916733b39aSJayamohan Kallickal { 45926733b39aSJayamohan Kallickal 45936733b39aSJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 45946733b39aSJayamohan Kallickal struct iscsi_conn *conn = task->conn; 45956733b39aSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn = conn->dd_data; 45966733b39aSJayamohan Kallickal struct beiscsi_hba *phba = beiscsi_conn->phba; 45976733b39aSJayamohan Kallickal struct iscsi_wrb *pwrb = NULL; 45986733b39aSJayamohan Kallickal unsigned int doorbell = 0; 45996733b39aSJayamohan Kallickal 46006733b39aSJayamohan Kallickal pwrb = io_task->pwrb_handle->pwrb; 46016733b39aSJayamohan Kallickal io_task->bhs_len = sizeof(struct be_cmd_bhs); 46026733b39aSJayamohan Kallickal 46036733b39aSJayamohan Kallickal if (writedir) { 4604bfead3b2SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 4605bfead3b2SJayamohan Kallickal INI_WR_CMD); 46066733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1); 46076733b39aSJayamohan Kallickal } else { 4608bfead3b2SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 4609bfead3b2SJayamohan Kallickal INI_RD_CMD); 46106733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); 46116733b39aSJayamohan Kallickal } 46126733b39aSJayamohan Kallickal 461309a1093aSJohn Soni Jose io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb, 461409a1093aSJohn Soni Jose type, pwrb); 461509a1093aSJohn Soni Jose 46166733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb, 4617dc63aac6SJayamohan Kallickal cpu_to_be16(*(unsigned short *) 4618dc63aac6SJayamohan Kallickal &io_task->cmd_bhs->iscsi_hdr.lun)); 46196733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen); 46206733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb, 46216733b39aSJayamohan Kallickal io_task->pwrb_handle->wrb_index); 46226733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 46236733b39aSJayamohan Kallickal be32_to_cpu(task->cmdsn)); 46246733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb, 46256733b39aSJayamohan Kallickal io_task->psgl_handle->sgl_index); 46266733b39aSJayamohan Kallickal 46276733b39aSJayamohan Kallickal hwi_write_sgl(pwrb, sg, num_sg, io_task); 46286733b39aSJayamohan Kallickal 46296733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb, 4630340c99e9SJohn Soni Jose io_task->pwrb_handle->wrb_index); 4631340c99e9SJohn Soni Jose if (io_task->pwrb_context->plast_wrb) 4632340c99e9SJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, 4633340c99e9SJohn Soni Jose io_task->pwrb_context->plast_wrb, 4634340c99e9SJohn Soni Jose io_task->pwrb_handle->wrb_index); 4635340c99e9SJohn Soni Jose io_task->pwrb_context->plast_wrb = pwrb; 4636340c99e9SJohn Soni Jose 46376733b39aSJayamohan Kallickal be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb)); 46386733b39aSJayamohan Kallickal 46396733b39aSJayamohan Kallickal doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK; 46406733b39aSJayamohan Kallickal doorbell |= (io_task->pwrb_handle->wrb_index & 46416733b39aSJayamohan Kallickal DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT; 46426733b39aSJayamohan Kallickal doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 46436733b39aSJayamohan Kallickal 46441e4be6ffSJayamohan Kallickal iowrite32(doorbell, phba->db_va + 46451e4be6ffSJayamohan Kallickal beiscsi_conn->doorbell_offset); 46466733b39aSJayamohan Kallickal return 0; 46476733b39aSJayamohan Kallickal } 46486733b39aSJayamohan Kallickal 46496733b39aSJayamohan Kallickal static int beiscsi_mtask(struct iscsi_task *task) 46506733b39aSJayamohan Kallickal { 4651dafab8e0SJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 46526733b39aSJayamohan Kallickal struct iscsi_conn *conn = task->conn; 46536733b39aSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn = conn->dd_data; 46546733b39aSJayamohan Kallickal struct beiscsi_hba *phba = beiscsi_conn->phba; 46556733b39aSJayamohan Kallickal struct iscsi_wrb *pwrb = NULL; 46566733b39aSJayamohan Kallickal unsigned int doorbell = 0; 4657dafab8e0SJayamohan Kallickal unsigned int cid; 465809a1093aSJohn Soni Jose unsigned int pwrb_typeoffset = 0; 4659e0493627SAlexey Khoroshilov int ret = 0; 46606733b39aSJayamohan Kallickal 4661bfead3b2SJayamohan Kallickal cid = beiscsi_conn->beiscsi_conn_cid; 46626733b39aSJayamohan Kallickal pwrb = io_task->pwrb_handle->pwrb; 466309a1093aSJohn Soni Jose 46642c9dfd36SJayamohan Kallickal if (is_chip_be2_be3r(phba)) { 46656733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 46666733b39aSJayamohan Kallickal be32_to_cpu(task->cmdsn)); 46676733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb, 46686733b39aSJayamohan Kallickal io_task->pwrb_handle->wrb_index); 46696733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb, 46706733b39aSJayamohan Kallickal io_task->psgl_handle->sgl_index); 467109a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, 467209a1093aSJohn Soni Jose task->data_count); 467309a1093aSJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb, 4674340c99e9SJohn Soni Jose io_task->pwrb_handle->wrb_index); 4675340c99e9SJohn Soni Jose if (io_task->pwrb_context->plast_wrb) 4676340c99e9SJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, 4677340c99e9SJohn Soni Jose io_task->pwrb_context->plast_wrb, 4678340c99e9SJohn Soni Jose io_task->pwrb_handle->wrb_index); 4679340c99e9SJohn Soni Jose io_task->pwrb_context->plast_wrb = pwrb; 4680340c99e9SJohn Soni Jose 468109a1093aSJohn Soni Jose pwrb_typeoffset = BE_WRB_TYPE_OFFSET; 46822c9dfd36SJayamohan Kallickal } else { 46832c9dfd36SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb, 46842c9dfd36SJayamohan Kallickal be32_to_cpu(task->cmdsn)); 46852c9dfd36SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb, 46862c9dfd36SJayamohan Kallickal io_task->pwrb_handle->wrb_index); 46872c9dfd36SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb, 46882c9dfd36SJayamohan Kallickal io_task->psgl_handle->sgl_index); 46892c9dfd36SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb, 46902c9dfd36SJayamohan Kallickal task->data_count); 46912c9dfd36SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb, 4692340c99e9SJohn Soni Jose io_task->pwrb_handle->wrb_index); 4693340c99e9SJohn Soni Jose if (io_task->pwrb_context->plast_wrb) 4694340c99e9SJohn Soni Jose AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, 4695340c99e9SJohn Soni Jose io_task->pwrb_context->plast_wrb, 4696340c99e9SJohn Soni Jose io_task->pwrb_handle->wrb_index); 4697340c99e9SJohn Soni Jose io_task->pwrb_context->plast_wrb = pwrb; 4698340c99e9SJohn Soni Jose 46992c9dfd36SJayamohan Kallickal pwrb_typeoffset = SKH_WRB_TYPE_OFFSET; 470009a1093aSJohn Soni Jose } 470109a1093aSJohn Soni Jose 4702dafab8e0SJayamohan Kallickal 47036733b39aSJayamohan Kallickal switch (task->hdr->opcode & ISCSI_OPCODE_MASK) { 47046733b39aSJayamohan Kallickal case ISCSI_OP_LOGIN: 47056733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1); 470609a1093aSJohn Soni Jose ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset); 4707e0493627SAlexey Khoroshilov ret = hwi_write_buffer(pwrb, task); 47086733b39aSJayamohan Kallickal break; 47096733b39aSJayamohan Kallickal case ISCSI_OP_NOOP_OUT: 47101390b01bSJayamohan Kallickal if (task->hdr->ttt != ISCSI_RESERVED_TAG) { 471109a1093aSJohn Soni Jose ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset); 47122c9dfd36SJayamohan Kallickal if (is_chip_be2_be3r(phba)) 47132c9dfd36SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, 471409a1093aSJohn Soni Jose dmsg, pwrb, 1); 471509a1093aSJohn Soni Jose else 47162c9dfd36SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 471709a1093aSJohn Soni Jose dmsg, pwrb, 1); 47181390b01bSJayamohan Kallickal } else { 471909a1093aSJohn Soni Jose ADAPTER_SET_WRB_TYPE(pwrb, INI_RD_CMD, pwrb_typeoffset); 47202c9dfd36SJayamohan Kallickal if (is_chip_be2_be3r(phba)) 47212c9dfd36SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, 472209a1093aSJohn Soni Jose dmsg, pwrb, 0); 472309a1093aSJohn Soni Jose else 47242c9dfd36SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 472509a1093aSJohn Soni Jose dmsg, pwrb, 0); 47261390b01bSJayamohan Kallickal } 4727e0493627SAlexey Khoroshilov ret = hwi_write_buffer(pwrb, task); 47286733b39aSJayamohan Kallickal break; 47296733b39aSJayamohan Kallickal case ISCSI_OP_TEXT: 473009a1093aSJohn Soni Jose ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset); 4731e0493627SAlexey Khoroshilov ret = hwi_write_buffer(pwrb, task); 47326733b39aSJayamohan Kallickal break; 47336733b39aSJayamohan Kallickal case ISCSI_OP_SCSI_TMFUNC: 473409a1093aSJohn Soni Jose ADAPTER_SET_WRB_TYPE(pwrb, INI_TMF_CMD, pwrb_typeoffset); 4735e0493627SAlexey Khoroshilov ret = hwi_write_buffer(pwrb, task); 47366733b39aSJayamohan Kallickal break; 47376733b39aSJayamohan Kallickal case ISCSI_OP_LOGOUT: 473809a1093aSJohn Soni Jose ADAPTER_SET_WRB_TYPE(pwrb, HWH_TYPE_LOGOUT, pwrb_typeoffset); 4739e0493627SAlexey Khoroshilov ret = hwi_write_buffer(pwrb, task); 47406733b39aSJayamohan Kallickal break; 47416733b39aSJayamohan Kallickal 47426733b39aSJayamohan Kallickal default: 474399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 474499bc5d55SJohn Soni Jose "BM_%d : opcode =%d Not supported\n", 47456733b39aSJayamohan Kallickal task->hdr->opcode & ISCSI_OPCODE_MASK); 474699bc5d55SJohn Soni Jose 47476733b39aSJayamohan Kallickal return -EINVAL; 47486733b39aSJayamohan Kallickal } 47496733b39aSJayamohan Kallickal 4750e0493627SAlexey Khoroshilov if (ret) 4751e0493627SAlexey Khoroshilov return ret; 4752e0493627SAlexey Khoroshilov 475309a1093aSJohn Soni Jose /* Set the task type */ 47542c9dfd36SJayamohan Kallickal io_task->wrb_type = (is_chip_be2_be3r(phba)) ? 47552c9dfd36SJayamohan Kallickal AMAP_GET_BITS(struct amap_iscsi_wrb, type, pwrb) : 47562c9dfd36SJayamohan Kallickal AMAP_GET_BITS(struct amap_iscsi_wrb_v2, type, pwrb); 47576733b39aSJayamohan Kallickal 4758bfead3b2SJayamohan Kallickal doorbell |= cid & DB_WRB_POST_CID_MASK; 47596733b39aSJayamohan Kallickal doorbell |= (io_task->pwrb_handle->wrb_index & 47606733b39aSJayamohan Kallickal DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT; 47616733b39aSJayamohan Kallickal doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 47621e4be6ffSJayamohan Kallickal iowrite32(doorbell, phba->db_va + 47631e4be6ffSJayamohan Kallickal beiscsi_conn->doorbell_offset); 47646733b39aSJayamohan Kallickal return 0; 47656733b39aSJayamohan Kallickal } 47666733b39aSJayamohan Kallickal 47676733b39aSJayamohan Kallickal static int beiscsi_task_xmit(struct iscsi_task *task) 47686733b39aSJayamohan Kallickal { 47696733b39aSJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 47706733b39aSJayamohan Kallickal struct scsi_cmnd *sc = task->sc; 47711868379bSJitendra Bhivare struct beiscsi_hba *phba; 47726733b39aSJayamohan Kallickal struct scatterlist *sg; 47736733b39aSJayamohan Kallickal int num_sg; 47746733b39aSJayamohan Kallickal unsigned int writedir = 0, xferlen = 0; 47756733b39aSJayamohan Kallickal 47769122e991SJitendra Bhivare phba = io_task->conn->phba; 47779122e991SJitendra Bhivare /** 47789122e991SJitendra Bhivare * HBA in error includes BEISCSI_HBA_FW_TIMEOUT. IO path might be 47799122e991SJitendra Bhivare * operational if FW still gets heartbeat from EP FW. Is management 47809122e991SJitendra Bhivare * path really needed to continue further? 47819122e991SJitendra Bhivare */ 4782d1d5ca88SJitendra Bhivare if (!beiscsi_hba_is_online(phba)) 47839122e991SJitendra Bhivare return -EIO; 47849122e991SJitendra Bhivare 47851868379bSJitendra Bhivare if (!io_task->conn->login_in_progress) 47861868379bSJitendra Bhivare task->hdr->exp_statsn = 0; 478709a1093aSJohn Soni Jose 47886733b39aSJayamohan Kallickal if (!sc) 47896733b39aSJayamohan Kallickal return beiscsi_mtask(task); 47906733b39aSJayamohan Kallickal 47916733b39aSJayamohan Kallickal io_task->scsi_cmnd = sc; 47929122e991SJitendra Bhivare io_task->num_sg = 0; 47936733b39aSJayamohan Kallickal num_sg = scsi_dma_map(sc); 47946733b39aSJayamohan Kallickal if (num_sg < 0) { 4795afb96058SJayamohan Kallickal beiscsi_log(phba, KERN_ERR, 4796afb96058SJayamohan Kallickal BEISCSI_LOG_IO | BEISCSI_LOG_ISCSI, 4797afb96058SJayamohan Kallickal "BM_%d : scsi_dma_map Failed " 4798afb96058SJayamohan Kallickal "Driver_ITT : 0x%x ITT : 0x%x Xferlen : 0x%x\n", 4799afb96058SJayamohan Kallickal be32_to_cpu(io_task->cmd_bhs->iscsi_hdr.itt), 4800afb96058SJayamohan Kallickal io_task->libiscsi_itt, scsi_bufflen(sc)); 480199bc5d55SJohn Soni Jose 48026733b39aSJayamohan Kallickal return num_sg; 48036733b39aSJayamohan Kallickal } 48049122e991SJitendra Bhivare /** 48059122e991SJitendra Bhivare * For scsi cmd task, check num_sg before unmapping in cleanup_task. 48069122e991SJitendra Bhivare * For management task, cleanup_task checks mtask_addr before unmapping. 48079122e991SJitendra Bhivare */ 48089122e991SJitendra Bhivare io_task->num_sg = num_sg; 48096733b39aSJayamohan Kallickal xferlen = scsi_bufflen(sc); 48106733b39aSJayamohan Kallickal sg = scsi_sglist(sc); 481199bc5d55SJohn Soni Jose if (sc->sc_data_direction == DMA_TO_DEVICE) 48126733b39aSJayamohan Kallickal writedir = 1; 481399bc5d55SJohn Soni Jose else 48146733b39aSJayamohan Kallickal writedir = 0; 481599bc5d55SJohn Soni Jose 481609a1093aSJohn Soni Jose return phba->iotask_fn(task, sg, num_sg, xferlen, writedir); 48176733b39aSJayamohan Kallickal } 48186733b39aSJayamohan Kallickal 4819ffce3e2eSJayamohan Kallickal /** 4820ffce3e2eSJayamohan Kallickal * beiscsi_bsg_request - handle bsg request from ISCSI transport 4821ffce3e2eSJayamohan Kallickal * @job: job to handle 4822ffce3e2eSJayamohan Kallickal */ 4823ffce3e2eSJayamohan Kallickal static int beiscsi_bsg_request(struct bsg_job *job) 4824ffce3e2eSJayamohan Kallickal { 4825ffce3e2eSJayamohan Kallickal struct Scsi_Host *shost; 4826ffce3e2eSJayamohan Kallickal struct beiscsi_hba *phba; 4827ffce3e2eSJayamohan Kallickal struct iscsi_bsg_request *bsg_req = job->request; 4828ffce3e2eSJayamohan Kallickal int rc = -EINVAL; 4829ffce3e2eSJayamohan Kallickal unsigned int tag; 4830ffce3e2eSJayamohan Kallickal struct be_dma_mem nonemb_cmd; 4831ffce3e2eSJayamohan Kallickal struct be_cmd_resp_hdr *resp; 4832ffce3e2eSJayamohan Kallickal struct iscsi_bsg_reply *bsg_reply = job->reply; 4833ffce3e2eSJayamohan Kallickal unsigned short status, extd_status; 4834ffce3e2eSJayamohan Kallickal 4835ffce3e2eSJayamohan Kallickal shost = iscsi_job_to_shost(job); 4836ffce3e2eSJayamohan Kallickal phba = iscsi_host_priv(shost); 4837ffce3e2eSJayamohan Kallickal 4838d1d5ca88SJitendra Bhivare if (!beiscsi_hba_is_online(phba)) { 48399122e991SJitendra Bhivare beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG, 48409122e991SJitendra Bhivare "BM_%d : HBA in error 0x%lx\n", phba->state); 48419122e991SJitendra Bhivare return -ENXIO; 48429122e991SJitendra Bhivare } 48439122e991SJitendra Bhivare 4844ffce3e2eSJayamohan Kallickal switch (bsg_req->msgcode) { 4845ffce3e2eSJayamohan Kallickal case ISCSI_BSG_HST_VENDOR: 4846ffce3e2eSJayamohan Kallickal nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev, 4847ffce3e2eSJayamohan Kallickal job->request_payload.payload_len, 4848ffce3e2eSJayamohan Kallickal &nonemb_cmd.dma); 4849ffce3e2eSJayamohan Kallickal if (nonemb_cmd.va == NULL) { 485099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 485199bc5d55SJohn Soni Jose "BM_%d : Failed to allocate memory for " 4852ffce3e2eSJayamohan Kallickal "beiscsi_bsg_request\n"); 48538359c79bSJohn Soni Jose return -ENOMEM; 4854ffce3e2eSJayamohan Kallickal } 4855ffce3e2eSJayamohan Kallickal tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job, 4856ffce3e2eSJayamohan Kallickal &nonemb_cmd); 4857ffce3e2eSJayamohan Kallickal if (!tag) { 485899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 48598359c79bSJohn Soni Jose "BM_%d : MBX Tag Allocation Failed\n"); 486099bc5d55SJohn Soni Jose 4861ffce3e2eSJayamohan Kallickal pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 4862ffce3e2eSJayamohan Kallickal nonemb_cmd.va, nonemb_cmd.dma); 4863ffce3e2eSJayamohan Kallickal return -EAGAIN; 4864e175defeSJohn Soni Jose } 4865e175defeSJohn Soni Jose 4866e175defeSJohn Soni Jose rc = wait_event_interruptible_timeout( 4867e175defeSJohn Soni Jose phba->ctrl.mcc_wait[tag], 486867296ad9SJitendra Bhivare phba->ctrl.mcc_tag_status[tag], 4869e175defeSJohn Soni Jose msecs_to_jiffies( 4870e175defeSJohn Soni Jose BEISCSI_HOST_MBX_TIMEOUT)); 4871d1d5ca88SJitendra Bhivare 4872d1d5ca88SJitendra Bhivare if (!test_bit(BEISCSI_HBA_ONLINE, &phba->state)) { 4873d1d5ca88SJitendra Bhivare clear_bit(MCC_TAG_STATE_RUNNING, 4874d1d5ca88SJitendra Bhivare &phba->ctrl.ptag_state[tag].tag_state); 4875d1d5ca88SJitendra Bhivare pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 4876d1d5ca88SJitendra Bhivare nonemb_cmd.va, nonemb_cmd.dma); 4877d1d5ca88SJitendra Bhivare return -EIO; 4878d1d5ca88SJitendra Bhivare } 487967296ad9SJitendra Bhivare extd_status = (phba->ctrl.mcc_tag_status[tag] & 488067296ad9SJitendra Bhivare CQE_STATUS_ADDL_MASK) >> CQE_STATUS_ADDL_SHIFT; 488167296ad9SJitendra Bhivare status = phba->ctrl.mcc_tag_status[tag] & CQE_STATUS_MASK; 4882090e2184SJitendra Bhivare free_mcc_wrb(&phba->ctrl, tag); 4883ffce3e2eSJayamohan Kallickal resp = (struct be_cmd_resp_hdr *)nonemb_cmd.va; 4884ffce3e2eSJayamohan Kallickal sg_copy_from_buffer(job->reply_payload.sg_list, 4885ffce3e2eSJayamohan Kallickal job->reply_payload.sg_cnt, 4886ffce3e2eSJayamohan Kallickal nonemb_cmd.va, (resp->response_length 4887ffce3e2eSJayamohan Kallickal + sizeof(*resp))); 4888ffce3e2eSJayamohan Kallickal bsg_reply->reply_payload_rcv_len = resp->response_length; 4889ffce3e2eSJayamohan Kallickal bsg_reply->result = status; 4890ffce3e2eSJayamohan Kallickal bsg_job_done(job, bsg_reply->result, 4891ffce3e2eSJayamohan Kallickal bsg_reply->reply_payload_rcv_len); 4892ffce3e2eSJayamohan Kallickal pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 4893ffce3e2eSJayamohan Kallickal nonemb_cmd.va, nonemb_cmd.dma); 4894ffce3e2eSJayamohan Kallickal if (status || extd_status) { 489599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 48968359c79bSJohn Soni Jose "BM_%d : MBX Cmd Failed" 4897ffce3e2eSJayamohan Kallickal " status = %d extd_status = %d\n", 4898ffce3e2eSJayamohan Kallickal status, extd_status); 489999bc5d55SJohn Soni Jose 4900ffce3e2eSJayamohan Kallickal return -EIO; 49018359c79bSJohn Soni Jose } else { 49028359c79bSJohn Soni Jose rc = 0; 4903ffce3e2eSJayamohan Kallickal } 4904ffce3e2eSJayamohan Kallickal break; 4905ffce3e2eSJayamohan Kallickal 4906ffce3e2eSJayamohan Kallickal default: 490799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 490899bc5d55SJohn Soni Jose "BM_%d : Unsupported bsg command: 0x%x\n", 4909ffce3e2eSJayamohan Kallickal bsg_req->msgcode); 4910ffce3e2eSJayamohan Kallickal break; 4911ffce3e2eSJayamohan Kallickal } 4912ffce3e2eSJayamohan Kallickal 4913ffce3e2eSJayamohan Kallickal return rc; 4914ffce3e2eSJayamohan Kallickal } 4915ffce3e2eSJayamohan Kallickal 49160825b8eeSBaoyou Xie static void beiscsi_hba_attrs_init(struct beiscsi_hba *phba) 491799bc5d55SJohn Soni Jose { 491899bc5d55SJohn Soni Jose /* Set the logging parameter */ 491999bc5d55SJohn Soni Jose beiscsi_log_enable_init(phba, beiscsi_log_enable); 492099bc5d55SJohn Soni Jose } 492199bc5d55SJohn Soni Jose 492250a4b824SJitendra Bhivare void beiscsi_start_boot_work(struct beiscsi_hba *phba, unsigned int s_handle) 492350a4b824SJitendra Bhivare { 492450a4b824SJitendra Bhivare if (phba->boot_struct.boot_kset) 492550a4b824SJitendra Bhivare return; 492650a4b824SJitendra Bhivare 492750a4b824SJitendra Bhivare /* skip if boot work is already in progress */ 492850a4b824SJitendra Bhivare if (test_and_set_bit(BEISCSI_HBA_BOOT_WORK, &phba->state)) 492950a4b824SJitendra Bhivare return; 493050a4b824SJitendra Bhivare 493150a4b824SJitendra Bhivare phba->boot_struct.retry = 3; 493250a4b824SJitendra Bhivare phba->boot_struct.tag = 0; 493350a4b824SJitendra Bhivare phba->boot_struct.s_handle = s_handle; 493450a4b824SJitendra Bhivare phba->boot_struct.action = BEISCSI_BOOT_GET_SHANDLE; 493550a4b824SJitendra Bhivare schedule_work(&phba->boot_work); 493650a4b824SJitendra Bhivare } 493750a4b824SJitendra Bhivare 49388d9ecd49SJitendra Bhivare /** 49398d9ecd49SJitendra Bhivare * Boot flag info for iscsi-utilities 49408d9ecd49SJitendra Bhivare * Bit 0 Block valid flag 49418d9ecd49SJitendra Bhivare * Bit 1 Firmware booting selected 49428d9ecd49SJitendra Bhivare */ 49438d9ecd49SJitendra Bhivare #define BEISCSI_SYSFS_ISCSI_BOOT_FLAGS 3 49448d9ecd49SJitendra Bhivare 494550a4b824SJitendra Bhivare static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf) 494650a4b824SJitendra Bhivare { 494750a4b824SJitendra Bhivare struct beiscsi_hba *phba = data; 494850a4b824SJitendra Bhivare struct mgmt_session_info *boot_sess = &phba->boot_struct.boot_sess; 494950a4b824SJitendra Bhivare struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0]; 495050a4b824SJitendra Bhivare char *str = buf; 495150a4b824SJitendra Bhivare int rc = -EPERM; 495250a4b824SJitendra Bhivare 495350a4b824SJitendra Bhivare switch (type) { 495450a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_NAME: 495550a4b824SJitendra Bhivare rc = sprintf(buf, "%.*s\n", 495650a4b824SJitendra Bhivare (int)strlen(boot_sess->target_name), 495750a4b824SJitendra Bhivare (char *)&boot_sess->target_name); 495850a4b824SJitendra Bhivare break; 495950a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_IP_ADDR: 496050a4b824SJitendra Bhivare if (boot_conn->dest_ipaddr.ip_type == BEISCSI_IP_TYPE_V4) 496150a4b824SJitendra Bhivare rc = sprintf(buf, "%pI4\n", 496250a4b824SJitendra Bhivare (char *)&boot_conn->dest_ipaddr.addr); 496350a4b824SJitendra Bhivare else 496450a4b824SJitendra Bhivare rc = sprintf(str, "%pI6\n", 496550a4b824SJitendra Bhivare (char *)&boot_conn->dest_ipaddr.addr); 496650a4b824SJitendra Bhivare break; 496750a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_PORT: 496850a4b824SJitendra Bhivare rc = sprintf(str, "%d\n", boot_conn->dest_port); 496950a4b824SJitendra Bhivare break; 497050a4b824SJitendra Bhivare 497150a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_CHAP_NAME: 497250a4b824SJitendra Bhivare rc = sprintf(str, "%.*s\n", 497350a4b824SJitendra Bhivare boot_conn->negotiated_login_options.auth_data.chap. 497450a4b824SJitendra Bhivare target_chap_name_length, 497550a4b824SJitendra Bhivare (char *)&boot_conn->negotiated_login_options. 497650a4b824SJitendra Bhivare auth_data.chap.target_chap_name); 497750a4b824SJitendra Bhivare break; 497850a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_CHAP_SECRET: 497950a4b824SJitendra Bhivare rc = sprintf(str, "%.*s\n", 498050a4b824SJitendra Bhivare boot_conn->negotiated_login_options.auth_data.chap. 498150a4b824SJitendra Bhivare target_secret_length, 498250a4b824SJitendra Bhivare (char *)&boot_conn->negotiated_login_options. 498350a4b824SJitendra Bhivare auth_data.chap.target_secret); 498450a4b824SJitendra Bhivare break; 498550a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_REV_CHAP_NAME: 498650a4b824SJitendra Bhivare rc = sprintf(str, "%.*s\n", 498750a4b824SJitendra Bhivare boot_conn->negotiated_login_options.auth_data.chap. 498850a4b824SJitendra Bhivare intr_chap_name_length, 498950a4b824SJitendra Bhivare (char *)&boot_conn->negotiated_login_options. 499050a4b824SJitendra Bhivare auth_data.chap.intr_chap_name); 499150a4b824SJitendra Bhivare break; 499250a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_REV_CHAP_SECRET: 499350a4b824SJitendra Bhivare rc = sprintf(str, "%.*s\n", 499450a4b824SJitendra Bhivare boot_conn->negotiated_login_options.auth_data.chap. 499550a4b824SJitendra Bhivare intr_secret_length, 499650a4b824SJitendra Bhivare (char *)&boot_conn->negotiated_login_options. 499750a4b824SJitendra Bhivare auth_data.chap.intr_secret); 499850a4b824SJitendra Bhivare break; 499950a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_FLAGS: 50008d9ecd49SJitendra Bhivare rc = sprintf(str, "%d\n", BEISCSI_SYSFS_ISCSI_BOOT_FLAGS); 500150a4b824SJitendra Bhivare break; 500250a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_NIC_ASSOC: 500350a4b824SJitendra Bhivare rc = sprintf(str, "0\n"); 500450a4b824SJitendra Bhivare break; 500550a4b824SJitendra Bhivare } 500650a4b824SJitendra Bhivare return rc; 500750a4b824SJitendra Bhivare } 500850a4b824SJitendra Bhivare 500950a4b824SJitendra Bhivare static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf) 501050a4b824SJitendra Bhivare { 501150a4b824SJitendra Bhivare struct beiscsi_hba *phba = data; 501250a4b824SJitendra Bhivare char *str = buf; 501350a4b824SJitendra Bhivare int rc = -EPERM; 501450a4b824SJitendra Bhivare 501550a4b824SJitendra Bhivare switch (type) { 501650a4b824SJitendra Bhivare case ISCSI_BOOT_INI_INITIATOR_NAME: 501750a4b824SJitendra Bhivare rc = sprintf(str, "%s\n", 501850a4b824SJitendra Bhivare phba->boot_struct.boot_sess.initiator_iscsiname); 501950a4b824SJitendra Bhivare break; 502050a4b824SJitendra Bhivare } 502150a4b824SJitendra Bhivare return rc; 502250a4b824SJitendra Bhivare } 502350a4b824SJitendra Bhivare 502450a4b824SJitendra Bhivare static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf) 502550a4b824SJitendra Bhivare { 502650a4b824SJitendra Bhivare struct beiscsi_hba *phba = data; 502750a4b824SJitendra Bhivare char *str = buf; 502850a4b824SJitendra Bhivare int rc = -EPERM; 502950a4b824SJitendra Bhivare 503050a4b824SJitendra Bhivare switch (type) { 503150a4b824SJitendra Bhivare case ISCSI_BOOT_ETH_FLAGS: 50328d9ecd49SJitendra Bhivare rc = sprintf(str, "%d\n", BEISCSI_SYSFS_ISCSI_BOOT_FLAGS); 503350a4b824SJitendra Bhivare break; 503450a4b824SJitendra Bhivare case ISCSI_BOOT_ETH_INDEX: 503550a4b824SJitendra Bhivare rc = sprintf(str, "0\n"); 503650a4b824SJitendra Bhivare break; 503750a4b824SJitendra Bhivare case ISCSI_BOOT_ETH_MAC: 503850a4b824SJitendra Bhivare rc = beiscsi_get_macaddr(str, phba); 503950a4b824SJitendra Bhivare break; 504050a4b824SJitendra Bhivare } 504150a4b824SJitendra Bhivare return rc; 504250a4b824SJitendra Bhivare } 504350a4b824SJitendra Bhivare 504450a4b824SJitendra Bhivare static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type) 504550a4b824SJitendra Bhivare { 504650a4b824SJitendra Bhivare umode_t rc = 0; 504750a4b824SJitendra Bhivare 504850a4b824SJitendra Bhivare switch (type) { 504950a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_NAME: 505050a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_IP_ADDR: 505150a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_PORT: 505250a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_CHAP_NAME: 505350a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_CHAP_SECRET: 505450a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_REV_CHAP_NAME: 505550a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_REV_CHAP_SECRET: 505650a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_NIC_ASSOC: 505750a4b824SJitendra Bhivare case ISCSI_BOOT_TGT_FLAGS: 505850a4b824SJitendra Bhivare rc = S_IRUGO; 505950a4b824SJitendra Bhivare break; 506050a4b824SJitendra Bhivare } 506150a4b824SJitendra Bhivare return rc; 506250a4b824SJitendra Bhivare } 506350a4b824SJitendra Bhivare 506450a4b824SJitendra Bhivare static umode_t beiscsi_ini_get_attr_visibility(void *data, int type) 506550a4b824SJitendra Bhivare { 506650a4b824SJitendra Bhivare umode_t rc = 0; 506750a4b824SJitendra Bhivare 506850a4b824SJitendra Bhivare switch (type) { 506950a4b824SJitendra Bhivare case ISCSI_BOOT_INI_INITIATOR_NAME: 507050a4b824SJitendra Bhivare rc = S_IRUGO; 507150a4b824SJitendra Bhivare break; 507250a4b824SJitendra Bhivare } 507350a4b824SJitendra Bhivare return rc; 507450a4b824SJitendra Bhivare } 507550a4b824SJitendra Bhivare 507650a4b824SJitendra Bhivare static umode_t beiscsi_eth_get_attr_visibility(void *data, int type) 507750a4b824SJitendra Bhivare { 507850a4b824SJitendra Bhivare umode_t rc = 0; 507950a4b824SJitendra Bhivare 508050a4b824SJitendra Bhivare switch (type) { 508150a4b824SJitendra Bhivare case ISCSI_BOOT_ETH_FLAGS: 508250a4b824SJitendra Bhivare case ISCSI_BOOT_ETH_MAC: 508350a4b824SJitendra Bhivare case ISCSI_BOOT_ETH_INDEX: 508450a4b824SJitendra Bhivare rc = S_IRUGO; 508550a4b824SJitendra Bhivare break; 508650a4b824SJitendra Bhivare } 508750a4b824SJitendra Bhivare return rc; 508850a4b824SJitendra Bhivare } 508950a4b824SJitendra Bhivare 509050a4b824SJitendra Bhivare static void beiscsi_boot_kobj_release(void *data) 509150a4b824SJitendra Bhivare { 509250a4b824SJitendra Bhivare struct beiscsi_hba *phba = data; 509350a4b824SJitendra Bhivare 509450a4b824SJitendra Bhivare scsi_host_put(phba->shost); 509550a4b824SJitendra Bhivare } 509650a4b824SJitendra Bhivare 509750a4b824SJitendra Bhivare static int beiscsi_boot_create_kset(struct beiscsi_hba *phba) 509850a4b824SJitendra Bhivare { 509950a4b824SJitendra Bhivare struct boot_struct *bs = &phba->boot_struct; 510050a4b824SJitendra Bhivare struct iscsi_boot_kobj *boot_kobj; 510150a4b824SJitendra Bhivare 510250a4b824SJitendra Bhivare if (bs->boot_kset) { 510350a4b824SJitendra Bhivare __beiscsi_log(phba, KERN_ERR, 510450a4b824SJitendra Bhivare "BM_%d: boot_kset already created\n"); 510550a4b824SJitendra Bhivare return 0; 510650a4b824SJitendra Bhivare } 510750a4b824SJitendra Bhivare 510850a4b824SJitendra Bhivare bs->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no); 510950a4b824SJitendra Bhivare if (!bs->boot_kset) { 511050a4b824SJitendra Bhivare __beiscsi_log(phba, KERN_ERR, 511150a4b824SJitendra Bhivare "BM_%d: boot_kset alloc failed\n"); 511250a4b824SJitendra Bhivare return -ENOMEM; 511350a4b824SJitendra Bhivare } 511450a4b824SJitendra Bhivare 511550a4b824SJitendra Bhivare /* get shost ref because the show function will refer phba */ 511650a4b824SJitendra Bhivare if (!scsi_host_get(phba->shost)) 511750a4b824SJitendra Bhivare goto free_kset; 511850a4b824SJitendra Bhivare 511950a4b824SJitendra Bhivare boot_kobj = iscsi_boot_create_target(bs->boot_kset, 0, phba, 512050a4b824SJitendra Bhivare beiscsi_show_boot_tgt_info, 512150a4b824SJitendra Bhivare beiscsi_tgt_get_attr_visibility, 512250a4b824SJitendra Bhivare beiscsi_boot_kobj_release); 512350a4b824SJitendra Bhivare if (!boot_kobj) 512450a4b824SJitendra Bhivare goto put_shost; 512550a4b824SJitendra Bhivare 512650a4b824SJitendra Bhivare if (!scsi_host_get(phba->shost)) 512750a4b824SJitendra Bhivare goto free_kset; 512850a4b824SJitendra Bhivare 512950a4b824SJitendra Bhivare boot_kobj = iscsi_boot_create_initiator(bs->boot_kset, 0, phba, 513050a4b824SJitendra Bhivare beiscsi_show_boot_ini_info, 513150a4b824SJitendra Bhivare beiscsi_ini_get_attr_visibility, 513250a4b824SJitendra Bhivare beiscsi_boot_kobj_release); 513350a4b824SJitendra Bhivare if (!boot_kobj) 513450a4b824SJitendra Bhivare goto put_shost; 513550a4b824SJitendra Bhivare 513650a4b824SJitendra Bhivare if (!scsi_host_get(phba->shost)) 513750a4b824SJitendra Bhivare goto free_kset; 513850a4b824SJitendra Bhivare 513950a4b824SJitendra Bhivare boot_kobj = iscsi_boot_create_ethernet(bs->boot_kset, 0, phba, 514050a4b824SJitendra Bhivare beiscsi_show_boot_eth_info, 514150a4b824SJitendra Bhivare beiscsi_eth_get_attr_visibility, 514250a4b824SJitendra Bhivare beiscsi_boot_kobj_release); 514350a4b824SJitendra Bhivare if (!boot_kobj) 514450a4b824SJitendra Bhivare goto put_shost; 514550a4b824SJitendra Bhivare 514650a4b824SJitendra Bhivare return 0; 514750a4b824SJitendra Bhivare 514850a4b824SJitendra Bhivare put_shost: 514950a4b824SJitendra Bhivare scsi_host_put(phba->shost); 515050a4b824SJitendra Bhivare free_kset: 515150a4b824SJitendra Bhivare iscsi_boot_destroy_kset(bs->boot_kset); 515250a4b824SJitendra Bhivare bs->boot_kset = NULL; 515350a4b824SJitendra Bhivare return -ENOMEM; 515450a4b824SJitendra Bhivare } 515550a4b824SJitendra Bhivare 515650a4b824SJitendra Bhivare static void beiscsi_boot_work(struct work_struct *work) 515750a4b824SJitendra Bhivare { 515850a4b824SJitendra Bhivare struct beiscsi_hba *phba = 515950a4b824SJitendra Bhivare container_of(work, struct beiscsi_hba, boot_work); 516050a4b824SJitendra Bhivare struct boot_struct *bs = &phba->boot_struct; 516150a4b824SJitendra Bhivare unsigned int tag = 0; 516250a4b824SJitendra Bhivare 5163d1d5ca88SJitendra Bhivare if (!beiscsi_hba_is_online(phba)) 516450a4b824SJitendra Bhivare return; 516550a4b824SJitendra Bhivare 516650a4b824SJitendra Bhivare beiscsi_log(phba, KERN_INFO, 516750a4b824SJitendra Bhivare BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX, 516850a4b824SJitendra Bhivare "BM_%d : %s action %d\n", 516950a4b824SJitendra Bhivare __func__, phba->boot_struct.action); 517050a4b824SJitendra Bhivare 517150a4b824SJitendra Bhivare switch (phba->boot_struct.action) { 517250a4b824SJitendra Bhivare case BEISCSI_BOOT_REOPEN_SESS: 517350a4b824SJitendra Bhivare tag = beiscsi_boot_reopen_sess(phba); 517450a4b824SJitendra Bhivare break; 517550a4b824SJitendra Bhivare case BEISCSI_BOOT_GET_SHANDLE: 517650a4b824SJitendra Bhivare tag = __beiscsi_boot_get_shandle(phba, 1); 517750a4b824SJitendra Bhivare break; 517850a4b824SJitendra Bhivare case BEISCSI_BOOT_GET_SINFO: 517950a4b824SJitendra Bhivare tag = beiscsi_boot_get_sinfo(phba); 518050a4b824SJitendra Bhivare break; 518150a4b824SJitendra Bhivare case BEISCSI_BOOT_LOGOUT_SESS: 518250a4b824SJitendra Bhivare tag = beiscsi_boot_logout_sess(phba); 518350a4b824SJitendra Bhivare break; 518450a4b824SJitendra Bhivare case BEISCSI_BOOT_CREATE_KSET: 518550a4b824SJitendra Bhivare beiscsi_boot_create_kset(phba); 518650a4b824SJitendra Bhivare /** 518750a4b824SJitendra Bhivare * updated boot_kset is made visible to all before 518850a4b824SJitendra Bhivare * ending the boot work. 518950a4b824SJitendra Bhivare */ 519050a4b824SJitendra Bhivare mb(); 519150a4b824SJitendra Bhivare clear_bit(BEISCSI_HBA_BOOT_WORK, &phba->state); 519250a4b824SJitendra Bhivare return; 519350a4b824SJitendra Bhivare } 519450a4b824SJitendra Bhivare if (!tag) { 519550a4b824SJitendra Bhivare if (bs->retry--) 519650a4b824SJitendra Bhivare schedule_work(&phba->boot_work); 519750a4b824SJitendra Bhivare else 519850a4b824SJitendra Bhivare clear_bit(BEISCSI_HBA_BOOT_WORK, &phba->state); 519950a4b824SJitendra Bhivare } 520050a4b824SJitendra Bhivare } 520150a4b824SJitendra Bhivare 520210bcd47dSJitendra Bhivare static void beiscsi_eqd_update_work(struct work_struct *work) 520310bcd47dSJitendra Bhivare { 520410bcd47dSJitendra Bhivare struct hwi_context_memory *phwi_context; 520510bcd47dSJitendra Bhivare struct be_set_eqd set_eqd[MAX_CPUS]; 520610bcd47dSJitendra Bhivare struct hwi_controller *phwi_ctrlr; 520710bcd47dSJitendra Bhivare struct be_eq_obj *pbe_eq; 520810bcd47dSJitendra Bhivare struct beiscsi_hba *phba; 520910bcd47dSJitendra Bhivare unsigned int pps, delta; 521010bcd47dSJitendra Bhivare struct be_aic_obj *aic; 521110bcd47dSJitendra Bhivare int eqd, i, num = 0; 521210bcd47dSJitendra Bhivare unsigned long now; 521310bcd47dSJitendra Bhivare 521410bcd47dSJitendra Bhivare phba = container_of(work, struct beiscsi_hba, eqd_update.work); 5215d1d5ca88SJitendra Bhivare if (!beiscsi_hba_is_online(phba)) 52169122e991SJitendra Bhivare return; 52179122e991SJitendra Bhivare 521873af08e1SJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 521973af08e1SJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 522073af08e1SJayamohan Kallickal 522173af08e1SJayamohan Kallickal for (i = 0; i <= phba->num_cpus; i++) { 522273af08e1SJayamohan Kallickal aic = &phba->aic_obj[i]; 522373af08e1SJayamohan Kallickal pbe_eq = &phwi_context->be_eq[i]; 522473af08e1SJayamohan Kallickal now = jiffies; 522510bcd47dSJitendra Bhivare if (!aic->jiffies || time_before(now, aic->jiffies) || 522673af08e1SJayamohan Kallickal pbe_eq->cq_count < aic->eq_prev) { 522710bcd47dSJitendra Bhivare aic->jiffies = now; 522873af08e1SJayamohan Kallickal aic->eq_prev = pbe_eq->cq_count; 522973af08e1SJayamohan Kallickal continue; 523073af08e1SJayamohan Kallickal } 523110bcd47dSJitendra Bhivare delta = jiffies_to_msecs(now - aic->jiffies); 523273af08e1SJayamohan Kallickal pps = (((u32)(pbe_eq->cq_count - aic->eq_prev) * 1000) / delta); 523373af08e1SJayamohan Kallickal eqd = (pps / 1500) << 2; 523473af08e1SJayamohan Kallickal 523573af08e1SJayamohan Kallickal if (eqd < 8) 523673af08e1SJayamohan Kallickal eqd = 0; 523773af08e1SJayamohan Kallickal eqd = min_t(u32, eqd, phwi_context->max_eqd); 523873af08e1SJayamohan Kallickal eqd = max_t(u32, eqd, phwi_context->min_eqd); 523973af08e1SJayamohan Kallickal 524010bcd47dSJitendra Bhivare aic->jiffies = now; 524173af08e1SJayamohan Kallickal aic->eq_prev = pbe_eq->cq_count; 524273af08e1SJayamohan Kallickal 524373af08e1SJayamohan Kallickal if (eqd != aic->prev_eqd) { 524473af08e1SJayamohan Kallickal set_eqd[num].delay_multiplier = (eqd * 65)/100; 524573af08e1SJayamohan Kallickal set_eqd[num].eq_id = pbe_eq->q.id; 524673af08e1SJayamohan Kallickal aic->prev_eqd = eqd; 524773af08e1SJayamohan Kallickal num++; 524873af08e1SJayamohan Kallickal } 524973af08e1SJayamohan Kallickal } 525010bcd47dSJitendra Bhivare if (num) 525110bcd47dSJitendra Bhivare /* completion of this is ignored */ 525210bcd47dSJitendra Bhivare beiscsi_modify_eq_delay(phba, set_eqd, num); 525373af08e1SJayamohan Kallickal 525410bcd47dSJitendra Bhivare schedule_delayed_work(&phba->eqd_update, 525510bcd47dSJitendra Bhivare msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL)); 52567a158003SJohn Soni Jose } 52577a158003SJohn Soni Jose 5258d1d5ca88SJitendra Bhivare static void beiscsi_hw_tpe_check(unsigned long ptr) 5259d1d5ca88SJitendra Bhivare { 5260d1d5ca88SJitendra Bhivare struct beiscsi_hba *phba; 5261d1d5ca88SJitendra Bhivare u32 wait; 5262d1d5ca88SJitendra Bhivare 5263d1d5ca88SJitendra Bhivare phba = (struct beiscsi_hba *)ptr; 5264d1d5ca88SJitendra Bhivare /* if not TPE, do nothing */ 5265d1d5ca88SJitendra Bhivare if (!beiscsi_detect_tpe(phba)) 5266d1d5ca88SJitendra Bhivare return; 5267d1d5ca88SJitendra Bhivare 5268d1d5ca88SJitendra Bhivare /* wait default 4000ms before recovering */ 5269d1d5ca88SJitendra Bhivare wait = 4000; 5270d1d5ca88SJitendra Bhivare if (phba->ue2rp > BEISCSI_UE_DETECT_INTERVAL) 5271d1d5ca88SJitendra Bhivare wait = phba->ue2rp - BEISCSI_UE_DETECT_INTERVAL; 5272d1d5ca88SJitendra Bhivare queue_delayed_work(phba->wq, &phba->recover_port, 5273d1d5ca88SJitendra Bhivare msecs_to_jiffies(wait)); 5274d1d5ca88SJitendra Bhivare } 5275d1d5ca88SJitendra Bhivare 5276d1d5ca88SJitendra Bhivare static void beiscsi_hw_health_check(unsigned long ptr) 5277d1d5ca88SJitendra Bhivare { 5278d1d5ca88SJitendra Bhivare struct beiscsi_hba *phba; 5279d1d5ca88SJitendra Bhivare 5280d1d5ca88SJitendra Bhivare phba = (struct beiscsi_hba *)ptr; 5281d1d5ca88SJitendra Bhivare beiscsi_detect_ue(phba); 5282d1d5ca88SJitendra Bhivare if (beiscsi_detect_ue(phba)) { 5283d1d5ca88SJitendra Bhivare __beiscsi_log(phba, KERN_ERR, 5284d1d5ca88SJitendra Bhivare "BM_%d : port in error: %lx\n", phba->state); 528510e1a44aSJitendra Bhivare /* sessions are no longer valid, so first fail the sessions */ 528610e1a44aSJitendra Bhivare queue_work(phba->wq, &phba->sess_work); 528710e1a44aSJitendra Bhivare 528810e1a44aSJitendra Bhivare /* detect UER supported */ 5289d1d5ca88SJitendra Bhivare if (!test_bit(BEISCSI_HBA_UER_SUPP, &phba->state)) 5290d1d5ca88SJitendra Bhivare return; 5291d1d5ca88SJitendra Bhivare /* modify this timer to check TPE */ 5292d1d5ca88SJitendra Bhivare phba->hw_check.function = beiscsi_hw_tpe_check; 5293d1d5ca88SJitendra Bhivare } 5294d1d5ca88SJitendra Bhivare 5295d1d5ca88SJitendra Bhivare mod_timer(&phba->hw_check, 5296d1d5ca88SJitendra Bhivare jiffies + msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL)); 5297d1d5ca88SJitendra Bhivare } 5298d1d5ca88SJitendra Bhivare 5299d1d5ca88SJitendra Bhivare /* 5300d1d5ca88SJitendra Bhivare * beiscsi_enable_port()- Enables the disabled port. 5301d1d5ca88SJitendra Bhivare * Only port resources freed in disable function are reallocated. 5302d1d5ca88SJitendra Bhivare * This is called in HBA error handling path. 5303d1d5ca88SJitendra Bhivare * 5304d1d5ca88SJitendra Bhivare * @phba: Instance of driver private structure 5305d1d5ca88SJitendra Bhivare * 5306d1d5ca88SJitendra Bhivare **/ 5307d1d5ca88SJitendra Bhivare static int beiscsi_enable_port(struct beiscsi_hba *phba) 5308d1d5ca88SJitendra Bhivare { 5309d1d5ca88SJitendra Bhivare struct hwi_context_memory *phwi_context; 5310d1d5ca88SJitendra Bhivare struct hwi_controller *phwi_ctrlr; 5311d1d5ca88SJitendra Bhivare struct be_eq_obj *pbe_eq; 5312d1d5ca88SJitendra Bhivare int ret, i; 5313d1d5ca88SJitendra Bhivare 5314d1d5ca88SJitendra Bhivare if (test_bit(BEISCSI_HBA_ONLINE, &phba->state)) { 5315d1d5ca88SJitendra Bhivare __beiscsi_log(phba, KERN_ERR, 5316d1d5ca88SJitendra Bhivare "BM_%d : %s : port is online %lx\n", 5317d1d5ca88SJitendra Bhivare __func__, phba->state); 5318d1d5ca88SJitendra Bhivare return 0; 5319d1d5ca88SJitendra Bhivare } 5320d1d5ca88SJitendra Bhivare 5321d1d5ca88SJitendra Bhivare ret = beiscsi_init_sliport(phba); 5322d1d5ca88SJitendra Bhivare if (ret) 5323d1d5ca88SJitendra Bhivare return ret; 5324d1d5ca88SJitendra Bhivare 532583148866SChristoph Hellwig be2iscsi_enable_msix(phba); 5326d1d5ca88SJitendra Bhivare 5327d1d5ca88SJitendra Bhivare beiscsi_get_params(phba); 53281cb3c3fdSJitendra Bhivare beiscsi_set_host_data(phba); 5329d1d5ca88SJitendra Bhivare /* Re-enable UER. If different TPE occurs then it is recoverable. */ 5330d1d5ca88SJitendra Bhivare beiscsi_set_uer_feature(phba); 5331d1d5ca88SJitendra Bhivare 5332d1d5ca88SJitendra Bhivare phba->shost->max_id = phba->params.cxns_per_ctrl; 5333d1d5ca88SJitendra Bhivare phba->shost->can_queue = phba->params.ios_per_ctrl; 5334dd940972SJitendra Bhivare ret = beiscsi_init_port(phba); 5335dd940972SJitendra Bhivare if (ret < 0) { 5336d1d5ca88SJitendra Bhivare __beiscsi_log(phba, KERN_ERR, 5337dd940972SJitendra Bhivare "BM_%d : init port failed\n"); 5338d1d5ca88SJitendra Bhivare goto disable_msix; 5339d1d5ca88SJitendra Bhivare } 5340d1d5ca88SJitendra Bhivare 5341d1d5ca88SJitendra Bhivare for (i = 0; i < MAX_MCC_CMD; i++) { 5342d1d5ca88SJitendra Bhivare init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]); 5343d1d5ca88SJitendra Bhivare phba->ctrl.mcc_tag[i] = i + 1; 5344d1d5ca88SJitendra Bhivare phba->ctrl.mcc_tag_status[i + 1] = 0; 5345d1d5ca88SJitendra Bhivare phba->ctrl.mcc_tag_available++; 5346d1d5ca88SJitendra Bhivare } 5347d1d5ca88SJitendra Bhivare 5348d1d5ca88SJitendra Bhivare phwi_ctrlr = phba->phwi_ctrlr; 5349d1d5ca88SJitendra Bhivare phwi_context = phwi_ctrlr->phwi_ctxt; 5350d1d5ca88SJitendra Bhivare for (i = 0; i < phba->num_cpus; i++) { 5351d1d5ca88SJitendra Bhivare pbe_eq = &phwi_context->be_eq[i]; 5352d1d5ca88SJitendra Bhivare irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget, be_iopoll); 5353d1d5ca88SJitendra Bhivare } 5354d1d5ca88SJitendra Bhivare 535583148866SChristoph Hellwig i = (phba->pcidev->msix_enabled) ? i : 0; 5356d1d5ca88SJitendra Bhivare /* Work item for MCC handling */ 5357d1d5ca88SJitendra Bhivare pbe_eq = &phwi_context->be_eq[i]; 5358d1d5ca88SJitendra Bhivare INIT_WORK(&pbe_eq->mcc_work, beiscsi_mcc_work); 5359d1d5ca88SJitendra Bhivare 5360d1d5ca88SJitendra Bhivare ret = beiscsi_init_irqs(phba); 5361d1d5ca88SJitendra Bhivare if (ret < 0) { 5362d1d5ca88SJitendra Bhivare __beiscsi_log(phba, KERN_ERR, 5363d1d5ca88SJitendra Bhivare "BM_%d : setup IRQs failed %d\n", ret); 5364d1d5ca88SJitendra Bhivare goto cleanup_port; 5365d1d5ca88SJitendra Bhivare } 5366d1d5ca88SJitendra Bhivare hwi_enable_intr(phba); 5367d1d5ca88SJitendra Bhivare /* port operational: clear all error bits */ 5368d1d5ca88SJitendra Bhivare set_bit(BEISCSI_HBA_ONLINE, &phba->state); 5369d1d5ca88SJitendra Bhivare __beiscsi_log(phba, KERN_INFO, 5370d1d5ca88SJitendra Bhivare "BM_%d : port online: 0x%lx\n", phba->state); 5371d1d5ca88SJitendra Bhivare 5372d1d5ca88SJitendra Bhivare /* start hw_check timer and eqd_update work */ 5373d1d5ca88SJitendra Bhivare schedule_delayed_work(&phba->eqd_update, 5374d1d5ca88SJitendra Bhivare msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL)); 5375d1d5ca88SJitendra Bhivare 5376d1d5ca88SJitendra Bhivare /** 5377d1d5ca88SJitendra Bhivare * Timer function gets modified for TPE detection. 5378d1d5ca88SJitendra Bhivare * Always reinit to do health check first. 5379d1d5ca88SJitendra Bhivare */ 5380d1d5ca88SJitendra Bhivare phba->hw_check.function = beiscsi_hw_health_check; 5381d1d5ca88SJitendra Bhivare mod_timer(&phba->hw_check, 5382d1d5ca88SJitendra Bhivare jiffies + msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL)); 5383d1d5ca88SJitendra Bhivare return 0; 5384d1d5ca88SJitendra Bhivare 5385d1d5ca88SJitendra Bhivare cleanup_port: 5386d1d5ca88SJitendra Bhivare for (i = 0; i < phba->num_cpus; i++) { 5387d1d5ca88SJitendra Bhivare pbe_eq = &phwi_context->be_eq[i]; 5388d1d5ca88SJitendra Bhivare irq_poll_disable(&pbe_eq->iopoll); 5389d1d5ca88SJitendra Bhivare } 5390d1d5ca88SJitendra Bhivare hwi_cleanup_port(phba); 5391d1d5ca88SJitendra Bhivare 5392d1d5ca88SJitendra Bhivare disable_msix: 539383148866SChristoph Hellwig pci_free_irq_vectors(phba->pcidev); 5394d1d5ca88SJitendra Bhivare return ret; 5395d1d5ca88SJitendra Bhivare } 5396d1d5ca88SJitendra Bhivare 5397d1d5ca88SJitendra Bhivare /* 5398d1d5ca88SJitendra Bhivare * beiscsi_disable_port()- Disable port and cleanup driver resources. 5399d1d5ca88SJitendra Bhivare * This is called in HBA error handling and driver removal. 5400d1d5ca88SJitendra Bhivare * @phba: Instance Priv structure 5401d1d5ca88SJitendra Bhivare * @unload: indicate driver is unloading 5402d1d5ca88SJitendra Bhivare * 5403d1d5ca88SJitendra Bhivare * Free the OS and HW resources held by the driver 5404d1d5ca88SJitendra Bhivare **/ 5405d1d5ca88SJitendra Bhivare static void beiscsi_disable_port(struct beiscsi_hba *phba, int unload) 5406d1d5ca88SJitendra Bhivare { 5407d1d5ca88SJitendra Bhivare struct hwi_context_memory *phwi_context; 5408d1d5ca88SJitendra Bhivare struct hwi_controller *phwi_ctrlr; 5409d1d5ca88SJitendra Bhivare struct be_eq_obj *pbe_eq; 541083148866SChristoph Hellwig unsigned int i; 5411d1d5ca88SJitendra Bhivare 5412d1d5ca88SJitendra Bhivare if (!test_and_clear_bit(BEISCSI_HBA_ONLINE, &phba->state)) 5413d1d5ca88SJitendra Bhivare return; 5414d1d5ca88SJitendra Bhivare 5415d1d5ca88SJitendra Bhivare phwi_ctrlr = phba->phwi_ctrlr; 5416d1d5ca88SJitendra Bhivare phwi_context = phwi_ctrlr->phwi_ctxt; 5417d1d5ca88SJitendra Bhivare hwi_disable_intr(phba); 541845371aa3SJitendra Bhivare beiscsi_free_irqs(phba); 541983148866SChristoph Hellwig pci_free_irq_vectors(phba->pcidev); 5420d1d5ca88SJitendra Bhivare 5421d1d5ca88SJitendra Bhivare for (i = 0; i < phba->num_cpus; i++) { 5422d1d5ca88SJitendra Bhivare pbe_eq = &phwi_context->be_eq[i]; 5423d1d5ca88SJitendra Bhivare irq_poll_disable(&pbe_eq->iopoll); 5424d1d5ca88SJitendra Bhivare } 5425d1d5ca88SJitendra Bhivare cancel_delayed_work_sync(&phba->eqd_update); 5426d1d5ca88SJitendra Bhivare cancel_work_sync(&phba->boot_work); 5427d1d5ca88SJitendra Bhivare /* WQ might be running cancel queued mcc_work if we are not exiting */ 5428d1d5ca88SJitendra Bhivare if (!unload && beiscsi_hba_in_error(phba)) { 5429d1d5ca88SJitendra Bhivare pbe_eq = &phwi_context->be_eq[i]; 5430d1d5ca88SJitendra Bhivare cancel_work_sync(&pbe_eq->mcc_work); 5431d1d5ca88SJitendra Bhivare } 5432d1d5ca88SJitendra Bhivare hwi_cleanup_port(phba); 5433dd940972SJitendra Bhivare beiscsi_cleanup_port(phba); 5434d1d5ca88SJitendra Bhivare } 5435d1d5ca88SJitendra Bhivare 543610e1a44aSJitendra Bhivare static void beiscsi_sess_work(struct work_struct *work) 543710e1a44aSJitendra Bhivare { 543810e1a44aSJitendra Bhivare struct beiscsi_hba *phba; 543910e1a44aSJitendra Bhivare 544010e1a44aSJitendra Bhivare phba = container_of(work, struct beiscsi_hba, sess_work); 544110e1a44aSJitendra Bhivare /* 544210e1a44aSJitendra Bhivare * This work gets scheduled only in case of HBA error. 544310e1a44aSJitendra Bhivare * Old sessions are gone so need to be re-established. 544410e1a44aSJitendra Bhivare * iscsi_session_failure needs process context hence this work. 544510e1a44aSJitendra Bhivare */ 544610e1a44aSJitendra Bhivare iscsi_host_for_each_session(phba->shost, beiscsi_session_fail); 544710e1a44aSJitendra Bhivare } 544810e1a44aSJitendra Bhivare 5449d1d5ca88SJitendra Bhivare static void beiscsi_recover_port(struct work_struct *work) 5450d1d5ca88SJitendra Bhivare { 5451d1d5ca88SJitendra Bhivare struct beiscsi_hba *phba; 5452d1d5ca88SJitendra Bhivare 5453d1d5ca88SJitendra Bhivare phba = container_of(work, struct beiscsi_hba, recover_port.work); 5454d1d5ca88SJitendra Bhivare beiscsi_disable_port(phba, 0); 5455d1d5ca88SJitendra Bhivare beiscsi_enable_port(phba); 5456d1d5ca88SJitendra Bhivare } 54573567f36aSJayamohan Kallickal 54583567f36aSJayamohan Kallickal static pci_ers_result_t beiscsi_eeh_err_detected(struct pci_dev *pdev, 54593567f36aSJayamohan Kallickal pci_channel_state_t state) 54603567f36aSJayamohan Kallickal { 54613567f36aSJayamohan Kallickal struct beiscsi_hba *phba = NULL; 54623567f36aSJayamohan Kallickal 54633567f36aSJayamohan Kallickal phba = (struct beiscsi_hba *)pci_get_drvdata(pdev); 54649122e991SJitendra Bhivare set_bit(BEISCSI_HBA_PCI_ERR, &phba->state); 54653567f36aSJayamohan Kallickal 54663567f36aSJayamohan Kallickal beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 54673567f36aSJayamohan Kallickal "BM_%d : EEH error detected\n"); 54683567f36aSJayamohan Kallickal 5469d1d5ca88SJitendra Bhivare /* first stop UE detection when PCI error detected */ 5470d1d5ca88SJitendra Bhivare del_timer_sync(&phba->hw_check); 5471d1d5ca88SJitendra Bhivare cancel_delayed_work_sync(&phba->recover_port); 5472d1d5ca88SJitendra Bhivare 547310e1a44aSJitendra Bhivare /* sessions are no longer valid, so first fail the sessions */ 547410e1a44aSJitendra Bhivare iscsi_host_for_each_session(phba->shost, beiscsi_session_fail); 5475d1d5ca88SJitendra Bhivare beiscsi_disable_port(phba, 0); 54763567f36aSJayamohan Kallickal 54773567f36aSJayamohan Kallickal if (state == pci_channel_io_perm_failure) { 54783567f36aSJayamohan Kallickal beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 54793567f36aSJayamohan Kallickal "BM_%d : EEH : State PERM Failure"); 54803567f36aSJayamohan Kallickal return PCI_ERS_RESULT_DISCONNECT; 54813567f36aSJayamohan Kallickal } 54823567f36aSJayamohan Kallickal 54833567f36aSJayamohan Kallickal pci_disable_device(pdev); 54843567f36aSJayamohan Kallickal 54853567f36aSJayamohan Kallickal /* The error could cause the FW to trigger a flash debug dump. 54863567f36aSJayamohan Kallickal * Resetting the card while flash dump is in progress 54873567f36aSJayamohan Kallickal * can cause it not to recover; wait for it to finish. 54883567f36aSJayamohan Kallickal * Wait only for first function as it is needed only once per 54893567f36aSJayamohan Kallickal * adapter. 54903567f36aSJayamohan Kallickal **/ 54913567f36aSJayamohan Kallickal if (pdev->devfn == 0) 54923567f36aSJayamohan Kallickal ssleep(30); 54933567f36aSJayamohan Kallickal 54943567f36aSJayamohan Kallickal return PCI_ERS_RESULT_NEED_RESET; 54953567f36aSJayamohan Kallickal } 54963567f36aSJayamohan Kallickal 54973567f36aSJayamohan Kallickal static pci_ers_result_t beiscsi_eeh_reset(struct pci_dev *pdev) 54983567f36aSJayamohan Kallickal { 54993567f36aSJayamohan Kallickal struct beiscsi_hba *phba = NULL; 55003567f36aSJayamohan Kallickal int status = 0; 55013567f36aSJayamohan Kallickal 55023567f36aSJayamohan Kallickal phba = (struct beiscsi_hba *)pci_get_drvdata(pdev); 55033567f36aSJayamohan Kallickal 55043567f36aSJayamohan Kallickal beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 55053567f36aSJayamohan Kallickal "BM_%d : EEH Reset\n"); 55063567f36aSJayamohan Kallickal 55073567f36aSJayamohan Kallickal status = pci_enable_device(pdev); 55083567f36aSJayamohan Kallickal if (status) 55093567f36aSJayamohan Kallickal return PCI_ERS_RESULT_DISCONNECT; 55103567f36aSJayamohan Kallickal 55113567f36aSJayamohan Kallickal pci_set_master(pdev); 55123567f36aSJayamohan Kallickal pci_set_power_state(pdev, PCI_D0); 55133567f36aSJayamohan Kallickal pci_restore_state(pdev); 55143567f36aSJayamohan Kallickal 55154d2ee1e6SJitendra Bhivare status = beiscsi_check_fw_rdy(phba); 55164d2ee1e6SJitendra Bhivare if (status) { 55173567f36aSJayamohan Kallickal beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 55183567f36aSJayamohan Kallickal "BM_%d : EEH Reset Completed\n"); 55193567f36aSJayamohan Kallickal } else { 55203567f36aSJayamohan Kallickal beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 55213567f36aSJayamohan Kallickal "BM_%d : EEH Reset Completion Failure\n"); 55223567f36aSJayamohan Kallickal return PCI_ERS_RESULT_DISCONNECT; 55233567f36aSJayamohan Kallickal } 55243567f36aSJayamohan Kallickal 55253567f36aSJayamohan Kallickal pci_cleanup_aer_uncorrect_error_status(pdev); 55263567f36aSJayamohan Kallickal return PCI_ERS_RESULT_RECOVERED; 55273567f36aSJayamohan Kallickal } 55283567f36aSJayamohan Kallickal 55293567f36aSJayamohan Kallickal static void beiscsi_eeh_resume(struct pci_dev *pdev) 55303567f36aSJayamohan Kallickal { 5531d1d5ca88SJitendra Bhivare struct beiscsi_hba *phba; 5532d1d5ca88SJitendra Bhivare int ret; 55333567f36aSJayamohan Kallickal 55343567f36aSJayamohan Kallickal phba = (struct beiscsi_hba *)pci_get_drvdata(pdev); 55353567f36aSJayamohan Kallickal pci_save_state(pdev); 55363567f36aSJayamohan Kallickal 5537d1d5ca88SJitendra Bhivare ret = beiscsi_enable_port(phba); 55384d2ee1e6SJitendra Bhivare if (ret) 5539d1d5ca88SJitendra Bhivare __beiscsi_log(phba, KERN_ERR, 5540d1d5ca88SJitendra Bhivare "BM_%d : AER EEH resume failed\n"); 55413567f36aSJayamohan Kallickal } 55423567f36aSJayamohan Kallickal 55436f039790SGreg Kroah-Hartman static int beiscsi_dev_probe(struct pci_dev *pcidev, 55446733b39aSJayamohan Kallickal const struct pci_device_id *id) 55456733b39aSJayamohan Kallickal { 5546bfead3b2SJayamohan Kallickal struct hwi_context_memory *phwi_context; 554729e80b7cSJitendra Bhivare struct hwi_controller *phwi_ctrlr; 554829e80b7cSJitendra Bhivare struct beiscsi_hba *phba = NULL; 5549bfead3b2SJayamohan Kallickal struct be_eq_obj *pbe_eq; 555050a4b824SJitendra Bhivare unsigned int s_handle; 555129e80b7cSJitendra Bhivare char wq_name[20]; 5552deeea8edSChristophe JAILLET int ret, i; 55536733b39aSJayamohan Kallickal 55546733b39aSJayamohan Kallickal ret = beiscsi_enable_pci(pcidev); 55556733b39aSJayamohan Kallickal if (ret < 0) { 555699bc5d55SJohn Soni Jose dev_err(&pcidev->dev, 555799bc5d55SJohn Soni Jose "beiscsi_dev_probe - Failed to enable pci device\n"); 55586733b39aSJayamohan Kallickal return ret; 55596733b39aSJayamohan Kallickal } 55606733b39aSJayamohan Kallickal 55616733b39aSJayamohan Kallickal phba = beiscsi_hba_alloc(pcidev); 55626733b39aSJayamohan Kallickal if (!phba) { 556399bc5d55SJohn Soni Jose dev_err(&pcidev->dev, 556499bc5d55SJohn Soni Jose "beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n"); 5565deeea8edSChristophe JAILLET ret = -ENOMEM; 55666733b39aSJayamohan Kallickal goto disable_pci; 55676733b39aSJayamohan Kallickal } 55686733b39aSJayamohan Kallickal 55693567f36aSJayamohan Kallickal /* Enable EEH reporting */ 55703567f36aSJayamohan Kallickal ret = pci_enable_pcie_error_reporting(pcidev); 55713567f36aSJayamohan Kallickal if (ret) 55723567f36aSJayamohan Kallickal beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 55733567f36aSJayamohan Kallickal "BM_%d : PCIe Error Reporting " 55743567f36aSJayamohan Kallickal "Enabling Failed\n"); 55753567f36aSJayamohan Kallickal 55763567f36aSJayamohan Kallickal pci_save_state(pcidev); 55773567f36aSJayamohan Kallickal 557899bc5d55SJohn Soni Jose /* Initialize Driver configuration Paramters */ 557999bc5d55SJohn Soni Jose beiscsi_hba_attrs_init(phba); 558099bc5d55SJohn Soni Jose 55816c83185aSJayamohan Kallickal phba->mac_addr_set = false; 5582e175defeSJohn Soni Jose 5583f98c96b0SJayamohan Kallickal switch (pcidev->device) { 5584f98c96b0SJayamohan Kallickal case BE_DEVICE_ID1: 5585f98c96b0SJayamohan Kallickal case OC_DEVICE_ID1: 5586f98c96b0SJayamohan Kallickal case OC_DEVICE_ID2: 5587f98c96b0SJayamohan Kallickal phba->generation = BE_GEN2; 558809a1093aSJohn Soni Jose phba->iotask_fn = beiscsi_iotask; 55895fa7db21SKetan Mukadam dev_warn(&pcidev->dev, 55905fa7db21SKetan Mukadam "Obsolete/Unsupported BE2 Adapter Family\n"); 5591f98c96b0SJayamohan Kallickal break; 5592f98c96b0SJayamohan Kallickal case BE_DEVICE_ID2: 5593f98c96b0SJayamohan Kallickal case OC_DEVICE_ID3: 5594f98c96b0SJayamohan Kallickal phba->generation = BE_GEN3; 559509a1093aSJohn Soni Jose phba->iotask_fn = beiscsi_iotask; 5596f98c96b0SJayamohan Kallickal break; 5597139a1b1eSJohn Soni Jose case OC_SKH_ID1: 5598139a1b1eSJohn Soni Jose phba->generation = BE_GEN4; 559909a1093aSJohn Soni Jose phba->iotask_fn = beiscsi_iotask_v2; 5600bf9131cbSJayamohan Kallickal break; 5601f98c96b0SJayamohan Kallickal default: 5602f98c96b0SJayamohan Kallickal phba->generation = 0; 5603f98c96b0SJayamohan Kallickal } 5604f98c96b0SJayamohan Kallickal 56056733b39aSJayamohan Kallickal ret = be_ctrl_init(phba, pcidev); 56066733b39aSJayamohan Kallickal if (ret) { 560799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 56084d2ee1e6SJitendra Bhivare "BM_%d : be_ctrl_init failed\n"); 560945371aa3SJitendra Bhivare goto free_hba; 56106733b39aSJayamohan Kallickal } 56116733b39aSJayamohan Kallickal 56124d2ee1e6SJitendra Bhivare ret = beiscsi_init_sliport(phba); 56134d2ee1e6SJitendra Bhivare if (ret) 561445371aa3SJitendra Bhivare goto free_hba; 56154d2ee1e6SJitendra Bhivare 56166733b39aSJayamohan Kallickal spin_lock_init(&phba->io_sgl_lock); 56176733b39aSJayamohan Kallickal spin_lock_init(&phba->mgmt_sgl_lock); 56188f09a3b9SJayamohan Kallickal spin_lock_init(&phba->async_pdu_lock); 5619480195c2SJitendra Bhivare ret = beiscsi_get_fw_config(&phba->ctrl, phba); 56207da50879SJayamohan Kallickal if (ret != 0) { 562199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 562299bc5d55SJohn Soni Jose "BM_%d : Error getting fw config\n"); 56237da50879SJayamohan Kallickal goto free_port; 56247da50879SJayamohan Kallickal } 5625480195c2SJitendra Bhivare beiscsi_get_port_name(&phba->ctrl, phba); 56264570f161SJitendra Bhivare beiscsi_get_params(phba); 56271cb3c3fdSJitendra Bhivare beiscsi_set_host_data(phba); 56286694095bSJitendra Bhivare beiscsi_set_uer_feature(phba); 562968c26a3aSJayamohan Kallickal 563083148866SChristoph Hellwig be2iscsi_enable_msix(phba); 563168c26a3aSJayamohan Kallickal 563268c26a3aSJayamohan Kallickal beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 563368c26a3aSJayamohan Kallickal "BM_%d : num_cpus = %d\n", 563468c26a3aSJayamohan Kallickal phba->num_cpus); 563568c26a3aSJayamohan Kallickal 5636843ae752SJayamohan Kallickal phba->shost->max_id = phba->params.cxns_per_ctrl; 5637aa874f07SJayamohan Kallickal phba->shost->can_queue = phba->params.ios_per_ctrl; 5638dd940972SJitendra Bhivare ret = beiscsi_get_memory(phba); 5639dd940972SJitendra Bhivare if (ret < 0) { 5640dd940972SJitendra Bhivare beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5641dd940972SJitendra Bhivare "BM_%d : alloc host mem failed\n"); 5642dd940972SJitendra Bhivare goto free_port; 5643dd940972SJitendra Bhivare } 5644dd940972SJitendra Bhivare 56456733b39aSJayamohan Kallickal ret = beiscsi_init_port(phba); 56466733b39aSJayamohan Kallickal if (ret < 0) { 564799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5648dd940972SJitendra Bhivare "BM_%d : init port failed\n"); 5649dd940972SJitendra Bhivare beiscsi_free_mem(phba); 56506733b39aSJayamohan Kallickal goto free_port; 56516733b39aSJayamohan Kallickal } 56526733b39aSJayamohan Kallickal 5653756d29c8SJayamohan Kallickal for (i = 0; i < MAX_MCC_CMD; i++) { 5654756d29c8SJayamohan Kallickal init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]); 5655756d29c8SJayamohan Kallickal phba->ctrl.mcc_tag[i] = i + 1; 565667296ad9SJitendra Bhivare phba->ctrl.mcc_tag_status[i + 1] = 0; 5657756d29c8SJayamohan Kallickal phba->ctrl.mcc_tag_available++; 56581957aa7fSJayamohan Kallickal memset(&phba->ctrl.ptag_state[i].tag_mem_state, 0, 56598fc01eaaSJayamohan Kallickal sizeof(struct be_dma_mem)); 5660756d29c8SJayamohan Kallickal } 5661756d29c8SJayamohan Kallickal 5662756d29c8SJayamohan Kallickal phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0; 5663756d29c8SJayamohan Kallickal 566429e80b7cSJitendra Bhivare snprintf(wq_name, sizeof(wq_name), "beiscsi_%02x_wq", 56656733b39aSJayamohan Kallickal phba->shost->host_no); 566629e80b7cSJitendra Bhivare phba->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 1, wq_name); 56676733b39aSJayamohan Kallickal if (!phba->wq) { 566899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 566999bc5d55SJohn Soni Jose "BM_%d : beiscsi_dev_probe-" 56706733b39aSJayamohan Kallickal "Failed to allocate work queue\n"); 5671deeea8edSChristophe JAILLET ret = -ENOMEM; 56726733b39aSJayamohan Kallickal goto free_twq; 56736733b39aSJayamohan Kallickal } 56746733b39aSJayamohan Kallickal 567510bcd47dSJitendra Bhivare INIT_DELAYED_WORK(&phba->eqd_update, beiscsi_eqd_update_work); 56766733b39aSJayamohan Kallickal 5677bfead3b2SJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 5678bfead3b2SJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 567972fb46a9SJohn Soni Jose 5680bfead3b2SJayamohan Kallickal for (i = 0; i < phba->num_cpus; i++) { 5681bfead3b2SJayamohan Kallickal pbe_eq = &phwi_context->be_eq[i]; 5682d1d5ca88SJitendra Bhivare irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget, be_iopoll); 56836733b39aSJayamohan Kallickal } 568472fb46a9SJohn Soni Jose 568583148866SChristoph Hellwig i = (phba->pcidev->msix_enabled) ? i : 0; 568672fb46a9SJohn Soni Jose /* Work item for MCC handling */ 568772fb46a9SJohn Soni Jose pbe_eq = &phwi_context->be_eq[i]; 5688a3095016SJitendra Bhivare INIT_WORK(&pbe_eq->mcc_work, beiscsi_mcc_work); 568972fb46a9SJohn Soni Jose 56906733b39aSJayamohan Kallickal ret = beiscsi_init_irqs(phba); 56916733b39aSJayamohan Kallickal if (ret < 0) { 569299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 569399bc5d55SJohn Soni Jose "BM_%d : beiscsi_dev_probe-" 56946733b39aSJayamohan Kallickal "Failed to beiscsi_init_irqs\n"); 569545371aa3SJitendra Bhivare goto disable_iopoll; 56966733b39aSJayamohan Kallickal } 5697238f6b72SJayamohan Kallickal hwi_enable_intr(phba); 5698f457a46fSMike Christie 5699d1d5ca88SJitendra Bhivare ret = iscsi_host_add(phba->shost, &phba->pcidev->dev); 5700d1d5ca88SJitendra Bhivare if (ret) 570145371aa3SJitendra Bhivare goto free_irqs; 57020598b8afSJayamohan Kallickal 5703d1d5ca88SJitendra Bhivare /* set online bit after port is operational */ 5704d1d5ca88SJitendra Bhivare set_bit(BEISCSI_HBA_ONLINE, &phba->state); 5705d1d5ca88SJitendra Bhivare __beiscsi_log(phba, KERN_INFO, 5706d1d5ca88SJitendra Bhivare "BM_%d : port online: 0x%lx\n", phba->state); 5707d1d5ca88SJitendra Bhivare 570850a4b824SJitendra Bhivare INIT_WORK(&phba->boot_work, beiscsi_boot_work); 570950a4b824SJitendra Bhivare ret = beiscsi_boot_get_shandle(phba, &s_handle); 571050a4b824SJitendra Bhivare if (ret > 0) { 571150a4b824SJitendra Bhivare beiscsi_start_boot_work(phba, s_handle); 571250a4b824SJitendra Bhivare /** 571350a4b824SJitendra Bhivare * Set this bit after starting the work to let 571450a4b824SJitendra Bhivare * probe handle it first. 571550a4b824SJitendra Bhivare * ASYNC event can too schedule this work. 5716f457a46fSMike Christie */ 571750a4b824SJitendra Bhivare set_bit(BEISCSI_HBA_BOOT_FOUND, &phba->state); 571850a4b824SJitendra Bhivare } 5719f457a46fSMike Christie 572096b48b92SJitendra Bhivare beiscsi_iface_create_default(phba); 572110bcd47dSJitendra Bhivare schedule_delayed_work(&phba->eqd_update, 572210bcd47dSJitendra Bhivare msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL)); 5723d1d5ca88SJitendra Bhivare 572410e1a44aSJitendra Bhivare INIT_WORK(&phba->sess_work, beiscsi_sess_work); 5725d1d5ca88SJitendra Bhivare INIT_DELAYED_WORK(&phba->recover_port, beiscsi_recover_port); 572610bcd47dSJitendra Bhivare /** 572710bcd47dSJitendra Bhivare * Start UE detection here. UE before this will cause stall in probe 572810bcd47dSJitendra Bhivare * and eventually fail the probe. 572910bcd47dSJitendra Bhivare */ 573010bcd47dSJitendra Bhivare init_timer(&phba->hw_check); 573110bcd47dSJitendra Bhivare phba->hw_check.function = beiscsi_hw_health_check; 573210bcd47dSJitendra Bhivare phba->hw_check.data = (unsigned long)phba; 573310bcd47dSJitendra Bhivare mod_timer(&phba->hw_check, 573410bcd47dSJitendra Bhivare jiffies + msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL)); 573599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 573699bc5d55SJohn Soni Jose "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n"); 57376733b39aSJayamohan Kallickal return 0; 57386733b39aSJayamohan Kallickal 573945371aa3SJitendra Bhivare free_irqs: 574045371aa3SJitendra Bhivare hwi_disable_intr(phba); 574145371aa3SJitendra Bhivare beiscsi_free_irqs(phba); 574245371aa3SJitendra Bhivare disable_iopoll: 5743bfead3b2SJayamohan Kallickal for (i = 0; i < phba->num_cpus; i++) { 5744bfead3b2SJayamohan Kallickal pbe_eq = &phwi_context->be_eq[i]; 5745511cbce2SChristoph Hellwig irq_poll_disable(&pbe_eq->iopoll); 5746bfead3b2SJayamohan Kallickal } 574745371aa3SJitendra Bhivare destroy_workqueue(phba->wq); 57486733b39aSJayamohan Kallickal free_twq: 5749d1d5ca88SJitendra Bhivare hwi_cleanup_port(phba); 57504d2ee1e6SJitendra Bhivare beiscsi_cleanup_port(phba); 57516733b39aSJayamohan Kallickal beiscsi_free_mem(phba); 57526733b39aSJayamohan Kallickal free_port: 57536733b39aSJayamohan Kallickal pci_free_consistent(phba->pcidev, 57546733b39aSJayamohan Kallickal phba->ctrl.mbox_mem_alloced.size, 57556733b39aSJayamohan Kallickal phba->ctrl.mbox_mem_alloced.va, 57566733b39aSJayamohan Kallickal phba->ctrl.mbox_mem_alloced.dma); 57576733b39aSJayamohan Kallickal beiscsi_unmap_pci_function(phba); 575845371aa3SJitendra Bhivare free_hba: 5759238f6b72SJayamohan Kallickal pci_disable_msix(phba->pcidev); 57606733b39aSJayamohan Kallickal pci_dev_put(phba->pcidev); 57616733b39aSJayamohan Kallickal iscsi_host_free(phba->shost); 57622e7cee02SJohn Soni Jose pci_set_drvdata(pcidev, NULL); 57636733b39aSJayamohan Kallickal disable_pci: 5764e307f3acSJohn Soni Jose pci_release_regions(pcidev); 57656733b39aSJayamohan Kallickal pci_disable_device(pcidev); 57666733b39aSJayamohan Kallickal return ret; 57676733b39aSJayamohan Kallickal } 57686733b39aSJayamohan Kallickal 5769d1d5ca88SJitendra Bhivare static void beiscsi_remove(struct pci_dev *pcidev) 5770d1d5ca88SJitendra Bhivare { 5771d1d5ca88SJitendra Bhivare struct beiscsi_hba *phba = NULL; 5772d1d5ca88SJitendra Bhivare 5773d1d5ca88SJitendra Bhivare phba = pci_get_drvdata(pcidev); 5774d1d5ca88SJitendra Bhivare if (!phba) { 5775d1d5ca88SJitendra Bhivare dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n"); 5776d1d5ca88SJitendra Bhivare return; 5777d1d5ca88SJitendra Bhivare } 5778d1d5ca88SJitendra Bhivare 5779d1d5ca88SJitendra Bhivare /* first stop UE detection before unloading */ 5780d1d5ca88SJitendra Bhivare del_timer_sync(&phba->hw_check); 5781d1d5ca88SJitendra Bhivare cancel_delayed_work_sync(&phba->recover_port); 578210e1a44aSJitendra Bhivare cancel_work_sync(&phba->sess_work); 5783d1d5ca88SJitendra Bhivare 5784d1d5ca88SJitendra Bhivare beiscsi_iface_destroy_default(phba); 5785d1d5ca88SJitendra Bhivare iscsi_host_remove(phba->shost); 5786d1d5ca88SJitendra Bhivare beiscsi_disable_port(phba, 1); 5787d1d5ca88SJitendra Bhivare 5788d1d5ca88SJitendra Bhivare /* after cancelling boot_work */ 5789d1d5ca88SJitendra Bhivare iscsi_boot_destroy_kset(phba->boot_struct.boot_kset); 5790d1d5ca88SJitendra Bhivare 5791d1d5ca88SJitendra Bhivare /* free all resources */ 5792d1d5ca88SJitendra Bhivare destroy_workqueue(phba->wq); 5793d1d5ca88SJitendra Bhivare beiscsi_free_mem(phba); 5794d1d5ca88SJitendra Bhivare 5795d1d5ca88SJitendra Bhivare /* ctrl uninit */ 5796d1d5ca88SJitendra Bhivare beiscsi_unmap_pci_function(phba); 5797d1d5ca88SJitendra Bhivare pci_free_consistent(phba->pcidev, 5798d1d5ca88SJitendra Bhivare phba->ctrl.mbox_mem_alloced.size, 5799d1d5ca88SJitendra Bhivare phba->ctrl.mbox_mem_alloced.va, 5800d1d5ca88SJitendra Bhivare phba->ctrl.mbox_mem_alloced.dma); 5801d1d5ca88SJitendra Bhivare 5802d1d5ca88SJitendra Bhivare pci_dev_put(phba->pcidev); 5803d1d5ca88SJitendra Bhivare iscsi_host_free(phba->shost); 5804d1d5ca88SJitendra Bhivare pci_disable_pcie_error_reporting(pcidev); 5805d1d5ca88SJitendra Bhivare pci_set_drvdata(pcidev, NULL); 5806d1d5ca88SJitendra Bhivare pci_release_regions(pcidev); 5807d1d5ca88SJitendra Bhivare pci_disable_device(pcidev); 5808d1d5ca88SJitendra Bhivare } 5809d1d5ca88SJitendra Bhivare 5810d1d5ca88SJitendra Bhivare 58113567f36aSJayamohan Kallickal static struct pci_error_handlers beiscsi_eeh_handlers = { 58123567f36aSJayamohan Kallickal .error_detected = beiscsi_eeh_err_detected, 58133567f36aSJayamohan Kallickal .slot_reset = beiscsi_eeh_reset, 58143567f36aSJayamohan Kallickal .resume = beiscsi_eeh_resume, 58153567f36aSJayamohan Kallickal }; 58163567f36aSJayamohan Kallickal 58176733b39aSJayamohan Kallickal struct iscsi_transport beiscsi_iscsi_transport = { 58186733b39aSJayamohan Kallickal .owner = THIS_MODULE, 58196733b39aSJayamohan Kallickal .name = DRV_NAME, 58209db0fb3aSJayamohan Kallickal .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO | 58216733b39aSJayamohan Kallickal CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD, 58226733b39aSJayamohan Kallickal .create_session = beiscsi_session_create, 58236733b39aSJayamohan Kallickal .destroy_session = beiscsi_session_destroy, 58246733b39aSJayamohan Kallickal .create_conn = beiscsi_conn_create, 58256733b39aSJayamohan Kallickal .bind_conn = beiscsi_conn_bind, 58266733b39aSJayamohan Kallickal .destroy_conn = iscsi_conn_teardown, 582796b48b92SJitendra Bhivare .attr_is_visible = beiscsi_attr_is_visible, 582896b48b92SJitendra Bhivare .set_iface_param = beiscsi_iface_set_param, 582996b48b92SJitendra Bhivare .get_iface_param = beiscsi_iface_get_param, 58306733b39aSJayamohan Kallickal .set_param = beiscsi_set_param, 5831c7f7fd5bSMike Christie .get_conn_param = iscsi_conn_get_param, 58326733b39aSJayamohan Kallickal .get_session_param = iscsi_session_get_param, 58336733b39aSJayamohan Kallickal .get_host_param = beiscsi_get_host_param, 58346733b39aSJayamohan Kallickal .start_conn = beiscsi_conn_start, 5835fa95d206SMike Christie .stop_conn = iscsi_conn_stop, 58366733b39aSJayamohan Kallickal .send_pdu = iscsi_conn_send_pdu, 58376733b39aSJayamohan Kallickal .xmit_task = beiscsi_task_xmit, 58386733b39aSJayamohan Kallickal .cleanup_task = beiscsi_cleanup_task, 58396733b39aSJayamohan Kallickal .alloc_pdu = beiscsi_alloc_pdu, 58406733b39aSJayamohan Kallickal .parse_pdu_itt = beiscsi_parse_pdu, 58416733b39aSJayamohan Kallickal .get_stats = beiscsi_conn_get_stats, 5842c7f7fd5bSMike Christie .get_ep_param = beiscsi_ep_get_param, 58436733b39aSJayamohan Kallickal .ep_connect = beiscsi_ep_connect, 58446733b39aSJayamohan Kallickal .ep_poll = beiscsi_ep_poll, 58456733b39aSJayamohan Kallickal .ep_disconnect = beiscsi_ep_disconnect, 58466733b39aSJayamohan Kallickal .session_recovery_timedout = iscsi_session_recovery_timedout, 5847ffce3e2eSJayamohan Kallickal .bsg_request = beiscsi_bsg_request, 58486733b39aSJayamohan Kallickal }; 58496733b39aSJayamohan Kallickal 58506733b39aSJayamohan Kallickal static struct pci_driver beiscsi_pci_driver = { 58516733b39aSJayamohan Kallickal .name = DRV_NAME, 58526733b39aSJayamohan Kallickal .probe = beiscsi_dev_probe, 58536733b39aSJayamohan Kallickal .remove = beiscsi_remove, 58543567f36aSJayamohan Kallickal .id_table = beiscsi_pci_id_table, 58553567f36aSJayamohan Kallickal .err_handler = &beiscsi_eeh_handlers 58566733b39aSJayamohan Kallickal }; 58576733b39aSJayamohan Kallickal 58586733b39aSJayamohan Kallickal static int __init beiscsi_module_init(void) 58596733b39aSJayamohan Kallickal { 58606733b39aSJayamohan Kallickal int ret; 58616733b39aSJayamohan Kallickal 58626733b39aSJayamohan Kallickal beiscsi_scsi_transport = 58636733b39aSJayamohan Kallickal iscsi_register_transport(&beiscsi_iscsi_transport); 58646733b39aSJayamohan Kallickal if (!beiscsi_scsi_transport) { 586599bc5d55SJohn Soni Jose printk(KERN_ERR 586699bc5d55SJohn Soni Jose "beiscsi_module_init - Unable to register beiscsi transport.\n"); 5867f55a24f2SJayamohan Kallickal return -ENOMEM; 58686733b39aSJayamohan Kallickal } 586999bc5d55SJohn Soni Jose printk(KERN_INFO "In beiscsi_module_init, tt=%p\n", 58706733b39aSJayamohan Kallickal &beiscsi_iscsi_transport); 58716733b39aSJayamohan Kallickal 58726733b39aSJayamohan Kallickal ret = pci_register_driver(&beiscsi_pci_driver); 58736733b39aSJayamohan Kallickal if (ret) { 587499bc5d55SJohn Soni Jose printk(KERN_ERR 587599bc5d55SJohn Soni Jose "beiscsi_module_init - Unable to register beiscsi pci driver.\n"); 58766733b39aSJayamohan Kallickal goto unregister_iscsi_transport; 58776733b39aSJayamohan Kallickal } 58786733b39aSJayamohan Kallickal return 0; 58796733b39aSJayamohan Kallickal 58806733b39aSJayamohan Kallickal unregister_iscsi_transport: 58816733b39aSJayamohan Kallickal iscsi_unregister_transport(&beiscsi_iscsi_transport); 58826733b39aSJayamohan Kallickal return ret; 58836733b39aSJayamohan Kallickal } 58846733b39aSJayamohan Kallickal 58856733b39aSJayamohan Kallickal static void __exit beiscsi_module_exit(void) 58866733b39aSJayamohan Kallickal { 58876733b39aSJayamohan Kallickal pci_unregister_driver(&beiscsi_pci_driver); 58886733b39aSJayamohan Kallickal iscsi_unregister_transport(&beiscsi_iscsi_transport); 58896733b39aSJayamohan Kallickal } 58906733b39aSJayamohan Kallickal 58916733b39aSJayamohan Kallickal module_init(beiscsi_module_init); 58926733b39aSJayamohan Kallickal module_exit(beiscsi_module_exit); 5893