16733b39aSJayamohan Kallickal /** 2255fa9a3SJayamohan Kallickal * Copyright (C) 2005 - 2011 Emulex 36733b39aSJayamohan Kallickal * All rights reserved. 46733b39aSJayamohan Kallickal * 56733b39aSJayamohan Kallickal * This program is free software; you can redistribute it and/or 66733b39aSJayamohan Kallickal * modify it under the terms of the GNU General Public License version 2 76733b39aSJayamohan Kallickal * as published by the Free Software Foundation. The full GNU General 86733b39aSJayamohan Kallickal * Public License is included in this distribution in the file called COPYING. 96733b39aSJayamohan Kallickal * 10255fa9a3SJayamohan Kallickal * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com) 116733b39aSJayamohan Kallickal * 126733b39aSJayamohan Kallickal * Contact Information: 13255fa9a3SJayamohan Kallickal * linux-drivers@emulex.com 146733b39aSJayamohan Kallickal * 15255fa9a3SJayamohan Kallickal * Emulex 16255fa9a3SJayamohan Kallickal * 3333 Susan Street 17255fa9a3SJayamohan Kallickal * Costa Mesa, CA 92626 186733b39aSJayamohan Kallickal */ 19255fa9a3SJayamohan Kallickal 206733b39aSJayamohan Kallickal #include <linux/reboot.h> 216733b39aSJayamohan Kallickal #include <linux/delay.h> 225a0e3ad6STejun Heo #include <linux/slab.h> 236733b39aSJayamohan Kallickal #include <linux/interrupt.h> 246733b39aSJayamohan Kallickal #include <linux/blkdev.h> 256733b39aSJayamohan Kallickal #include <linux/pci.h> 266733b39aSJayamohan Kallickal #include <linux/string.h> 276733b39aSJayamohan Kallickal #include <linux/kernel.h> 286733b39aSJayamohan Kallickal #include <linux/semaphore.h> 29c7acc5b8SJayamohan Kallickal #include <linux/iscsi_boot_sysfs.h> 30acf3368fSPaul Gortmaker #include <linux/module.h> 31ffce3e2eSJayamohan Kallickal #include <linux/bsg-lib.h> 326733b39aSJayamohan Kallickal 336733b39aSJayamohan Kallickal #include <scsi/libiscsi.h> 34ffce3e2eSJayamohan Kallickal #include <scsi/scsi_bsg_iscsi.h> 35ffce3e2eSJayamohan Kallickal #include <scsi/scsi_netlink.h> 366733b39aSJayamohan Kallickal #include <scsi/scsi_transport_iscsi.h> 376733b39aSJayamohan Kallickal #include <scsi/scsi_transport.h> 386733b39aSJayamohan Kallickal #include <scsi/scsi_cmnd.h> 396733b39aSJayamohan Kallickal #include <scsi/scsi_device.h> 406733b39aSJayamohan Kallickal #include <scsi/scsi_host.h> 416733b39aSJayamohan Kallickal #include <scsi/scsi.h> 426733b39aSJayamohan Kallickal #include "be_main.h" 436733b39aSJayamohan Kallickal #include "be_iscsi.h" 446733b39aSJayamohan Kallickal #include "be_mgmt.h" 450a513dd8SJohn Soni Jose #include "be_cmds.h" 466733b39aSJayamohan Kallickal 476733b39aSJayamohan Kallickal static unsigned int be_iopoll_budget = 10; 486733b39aSJayamohan Kallickal static unsigned int be_max_phys_size = 64; 49bfead3b2SJayamohan Kallickal static unsigned int enable_msix = 1; 50e9b91193SJayamohan Kallickal static unsigned int gcrashmode = 0; 51e9b91193SJayamohan Kallickal static unsigned int num_hba = 0; 526733b39aSJayamohan Kallickal 536733b39aSJayamohan Kallickal MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table); 546733b39aSJayamohan Kallickal MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR); 5576d15dbdSJayamohan Kallickal MODULE_VERSION(BUILD_STR); 562f635883SJayamohan Kallickal MODULE_AUTHOR("Emulex Corporation"); 576733b39aSJayamohan Kallickal MODULE_LICENSE("GPL"); 586733b39aSJayamohan Kallickal module_param(be_iopoll_budget, int, 0); 596733b39aSJayamohan Kallickal module_param(enable_msix, int, 0); 606733b39aSJayamohan Kallickal module_param(be_max_phys_size, uint, S_IRUGO); 6199bc5d55SJohn Soni Jose MODULE_PARM_DESC(be_max_phys_size, 6299bc5d55SJohn Soni Jose "Maximum Size (In Kilobytes) of physically contiguous " 6399bc5d55SJohn Soni Jose "memory that can be allocated. Range is 16 - 128"); 6499bc5d55SJohn Soni Jose 6599bc5d55SJohn Soni Jose #define beiscsi_disp_param(_name)\ 6699bc5d55SJohn Soni Jose ssize_t \ 6799bc5d55SJohn Soni Jose beiscsi_##_name##_disp(struct device *dev,\ 6899bc5d55SJohn Soni Jose struct device_attribute *attrib, char *buf) \ 6999bc5d55SJohn Soni Jose { \ 7099bc5d55SJohn Soni Jose struct Scsi_Host *shost = class_to_shost(dev);\ 7199bc5d55SJohn Soni Jose struct beiscsi_hba *phba = iscsi_host_priv(shost); \ 7299bc5d55SJohn Soni Jose uint32_t param_val = 0; \ 7399bc5d55SJohn Soni Jose param_val = phba->attr_##_name;\ 7499bc5d55SJohn Soni Jose return snprintf(buf, PAGE_SIZE, "%d\n",\ 7599bc5d55SJohn Soni Jose phba->attr_##_name);\ 7699bc5d55SJohn Soni Jose } 7799bc5d55SJohn Soni Jose 7899bc5d55SJohn Soni Jose #define beiscsi_change_param(_name, _minval, _maxval, _defaval)\ 7999bc5d55SJohn Soni Jose int \ 8099bc5d55SJohn Soni Jose beiscsi_##_name##_change(struct beiscsi_hba *phba, uint32_t val)\ 8199bc5d55SJohn Soni Jose {\ 8299bc5d55SJohn Soni Jose if (val >= _minval && val <= _maxval) {\ 8399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\ 8499bc5d55SJohn Soni Jose "BA_%d : beiscsi_"#_name" updated "\ 8599bc5d55SJohn Soni Jose "from 0x%x ==> 0x%x\n",\ 8699bc5d55SJohn Soni Jose phba->attr_##_name, val); \ 8799bc5d55SJohn Soni Jose phba->attr_##_name = val;\ 8899bc5d55SJohn Soni Jose return 0;\ 8999bc5d55SJohn Soni Jose } \ 9099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, \ 9199bc5d55SJohn Soni Jose "BA_%d beiscsi_"#_name" attribute "\ 9299bc5d55SJohn Soni Jose "cannot be updated to 0x%x, "\ 9399bc5d55SJohn Soni Jose "range allowed is ["#_minval" - "#_maxval"]\n", val);\ 9499bc5d55SJohn Soni Jose return -EINVAL;\ 9599bc5d55SJohn Soni Jose } 9699bc5d55SJohn Soni Jose 9799bc5d55SJohn Soni Jose #define beiscsi_store_param(_name) \ 9899bc5d55SJohn Soni Jose ssize_t \ 9999bc5d55SJohn Soni Jose beiscsi_##_name##_store(struct device *dev,\ 10099bc5d55SJohn Soni Jose struct device_attribute *attr, const char *buf,\ 10199bc5d55SJohn Soni Jose size_t count) \ 10299bc5d55SJohn Soni Jose { \ 10399bc5d55SJohn Soni Jose struct Scsi_Host *shost = class_to_shost(dev);\ 10499bc5d55SJohn Soni Jose struct beiscsi_hba *phba = iscsi_host_priv(shost);\ 10599bc5d55SJohn Soni Jose uint32_t param_val = 0;\ 10699bc5d55SJohn Soni Jose if (!isdigit(buf[0]))\ 10799bc5d55SJohn Soni Jose return -EINVAL;\ 10899bc5d55SJohn Soni Jose if (sscanf(buf, "%i", ¶m_val) != 1)\ 10999bc5d55SJohn Soni Jose return -EINVAL;\ 11099bc5d55SJohn Soni Jose if (beiscsi_##_name##_change(phba, param_val) == 0) \ 11199bc5d55SJohn Soni Jose return strlen(buf);\ 11299bc5d55SJohn Soni Jose else \ 11399bc5d55SJohn Soni Jose return -EINVAL;\ 11499bc5d55SJohn Soni Jose } 11599bc5d55SJohn Soni Jose 11699bc5d55SJohn Soni Jose #define beiscsi_init_param(_name, _minval, _maxval, _defval) \ 11799bc5d55SJohn Soni Jose int \ 11899bc5d55SJohn Soni Jose beiscsi_##_name##_init(struct beiscsi_hba *phba, uint32_t val) \ 11999bc5d55SJohn Soni Jose { \ 12099bc5d55SJohn Soni Jose if (val >= _minval && val <= _maxval) {\ 12199bc5d55SJohn Soni Jose phba->attr_##_name = val;\ 12299bc5d55SJohn Soni Jose return 0;\ 12399bc5d55SJohn Soni Jose } \ 12499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\ 12599bc5d55SJohn Soni Jose "BA_%d beiscsi_"#_name" attribute " \ 12699bc5d55SJohn Soni Jose "cannot be updated to 0x%x, "\ 12799bc5d55SJohn Soni Jose "range allowed is ["#_minval" - "#_maxval"]\n", val);\ 12899bc5d55SJohn Soni Jose phba->attr_##_name = _defval;\ 12999bc5d55SJohn Soni Jose return -EINVAL;\ 13099bc5d55SJohn Soni Jose } 13199bc5d55SJohn Soni Jose 13299bc5d55SJohn Soni Jose #define BEISCSI_RW_ATTR(_name, _minval, _maxval, _defval, _descp) \ 13399bc5d55SJohn Soni Jose static uint beiscsi_##_name = _defval;\ 13499bc5d55SJohn Soni Jose module_param(beiscsi_##_name, uint, S_IRUGO);\ 13599bc5d55SJohn Soni Jose MODULE_PARM_DESC(beiscsi_##_name, _descp);\ 13699bc5d55SJohn Soni Jose beiscsi_disp_param(_name)\ 13799bc5d55SJohn Soni Jose beiscsi_change_param(_name, _minval, _maxval, _defval)\ 13899bc5d55SJohn Soni Jose beiscsi_store_param(_name)\ 13999bc5d55SJohn Soni Jose beiscsi_init_param(_name, _minval, _maxval, _defval)\ 14099bc5d55SJohn Soni Jose DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\ 14199bc5d55SJohn Soni Jose beiscsi_##_name##_disp, beiscsi_##_name##_store) 14299bc5d55SJohn Soni Jose 14399bc5d55SJohn Soni Jose /* 14499bc5d55SJohn Soni Jose * When new log level added update the 14599bc5d55SJohn Soni Jose * the MAX allowed value for log_enable 14699bc5d55SJohn Soni Jose */ 14799bc5d55SJohn Soni Jose BEISCSI_RW_ATTR(log_enable, 0x00, 14899bc5d55SJohn Soni Jose 0xFF, 0x00, "Enable logging Bit Mask\n" 14999bc5d55SJohn Soni Jose "\t\t\t\tInitialization Events : 0x01\n" 15099bc5d55SJohn Soni Jose "\t\t\t\tMailbox Events : 0x02\n" 15199bc5d55SJohn Soni Jose "\t\t\t\tMiscellaneous Events : 0x04\n" 15299bc5d55SJohn Soni Jose "\t\t\t\tError Handling : 0x08\n" 15399bc5d55SJohn Soni Jose "\t\t\t\tIO Path Events : 0x10\n" 15499bc5d55SJohn Soni Jose "\t\t\t\tConfiguration Path : 0x20\n"); 15599bc5d55SJohn Soni Jose 15699bc5d55SJohn Soni Jose struct device_attribute *beiscsi_attrs[] = { 15799bc5d55SJohn Soni Jose &dev_attr_beiscsi_log_enable, 15899bc5d55SJohn Soni Jose NULL, 15999bc5d55SJohn Soni Jose }; 1606733b39aSJayamohan Kallickal 1616763daaeSJohn Soni Jose static char const *cqe_desc[] = { 1626763daaeSJohn Soni Jose "RESERVED_DESC", 1636763daaeSJohn Soni Jose "SOL_CMD_COMPLETE", 1646763daaeSJohn Soni Jose "SOL_CMD_KILLED_DATA_DIGEST_ERR", 1656763daaeSJohn Soni Jose "CXN_KILLED_PDU_SIZE_EXCEEDS_DSL", 1666763daaeSJohn Soni Jose "CXN_KILLED_BURST_LEN_MISMATCH", 1676763daaeSJohn Soni Jose "CXN_KILLED_AHS_RCVD", 1686763daaeSJohn Soni Jose "CXN_KILLED_HDR_DIGEST_ERR", 1696763daaeSJohn Soni Jose "CXN_KILLED_UNKNOWN_HDR", 1706763daaeSJohn Soni Jose "CXN_KILLED_STALE_ITT_TTT_RCVD", 1716763daaeSJohn Soni Jose "CXN_KILLED_INVALID_ITT_TTT_RCVD", 1726763daaeSJohn Soni Jose "CXN_KILLED_RST_RCVD", 1736763daaeSJohn Soni Jose "CXN_KILLED_TIMED_OUT", 1746763daaeSJohn Soni Jose "CXN_KILLED_RST_SENT", 1756763daaeSJohn Soni Jose "CXN_KILLED_FIN_RCVD", 1766763daaeSJohn Soni Jose "CXN_KILLED_BAD_UNSOL_PDU_RCVD", 1776763daaeSJohn Soni Jose "CXN_KILLED_BAD_WRB_INDEX_ERROR", 1786763daaeSJohn Soni Jose "CXN_KILLED_OVER_RUN_RESIDUAL", 1796763daaeSJohn Soni Jose "CXN_KILLED_UNDER_RUN_RESIDUAL", 1806763daaeSJohn Soni Jose "CMD_KILLED_INVALID_STATSN_RCVD", 1816763daaeSJohn Soni Jose "CMD_KILLED_INVALID_R2T_RCVD", 1826763daaeSJohn Soni Jose "CMD_CXN_KILLED_LUN_INVALID", 1836763daaeSJohn Soni Jose "CMD_CXN_KILLED_ICD_INVALID", 1846763daaeSJohn Soni Jose "CMD_CXN_KILLED_ITT_INVALID", 1856763daaeSJohn Soni Jose "CMD_CXN_KILLED_SEQ_OUTOFORDER", 1866763daaeSJohn Soni Jose "CMD_CXN_KILLED_INVALID_DATASN_RCVD", 1876763daaeSJohn Soni Jose "CXN_INVALIDATE_NOTIFY", 1886763daaeSJohn Soni Jose "CXN_INVALIDATE_INDEX_NOTIFY", 1896763daaeSJohn Soni Jose "CMD_INVALIDATED_NOTIFY", 1906763daaeSJohn Soni Jose "UNSOL_HDR_NOTIFY", 1916763daaeSJohn Soni Jose "UNSOL_DATA_NOTIFY", 1926763daaeSJohn Soni Jose "UNSOL_DATA_DIGEST_ERROR_NOTIFY", 1936763daaeSJohn Soni Jose "DRIVERMSG_NOTIFY", 1946763daaeSJohn Soni Jose "CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN", 1956763daaeSJohn Soni Jose "SOL_CMD_KILLED_DIF_ERR", 1966763daaeSJohn Soni Jose "CXN_KILLED_SYN_RCVD", 1976763daaeSJohn Soni Jose "CXN_KILLED_IMM_DATA_RCVD" 1986763daaeSJohn Soni Jose }; 1996763daaeSJohn Soni Jose 2006733b39aSJayamohan Kallickal static int beiscsi_slave_configure(struct scsi_device *sdev) 2016733b39aSJayamohan Kallickal { 2026733b39aSJayamohan Kallickal blk_queue_max_segment_size(sdev->request_queue, 65536); 2036733b39aSJayamohan Kallickal return 0; 2046733b39aSJayamohan Kallickal } 2056733b39aSJayamohan Kallickal 2064183122dSJayamohan Kallickal static int beiscsi_eh_abort(struct scsi_cmnd *sc) 2074183122dSJayamohan Kallickal { 2084183122dSJayamohan Kallickal struct iscsi_cls_session *cls_session; 2094183122dSJayamohan Kallickal struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr; 2104183122dSJayamohan Kallickal struct beiscsi_io_task *aborted_io_task; 2114183122dSJayamohan Kallickal struct iscsi_conn *conn; 2124183122dSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn; 2134183122dSJayamohan Kallickal struct beiscsi_hba *phba; 2144183122dSJayamohan Kallickal struct iscsi_session *session; 2154183122dSJayamohan Kallickal struct invalidate_command_table *inv_tbl; 2163cbb7a74SJayamohan Kallickal struct be_dma_mem nonemb_cmd; 2174183122dSJayamohan Kallickal unsigned int cid, tag, num_invalidate; 2184183122dSJayamohan Kallickal 2194183122dSJayamohan Kallickal cls_session = starget_to_session(scsi_target(sc->device)); 2204183122dSJayamohan Kallickal session = cls_session->dd_data; 2214183122dSJayamohan Kallickal 2224183122dSJayamohan Kallickal spin_lock_bh(&session->lock); 2234183122dSJayamohan Kallickal if (!aborted_task || !aborted_task->sc) { 2244183122dSJayamohan Kallickal /* we raced */ 2254183122dSJayamohan Kallickal spin_unlock_bh(&session->lock); 2264183122dSJayamohan Kallickal return SUCCESS; 2274183122dSJayamohan Kallickal } 2284183122dSJayamohan Kallickal 2294183122dSJayamohan Kallickal aborted_io_task = aborted_task->dd_data; 2304183122dSJayamohan Kallickal if (!aborted_io_task->scsi_cmnd) { 2314183122dSJayamohan Kallickal /* raced or invalid command */ 2324183122dSJayamohan Kallickal spin_unlock_bh(&session->lock); 2334183122dSJayamohan Kallickal return SUCCESS; 2344183122dSJayamohan Kallickal } 2354183122dSJayamohan Kallickal spin_unlock_bh(&session->lock); 2364183122dSJayamohan Kallickal conn = aborted_task->conn; 2374183122dSJayamohan Kallickal beiscsi_conn = conn->dd_data; 2384183122dSJayamohan Kallickal phba = beiscsi_conn->phba; 2394183122dSJayamohan Kallickal 2404183122dSJayamohan Kallickal /* invalidate iocb */ 2414183122dSJayamohan Kallickal cid = beiscsi_conn->beiscsi_conn_cid; 2424183122dSJayamohan Kallickal inv_tbl = phba->inv_tbl; 2434183122dSJayamohan Kallickal memset(inv_tbl, 0x0, sizeof(*inv_tbl)); 2444183122dSJayamohan Kallickal inv_tbl->cid = cid; 2454183122dSJayamohan Kallickal inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index; 2464183122dSJayamohan Kallickal num_invalidate = 1; 2473cbb7a74SJayamohan Kallickal nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev, 2483cbb7a74SJayamohan Kallickal sizeof(struct invalidate_commands_params_in), 2493cbb7a74SJayamohan Kallickal &nonemb_cmd.dma); 2503cbb7a74SJayamohan Kallickal if (nonemb_cmd.va == NULL) { 25199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH, 25299bc5d55SJohn Soni Jose "BM_%d : Failed to allocate memory for" 2533cbb7a74SJayamohan Kallickal "mgmt_invalidate_icds\n"); 2543cbb7a74SJayamohan Kallickal return FAILED; 2553cbb7a74SJayamohan Kallickal } 2563cbb7a74SJayamohan Kallickal nonemb_cmd.size = sizeof(struct invalidate_commands_params_in); 2573cbb7a74SJayamohan Kallickal 2583cbb7a74SJayamohan Kallickal tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate, 2593cbb7a74SJayamohan Kallickal cid, &nonemb_cmd); 2604183122dSJayamohan Kallickal if (!tag) { 26199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH, 26299bc5d55SJohn Soni Jose "BM_%d : mgmt_invalidate_icds could not be" 2634183122dSJayamohan Kallickal "submitted\n"); 2643cbb7a74SJayamohan Kallickal pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 2653cbb7a74SJayamohan Kallickal nonemb_cmd.va, nonemb_cmd.dma); 2663cbb7a74SJayamohan Kallickal 2674183122dSJayamohan Kallickal return FAILED; 2684183122dSJayamohan Kallickal } else { 2694183122dSJayamohan Kallickal wait_event_interruptible(phba->ctrl.mcc_wait[tag], 2704183122dSJayamohan Kallickal phba->ctrl.mcc_numtag[tag]); 2714183122dSJayamohan Kallickal free_mcc_tag(&phba->ctrl, tag); 2724183122dSJayamohan Kallickal } 2733cbb7a74SJayamohan Kallickal pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 2743cbb7a74SJayamohan Kallickal nonemb_cmd.va, nonemb_cmd.dma); 2754183122dSJayamohan Kallickal return iscsi_eh_abort(sc); 2764183122dSJayamohan Kallickal } 2774183122dSJayamohan Kallickal 2784183122dSJayamohan Kallickal static int beiscsi_eh_device_reset(struct scsi_cmnd *sc) 2794183122dSJayamohan Kallickal { 2804183122dSJayamohan Kallickal struct iscsi_task *abrt_task; 2814183122dSJayamohan Kallickal struct beiscsi_io_task *abrt_io_task; 2824183122dSJayamohan Kallickal struct iscsi_conn *conn; 2834183122dSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn; 2844183122dSJayamohan Kallickal struct beiscsi_hba *phba; 2854183122dSJayamohan Kallickal struct iscsi_session *session; 2864183122dSJayamohan Kallickal struct iscsi_cls_session *cls_session; 2874183122dSJayamohan Kallickal struct invalidate_command_table *inv_tbl; 2883cbb7a74SJayamohan Kallickal struct be_dma_mem nonemb_cmd; 2894183122dSJayamohan Kallickal unsigned int cid, tag, i, num_invalidate; 2904183122dSJayamohan Kallickal 2914183122dSJayamohan Kallickal /* invalidate iocbs */ 2924183122dSJayamohan Kallickal cls_session = starget_to_session(scsi_target(sc->device)); 2934183122dSJayamohan Kallickal session = cls_session->dd_data; 2944183122dSJayamohan Kallickal spin_lock_bh(&session->lock); 295db7f7709SJayamohan Kallickal if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) { 296db7f7709SJayamohan Kallickal spin_unlock_bh(&session->lock); 297db7f7709SJayamohan Kallickal return FAILED; 298db7f7709SJayamohan Kallickal } 2994183122dSJayamohan Kallickal conn = session->leadconn; 3004183122dSJayamohan Kallickal beiscsi_conn = conn->dd_data; 3014183122dSJayamohan Kallickal phba = beiscsi_conn->phba; 3024183122dSJayamohan Kallickal cid = beiscsi_conn->beiscsi_conn_cid; 3034183122dSJayamohan Kallickal inv_tbl = phba->inv_tbl; 3044183122dSJayamohan Kallickal memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN); 3054183122dSJayamohan Kallickal num_invalidate = 0; 3064183122dSJayamohan Kallickal for (i = 0; i < conn->session->cmds_max; i++) { 3074183122dSJayamohan Kallickal abrt_task = conn->session->cmds[i]; 3084183122dSJayamohan Kallickal abrt_io_task = abrt_task->dd_data; 3094183122dSJayamohan Kallickal if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE) 3104183122dSJayamohan Kallickal continue; 3114183122dSJayamohan Kallickal 3124183122dSJayamohan Kallickal if (abrt_task->sc->device->lun != abrt_task->sc->device->lun) 3134183122dSJayamohan Kallickal continue; 3144183122dSJayamohan Kallickal 3154183122dSJayamohan Kallickal inv_tbl->cid = cid; 3164183122dSJayamohan Kallickal inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index; 3174183122dSJayamohan Kallickal num_invalidate++; 3184183122dSJayamohan Kallickal inv_tbl++; 3194183122dSJayamohan Kallickal } 3204183122dSJayamohan Kallickal spin_unlock_bh(&session->lock); 3214183122dSJayamohan Kallickal inv_tbl = phba->inv_tbl; 3224183122dSJayamohan Kallickal 3233cbb7a74SJayamohan Kallickal nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev, 3243cbb7a74SJayamohan Kallickal sizeof(struct invalidate_commands_params_in), 3253cbb7a74SJayamohan Kallickal &nonemb_cmd.dma); 3263cbb7a74SJayamohan Kallickal if (nonemb_cmd.va == NULL) { 32799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH, 32899bc5d55SJohn Soni Jose "BM_%d : Failed to allocate memory for" 3293cbb7a74SJayamohan Kallickal "mgmt_invalidate_icds\n"); 3303cbb7a74SJayamohan Kallickal return FAILED; 3313cbb7a74SJayamohan Kallickal } 3323cbb7a74SJayamohan Kallickal nonemb_cmd.size = sizeof(struct invalidate_commands_params_in); 3333cbb7a74SJayamohan Kallickal memset(nonemb_cmd.va, 0, nonemb_cmd.size); 3343cbb7a74SJayamohan Kallickal tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate, 3353cbb7a74SJayamohan Kallickal cid, &nonemb_cmd); 3364183122dSJayamohan Kallickal if (!tag) { 33799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH, 33899bc5d55SJohn Soni Jose "BM_%d : mgmt_invalidate_icds could not be" 3394183122dSJayamohan Kallickal " submitted\n"); 3403cbb7a74SJayamohan Kallickal pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 3413cbb7a74SJayamohan Kallickal nonemb_cmd.va, nonemb_cmd.dma); 3424183122dSJayamohan Kallickal return FAILED; 3434183122dSJayamohan Kallickal } else { 3444183122dSJayamohan Kallickal wait_event_interruptible(phba->ctrl.mcc_wait[tag], 3454183122dSJayamohan Kallickal phba->ctrl.mcc_numtag[tag]); 3464183122dSJayamohan Kallickal free_mcc_tag(&phba->ctrl, tag); 3474183122dSJayamohan Kallickal } 3483cbb7a74SJayamohan Kallickal pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 3493cbb7a74SJayamohan Kallickal nonemb_cmd.va, nonemb_cmd.dma); 3504183122dSJayamohan Kallickal return iscsi_eh_device_reset(sc); 3514183122dSJayamohan Kallickal } 3524183122dSJayamohan Kallickal 353c7acc5b8SJayamohan Kallickal static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf) 354c7acc5b8SJayamohan Kallickal { 355c7acc5b8SJayamohan Kallickal struct beiscsi_hba *phba = data; 356f457a46fSMike Christie struct mgmt_session_info *boot_sess = &phba->boot_sess; 357f457a46fSMike Christie struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0]; 358c7acc5b8SJayamohan Kallickal char *str = buf; 359c7acc5b8SJayamohan Kallickal int rc; 360c7acc5b8SJayamohan Kallickal 361c7acc5b8SJayamohan Kallickal switch (type) { 362c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_NAME: 363c7acc5b8SJayamohan Kallickal rc = sprintf(buf, "%.*s\n", 364f457a46fSMike Christie (int)strlen(boot_sess->target_name), 365f457a46fSMike Christie (char *)&boot_sess->target_name); 366c7acc5b8SJayamohan Kallickal break; 367c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_IP_ADDR: 368f457a46fSMike Christie if (boot_conn->dest_ipaddr.ip_type == 0x1) 369c7acc5b8SJayamohan Kallickal rc = sprintf(buf, "%pI4\n", 3700e43895eSMike Christie (char *)&boot_conn->dest_ipaddr.addr); 371c7acc5b8SJayamohan Kallickal else 372c7acc5b8SJayamohan Kallickal rc = sprintf(str, "%pI6\n", 3730e43895eSMike Christie (char *)&boot_conn->dest_ipaddr.addr); 374c7acc5b8SJayamohan Kallickal break; 375c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_PORT: 376f457a46fSMike Christie rc = sprintf(str, "%d\n", boot_conn->dest_port); 377c7acc5b8SJayamohan Kallickal break; 378c7acc5b8SJayamohan Kallickal 379c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_CHAP_NAME: 380c7acc5b8SJayamohan Kallickal rc = sprintf(str, "%.*s\n", 381f457a46fSMike Christie boot_conn->negotiated_login_options.auth_data.chap. 382c7acc5b8SJayamohan Kallickal target_chap_name_length, 383f457a46fSMike Christie (char *)&boot_conn->negotiated_login_options. 384f457a46fSMike Christie auth_data.chap.target_chap_name); 385c7acc5b8SJayamohan Kallickal break; 386c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_CHAP_SECRET: 387c7acc5b8SJayamohan Kallickal rc = sprintf(str, "%.*s\n", 388f457a46fSMike Christie boot_conn->negotiated_login_options.auth_data.chap. 389c7acc5b8SJayamohan Kallickal target_secret_length, 390f457a46fSMike Christie (char *)&boot_conn->negotiated_login_options. 391f457a46fSMike Christie auth_data.chap.target_secret); 392c7acc5b8SJayamohan Kallickal break; 393c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_REV_CHAP_NAME: 394c7acc5b8SJayamohan Kallickal rc = sprintf(str, "%.*s\n", 395f457a46fSMike Christie boot_conn->negotiated_login_options.auth_data.chap. 396c7acc5b8SJayamohan Kallickal intr_chap_name_length, 397f457a46fSMike Christie (char *)&boot_conn->negotiated_login_options. 398f457a46fSMike Christie auth_data.chap.intr_chap_name); 399c7acc5b8SJayamohan Kallickal break; 400c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_REV_CHAP_SECRET: 401c7acc5b8SJayamohan Kallickal rc = sprintf(str, "%.*s\n", 402f457a46fSMike Christie boot_conn->negotiated_login_options.auth_data.chap. 403c7acc5b8SJayamohan Kallickal intr_secret_length, 404f457a46fSMike Christie (char *)&boot_conn->negotiated_login_options. 405f457a46fSMike Christie auth_data.chap.intr_secret); 406c7acc5b8SJayamohan Kallickal break; 407c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_FLAGS: 408c7acc5b8SJayamohan Kallickal rc = sprintf(str, "2\n"); 409c7acc5b8SJayamohan Kallickal break; 410c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_NIC_ASSOC: 411c7acc5b8SJayamohan Kallickal rc = sprintf(str, "0\n"); 412c7acc5b8SJayamohan Kallickal break; 413c7acc5b8SJayamohan Kallickal default: 414c7acc5b8SJayamohan Kallickal rc = -ENOSYS; 415c7acc5b8SJayamohan Kallickal break; 416c7acc5b8SJayamohan Kallickal } 417c7acc5b8SJayamohan Kallickal return rc; 418c7acc5b8SJayamohan Kallickal } 419c7acc5b8SJayamohan Kallickal 420c7acc5b8SJayamohan Kallickal static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf) 421c7acc5b8SJayamohan Kallickal { 422c7acc5b8SJayamohan Kallickal struct beiscsi_hba *phba = data; 423c7acc5b8SJayamohan Kallickal char *str = buf; 424c7acc5b8SJayamohan Kallickal int rc; 425c7acc5b8SJayamohan Kallickal 426c7acc5b8SJayamohan Kallickal switch (type) { 427c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_INI_INITIATOR_NAME: 428c7acc5b8SJayamohan Kallickal rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname); 429c7acc5b8SJayamohan Kallickal break; 430c7acc5b8SJayamohan Kallickal default: 431c7acc5b8SJayamohan Kallickal rc = -ENOSYS; 432c7acc5b8SJayamohan Kallickal break; 433c7acc5b8SJayamohan Kallickal } 434c7acc5b8SJayamohan Kallickal return rc; 435c7acc5b8SJayamohan Kallickal } 436c7acc5b8SJayamohan Kallickal 437c7acc5b8SJayamohan Kallickal static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf) 438c7acc5b8SJayamohan Kallickal { 439c7acc5b8SJayamohan Kallickal struct beiscsi_hba *phba = data; 440c7acc5b8SJayamohan Kallickal char *str = buf; 441c7acc5b8SJayamohan Kallickal int rc; 442c7acc5b8SJayamohan Kallickal 443c7acc5b8SJayamohan Kallickal switch (type) { 444c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_ETH_FLAGS: 445c7acc5b8SJayamohan Kallickal rc = sprintf(str, "2\n"); 446c7acc5b8SJayamohan Kallickal break; 447c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_ETH_INDEX: 448c7acc5b8SJayamohan Kallickal rc = sprintf(str, "0\n"); 449c7acc5b8SJayamohan Kallickal break; 450c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_ETH_MAC: 4510e43895eSMike Christie rc = beiscsi_get_macaddr(str, phba); 452c7acc5b8SJayamohan Kallickal break; 453c7acc5b8SJayamohan Kallickal default: 454c7acc5b8SJayamohan Kallickal rc = -ENOSYS; 455c7acc5b8SJayamohan Kallickal break; 456c7acc5b8SJayamohan Kallickal } 457c7acc5b8SJayamohan Kallickal return rc; 458c7acc5b8SJayamohan Kallickal } 459c7acc5b8SJayamohan Kallickal 460c7acc5b8SJayamohan Kallickal 461587a1f16SAl Viro static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type) 462c7acc5b8SJayamohan Kallickal { 463587a1f16SAl Viro umode_t rc; 464c7acc5b8SJayamohan Kallickal 465c7acc5b8SJayamohan Kallickal switch (type) { 466c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_NAME: 467c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_IP_ADDR: 468c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_PORT: 469c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_CHAP_NAME: 470c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_CHAP_SECRET: 471c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_REV_CHAP_NAME: 472c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_REV_CHAP_SECRET: 473c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_NIC_ASSOC: 474c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_TGT_FLAGS: 475c7acc5b8SJayamohan Kallickal rc = S_IRUGO; 476c7acc5b8SJayamohan Kallickal break; 477c7acc5b8SJayamohan Kallickal default: 478c7acc5b8SJayamohan Kallickal rc = 0; 479c7acc5b8SJayamohan Kallickal break; 480c7acc5b8SJayamohan Kallickal } 481c7acc5b8SJayamohan Kallickal return rc; 482c7acc5b8SJayamohan Kallickal } 483c7acc5b8SJayamohan Kallickal 484587a1f16SAl Viro static umode_t beiscsi_ini_get_attr_visibility(void *data, int type) 485c7acc5b8SJayamohan Kallickal { 486587a1f16SAl Viro umode_t rc; 487c7acc5b8SJayamohan Kallickal 488c7acc5b8SJayamohan Kallickal switch (type) { 489c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_INI_INITIATOR_NAME: 490c7acc5b8SJayamohan Kallickal rc = S_IRUGO; 491c7acc5b8SJayamohan Kallickal break; 492c7acc5b8SJayamohan Kallickal default: 493c7acc5b8SJayamohan Kallickal rc = 0; 494c7acc5b8SJayamohan Kallickal break; 495c7acc5b8SJayamohan Kallickal } 496c7acc5b8SJayamohan Kallickal return rc; 497c7acc5b8SJayamohan Kallickal } 498c7acc5b8SJayamohan Kallickal 499c7acc5b8SJayamohan Kallickal 500587a1f16SAl Viro static umode_t beiscsi_eth_get_attr_visibility(void *data, int type) 501c7acc5b8SJayamohan Kallickal { 502587a1f16SAl Viro umode_t rc; 503c7acc5b8SJayamohan Kallickal 504c7acc5b8SJayamohan Kallickal switch (type) { 505c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_ETH_FLAGS: 506c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_ETH_MAC: 507c7acc5b8SJayamohan Kallickal case ISCSI_BOOT_ETH_INDEX: 508c7acc5b8SJayamohan Kallickal rc = S_IRUGO; 509c7acc5b8SJayamohan Kallickal break; 510c7acc5b8SJayamohan Kallickal default: 511c7acc5b8SJayamohan Kallickal rc = 0; 512c7acc5b8SJayamohan Kallickal break; 513c7acc5b8SJayamohan Kallickal } 514c7acc5b8SJayamohan Kallickal return rc; 515c7acc5b8SJayamohan Kallickal } 516c7acc5b8SJayamohan Kallickal 517bfead3b2SJayamohan Kallickal /*------------------- PCI Driver operations and data ----------------- */ 518bfead3b2SJayamohan Kallickal static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = { 519bfead3b2SJayamohan Kallickal { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) }, 520f98c96b0SJayamohan Kallickal { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) }, 521bfead3b2SJayamohan Kallickal { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) }, 522bfead3b2SJayamohan Kallickal { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) }, 523bfead3b2SJayamohan Kallickal { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) }, 524bfead3b2SJayamohan Kallickal { 0 } 525bfead3b2SJayamohan Kallickal }; 526bfead3b2SJayamohan Kallickal MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table); 527bfead3b2SJayamohan Kallickal 52899bc5d55SJohn Soni Jose 5296733b39aSJayamohan Kallickal static struct scsi_host_template beiscsi_sht = { 5306733b39aSJayamohan Kallickal .module = THIS_MODULE, 5312f635883SJayamohan Kallickal .name = "Emulex 10Gbe open-iscsi Initiator Driver", 5326733b39aSJayamohan Kallickal .proc_name = DRV_NAME, 5336733b39aSJayamohan Kallickal .queuecommand = iscsi_queuecommand, 5346733b39aSJayamohan Kallickal .change_queue_depth = iscsi_change_queue_depth, 5356733b39aSJayamohan Kallickal .slave_configure = beiscsi_slave_configure, 5366733b39aSJayamohan Kallickal .target_alloc = iscsi_target_alloc, 5374183122dSJayamohan Kallickal .eh_abort_handler = beiscsi_eh_abort, 5384183122dSJayamohan Kallickal .eh_device_reset_handler = beiscsi_eh_device_reset, 539309ce156SJayamohan Kallickal .eh_target_reset_handler = iscsi_eh_session_reset, 54099bc5d55SJohn Soni Jose .shost_attrs = beiscsi_attrs, 5416733b39aSJayamohan Kallickal .sg_tablesize = BEISCSI_SGLIST_ELEMENTS, 5426733b39aSJayamohan Kallickal .can_queue = BE2_IO_DEPTH, 5436733b39aSJayamohan Kallickal .this_id = -1, 5446733b39aSJayamohan Kallickal .max_sectors = BEISCSI_MAX_SECTORS, 5456733b39aSJayamohan Kallickal .cmd_per_lun = BEISCSI_CMD_PER_LUN, 5466733b39aSJayamohan Kallickal .use_clustering = ENABLE_CLUSTERING, 547ffce3e2eSJayamohan Kallickal .vendor_id = SCSI_NL_VID_TYPE_PCI | BE_VENDOR_ID, 548ffce3e2eSJayamohan Kallickal 5496733b39aSJayamohan Kallickal }; 5506733b39aSJayamohan Kallickal 551bfead3b2SJayamohan Kallickal static struct scsi_transport_template *beiscsi_scsi_transport; 5526733b39aSJayamohan Kallickal 5536733b39aSJayamohan Kallickal static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev) 5546733b39aSJayamohan Kallickal { 5556733b39aSJayamohan Kallickal struct beiscsi_hba *phba; 5566733b39aSJayamohan Kallickal struct Scsi_Host *shost; 5576733b39aSJayamohan Kallickal 5586733b39aSJayamohan Kallickal shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0); 5596733b39aSJayamohan Kallickal if (!shost) { 56099bc5d55SJohn Soni Jose dev_err(&pcidev->dev, 56199bc5d55SJohn Soni Jose "beiscsi_hba_alloc - iscsi_host_alloc failed\n"); 5626733b39aSJayamohan Kallickal return NULL; 5636733b39aSJayamohan Kallickal } 5646733b39aSJayamohan Kallickal shost->dma_boundary = pcidev->dma_mask; 5656733b39aSJayamohan Kallickal shost->max_id = BE2_MAX_SESSIONS; 5666733b39aSJayamohan Kallickal shost->max_channel = 0; 5676733b39aSJayamohan Kallickal shost->max_cmd_len = BEISCSI_MAX_CMD_LEN; 5686733b39aSJayamohan Kallickal shost->max_lun = BEISCSI_NUM_MAX_LUN; 5696733b39aSJayamohan Kallickal shost->transportt = beiscsi_scsi_transport; 5706733b39aSJayamohan Kallickal phba = iscsi_host_priv(shost); 5716733b39aSJayamohan Kallickal memset(phba, 0, sizeof(*phba)); 5726733b39aSJayamohan Kallickal phba->shost = shost; 5736733b39aSJayamohan Kallickal phba->pcidev = pci_dev_get(pcidev); 5742807afb7SJayamohan Kallickal pci_set_drvdata(pcidev, phba); 5750e43895eSMike Christie phba->interface_handle = 0xFFFFFFFF; 5766733b39aSJayamohan Kallickal 5776733b39aSJayamohan Kallickal if (iscsi_host_add(shost, &phba->pcidev->dev)) 5786733b39aSJayamohan Kallickal goto free_devices; 579c7acc5b8SJayamohan Kallickal 5806733b39aSJayamohan Kallickal return phba; 5816733b39aSJayamohan Kallickal 5826733b39aSJayamohan Kallickal free_devices: 5836733b39aSJayamohan Kallickal pci_dev_put(phba->pcidev); 5846733b39aSJayamohan Kallickal iscsi_host_free(phba->shost); 5856733b39aSJayamohan Kallickal return NULL; 5866733b39aSJayamohan Kallickal } 5876733b39aSJayamohan Kallickal 5886733b39aSJayamohan Kallickal static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba) 5896733b39aSJayamohan Kallickal { 5906733b39aSJayamohan Kallickal if (phba->csr_va) { 5916733b39aSJayamohan Kallickal iounmap(phba->csr_va); 5926733b39aSJayamohan Kallickal phba->csr_va = NULL; 5936733b39aSJayamohan Kallickal } 5946733b39aSJayamohan Kallickal if (phba->db_va) { 5956733b39aSJayamohan Kallickal iounmap(phba->db_va); 5966733b39aSJayamohan Kallickal phba->db_va = NULL; 5976733b39aSJayamohan Kallickal } 5986733b39aSJayamohan Kallickal if (phba->pci_va) { 5996733b39aSJayamohan Kallickal iounmap(phba->pci_va); 6006733b39aSJayamohan Kallickal phba->pci_va = NULL; 6016733b39aSJayamohan Kallickal } 6026733b39aSJayamohan Kallickal } 6036733b39aSJayamohan Kallickal 6046733b39aSJayamohan Kallickal static int beiscsi_map_pci_bars(struct beiscsi_hba *phba, 6056733b39aSJayamohan Kallickal struct pci_dev *pcidev) 6066733b39aSJayamohan Kallickal { 6076733b39aSJayamohan Kallickal u8 __iomem *addr; 608f98c96b0SJayamohan Kallickal int pcicfg_reg; 6096733b39aSJayamohan Kallickal 6106733b39aSJayamohan Kallickal addr = ioremap_nocache(pci_resource_start(pcidev, 2), 6116733b39aSJayamohan Kallickal pci_resource_len(pcidev, 2)); 6126733b39aSJayamohan Kallickal if (addr == NULL) 6136733b39aSJayamohan Kallickal return -ENOMEM; 6146733b39aSJayamohan Kallickal phba->ctrl.csr = addr; 6156733b39aSJayamohan Kallickal phba->csr_va = addr; 6166733b39aSJayamohan Kallickal phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2); 6176733b39aSJayamohan Kallickal 6186733b39aSJayamohan Kallickal addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024); 6196733b39aSJayamohan Kallickal if (addr == NULL) 6206733b39aSJayamohan Kallickal goto pci_map_err; 6216733b39aSJayamohan Kallickal phba->ctrl.db = addr; 6226733b39aSJayamohan Kallickal phba->db_va = addr; 6236733b39aSJayamohan Kallickal phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4); 6246733b39aSJayamohan Kallickal 625f98c96b0SJayamohan Kallickal if (phba->generation == BE_GEN2) 626f98c96b0SJayamohan Kallickal pcicfg_reg = 1; 627f98c96b0SJayamohan Kallickal else 628f98c96b0SJayamohan Kallickal pcicfg_reg = 0; 629f98c96b0SJayamohan Kallickal 630f98c96b0SJayamohan Kallickal addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg), 631f98c96b0SJayamohan Kallickal pci_resource_len(pcidev, pcicfg_reg)); 632f98c96b0SJayamohan Kallickal 6336733b39aSJayamohan Kallickal if (addr == NULL) 6346733b39aSJayamohan Kallickal goto pci_map_err; 6356733b39aSJayamohan Kallickal phba->ctrl.pcicfg = addr; 6366733b39aSJayamohan Kallickal phba->pci_va = addr; 637f98c96b0SJayamohan Kallickal phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg); 6386733b39aSJayamohan Kallickal return 0; 6396733b39aSJayamohan Kallickal 6406733b39aSJayamohan Kallickal pci_map_err: 6416733b39aSJayamohan Kallickal beiscsi_unmap_pci_function(phba); 6426733b39aSJayamohan Kallickal return -ENOMEM; 6436733b39aSJayamohan Kallickal } 6446733b39aSJayamohan Kallickal 6456733b39aSJayamohan Kallickal static int beiscsi_enable_pci(struct pci_dev *pcidev) 6466733b39aSJayamohan Kallickal { 6476733b39aSJayamohan Kallickal int ret; 6486733b39aSJayamohan Kallickal 6496733b39aSJayamohan Kallickal ret = pci_enable_device(pcidev); 6506733b39aSJayamohan Kallickal if (ret) { 65199bc5d55SJohn Soni Jose dev_err(&pcidev->dev, 65299bc5d55SJohn Soni Jose "beiscsi_enable_pci - enable device failed\n"); 6536733b39aSJayamohan Kallickal return ret; 6546733b39aSJayamohan Kallickal } 6556733b39aSJayamohan Kallickal 656bfead3b2SJayamohan Kallickal pci_set_master(pcidev); 6576733b39aSJayamohan Kallickal if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) { 6586733b39aSJayamohan Kallickal ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32)); 6596733b39aSJayamohan Kallickal if (ret) { 6606733b39aSJayamohan Kallickal dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n"); 6616733b39aSJayamohan Kallickal pci_disable_device(pcidev); 6626733b39aSJayamohan Kallickal return ret; 6636733b39aSJayamohan Kallickal } 6646733b39aSJayamohan Kallickal } 6656733b39aSJayamohan Kallickal return 0; 6666733b39aSJayamohan Kallickal } 6676733b39aSJayamohan Kallickal 6686733b39aSJayamohan Kallickal static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev) 6696733b39aSJayamohan Kallickal { 6706733b39aSJayamohan Kallickal struct be_ctrl_info *ctrl = &phba->ctrl; 6716733b39aSJayamohan Kallickal struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced; 6726733b39aSJayamohan Kallickal struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem; 6736733b39aSJayamohan Kallickal int status = 0; 6746733b39aSJayamohan Kallickal 6756733b39aSJayamohan Kallickal ctrl->pdev = pdev; 6766733b39aSJayamohan Kallickal status = beiscsi_map_pci_bars(phba, pdev); 6776733b39aSJayamohan Kallickal if (status) 6786733b39aSJayamohan Kallickal return status; 6796733b39aSJayamohan Kallickal mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16; 6806733b39aSJayamohan Kallickal mbox_mem_alloc->va = pci_alloc_consistent(pdev, 6816733b39aSJayamohan Kallickal mbox_mem_alloc->size, 6826733b39aSJayamohan Kallickal &mbox_mem_alloc->dma); 6836733b39aSJayamohan Kallickal if (!mbox_mem_alloc->va) { 6846733b39aSJayamohan Kallickal beiscsi_unmap_pci_function(phba); 685a49e06d5SJayamohan Kallickal return -ENOMEM; 6866733b39aSJayamohan Kallickal } 6876733b39aSJayamohan Kallickal 6886733b39aSJayamohan Kallickal mbox_mem_align->size = sizeof(struct be_mcc_mailbox); 6896733b39aSJayamohan Kallickal mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16); 6906733b39aSJayamohan Kallickal mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16); 6916733b39aSJayamohan Kallickal memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox)); 6926733b39aSJayamohan Kallickal spin_lock_init(&ctrl->mbox_lock); 693bfead3b2SJayamohan Kallickal spin_lock_init(&phba->ctrl.mcc_lock); 694bfead3b2SJayamohan Kallickal spin_lock_init(&phba->ctrl.mcc_cq_lock); 695bfead3b2SJayamohan Kallickal 6966733b39aSJayamohan Kallickal return status; 6976733b39aSJayamohan Kallickal } 6986733b39aSJayamohan Kallickal 6996733b39aSJayamohan Kallickal static void beiscsi_get_params(struct beiscsi_hba *phba) 7006733b39aSJayamohan Kallickal { 7017da50879SJayamohan Kallickal phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count 7027da50879SJayamohan Kallickal - (phba->fw_config.iscsi_cid_count 7037da50879SJayamohan Kallickal + BE2_TMFS 7047da50879SJayamohan Kallickal + BE2_NOPOUT_REQ)); 7057da50879SJayamohan Kallickal phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count; 706ed58ea2aSJayamohan Kallickal phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2; 7076eab04a8SJustin P. Mattock phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count; 7086733b39aSJayamohan Kallickal phba->params.num_sge_per_io = BE2_SGE; 7096733b39aSJayamohan Kallickal phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ; 7106733b39aSJayamohan Kallickal phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ; 7116733b39aSJayamohan Kallickal phba->params.eq_timer = 64; 7126733b39aSJayamohan Kallickal phba->params.num_eq_entries = 7137da50879SJayamohan Kallickal (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2 7147da50879SJayamohan Kallickal + BE2_TMFS) / 512) + 1) * 512; 7156733b39aSJayamohan Kallickal phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024) 7166733b39aSJayamohan Kallickal ? 1024 : phba->params.num_eq_entries; 71799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 71899bc5d55SJohn Soni Jose "BM_%d : phba->params.num_eq_entries=%d\n", 7196733b39aSJayamohan Kallickal phba->params.num_eq_entries); 7206733b39aSJayamohan Kallickal phba->params.num_cq_entries = 7217da50879SJayamohan Kallickal (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2 7227da50879SJayamohan Kallickal + BE2_TMFS) / 512) + 1) * 512; 7236733b39aSJayamohan Kallickal phba->params.wrbs_per_cxn = 256; 7246733b39aSJayamohan Kallickal } 7256733b39aSJayamohan Kallickal 7266733b39aSJayamohan Kallickal static void hwi_ring_eq_db(struct beiscsi_hba *phba, 7276733b39aSJayamohan Kallickal unsigned int id, unsigned int clr_interrupt, 7286733b39aSJayamohan Kallickal unsigned int num_processed, 7296733b39aSJayamohan Kallickal unsigned char rearm, unsigned char event) 7306733b39aSJayamohan Kallickal { 7316733b39aSJayamohan Kallickal u32 val = 0; 7326733b39aSJayamohan Kallickal val |= id & DB_EQ_RING_ID_MASK; 7336733b39aSJayamohan Kallickal if (rearm) 7346733b39aSJayamohan Kallickal val |= 1 << DB_EQ_REARM_SHIFT; 7356733b39aSJayamohan Kallickal if (clr_interrupt) 7366733b39aSJayamohan Kallickal val |= 1 << DB_EQ_CLR_SHIFT; 7376733b39aSJayamohan Kallickal if (event) 7386733b39aSJayamohan Kallickal val |= 1 << DB_EQ_EVNT_SHIFT; 7396733b39aSJayamohan Kallickal val |= num_processed << DB_EQ_NUM_POPPED_SHIFT; 7406733b39aSJayamohan Kallickal iowrite32(val, phba->db_va + DB_EQ_OFFSET); 7416733b39aSJayamohan Kallickal } 7426733b39aSJayamohan Kallickal 7436733b39aSJayamohan Kallickal /** 744bfead3b2SJayamohan Kallickal * be_isr_mcc - The isr routine of the driver. 745bfead3b2SJayamohan Kallickal * @irq: Not used 746bfead3b2SJayamohan Kallickal * @dev_id: Pointer to host adapter structure 747bfead3b2SJayamohan Kallickal */ 748bfead3b2SJayamohan Kallickal static irqreturn_t be_isr_mcc(int irq, void *dev_id) 749bfead3b2SJayamohan Kallickal { 750bfead3b2SJayamohan Kallickal struct beiscsi_hba *phba; 751bfead3b2SJayamohan Kallickal struct be_eq_entry *eqe = NULL; 752bfead3b2SJayamohan Kallickal struct be_queue_info *eq; 753bfead3b2SJayamohan Kallickal struct be_queue_info *mcc; 754bfead3b2SJayamohan Kallickal unsigned int num_eq_processed; 755bfead3b2SJayamohan Kallickal struct be_eq_obj *pbe_eq; 756bfead3b2SJayamohan Kallickal unsigned long flags; 757bfead3b2SJayamohan Kallickal 758bfead3b2SJayamohan Kallickal pbe_eq = dev_id; 759bfead3b2SJayamohan Kallickal eq = &pbe_eq->q; 760bfead3b2SJayamohan Kallickal phba = pbe_eq->phba; 761bfead3b2SJayamohan Kallickal mcc = &phba->ctrl.mcc_obj.cq; 762bfead3b2SJayamohan Kallickal eqe = queue_tail_node(eq); 763bfead3b2SJayamohan Kallickal 764bfead3b2SJayamohan Kallickal num_eq_processed = 0; 765bfead3b2SJayamohan Kallickal 766bfead3b2SJayamohan Kallickal while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 767bfead3b2SJayamohan Kallickal & EQE_VALID_MASK) { 768bfead3b2SJayamohan Kallickal if (((eqe->dw[offsetof(struct amap_eq_entry, 769bfead3b2SJayamohan Kallickal resource_id) / 32] & 770bfead3b2SJayamohan Kallickal EQE_RESID_MASK) >> 16) == mcc->id) { 771bfead3b2SJayamohan Kallickal spin_lock_irqsave(&phba->isr_lock, flags); 772bfead3b2SJayamohan Kallickal phba->todo_mcc_cq = 1; 773bfead3b2SJayamohan Kallickal spin_unlock_irqrestore(&phba->isr_lock, flags); 774bfead3b2SJayamohan Kallickal } 775bfead3b2SJayamohan Kallickal AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 776bfead3b2SJayamohan Kallickal queue_tail_inc(eq); 777bfead3b2SJayamohan Kallickal eqe = queue_tail_node(eq); 778bfead3b2SJayamohan Kallickal num_eq_processed++; 779bfead3b2SJayamohan Kallickal } 780bfead3b2SJayamohan Kallickal if (phba->todo_mcc_cq) 781bfead3b2SJayamohan Kallickal queue_work(phba->wq, &phba->work_cqs); 782bfead3b2SJayamohan Kallickal if (num_eq_processed) 783bfead3b2SJayamohan Kallickal hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1); 784bfead3b2SJayamohan Kallickal 785bfead3b2SJayamohan Kallickal return IRQ_HANDLED; 786bfead3b2SJayamohan Kallickal } 787bfead3b2SJayamohan Kallickal 788bfead3b2SJayamohan Kallickal /** 789bfead3b2SJayamohan Kallickal * be_isr_msix - The isr routine of the driver. 790bfead3b2SJayamohan Kallickal * @irq: Not used 791bfead3b2SJayamohan Kallickal * @dev_id: Pointer to host adapter structure 792bfead3b2SJayamohan Kallickal */ 793bfead3b2SJayamohan Kallickal static irqreturn_t be_isr_msix(int irq, void *dev_id) 794bfead3b2SJayamohan Kallickal { 795bfead3b2SJayamohan Kallickal struct beiscsi_hba *phba; 796bfead3b2SJayamohan Kallickal struct be_eq_entry *eqe = NULL; 797bfead3b2SJayamohan Kallickal struct be_queue_info *eq; 798bfead3b2SJayamohan Kallickal struct be_queue_info *cq; 799bfead3b2SJayamohan Kallickal unsigned int num_eq_processed; 800bfead3b2SJayamohan Kallickal struct be_eq_obj *pbe_eq; 801bfead3b2SJayamohan Kallickal unsigned long flags; 802bfead3b2SJayamohan Kallickal 803bfead3b2SJayamohan Kallickal pbe_eq = dev_id; 804bfead3b2SJayamohan Kallickal eq = &pbe_eq->q; 805bfead3b2SJayamohan Kallickal cq = pbe_eq->cq; 806bfead3b2SJayamohan Kallickal eqe = queue_tail_node(eq); 807bfead3b2SJayamohan Kallickal 808bfead3b2SJayamohan Kallickal phba = pbe_eq->phba; 809bfead3b2SJayamohan Kallickal num_eq_processed = 0; 810bfead3b2SJayamohan Kallickal if (blk_iopoll_enabled) { 811bfead3b2SJayamohan Kallickal while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 812bfead3b2SJayamohan Kallickal & EQE_VALID_MASK) { 813bfead3b2SJayamohan Kallickal if (!blk_iopoll_sched_prep(&pbe_eq->iopoll)) 814bfead3b2SJayamohan Kallickal blk_iopoll_sched(&pbe_eq->iopoll); 815bfead3b2SJayamohan Kallickal 816bfead3b2SJayamohan Kallickal AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 817bfead3b2SJayamohan Kallickal queue_tail_inc(eq); 818bfead3b2SJayamohan Kallickal eqe = queue_tail_node(eq); 819bfead3b2SJayamohan Kallickal num_eq_processed++; 820bfead3b2SJayamohan Kallickal } 821bfead3b2SJayamohan Kallickal if (num_eq_processed) 822bfead3b2SJayamohan Kallickal hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1); 823bfead3b2SJayamohan Kallickal 824bfead3b2SJayamohan Kallickal return IRQ_HANDLED; 825bfead3b2SJayamohan Kallickal } else { 826bfead3b2SJayamohan Kallickal while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 827bfead3b2SJayamohan Kallickal & EQE_VALID_MASK) { 828bfead3b2SJayamohan Kallickal spin_lock_irqsave(&phba->isr_lock, flags); 829bfead3b2SJayamohan Kallickal phba->todo_cq = 1; 830bfead3b2SJayamohan Kallickal spin_unlock_irqrestore(&phba->isr_lock, flags); 831bfead3b2SJayamohan Kallickal AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 832bfead3b2SJayamohan Kallickal queue_tail_inc(eq); 833bfead3b2SJayamohan Kallickal eqe = queue_tail_node(eq); 834bfead3b2SJayamohan Kallickal num_eq_processed++; 835bfead3b2SJayamohan Kallickal } 836bfead3b2SJayamohan Kallickal if (phba->todo_cq) 837bfead3b2SJayamohan Kallickal queue_work(phba->wq, &phba->work_cqs); 838bfead3b2SJayamohan Kallickal 839bfead3b2SJayamohan Kallickal if (num_eq_processed) 840bfead3b2SJayamohan Kallickal hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1); 841bfead3b2SJayamohan Kallickal 842bfead3b2SJayamohan Kallickal return IRQ_HANDLED; 843bfead3b2SJayamohan Kallickal } 844bfead3b2SJayamohan Kallickal } 845bfead3b2SJayamohan Kallickal 846bfead3b2SJayamohan Kallickal /** 8476733b39aSJayamohan Kallickal * be_isr - The isr routine of the driver. 8486733b39aSJayamohan Kallickal * @irq: Not used 8496733b39aSJayamohan Kallickal * @dev_id: Pointer to host adapter structure 8506733b39aSJayamohan Kallickal */ 8516733b39aSJayamohan Kallickal static irqreturn_t be_isr(int irq, void *dev_id) 8526733b39aSJayamohan Kallickal { 8536733b39aSJayamohan Kallickal struct beiscsi_hba *phba; 8546733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 8556733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context; 8566733b39aSJayamohan Kallickal struct be_eq_entry *eqe = NULL; 8576733b39aSJayamohan Kallickal struct be_queue_info *eq; 8586733b39aSJayamohan Kallickal struct be_queue_info *cq; 859bfead3b2SJayamohan Kallickal struct be_queue_info *mcc; 8606733b39aSJayamohan Kallickal unsigned long flags, index; 861bfead3b2SJayamohan Kallickal unsigned int num_mcceq_processed, num_ioeq_processed; 8626733b39aSJayamohan Kallickal struct be_ctrl_info *ctrl; 863bfead3b2SJayamohan Kallickal struct be_eq_obj *pbe_eq; 8646733b39aSJayamohan Kallickal int isr; 8656733b39aSJayamohan Kallickal 8666733b39aSJayamohan Kallickal phba = dev_id; 8676eab04a8SJustin P. Mattock ctrl = &phba->ctrl; 8686733b39aSJayamohan Kallickal isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET + 8696733b39aSJayamohan Kallickal (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE)); 8706733b39aSJayamohan Kallickal if (!isr) 8716733b39aSJayamohan Kallickal return IRQ_NONE; 8726733b39aSJayamohan Kallickal 8736733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 8746733b39aSJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 875bfead3b2SJayamohan Kallickal pbe_eq = &phwi_context->be_eq[0]; 876bfead3b2SJayamohan Kallickal 877bfead3b2SJayamohan Kallickal eq = &phwi_context->be_eq[0].q; 878bfead3b2SJayamohan Kallickal mcc = &phba->ctrl.mcc_obj.cq; 8796733b39aSJayamohan Kallickal index = 0; 8806733b39aSJayamohan Kallickal eqe = queue_tail_node(eq); 8816733b39aSJayamohan Kallickal 882bfead3b2SJayamohan Kallickal num_ioeq_processed = 0; 883bfead3b2SJayamohan Kallickal num_mcceq_processed = 0; 8846733b39aSJayamohan Kallickal if (blk_iopoll_enabled) { 8856733b39aSJayamohan Kallickal while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 8866733b39aSJayamohan Kallickal & EQE_VALID_MASK) { 887bfead3b2SJayamohan Kallickal if (((eqe->dw[offsetof(struct amap_eq_entry, 888bfead3b2SJayamohan Kallickal resource_id) / 32] & 889bfead3b2SJayamohan Kallickal EQE_RESID_MASK) >> 16) == mcc->id) { 890bfead3b2SJayamohan Kallickal spin_lock_irqsave(&phba->isr_lock, flags); 891bfead3b2SJayamohan Kallickal phba->todo_mcc_cq = 1; 892bfead3b2SJayamohan Kallickal spin_unlock_irqrestore(&phba->isr_lock, flags); 893bfead3b2SJayamohan Kallickal num_mcceq_processed++; 894bfead3b2SJayamohan Kallickal } else { 895bfead3b2SJayamohan Kallickal if (!blk_iopoll_sched_prep(&pbe_eq->iopoll)) 896bfead3b2SJayamohan Kallickal blk_iopoll_sched(&pbe_eq->iopoll); 897bfead3b2SJayamohan Kallickal num_ioeq_processed++; 898bfead3b2SJayamohan Kallickal } 8996733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 9006733b39aSJayamohan Kallickal queue_tail_inc(eq); 9016733b39aSJayamohan Kallickal eqe = queue_tail_node(eq); 9026733b39aSJayamohan Kallickal } 903bfead3b2SJayamohan Kallickal if (num_ioeq_processed || num_mcceq_processed) { 904bfead3b2SJayamohan Kallickal if (phba->todo_mcc_cq) 905bfead3b2SJayamohan Kallickal queue_work(phba->wq, &phba->work_cqs); 906bfead3b2SJayamohan Kallickal 907bfead3b2SJayamohan Kallickal if ((num_mcceq_processed) && (!num_ioeq_processed)) 908bfead3b2SJayamohan Kallickal hwi_ring_eq_db(phba, eq->id, 0, 909bfead3b2SJayamohan Kallickal (num_ioeq_processed + 910bfead3b2SJayamohan Kallickal num_mcceq_processed) , 1, 1); 911bfead3b2SJayamohan Kallickal else 912bfead3b2SJayamohan Kallickal hwi_ring_eq_db(phba, eq->id, 0, 913bfead3b2SJayamohan Kallickal (num_ioeq_processed + 914bfead3b2SJayamohan Kallickal num_mcceq_processed), 0, 1); 915bfead3b2SJayamohan Kallickal 9166733b39aSJayamohan Kallickal return IRQ_HANDLED; 9176733b39aSJayamohan Kallickal } else 9186733b39aSJayamohan Kallickal return IRQ_NONE; 9196733b39aSJayamohan Kallickal } else { 920bfead3b2SJayamohan Kallickal cq = &phwi_context->be_cq[0]; 9216733b39aSJayamohan Kallickal while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 9226733b39aSJayamohan Kallickal & EQE_VALID_MASK) { 9236733b39aSJayamohan Kallickal 9246733b39aSJayamohan Kallickal if (((eqe->dw[offsetof(struct amap_eq_entry, 9256733b39aSJayamohan Kallickal resource_id) / 32] & 9266733b39aSJayamohan Kallickal EQE_RESID_MASK) >> 16) != cq->id) { 9276733b39aSJayamohan Kallickal spin_lock_irqsave(&phba->isr_lock, flags); 9286733b39aSJayamohan Kallickal phba->todo_mcc_cq = 1; 9296733b39aSJayamohan Kallickal spin_unlock_irqrestore(&phba->isr_lock, flags); 9306733b39aSJayamohan Kallickal } else { 9316733b39aSJayamohan Kallickal spin_lock_irqsave(&phba->isr_lock, flags); 9326733b39aSJayamohan Kallickal phba->todo_cq = 1; 9336733b39aSJayamohan Kallickal spin_unlock_irqrestore(&phba->isr_lock, flags); 9346733b39aSJayamohan Kallickal } 9356733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 9366733b39aSJayamohan Kallickal queue_tail_inc(eq); 9376733b39aSJayamohan Kallickal eqe = queue_tail_node(eq); 938bfead3b2SJayamohan Kallickal num_ioeq_processed++; 9396733b39aSJayamohan Kallickal } 9406733b39aSJayamohan Kallickal if (phba->todo_cq || phba->todo_mcc_cq) 9416733b39aSJayamohan Kallickal queue_work(phba->wq, &phba->work_cqs); 9426733b39aSJayamohan Kallickal 943bfead3b2SJayamohan Kallickal if (num_ioeq_processed) { 944bfead3b2SJayamohan Kallickal hwi_ring_eq_db(phba, eq->id, 0, 945bfead3b2SJayamohan Kallickal num_ioeq_processed, 1, 1); 9466733b39aSJayamohan Kallickal return IRQ_HANDLED; 9476733b39aSJayamohan Kallickal } else 9486733b39aSJayamohan Kallickal return IRQ_NONE; 9496733b39aSJayamohan Kallickal } 9506733b39aSJayamohan Kallickal } 9516733b39aSJayamohan Kallickal 9526733b39aSJayamohan Kallickal static int beiscsi_init_irqs(struct beiscsi_hba *phba) 9536733b39aSJayamohan Kallickal { 9546733b39aSJayamohan Kallickal struct pci_dev *pcidev = phba->pcidev; 955bfead3b2SJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 956bfead3b2SJayamohan Kallickal struct hwi_context_memory *phwi_context; 9574f5af07eSJayamohan Kallickal int ret, msix_vec, i, j; 9586733b39aSJayamohan Kallickal 959bfead3b2SJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 960bfead3b2SJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 961bfead3b2SJayamohan Kallickal 962bfead3b2SJayamohan Kallickal if (phba->msix_enabled) { 963bfead3b2SJayamohan Kallickal for (i = 0; i < phba->num_cpus; i++) { 9648fcfb210SJayamohan Kallickal phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, 9658fcfb210SJayamohan Kallickal GFP_KERNEL); 9668fcfb210SJayamohan Kallickal if (!phba->msi_name[i]) { 9678fcfb210SJayamohan Kallickal ret = -ENOMEM; 9688fcfb210SJayamohan Kallickal goto free_msix_irqs; 9698fcfb210SJayamohan Kallickal } 9708fcfb210SJayamohan Kallickal 9718fcfb210SJayamohan Kallickal sprintf(phba->msi_name[i], "beiscsi_%02x_%02x", 9728fcfb210SJayamohan Kallickal phba->shost->host_no, i); 973bfead3b2SJayamohan Kallickal msix_vec = phba->msix_entries[i].vector; 9748fcfb210SJayamohan Kallickal ret = request_irq(msix_vec, be_isr_msix, 0, 9758fcfb210SJayamohan Kallickal phba->msi_name[i], 976bfead3b2SJayamohan Kallickal &phwi_context->be_eq[i]); 9774f5af07eSJayamohan Kallickal if (ret) { 97899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 97999bc5d55SJohn Soni Jose "BM_%d : beiscsi_init_irqs-Failed to" 98099bc5d55SJohn Soni Jose "register msix for i = %d\n", 98199bc5d55SJohn Soni Jose i); 9828fcfb210SJayamohan Kallickal kfree(phba->msi_name[i]); 9834f5af07eSJayamohan Kallickal goto free_msix_irqs; 9844f5af07eSJayamohan Kallickal } 985bfead3b2SJayamohan Kallickal } 9868fcfb210SJayamohan Kallickal phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL); 9878fcfb210SJayamohan Kallickal if (!phba->msi_name[i]) { 9888fcfb210SJayamohan Kallickal ret = -ENOMEM; 9898fcfb210SJayamohan Kallickal goto free_msix_irqs; 9908fcfb210SJayamohan Kallickal } 9918fcfb210SJayamohan Kallickal sprintf(phba->msi_name[i], "beiscsi_mcc_%02x", 9928fcfb210SJayamohan Kallickal phba->shost->host_no); 993bfead3b2SJayamohan Kallickal msix_vec = phba->msix_entries[i].vector; 9948fcfb210SJayamohan Kallickal ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i], 995bfead3b2SJayamohan Kallickal &phwi_context->be_eq[i]); 9964f5af07eSJayamohan Kallickal if (ret) { 99799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT , 99899bc5d55SJohn Soni Jose "BM_%d : beiscsi_init_irqs-" 9994f5af07eSJayamohan Kallickal "Failed to register beiscsi_msix_mcc\n"); 10008fcfb210SJayamohan Kallickal kfree(phba->msi_name[i]); 10014f5af07eSJayamohan Kallickal goto free_msix_irqs; 10024f5af07eSJayamohan Kallickal } 10034f5af07eSJayamohan Kallickal 1004bfead3b2SJayamohan Kallickal } else { 1005bfead3b2SJayamohan Kallickal ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED, 1006bfead3b2SJayamohan Kallickal "beiscsi", phba); 10076733b39aSJayamohan Kallickal if (ret) { 100899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 100999bc5d55SJohn Soni Jose "BM_%d : beiscsi_init_irqs-" 10106733b39aSJayamohan Kallickal "Failed to register irq\\n"); 10116733b39aSJayamohan Kallickal return ret; 10126733b39aSJayamohan Kallickal } 1013bfead3b2SJayamohan Kallickal } 10146733b39aSJayamohan Kallickal return 0; 10154f5af07eSJayamohan Kallickal free_msix_irqs: 10168fcfb210SJayamohan Kallickal for (j = i - 1; j >= 0; j--) { 10178fcfb210SJayamohan Kallickal kfree(phba->msi_name[j]); 10188fcfb210SJayamohan Kallickal msix_vec = phba->msix_entries[j].vector; 10194f5af07eSJayamohan Kallickal free_irq(msix_vec, &phwi_context->be_eq[j]); 10208fcfb210SJayamohan Kallickal } 10214f5af07eSJayamohan Kallickal return ret; 10226733b39aSJayamohan Kallickal } 10236733b39aSJayamohan Kallickal 10246733b39aSJayamohan Kallickal static void hwi_ring_cq_db(struct beiscsi_hba *phba, 10256733b39aSJayamohan Kallickal unsigned int id, unsigned int num_processed, 10266733b39aSJayamohan Kallickal unsigned char rearm, unsigned char event) 10276733b39aSJayamohan Kallickal { 10286733b39aSJayamohan Kallickal u32 val = 0; 10296733b39aSJayamohan Kallickal val |= id & DB_CQ_RING_ID_MASK; 10306733b39aSJayamohan Kallickal if (rearm) 10316733b39aSJayamohan Kallickal val |= 1 << DB_CQ_REARM_SHIFT; 10326733b39aSJayamohan Kallickal val |= num_processed << DB_CQ_NUM_POPPED_SHIFT; 10336733b39aSJayamohan Kallickal iowrite32(val, phba->db_va + DB_CQ_OFFSET); 10346733b39aSJayamohan Kallickal } 10356733b39aSJayamohan Kallickal 10366733b39aSJayamohan Kallickal static unsigned int 10376733b39aSJayamohan Kallickal beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn, 10386733b39aSJayamohan Kallickal struct beiscsi_hba *phba, 10396733b39aSJayamohan Kallickal unsigned short cid, 10406733b39aSJayamohan Kallickal struct pdu_base *ppdu, 10416733b39aSJayamohan Kallickal unsigned long pdu_len, 10426733b39aSJayamohan Kallickal void *pbuffer, unsigned long buf_len) 10436733b39aSJayamohan Kallickal { 10446733b39aSJayamohan Kallickal struct iscsi_conn *conn = beiscsi_conn->conn; 10456733b39aSJayamohan Kallickal struct iscsi_session *session = conn->session; 1046bfead3b2SJayamohan Kallickal struct iscsi_task *task; 1047bfead3b2SJayamohan Kallickal struct beiscsi_io_task *io_task; 1048bfead3b2SJayamohan Kallickal struct iscsi_hdr *login_hdr; 10496733b39aSJayamohan Kallickal 10506733b39aSJayamohan Kallickal switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] & 10516733b39aSJayamohan Kallickal PDUBASE_OPCODE_MASK) { 10526733b39aSJayamohan Kallickal case ISCSI_OP_NOOP_IN: 10536733b39aSJayamohan Kallickal pbuffer = NULL; 10546733b39aSJayamohan Kallickal buf_len = 0; 10556733b39aSJayamohan Kallickal break; 10566733b39aSJayamohan Kallickal case ISCSI_OP_ASYNC_EVENT: 10576733b39aSJayamohan Kallickal break; 10586733b39aSJayamohan Kallickal case ISCSI_OP_REJECT: 10596733b39aSJayamohan Kallickal WARN_ON(!pbuffer); 10606733b39aSJayamohan Kallickal WARN_ON(!(buf_len == 48)); 106199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 106299bc5d55SJohn Soni Jose BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 106399bc5d55SJohn Soni Jose "BM_%d : In ISCSI_OP_REJECT\n"); 10646733b39aSJayamohan Kallickal break; 10656733b39aSJayamohan Kallickal case ISCSI_OP_LOGIN_RSP: 10667bd6e25cSJayamohan Kallickal case ISCSI_OP_TEXT_RSP: 1067bfead3b2SJayamohan Kallickal task = conn->login_task; 1068bfead3b2SJayamohan Kallickal io_task = task->dd_data; 1069bfead3b2SJayamohan Kallickal login_hdr = (struct iscsi_hdr *)ppdu; 1070bfead3b2SJayamohan Kallickal login_hdr->itt = io_task->libiscsi_itt; 10716733b39aSJayamohan Kallickal break; 10726733b39aSJayamohan Kallickal default: 107399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, 107499bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 107599bc5d55SJohn Soni Jose "BM_%d : Unrecognized opcode 0x%x in async msg\n", 10766733b39aSJayamohan Kallickal (ppdu-> 10776733b39aSJayamohan Kallickal dw[offsetof(struct amap_pdu_base, opcode) / 32] 10786733b39aSJayamohan Kallickal & PDUBASE_OPCODE_MASK)); 10796733b39aSJayamohan Kallickal return 1; 10806733b39aSJayamohan Kallickal } 10816733b39aSJayamohan Kallickal 10826733b39aSJayamohan Kallickal spin_lock_bh(&session->lock); 10836733b39aSJayamohan Kallickal __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len); 10846733b39aSJayamohan Kallickal spin_unlock_bh(&session->lock); 10856733b39aSJayamohan Kallickal return 0; 10866733b39aSJayamohan Kallickal } 10876733b39aSJayamohan Kallickal 10886733b39aSJayamohan Kallickal static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba) 10896733b39aSJayamohan Kallickal { 10906733b39aSJayamohan Kallickal struct sgl_handle *psgl_handle; 10916733b39aSJayamohan Kallickal 10926733b39aSJayamohan Kallickal if (phba->io_sgl_hndl_avbl) { 109399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO, 109499bc5d55SJohn Soni Jose "BM_%d : In alloc_io_sgl_handle," 109599bc5d55SJohn Soni Jose " io_sgl_alloc_index=%d\n", 10966733b39aSJayamohan Kallickal phba->io_sgl_alloc_index); 109799bc5d55SJohn Soni Jose 10986733b39aSJayamohan Kallickal psgl_handle = phba->io_sgl_hndl_base[phba-> 10996733b39aSJayamohan Kallickal io_sgl_alloc_index]; 11006733b39aSJayamohan Kallickal phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL; 11016733b39aSJayamohan Kallickal phba->io_sgl_hndl_avbl--; 1102bfead3b2SJayamohan Kallickal if (phba->io_sgl_alloc_index == (phba->params. 1103bfead3b2SJayamohan Kallickal ios_per_ctrl - 1)) 11046733b39aSJayamohan Kallickal phba->io_sgl_alloc_index = 0; 11056733b39aSJayamohan Kallickal else 11066733b39aSJayamohan Kallickal phba->io_sgl_alloc_index++; 11076733b39aSJayamohan Kallickal } else 11086733b39aSJayamohan Kallickal psgl_handle = NULL; 11096733b39aSJayamohan Kallickal return psgl_handle; 11106733b39aSJayamohan Kallickal } 11116733b39aSJayamohan Kallickal 11126733b39aSJayamohan Kallickal static void 11136733b39aSJayamohan Kallickal free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) 11146733b39aSJayamohan Kallickal { 111599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO, 111699bc5d55SJohn Soni Jose "BM_%d : In free_,io_sgl_free_index=%d\n", 11176733b39aSJayamohan Kallickal phba->io_sgl_free_index); 111899bc5d55SJohn Soni Jose 11196733b39aSJayamohan Kallickal if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) { 11206733b39aSJayamohan Kallickal /* 11216733b39aSJayamohan Kallickal * this can happen if clean_task is called on a task that 11226733b39aSJayamohan Kallickal * failed in xmit_task or alloc_pdu. 11236733b39aSJayamohan Kallickal */ 112499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO, 112599bc5d55SJohn Soni Jose "BM_%d : Double Free in IO SGL io_sgl_free_index=%d," 11266733b39aSJayamohan Kallickal "value there=%p\n", phba->io_sgl_free_index, 112799bc5d55SJohn Soni Jose phba->io_sgl_hndl_base 112899bc5d55SJohn Soni Jose [phba->io_sgl_free_index]); 11296733b39aSJayamohan Kallickal return; 11306733b39aSJayamohan Kallickal } 11316733b39aSJayamohan Kallickal phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle; 11326733b39aSJayamohan Kallickal phba->io_sgl_hndl_avbl++; 11336733b39aSJayamohan Kallickal if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1)) 11346733b39aSJayamohan Kallickal phba->io_sgl_free_index = 0; 11356733b39aSJayamohan Kallickal else 11366733b39aSJayamohan Kallickal phba->io_sgl_free_index++; 11376733b39aSJayamohan Kallickal } 11386733b39aSJayamohan Kallickal 11396733b39aSJayamohan Kallickal /** 11406733b39aSJayamohan Kallickal * alloc_wrb_handle - To allocate a wrb handle 11416733b39aSJayamohan Kallickal * @phba: The hba pointer 11426733b39aSJayamohan Kallickal * @cid: The cid to use for allocation 11436733b39aSJayamohan Kallickal * 11446733b39aSJayamohan Kallickal * This happens under session_lock until submission to chip 11456733b39aSJayamohan Kallickal */ 1146d5431488SJayamohan Kallickal struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid) 11476733b39aSJayamohan Kallickal { 11486733b39aSJayamohan Kallickal struct hwi_wrb_context *pwrb_context; 11496733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 1150d5431488SJayamohan Kallickal struct wrb_handle *pwrb_handle, *pwrb_handle_tmp; 11516733b39aSJayamohan Kallickal 11526733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 11536733b39aSJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[cid]; 1154d5431488SJayamohan Kallickal if (pwrb_context->wrb_handles_available >= 2) { 1155bfead3b2SJayamohan Kallickal pwrb_handle = pwrb_context->pwrb_handle_base[ 1156bfead3b2SJayamohan Kallickal pwrb_context->alloc_index]; 1157bfead3b2SJayamohan Kallickal pwrb_context->wrb_handles_available--; 1158bfead3b2SJayamohan Kallickal if (pwrb_context->alloc_index == 1159bfead3b2SJayamohan Kallickal (phba->params.wrbs_per_cxn - 1)) 1160bfead3b2SJayamohan Kallickal pwrb_context->alloc_index = 0; 1161bfead3b2SJayamohan Kallickal else 1162bfead3b2SJayamohan Kallickal pwrb_context->alloc_index++; 1163d5431488SJayamohan Kallickal pwrb_handle_tmp = pwrb_context->pwrb_handle_base[ 1164d5431488SJayamohan Kallickal pwrb_context->alloc_index]; 1165d5431488SJayamohan Kallickal pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index; 1166bfead3b2SJayamohan Kallickal } else 1167bfead3b2SJayamohan Kallickal pwrb_handle = NULL; 11686733b39aSJayamohan Kallickal return pwrb_handle; 11696733b39aSJayamohan Kallickal } 11706733b39aSJayamohan Kallickal 11716733b39aSJayamohan Kallickal /** 11726733b39aSJayamohan Kallickal * free_wrb_handle - To free the wrb handle back to pool 11736733b39aSJayamohan Kallickal * @phba: The hba pointer 11746733b39aSJayamohan Kallickal * @pwrb_context: The context to free from 11756733b39aSJayamohan Kallickal * @pwrb_handle: The wrb_handle to free 11766733b39aSJayamohan Kallickal * 11776733b39aSJayamohan Kallickal * This happens under session_lock until submission to chip 11786733b39aSJayamohan Kallickal */ 11796733b39aSJayamohan Kallickal static void 11806733b39aSJayamohan Kallickal free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context, 11816733b39aSJayamohan Kallickal struct wrb_handle *pwrb_handle) 11826733b39aSJayamohan Kallickal { 118332951dd8SJayamohan Kallickal pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle; 1184bfead3b2SJayamohan Kallickal pwrb_context->wrb_handles_available++; 1185bfead3b2SJayamohan Kallickal if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1)) 1186bfead3b2SJayamohan Kallickal pwrb_context->free_index = 0; 1187bfead3b2SJayamohan Kallickal else 1188bfead3b2SJayamohan Kallickal pwrb_context->free_index++; 1189bfead3b2SJayamohan Kallickal 119099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, 119199bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 119299bc5d55SJohn Soni Jose "BM_%d : FREE WRB: pwrb_handle=%p free_index=0x%x" 11936733b39aSJayamohan Kallickal "wrb_handles_available=%d\n", 11946733b39aSJayamohan Kallickal pwrb_handle, pwrb_context->free_index, 1195bfead3b2SJayamohan Kallickal pwrb_context->wrb_handles_available); 11966733b39aSJayamohan Kallickal } 11976733b39aSJayamohan Kallickal 11986733b39aSJayamohan Kallickal static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba) 11996733b39aSJayamohan Kallickal { 12006733b39aSJayamohan Kallickal struct sgl_handle *psgl_handle; 12016733b39aSJayamohan Kallickal 12026733b39aSJayamohan Kallickal if (phba->eh_sgl_hndl_avbl) { 12036733b39aSJayamohan Kallickal psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index]; 12046733b39aSJayamohan Kallickal phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL; 120599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG, 120699bc5d55SJohn Soni Jose "BM_%d : mgmt_sgl_alloc_index=%d=0x%x\n", 120799bc5d55SJohn Soni Jose phba->eh_sgl_alloc_index, 120899bc5d55SJohn Soni Jose phba->eh_sgl_alloc_index); 120999bc5d55SJohn Soni Jose 12106733b39aSJayamohan Kallickal phba->eh_sgl_hndl_avbl--; 12116733b39aSJayamohan Kallickal if (phba->eh_sgl_alloc_index == 12126733b39aSJayamohan Kallickal (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 12136733b39aSJayamohan Kallickal 1)) 12146733b39aSJayamohan Kallickal phba->eh_sgl_alloc_index = 0; 12156733b39aSJayamohan Kallickal else 12166733b39aSJayamohan Kallickal phba->eh_sgl_alloc_index++; 12176733b39aSJayamohan Kallickal } else 12186733b39aSJayamohan Kallickal psgl_handle = NULL; 12196733b39aSJayamohan Kallickal return psgl_handle; 12206733b39aSJayamohan Kallickal } 12216733b39aSJayamohan Kallickal 12226733b39aSJayamohan Kallickal void 12236733b39aSJayamohan Kallickal free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) 12246733b39aSJayamohan Kallickal { 12256733b39aSJayamohan Kallickal 122699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG, 122799bc5d55SJohn Soni Jose "BM_%d : In free_mgmt_sgl_handle," 122899bc5d55SJohn Soni Jose "eh_sgl_free_index=%d\n", 1229bfead3b2SJayamohan Kallickal phba->eh_sgl_free_index); 123099bc5d55SJohn Soni Jose 12316733b39aSJayamohan Kallickal if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) { 12326733b39aSJayamohan Kallickal /* 12336733b39aSJayamohan Kallickal * this can happen if clean_task is called on a task that 12346733b39aSJayamohan Kallickal * failed in xmit_task or alloc_pdu. 12356733b39aSJayamohan Kallickal */ 123699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG, 123799bc5d55SJohn Soni Jose "BM_%d : Double Free in eh SGL ," 123899bc5d55SJohn Soni Jose "eh_sgl_free_index=%d\n", 12396733b39aSJayamohan Kallickal phba->eh_sgl_free_index); 12406733b39aSJayamohan Kallickal return; 12416733b39aSJayamohan Kallickal } 12426733b39aSJayamohan Kallickal phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle; 12436733b39aSJayamohan Kallickal phba->eh_sgl_hndl_avbl++; 12446733b39aSJayamohan Kallickal if (phba->eh_sgl_free_index == 12456733b39aSJayamohan Kallickal (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1)) 12466733b39aSJayamohan Kallickal phba->eh_sgl_free_index = 0; 12476733b39aSJayamohan Kallickal else 12486733b39aSJayamohan Kallickal phba->eh_sgl_free_index++; 12496733b39aSJayamohan Kallickal } 12506733b39aSJayamohan Kallickal 12516733b39aSJayamohan Kallickal static void 12526733b39aSJayamohan Kallickal be_complete_io(struct beiscsi_conn *beiscsi_conn, 12536733b39aSJayamohan Kallickal struct iscsi_task *task, struct sol_cqe *psol) 12546733b39aSJayamohan Kallickal { 12556733b39aSJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 12566733b39aSJayamohan Kallickal struct be_status_bhs *sts_bhs = 12576733b39aSJayamohan Kallickal (struct be_status_bhs *)io_task->cmd_bhs; 12586733b39aSJayamohan Kallickal struct iscsi_conn *conn = beiscsi_conn->conn; 12596733b39aSJayamohan Kallickal unsigned char *sense; 12606733b39aSJayamohan Kallickal u32 resid = 0, exp_cmdsn, max_cmdsn; 12616733b39aSJayamohan Kallickal u8 rsp, status, flags; 12626733b39aSJayamohan Kallickal 1263bfead3b2SJayamohan Kallickal exp_cmdsn = (psol-> 12646733b39aSJayamohan Kallickal dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32] 12656733b39aSJayamohan Kallickal & SOL_EXP_CMD_SN_MASK); 1266bfead3b2SJayamohan Kallickal max_cmdsn = ((psol-> 12676733b39aSJayamohan Kallickal dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32] 12686733b39aSJayamohan Kallickal & SOL_EXP_CMD_SN_MASK) + 12696733b39aSJayamohan Kallickal ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) 12706733b39aSJayamohan Kallickal / 32] & SOL_CMD_WND_MASK) >> 24) - 1); 12716733b39aSJayamohan Kallickal rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32] 12726733b39aSJayamohan Kallickal & SOL_RESP_MASK) >> 16); 12736733b39aSJayamohan Kallickal status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32] 12746733b39aSJayamohan Kallickal & SOL_STS_MASK) >> 8); 12756733b39aSJayamohan Kallickal flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32] 12766733b39aSJayamohan Kallickal & SOL_FLAGS_MASK) >> 24) | 0x80; 1277bd535451SJayamohan Kallickal if (!task->sc) { 1278bd535451SJayamohan Kallickal if (io_task->scsi_cmnd) 1279bd535451SJayamohan Kallickal scsi_dma_unmap(io_task->scsi_cmnd); 12806733b39aSJayamohan Kallickal 1281bd535451SJayamohan Kallickal return; 1282bd535451SJayamohan Kallickal } 12836733b39aSJayamohan Kallickal task->sc->result = (DID_OK << 16) | status; 12846733b39aSJayamohan Kallickal if (rsp != ISCSI_STATUS_CMD_COMPLETED) { 12856733b39aSJayamohan Kallickal task->sc->result = DID_ERROR << 16; 12866733b39aSJayamohan Kallickal goto unmap; 12876733b39aSJayamohan Kallickal } 12886733b39aSJayamohan Kallickal 12896733b39aSJayamohan Kallickal /* bidi not initially supported */ 12906733b39aSJayamohan Kallickal if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) { 12916733b39aSJayamohan Kallickal resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 12926733b39aSJayamohan Kallickal 32] & SOL_RES_CNT_MASK); 12936733b39aSJayamohan Kallickal 12946733b39aSJayamohan Kallickal if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW)) 12956733b39aSJayamohan Kallickal task->sc->result = DID_ERROR << 16; 12966733b39aSJayamohan Kallickal 12976733b39aSJayamohan Kallickal if (flags & ISCSI_FLAG_CMD_UNDERFLOW) { 12986733b39aSJayamohan Kallickal scsi_set_resid(task->sc, resid); 12996733b39aSJayamohan Kallickal if (!status && (scsi_bufflen(task->sc) - resid < 13006733b39aSJayamohan Kallickal task->sc->underflow)) 13016733b39aSJayamohan Kallickal task->sc->result = DID_ERROR << 16; 13026733b39aSJayamohan Kallickal } 13036733b39aSJayamohan Kallickal } 13046733b39aSJayamohan Kallickal 13056733b39aSJayamohan Kallickal if (status == SAM_STAT_CHECK_CONDITION) { 13064053a4beSDan Carpenter u16 sense_len; 1307bfead3b2SJayamohan Kallickal unsigned short *slen = (unsigned short *)sts_bhs->sense_info; 13084053a4beSDan Carpenter 13096733b39aSJayamohan Kallickal sense = sts_bhs->sense_info + sizeof(unsigned short); 13104053a4beSDan Carpenter sense_len = be16_to_cpu(*slen); 13116733b39aSJayamohan Kallickal memcpy(task->sc->sense_buffer, sense, 13126733b39aSJayamohan Kallickal min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE)); 13136733b39aSJayamohan Kallickal } 1314756d29c8SJayamohan Kallickal 13156733b39aSJayamohan Kallickal if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) { 13166733b39aSJayamohan Kallickal if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32] 13176733b39aSJayamohan Kallickal & SOL_RES_CNT_MASK) 13186733b39aSJayamohan Kallickal conn->rxdata_octets += (psol-> 13196733b39aSJayamohan Kallickal dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32] 13206733b39aSJayamohan Kallickal & SOL_RES_CNT_MASK); 13216733b39aSJayamohan Kallickal } 13226733b39aSJayamohan Kallickal unmap: 13236733b39aSJayamohan Kallickal scsi_dma_unmap(io_task->scsi_cmnd); 13246733b39aSJayamohan Kallickal iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn); 13256733b39aSJayamohan Kallickal } 13266733b39aSJayamohan Kallickal 13276733b39aSJayamohan Kallickal static void 13286733b39aSJayamohan Kallickal be_complete_logout(struct beiscsi_conn *beiscsi_conn, 13296733b39aSJayamohan Kallickal struct iscsi_task *task, struct sol_cqe *psol) 13306733b39aSJayamohan Kallickal { 13316733b39aSJayamohan Kallickal struct iscsi_logout_rsp *hdr; 1332bfead3b2SJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 13336733b39aSJayamohan Kallickal struct iscsi_conn *conn = beiscsi_conn->conn; 13346733b39aSJayamohan Kallickal 13356733b39aSJayamohan Kallickal hdr = (struct iscsi_logout_rsp *)task->hdr; 13367bd6e25cSJayamohan Kallickal hdr->opcode = ISCSI_OP_LOGOUT_RSP; 13376733b39aSJayamohan Kallickal hdr->t2wait = 5; 13386733b39aSJayamohan Kallickal hdr->t2retain = 0; 13396733b39aSJayamohan Kallickal hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32] 13406733b39aSJayamohan Kallickal & SOL_FLAGS_MASK) >> 24) | 0x80; 13416733b39aSJayamohan Kallickal hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 13426733b39aSJayamohan Kallickal 32] & SOL_RESP_MASK); 13436733b39aSJayamohan Kallickal hdr->exp_cmdsn = cpu_to_be32(psol-> 13446733b39aSJayamohan Kallickal dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32] 13456733b39aSJayamohan Kallickal & SOL_EXP_CMD_SN_MASK); 13466733b39aSJayamohan Kallickal hdr->max_cmdsn = be32_to_cpu((psol-> 13476733b39aSJayamohan Kallickal dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32] 13486733b39aSJayamohan Kallickal & SOL_EXP_CMD_SN_MASK) + 13496733b39aSJayamohan Kallickal ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) 13506733b39aSJayamohan Kallickal / 32] & SOL_CMD_WND_MASK) >> 24) - 1); 13517bd6e25cSJayamohan Kallickal hdr->dlength[0] = 0; 13527bd6e25cSJayamohan Kallickal hdr->dlength[1] = 0; 13537bd6e25cSJayamohan Kallickal hdr->dlength[2] = 0; 13546733b39aSJayamohan Kallickal hdr->hlength = 0; 1355bfead3b2SJayamohan Kallickal hdr->itt = io_task->libiscsi_itt; 13566733b39aSJayamohan Kallickal __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 13576733b39aSJayamohan Kallickal } 13586733b39aSJayamohan Kallickal 13596733b39aSJayamohan Kallickal static void 13606733b39aSJayamohan Kallickal be_complete_tmf(struct beiscsi_conn *beiscsi_conn, 13616733b39aSJayamohan Kallickal struct iscsi_task *task, struct sol_cqe *psol) 13626733b39aSJayamohan Kallickal { 13636733b39aSJayamohan Kallickal struct iscsi_tm_rsp *hdr; 13646733b39aSJayamohan Kallickal struct iscsi_conn *conn = beiscsi_conn->conn; 1365bfead3b2SJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 13666733b39aSJayamohan Kallickal 13676733b39aSJayamohan Kallickal hdr = (struct iscsi_tm_rsp *)task->hdr; 13687bd6e25cSJayamohan Kallickal hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP; 13696733b39aSJayamohan Kallickal hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32] 13706733b39aSJayamohan Kallickal & SOL_FLAGS_MASK) >> 24) | 0x80; 13716733b39aSJayamohan Kallickal hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 13726733b39aSJayamohan Kallickal 32] & SOL_RESP_MASK); 13736733b39aSJayamohan Kallickal hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe, 13746733b39aSJayamohan Kallickal i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK); 13756733b39aSJayamohan Kallickal hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe, 13766733b39aSJayamohan Kallickal i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) + 13776733b39aSJayamohan Kallickal ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) 13786733b39aSJayamohan Kallickal / 32] & SOL_CMD_WND_MASK) >> 24) - 1); 1379bfead3b2SJayamohan Kallickal hdr->itt = io_task->libiscsi_itt; 13806733b39aSJayamohan Kallickal __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 13816733b39aSJayamohan Kallickal } 13826733b39aSJayamohan Kallickal 13836733b39aSJayamohan Kallickal static void 13846733b39aSJayamohan Kallickal hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn, 13856733b39aSJayamohan Kallickal struct beiscsi_hba *phba, struct sol_cqe *psol) 13866733b39aSJayamohan Kallickal { 13876733b39aSJayamohan Kallickal struct hwi_wrb_context *pwrb_context; 1388bfead3b2SJayamohan Kallickal struct wrb_handle *pwrb_handle = NULL; 13896733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 1390bfead3b2SJayamohan Kallickal struct iscsi_task *task; 1391bfead3b2SJayamohan Kallickal struct beiscsi_io_task *io_task; 13926733b39aSJayamohan Kallickal struct iscsi_conn *conn = beiscsi_conn->conn; 13936733b39aSJayamohan Kallickal struct iscsi_session *session = conn->session; 13946733b39aSJayamohan Kallickal 13956733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 13966733b39aSJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[((psol-> 13976733b39aSJayamohan Kallickal dw[offsetof(struct amap_sol_cqe, cid) / 32] & 13987da50879SJayamohan Kallickal SOL_CID_MASK) >> 6) - 13997da50879SJayamohan Kallickal phba->fw_config.iscsi_cid_start]; 14006733b39aSJayamohan Kallickal pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol-> 14016733b39aSJayamohan Kallickal dw[offsetof(struct amap_sol_cqe, wrb_index) / 14026733b39aSJayamohan Kallickal 32] & SOL_WRB_INDEX_MASK) >> 16)]; 1403bfead3b2SJayamohan Kallickal task = pwrb_handle->pio_handle; 140435e66019SJayamohan Kallickal 1405bfead3b2SJayamohan Kallickal io_task = task->dd_data; 14061282ab76SMike Christie spin_lock_bh(&phba->mgmt_sgl_lock); 1407bfead3b2SJayamohan Kallickal free_mgmt_sgl_handle(phba, io_task->psgl_handle); 14081282ab76SMike Christie spin_unlock_bh(&phba->mgmt_sgl_lock); 14096733b39aSJayamohan Kallickal spin_lock_bh(&session->lock); 14106733b39aSJayamohan Kallickal free_wrb_handle(phba, pwrb_context, pwrb_handle); 14116733b39aSJayamohan Kallickal spin_unlock_bh(&session->lock); 14126733b39aSJayamohan Kallickal } 14136733b39aSJayamohan Kallickal 14146733b39aSJayamohan Kallickal static void 14156733b39aSJayamohan Kallickal be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn, 14166733b39aSJayamohan Kallickal struct iscsi_task *task, struct sol_cqe *psol) 14176733b39aSJayamohan Kallickal { 14186733b39aSJayamohan Kallickal struct iscsi_nopin *hdr; 14196733b39aSJayamohan Kallickal struct iscsi_conn *conn = beiscsi_conn->conn; 1420bfead3b2SJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 14216733b39aSJayamohan Kallickal 14226733b39aSJayamohan Kallickal hdr = (struct iscsi_nopin *)task->hdr; 14236733b39aSJayamohan Kallickal hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32] 14246733b39aSJayamohan Kallickal & SOL_FLAGS_MASK) >> 24) | 0x80; 14256733b39aSJayamohan Kallickal hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe, 14266733b39aSJayamohan Kallickal i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK); 14276733b39aSJayamohan Kallickal hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe, 14286733b39aSJayamohan Kallickal i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) + 14296733b39aSJayamohan Kallickal ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) 14306733b39aSJayamohan Kallickal / 32] & SOL_CMD_WND_MASK) >> 24) - 1); 14316733b39aSJayamohan Kallickal hdr->opcode = ISCSI_OP_NOOP_IN; 1432bfead3b2SJayamohan Kallickal hdr->itt = io_task->libiscsi_itt; 14336733b39aSJayamohan Kallickal __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 14346733b39aSJayamohan Kallickal } 14356733b39aSJayamohan Kallickal 14366733b39aSJayamohan Kallickal static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn, 14376733b39aSJayamohan Kallickal struct beiscsi_hba *phba, struct sol_cqe *psol) 14386733b39aSJayamohan Kallickal { 14396733b39aSJayamohan Kallickal struct hwi_wrb_context *pwrb_context; 14406733b39aSJayamohan Kallickal struct wrb_handle *pwrb_handle; 14416733b39aSJayamohan Kallickal struct iscsi_wrb *pwrb = NULL; 14426733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 14436733b39aSJayamohan Kallickal struct iscsi_task *task; 1444bfead3b2SJayamohan Kallickal unsigned int type; 14456733b39aSJayamohan Kallickal struct iscsi_conn *conn = beiscsi_conn->conn; 14466733b39aSJayamohan Kallickal struct iscsi_session *session = conn->session; 14476733b39aSJayamohan Kallickal 14486733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 144932951dd8SJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof 1450bfead3b2SJayamohan Kallickal (struct amap_sol_cqe, cid) / 32] 14517da50879SJayamohan Kallickal & SOL_CID_MASK) >> 6) - 14527da50879SJayamohan Kallickal phba->fw_config.iscsi_cid_start]; 14536733b39aSJayamohan Kallickal pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol-> 14546733b39aSJayamohan Kallickal dw[offsetof(struct amap_sol_cqe, wrb_index) / 14556733b39aSJayamohan Kallickal 32] & SOL_WRB_INDEX_MASK) >> 16)]; 14566733b39aSJayamohan Kallickal task = pwrb_handle->pio_handle; 14576733b39aSJayamohan Kallickal pwrb = pwrb_handle->pwrb; 1458bfead3b2SJayamohan Kallickal type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] & 1459bfead3b2SJayamohan Kallickal WRB_TYPE_MASK) >> 28; 146032951dd8SJayamohan Kallickal 1461bfead3b2SJayamohan Kallickal spin_lock_bh(&session->lock); 1462bfead3b2SJayamohan Kallickal switch (type) { 14636733b39aSJayamohan Kallickal case HWH_TYPE_IO: 14646733b39aSJayamohan Kallickal case HWH_TYPE_IO_RD: 14656733b39aSJayamohan Kallickal if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == 1466dafab8e0SJayamohan Kallickal ISCSI_OP_NOOP_OUT) 14676733b39aSJayamohan Kallickal be_complete_nopin_resp(beiscsi_conn, task, psol); 1468dafab8e0SJayamohan Kallickal else 14696733b39aSJayamohan Kallickal be_complete_io(beiscsi_conn, task, psol); 14706733b39aSJayamohan Kallickal break; 14716733b39aSJayamohan Kallickal 14726733b39aSJayamohan Kallickal case HWH_TYPE_LOGOUT: 1473dafab8e0SJayamohan Kallickal if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT) 14746733b39aSJayamohan Kallickal be_complete_logout(beiscsi_conn, task, psol); 1475dafab8e0SJayamohan Kallickal else 1476dafab8e0SJayamohan Kallickal be_complete_tmf(beiscsi_conn, task, psol); 1477dafab8e0SJayamohan Kallickal 14786733b39aSJayamohan Kallickal break; 14796733b39aSJayamohan Kallickal 14806733b39aSJayamohan Kallickal case HWH_TYPE_LOGIN: 148199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 148299bc5d55SJohn Soni Jose BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 148399bc5d55SJohn Soni Jose "BM_%d :\t\t No HWH_TYPE_LOGIN Expected in" 148499bc5d55SJohn Soni Jose " hwi_complete_cmd- Solicited path\n"); 14856733b39aSJayamohan Kallickal break; 14866733b39aSJayamohan Kallickal 14876733b39aSJayamohan Kallickal case HWH_TYPE_NOP: 14886733b39aSJayamohan Kallickal be_complete_nopin_resp(beiscsi_conn, task, psol); 14896733b39aSJayamohan Kallickal break; 14906733b39aSJayamohan Kallickal 14916733b39aSJayamohan Kallickal default: 149299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, 149399bc5d55SJohn Soni Jose BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 149499bc5d55SJohn Soni Jose "BM_%d : In hwi_complete_cmd, unknown type = %d" 1495bfead3b2SJayamohan Kallickal "wrb_index 0x%x CID 0x%x\n", type, 1496bfead3b2SJayamohan Kallickal ((psol->dw[offsetof(struct amap_iscsi_wrb, 1497bfead3b2SJayamohan Kallickal type) / 32] & SOL_WRB_INDEX_MASK) >> 16), 1498bfead3b2SJayamohan Kallickal ((psol->dw[offsetof(struct amap_sol_cqe, 1499bfead3b2SJayamohan Kallickal cid) / 32] & SOL_CID_MASK) >> 6)); 15006733b39aSJayamohan Kallickal break; 15016733b39aSJayamohan Kallickal } 150235e66019SJayamohan Kallickal 15036733b39aSJayamohan Kallickal spin_unlock_bh(&session->lock); 15046733b39aSJayamohan Kallickal } 15056733b39aSJayamohan Kallickal 15066733b39aSJayamohan Kallickal static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context 15076733b39aSJayamohan Kallickal *pasync_ctx, unsigned int is_header, 15086733b39aSJayamohan Kallickal unsigned int host_write_ptr) 15096733b39aSJayamohan Kallickal { 15106733b39aSJayamohan Kallickal if (is_header) 15116733b39aSJayamohan Kallickal return &pasync_ctx->async_entry[host_write_ptr]. 15126733b39aSJayamohan Kallickal header_busy_list; 15136733b39aSJayamohan Kallickal else 15146733b39aSJayamohan Kallickal return &pasync_ctx->async_entry[host_write_ptr].data_busy_list; 15156733b39aSJayamohan Kallickal } 15166733b39aSJayamohan Kallickal 15176733b39aSJayamohan Kallickal static struct async_pdu_handle * 15186733b39aSJayamohan Kallickal hwi_get_async_handle(struct beiscsi_hba *phba, 15196733b39aSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn, 15206733b39aSJayamohan Kallickal struct hwi_async_pdu_context *pasync_ctx, 15216733b39aSJayamohan Kallickal struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index) 15226733b39aSJayamohan Kallickal { 15236733b39aSJayamohan Kallickal struct be_bus_address phys_addr; 15246733b39aSJayamohan Kallickal struct list_head *pbusy_list; 15256733b39aSJayamohan Kallickal struct async_pdu_handle *pasync_handle = NULL; 15266733b39aSJayamohan Kallickal unsigned char is_header = 0; 15276733b39aSJayamohan Kallickal 15286733b39aSJayamohan Kallickal phys_addr.u.a32.address_lo = 15296733b39aSJayamohan Kallickal pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] - 15306733b39aSJayamohan Kallickal ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32] 15316733b39aSJayamohan Kallickal & PDUCQE_DPL_MASK) >> 16); 15326733b39aSJayamohan Kallickal phys_addr.u.a32.address_hi = 15336733b39aSJayamohan Kallickal pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32]; 15346733b39aSJayamohan Kallickal 15356733b39aSJayamohan Kallickal phys_addr.u.a64.address = 15366733b39aSJayamohan Kallickal *((unsigned long long *)(&phys_addr.u.a64.address)); 15376733b39aSJayamohan Kallickal 15386733b39aSJayamohan Kallickal switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32] 15396733b39aSJayamohan Kallickal & PDUCQE_CODE_MASK) { 15406733b39aSJayamohan Kallickal case UNSOL_HDR_NOTIFY: 15416733b39aSJayamohan Kallickal is_header = 1; 15426733b39aSJayamohan Kallickal 15436733b39aSJayamohan Kallickal pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1, 15446733b39aSJayamohan Kallickal (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, 15456733b39aSJayamohan Kallickal index) / 32] & PDUCQE_INDEX_MASK)); 15466733b39aSJayamohan Kallickal break; 15476733b39aSJayamohan Kallickal case UNSOL_DATA_NOTIFY: 15486733b39aSJayamohan Kallickal pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe-> 15496733b39aSJayamohan Kallickal dw[offsetof(struct amap_i_t_dpdu_cqe, 15506733b39aSJayamohan Kallickal index) / 32] & PDUCQE_INDEX_MASK)); 15516733b39aSJayamohan Kallickal break; 15526733b39aSJayamohan Kallickal default: 15536733b39aSJayamohan Kallickal pbusy_list = NULL; 155499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, 155599bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 155699bc5d55SJohn Soni Jose "BM_%d : Unexpected code=%d\n", 15576733b39aSJayamohan Kallickal pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, 15586733b39aSJayamohan Kallickal code) / 32] & PDUCQE_CODE_MASK); 15596733b39aSJayamohan Kallickal return NULL; 15606733b39aSJayamohan Kallickal } 15616733b39aSJayamohan Kallickal 15626733b39aSJayamohan Kallickal WARN_ON(list_empty(pbusy_list)); 15636733b39aSJayamohan Kallickal list_for_each_entry(pasync_handle, pbusy_list, link) { 1564dc63aac6SJayamohan Kallickal if (pasync_handle->pa.u.a64.address == phys_addr.u.a64.address) 15656733b39aSJayamohan Kallickal break; 15666733b39aSJayamohan Kallickal } 15676733b39aSJayamohan Kallickal 15686733b39aSJayamohan Kallickal WARN_ON(!pasync_handle); 15696733b39aSJayamohan Kallickal 15707da50879SJayamohan Kallickal pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid - 15717da50879SJayamohan Kallickal phba->fw_config.iscsi_cid_start; 15726733b39aSJayamohan Kallickal pasync_handle->is_header = is_header; 15736733b39aSJayamohan Kallickal pasync_handle->buffer_len = ((pdpdu_cqe-> 15746733b39aSJayamohan Kallickal dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32] 15756733b39aSJayamohan Kallickal & PDUCQE_DPL_MASK) >> 16); 15766733b39aSJayamohan Kallickal 15776733b39aSJayamohan Kallickal *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, 15786733b39aSJayamohan Kallickal index) / 32] & PDUCQE_INDEX_MASK); 15796733b39aSJayamohan Kallickal return pasync_handle; 15806733b39aSJayamohan Kallickal } 15816733b39aSJayamohan Kallickal 15826733b39aSJayamohan Kallickal static unsigned int 158399bc5d55SJohn Soni Jose hwi_update_async_writables(struct beiscsi_hba *phba, 158499bc5d55SJohn Soni Jose struct hwi_async_pdu_context *pasync_ctx, 15856733b39aSJayamohan Kallickal unsigned int is_header, unsigned int cq_index) 15866733b39aSJayamohan Kallickal { 15876733b39aSJayamohan Kallickal struct list_head *pbusy_list; 15886733b39aSJayamohan Kallickal struct async_pdu_handle *pasync_handle; 15896733b39aSJayamohan Kallickal unsigned int num_entries, writables = 0; 15906733b39aSJayamohan Kallickal unsigned int *pep_read_ptr, *pwritables; 15916733b39aSJayamohan Kallickal 1592dc63aac6SJayamohan Kallickal num_entries = pasync_ctx->num_entries; 15936733b39aSJayamohan Kallickal if (is_header) { 15946733b39aSJayamohan Kallickal pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr; 15956733b39aSJayamohan Kallickal pwritables = &pasync_ctx->async_header.writables; 15966733b39aSJayamohan Kallickal } else { 15976733b39aSJayamohan Kallickal pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr; 15986733b39aSJayamohan Kallickal pwritables = &pasync_ctx->async_data.writables; 15996733b39aSJayamohan Kallickal } 16006733b39aSJayamohan Kallickal 16016733b39aSJayamohan Kallickal while ((*pep_read_ptr) != cq_index) { 16026733b39aSJayamohan Kallickal (*pep_read_ptr)++; 16036733b39aSJayamohan Kallickal *pep_read_ptr = (*pep_read_ptr) % num_entries; 16046733b39aSJayamohan Kallickal 16056733b39aSJayamohan Kallickal pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header, 16066733b39aSJayamohan Kallickal *pep_read_ptr); 16076733b39aSJayamohan Kallickal if (writables == 0) 16086733b39aSJayamohan Kallickal WARN_ON(list_empty(pbusy_list)); 16096733b39aSJayamohan Kallickal 16106733b39aSJayamohan Kallickal if (!list_empty(pbusy_list)) { 16116733b39aSJayamohan Kallickal pasync_handle = list_entry(pbusy_list->next, 16126733b39aSJayamohan Kallickal struct async_pdu_handle, 16136733b39aSJayamohan Kallickal link); 16146733b39aSJayamohan Kallickal WARN_ON(!pasync_handle); 16156733b39aSJayamohan Kallickal pasync_handle->consumed = 1; 16166733b39aSJayamohan Kallickal } 16176733b39aSJayamohan Kallickal 16186733b39aSJayamohan Kallickal writables++; 16196733b39aSJayamohan Kallickal } 16206733b39aSJayamohan Kallickal 16216733b39aSJayamohan Kallickal if (!writables) { 162299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 162399bc5d55SJohn Soni Jose BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 162499bc5d55SJohn Soni Jose "BM_%d : Duplicate notification received - index 0x%x!!\n", 16256733b39aSJayamohan Kallickal cq_index); 16266733b39aSJayamohan Kallickal WARN_ON(1); 16276733b39aSJayamohan Kallickal } 16286733b39aSJayamohan Kallickal 16296733b39aSJayamohan Kallickal *pwritables = *pwritables + writables; 16306733b39aSJayamohan Kallickal return 0; 16316733b39aSJayamohan Kallickal } 16326733b39aSJayamohan Kallickal 16339728d8d0SJayamohan Kallickal static void hwi_free_async_msg(struct beiscsi_hba *phba, 16346733b39aSJayamohan Kallickal unsigned int cri) 16356733b39aSJayamohan Kallickal { 16366733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 16376733b39aSJayamohan Kallickal struct hwi_async_pdu_context *pasync_ctx; 16386733b39aSJayamohan Kallickal struct async_pdu_handle *pasync_handle, *tmp_handle; 16396733b39aSJayamohan Kallickal struct list_head *plist; 16406733b39aSJayamohan Kallickal 16416733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 16426733b39aSJayamohan Kallickal pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); 16436733b39aSJayamohan Kallickal 16446733b39aSJayamohan Kallickal plist = &pasync_ctx->async_entry[cri].wait_queue.list; 16456733b39aSJayamohan Kallickal 16466733b39aSJayamohan Kallickal list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) { 16476733b39aSJayamohan Kallickal list_del(&pasync_handle->link); 16486733b39aSJayamohan Kallickal 16499728d8d0SJayamohan Kallickal if (pasync_handle->is_header) { 16506733b39aSJayamohan Kallickal list_add_tail(&pasync_handle->link, 16516733b39aSJayamohan Kallickal &pasync_ctx->async_header.free_list); 16526733b39aSJayamohan Kallickal pasync_ctx->async_header.free_entries++; 16536733b39aSJayamohan Kallickal } else { 16546733b39aSJayamohan Kallickal list_add_tail(&pasync_handle->link, 16556733b39aSJayamohan Kallickal &pasync_ctx->async_data.free_list); 16566733b39aSJayamohan Kallickal pasync_ctx->async_data.free_entries++; 16576733b39aSJayamohan Kallickal } 16586733b39aSJayamohan Kallickal } 16596733b39aSJayamohan Kallickal 16606733b39aSJayamohan Kallickal INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list); 16616733b39aSJayamohan Kallickal pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0; 16626733b39aSJayamohan Kallickal pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0; 16636733b39aSJayamohan Kallickal } 16646733b39aSJayamohan Kallickal 16656733b39aSJayamohan Kallickal static struct phys_addr * 16666733b39aSJayamohan Kallickal hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx, 16676733b39aSJayamohan Kallickal unsigned int is_header, unsigned int host_write_ptr) 16686733b39aSJayamohan Kallickal { 16696733b39aSJayamohan Kallickal struct phys_addr *pasync_sge = NULL; 16706733b39aSJayamohan Kallickal 16716733b39aSJayamohan Kallickal if (is_header) 16726733b39aSJayamohan Kallickal pasync_sge = pasync_ctx->async_header.ring_base; 16736733b39aSJayamohan Kallickal else 16746733b39aSJayamohan Kallickal pasync_sge = pasync_ctx->async_data.ring_base; 16756733b39aSJayamohan Kallickal 16766733b39aSJayamohan Kallickal return pasync_sge + host_write_ptr; 16776733b39aSJayamohan Kallickal } 16786733b39aSJayamohan Kallickal 16796733b39aSJayamohan Kallickal static void hwi_post_async_buffers(struct beiscsi_hba *phba, 16806733b39aSJayamohan Kallickal unsigned int is_header) 16816733b39aSJayamohan Kallickal { 16826733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 16836733b39aSJayamohan Kallickal struct hwi_async_pdu_context *pasync_ctx; 16846733b39aSJayamohan Kallickal struct async_pdu_handle *pasync_handle; 16856733b39aSJayamohan Kallickal struct list_head *pfree_link, *pbusy_list; 16866733b39aSJayamohan Kallickal struct phys_addr *pasync_sge; 16876733b39aSJayamohan Kallickal unsigned int ring_id, num_entries; 16886733b39aSJayamohan Kallickal unsigned int host_write_num; 16896733b39aSJayamohan Kallickal unsigned int writables; 16906733b39aSJayamohan Kallickal unsigned int i = 0; 16916733b39aSJayamohan Kallickal u32 doorbell = 0; 16926733b39aSJayamohan Kallickal 16936733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 16946733b39aSJayamohan Kallickal pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); 1695dc63aac6SJayamohan Kallickal num_entries = pasync_ctx->num_entries; 16966733b39aSJayamohan Kallickal 16976733b39aSJayamohan Kallickal if (is_header) { 16986733b39aSJayamohan Kallickal writables = min(pasync_ctx->async_header.writables, 16996733b39aSJayamohan Kallickal pasync_ctx->async_header.free_entries); 17006733b39aSJayamohan Kallickal pfree_link = pasync_ctx->async_header.free_list.next; 17016733b39aSJayamohan Kallickal host_write_num = pasync_ctx->async_header.host_write_ptr; 17026733b39aSJayamohan Kallickal ring_id = phwi_ctrlr->default_pdu_hdr.id; 17036733b39aSJayamohan Kallickal } else { 17046733b39aSJayamohan Kallickal writables = min(pasync_ctx->async_data.writables, 17056733b39aSJayamohan Kallickal pasync_ctx->async_data.free_entries); 17066733b39aSJayamohan Kallickal pfree_link = pasync_ctx->async_data.free_list.next; 17076733b39aSJayamohan Kallickal host_write_num = pasync_ctx->async_data.host_write_ptr; 17086733b39aSJayamohan Kallickal ring_id = phwi_ctrlr->default_pdu_data.id; 17096733b39aSJayamohan Kallickal } 17106733b39aSJayamohan Kallickal 17116733b39aSJayamohan Kallickal writables = (writables / 8) * 8; 17126733b39aSJayamohan Kallickal if (writables) { 17136733b39aSJayamohan Kallickal for (i = 0; i < writables; i++) { 17146733b39aSJayamohan Kallickal pbusy_list = 17156733b39aSJayamohan Kallickal hwi_get_async_busy_list(pasync_ctx, is_header, 17166733b39aSJayamohan Kallickal host_write_num); 17176733b39aSJayamohan Kallickal pasync_handle = 17186733b39aSJayamohan Kallickal list_entry(pfree_link, struct async_pdu_handle, 17196733b39aSJayamohan Kallickal link); 17206733b39aSJayamohan Kallickal WARN_ON(!pasync_handle); 17216733b39aSJayamohan Kallickal pasync_handle->consumed = 0; 17226733b39aSJayamohan Kallickal 17236733b39aSJayamohan Kallickal pfree_link = pfree_link->next; 17246733b39aSJayamohan Kallickal 17256733b39aSJayamohan Kallickal pasync_sge = hwi_get_ring_address(pasync_ctx, 17266733b39aSJayamohan Kallickal is_header, host_write_num); 17276733b39aSJayamohan Kallickal 17286733b39aSJayamohan Kallickal pasync_sge->hi = pasync_handle->pa.u.a32.address_lo; 17296733b39aSJayamohan Kallickal pasync_sge->lo = pasync_handle->pa.u.a32.address_hi; 17306733b39aSJayamohan Kallickal 17316733b39aSJayamohan Kallickal list_move(&pasync_handle->link, pbusy_list); 17326733b39aSJayamohan Kallickal 17336733b39aSJayamohan Kallickal host_write_num++; 17346733b39aSJayamohan Kallickal host_write_num = host_write_num % num_entries; 17356733b39aSJayamohan Kallickal } 17366733b39aSJayamohan Kallickal 17376733b39aSJayamohan Kallickal if (is_header) { 17386733b39aSJayamohan Kallickal pasync_ctx->async_header.host_write_ptr = 17396733b39aSJayamohan Kallickal host_write_num; 17406733b39aSJayamohan Kallickal pasync_ctx->async_header.free_entries -= writables; 17416733b39aSJayamohan Kallickal pasync_ctx->async_header.writables -= writables; 17426733b39aSJayamohan Kallickal pasync_ctx->async_header.busy_entries += writables; 17436733b39aSJayamohan Kallickal } else { 17446733b39aSJayamohan Kallickal pasync_ctx->async_data.host_write_ptr = host_write_num; 17456733b39aSJayamohan Kallickal pasync_ctx->async_data.free_entries -= writables; 17466733b39aSJayamohan Kallickal pasync_ctx->async_data.writables -= writables; 17476733b39aSJayamohan Kallickal pasync_ctx->async_data.busy_entries += writables; 17486733b39aSJayamohan Kallickal } 17496733b39aSJayamohan Kallickal 17506733b39aSJayamohan Kallickal doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK; 17516733b39aSJayamohan Kallickal doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT; 17526733b39aSJayamohan Kallickal doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT; 17536733b39aSJayamohan Kallickal doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK) 17546733b39aSJayamohan Kallickal << DB_DEF_PDU_CQPROC_SHIFT; 17556733b39aSJayamohan Kallickal 17566733b39aSJayamohan Kallickal iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET); 17576733b39aSJayamohan Kallickal } 17586733b39aSJayamohan Kallickal } 17596733b39aSJayamohan Kallickal 17606733b39aSJayamohan Kallickal static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba, 17616733b39aSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn, 17626733b39aSJayamohan Kallickal struct i_t_dpdu_cqe *pdpdu_cqe) 17636733b39aSJayamohan Kallickal { 17646733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 17656733b39aSJayamohan Kallickal struct hwi_async_pdu_context *pasync_ctx; 17666733b39aSJayamohan Kallickal struct async_pdu_handle *pasync_handle = NULL; 17676733b39aSJayamohan Kallickal unsigned int cq_index = -1; 17686733b39aSJayamohan Kallickal 17696733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 17706733b39aSJayamohan Kallickal pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); 17716733b39aSJayamohan Kallickal 17726733b39aSJayamohan Kallickal pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx, 17736733b39aSJayamohan Kallickal pdpdu_cqe, &cq_index); 17746733b39aSJayamohan Kallickal BUG_ON(pasync_handle->is_header != 0); 17756733b39aSJayamohan Kallickal if (pasync_handle->consumed == 0) 177699bc5d55SJohn Soni Jose hwi_update_async_writables(phba, pasync_ctx, 177799bc5d55SJohn Soni Jose pasync_handle->is_header, cq_index); 17786733b39aSJayamohan Kallickal 17796733b39aSJayamohan Kallickal hwi_free_async_msg(phba, pasync_handle->cri); 17806733b39aSJayamohan Kallickal hwi_post_async_buffers(phba, pasync_handle->is_header); 17816733b39aSJayamohan Kallickal } 17826733b39aSJayamohan Kallickal 17836733b39aSJayamohan Kallickal static unsigned int 17846733b39aSJayamohan Kallickal hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn, 17856733b39aSJayamohan Kallickal struct beiscsi_hba *phba, 17866733b39aSJayamohan Kallickal struct hwi_async_pdu_context *pasync_ctx, unsigned short cri) 17876733b39aSJayamohan Kallickal { 17886733b39aSJayamohan Kallickal struct list_head *plist; 17896733b39aSJayamohan Kallickal struct async_pdu_handle *pasync_handle; 17906733b39aSJayamohan Kallickal void *phdr = NULL; 17916733b39aSJayamohan Kallickal unsigned int hdr_len = 0, buf_len = 0; 17926733b39aSJayamohan Kallickal unsigned int status, index = 0, offset = 0; 17936733b39aSJayamohan Kallickal void *pfirst_buffer = NULL; 17946733b39aSJayamohan Kallickal unsigned int num_buf = 0; 17956733b39aSJayamohan Kallickal 17966733b39aSJayamohan Kallickal plist = &pasync_ctx->async_entry[cri].wait_queue.list; 17976733b39aSJayamohan Kallickal 17986733b39aSJayamohan Kallickal list_for_each_entry(pasync_handle, plist, link) { 17996733b39aSJayamohan Kallickal if (index == 0) { 18006733b39aSJayamohan Kallickal phdr = pasync_handle->pbuffer; 18016733b39aSJayamohan Kallickal hdr_len = pasync_handle->buffer_len; 18026733b39aSJayamohan Kallickal } else { 18036733b39aSJayamohan Kallickal buf_len = pasync_handle->buffer_len; 18046733b39aSJayamohan Kallickal if (!num_buf) { 18056733b39aSJayamohan Kallickal pfirst_buffer = pasync_handle->pbuffer; 18066733b39aSJayamohan Kallickal num_buf++; 18076733b39aSJayamohan Kallickal } 18086733b39aSJayamohan Kallickal memcpy(pfirst_buffer + offset, 18096733b39aSJayamohan Kallickal pasync_handle->pbuffer, buf_len); 1810f2ba02b8SJayamohan Kallickal offset += buf_len; 18116733b39aSJayamohan Kallickal } 18126733b39aSJayamohan Kallickal index++; 18136733b39aSJayamohan Kallickal } 18146733b39aSJayamohan Kallickal 18156733b39aSJayamohan Kallickal status = beiscsi_process_async_pdu(beiscsi_conn, phba, 18167da50879SJayamohan Kallickal (beiscsi_conn->beiscsi_conn_cid - 18177da50879SJayamohan Kallickal phba->fw_config.iscsi_cid_start), 18186733b39aSJayamohan Kallickal phdr, hdr_len, pfirst_buffer, 1819f2ba02b8SJayamohan Kallickal offset); 18206733b39aSJayamohan Kallickal 18216733b39aSJayamohan Kallickal hwi_free_async_msg(phba, cri); 18226733b39aSJayamohan Kallickal return 0; 18236733b39aSJayamohan Kallickal } 18246733b39aSJayamohan Kallickal 18256733b39aSJayamohan Kallickal static unsigned int 18266733b39aSJayamohan Kallickal hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn, 18276733b39aSJayamohan Kallickal struct beiscsi_hba *phba, 18286733b39aSJayamohan Kallickal struct async_pdu_handle *pasync_handle) 18296733b39aSJayamohan Kallickal { 18306733b39aSJayamohan Kallickal struct hwi_async_pdu_context *pasync_ctx; 18316733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 18326733b39aSJayamohan Kallickal unsigned int bytes_needed = 0, status = 0; 18336733b39aSJayamohan Kallickal unsigned short cri = pasync_handle->cri; 18346733b39aSJayamohan Kallickal struct pdu_base *ppdu; 18356733b39aSJayamohan Kallickal 18366733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 18376733b39aSJayamohan Kallickal pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); 18386733b39aSJayamohan Kallickal 18396733b39aSJayamohan Kallickal list_del(&pasync_handle->link); 18406733b39aSJayamohan Kallickal if (pasync_handle->is_header) { 18416733b39aSJayamohan Kallickal pasync_ctx->async_header.busy_entries--; 18426733b39aSJayamohan Kallickal if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) { 18436733b39aSJayamohan Kallickal hwi_free_async_msg(phba, cri); 18446733b39aSJayamohan Kallickal BUG(); 18456733b39aSJayamohan Kallickal } 18466733b39aSJayamohan Kallickal 18476733b39aSJayamohan Kallickal pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0; 18486733b39aSJayamohan Kallickal pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1; 18496733b39aSJayamohan Kallickal pasync_ctx->async_entry[cri].wait_queue.hdr_len = 18506733b39aSJayamohan Kallickal (unsigned short)pasync_handle->buffer_len; 18516733b39aSJayamohan Kallickal list_add_tail(&pasync_handle->link, 18526733b39aSJayamohan Kallickal &pasync_ctx->async_entry[cri].wait_queue.list); 18536733b39aSJayamohan Kallickal 18546733b39aSJayamohan Kallickal ppdu = pasync_handle->pbuffer; 18556733b39aSJayamohan Kallickal bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base, 18566733b39aSJayamohan Kallickal data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) & 18576733b39aSJayamohan Kallickal 0xFFFF0000) | ((be16_to_cpu((ppdu-> 18586733b39aSJayamohan Kallickal dw[offsetof(struct amap_pdu_base, data_len_lo) / 32] 18596733b39aSJayamohan Kallickal & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF)); 18606733b39aSJayamohan Kallickal 18616733b39aSJayamohan Kallickal if (status == 0) { 18626733b39aSJayamohan Kallickal pasync_ctx->async_entry[cri].wait_queue.bytes_needed = 18636733b39aSJayamohan Kallickal bytes_needed; 18646733b39aSJayamohan Kallickal 18656733b39aSJayamohan Kallickal if (bytes_needed == 0) 18666733b39aSJayamohan Kallickal status = hwi_fwd_async_msg(beiscsi_conn, phba, 18676733b39aSJayamohan Kallickal pasync_ctx, cri); 18686733b39aSJayamohan Kallickal } 18696733b39aSJayamohan Kallickal } else { 18706733b39aSJayamohan Kallickal pasync_ctx->async_data.busy_entries--; 18716733b39aSJayamohan Kallickal if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) { 18726733b39aSJayamohan Kallickal list_add_tail(&pasync_handle->link, 18736733b39aSJayamohan Kallickal &pasync_ctx->async_entry[cri].wait_queue. 18746733b39aSJayamohan Kallickal list); 18756733b39aSJayamohan Kallickal pasync_ctx->async_entry[cri].wait_queue. 18766733b39aSJayamohan Kallickal bytes_received += 18776733b39aSJayamohan Kallickal (unsigned short)pasync_handle->buffer_len; 18786733b39aSJayamohan Kallickal 18796733b39aSJayamohan Kallickal if (pasync_ctx->async_entry[cri].wait_queue. 18806733b39aSJayamohan Kallickal bytes_received >= 18816733b39aSJayamohan Kallickal pasync_ctx->async_entry[cri].wait_queue. 18826733b39aSJayamohan Kallickal bytes_needed) 18836733b39aSJayamohan Kallickal status = hwi_fwd_async_msg(beiscsi_conn, phba, 18846733b39aSJayamohan Kallickal pasync_ctx, cri); 18856733b39aSJayamohan Kallickal } 18866733b39aSJayamohan Kallickal } 18876733b39aSJayamohan Kallickal return status; 18886733b39aSJayamohan Kallickal } 18896733b39aSJayamohan Kallickal 18906733b39aSJayamohan Kallickal static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn, 18916733b39aSJayamohan Kallickal struct beiscsi_hba *phba, 18926733b39aSJayamohan Kallickal struct i_t_dpdu_cqe *pdpdu_cqe) 18936733b39aSJayamohan Kallickal { 18946733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 18956733b39aSJayamohan Kallickal struct hwi_async_pdu_context *pasync_ctx; 18966733b39aSJayamohan Kallickal struct async_pdu_handle *pasync_handle = NULL; 18976733b39aSJayamohan Kallickal unsigned int cq_index = -1; 18986733b39aSJayamohan Kallickal 18996733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 19006733b39aSJayamohan Kallickal pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); 19016733b39aSJayamohan Kallickal pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx, 19026733b39aSJayamohan Kallickal pdpdu_cqe, &cq_index); 19036733b39aSJayamohan Kallickal 19046733b39aSJayamohan Kallickal if (pasync_handle->consumed == 0) 190599bc5d55SJohn Soni Jose hwi_update_async_writables(phba, pasync_ctx, 190699bc5d55SJohn Soni Jose pasync_handle->is_header, cq_index); 190799bc5d55SJohn Soni Jose 19086733b39aSJayamohan Kallickal hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle); 19096733b39aSJayamohan Kallickal hwi_post_async_buffers(phba, pasync_handle->is_header); 19106733b39aSJayamohan Kallickal } 19116733b39aSJayamohan Kallickal 1912756d29c8SJayamohan Kallickal static void beiscsi_process_mcc_isr(struct beiscsi_hba *phba) 1913756d29c8SJayamohan Kallickal { 1914756d29c8SJayamohan Kallickal struct be_queue_info *mcc_cq; 1915756d29c8SJayamohan Kallickal struct be_mcc_compl *mcc_compl; 1916756d29c8SJayamohan Kallickal unsigned int num_processed = 0; 1917756d29c8SJayamohan Kallickal 1918756d29c8SJayamohan Kallickal mcc_cq = &phba->ctrl.mcc_obj.cq; 1919756d29c8SJayamohan Kallickal mcc_compl = queue_tail_node(mcc_cq); 1920756d29c8SJayamohan Kallickal mcc_compl->flags = le32_to_cpu(mcc_compl->flags); 1921756d29c8SJayamohan Kallickal while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) { 1922756d29c8SJayamohan Kallickal 1923756d29c8SJayamohan Kallickal if (num_processed >= 32) { 1924756d29c8SJayamohan Kallickal hwi_ring_cq_db(phba, mcc_cq->id, 1925756d29c8SJayamohan Kallickal num_processed, 0, 0); 1926756d29c8SJayamohan Kallickal num_processed = 0; 1927756d29c8SJayamohan Kallickal } 1928756d29c8SJayamohan Kallickal if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) { 1929756d29c8SJayamohan Kallickal /* Interpret flags as an async trailer */ 1930756d29c8SJayamohan Kallickal if (is_link_state_evt(mcc_compl->flags)) 1931756d29c8SJayamohan Kallickal /* Interpret compl as a async link evt */ 1932756d29c8SJayamohan Kallickal beiscsi_async_link_state_process(phba, 1933756d29c8SJayamohan Kallickal (struct be_async_event_link_state *) mcc_compl); 1934756d29c8SJayamohan Kallickal else 193599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_MBOX, 193699bc5d55SJohn Soni Jose "BM_%d : Unsupported Async Event, flags" 193799bc5d55SJohn Soni Jose " = 0x%08x\n", 193899bc5d55SJohn Soni Jose mcc_compl->flags); 1939756d29c8SJayamohan Kallickal } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) { 1940756d29c8SJayamohan Kallickal be_mcc_compl_process_isr(&phba->ctrl, mcc_compl); 1941756d29c8SJayamohan Kallickal atomic_dec(&phba->ctrl.mcc_obj.q.used); 1942756d29c8SJayamohan Kallickal } 1943756d29c8SJayamohan Kallickal 1944756d29c8SJayamohan Kallickal mcc_compl->flags = 0; 1945756d29c8SJayamohan Kallickal queue_tail_inc(mcc_cq); 1946756d29c8SJayamohan Kallickal mcc_compl = queue_tail_node(mcc_cq); 1947756d29c8SJayamohan Kallickal mcc_compl->flags = le32_to_cpu(mcc_compl->flags); 1948756d29c8SJayamohan Kallickal num_processed++; 1949756d29c8SJayamohan Kallickal } 1950756d29c8SJayamohan Kallickal 1951756d29c8SJayamohan Kallickal if (num_processed > 0) 1952756d29c8SJayamohan Kallickal hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0); 1953756d29c8SJayamohan Kallickal 1954756d29c8SJayamohan Kallickal } 1955bfead3b2SJayamohan Kallickal 19566763daaeSJohn Soni Jose /** 19576763daaeSJohn Soni Jose * beiscsi_process_cq()- Process the Completion Queue 19586763daaeSJohn Soni Jose * @pbe_eq: Event Q on which the Completion has come 19596763daaeSJohn Soni Jose * 19606763daaeSJohn Soni Jose * return 19616763daaeSJohn Soni Jose * Number of Completion Entries processed. 19626763daaeSJohn Soni Jose **/ 1963bfead3b2SJayamohan Kallickal static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq) 19646733b39aSJayamohan Kallickal { 19656733b39aSJayamohan Kallickal struct be_queue_info *cq; 19666733b39aSJayamohan Kallickal struct sol_cqe *sol; 19676733b39aSJayamohan Kallickal struct dmsg_cqe *dmsg; 19686733b39aSJayamohan Kallickal unsigned int num_processed = 0; 19696733b39aSJayamohan Kallickal unsigned int tot_nump = 0; 19700a513dd8SJohn Soni Jose unsigned short code = 0, cid = 0; 19716733b39aSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn; 1972c2462288SJayamohan Kallickal struct beiscsi_endpoint *beiscsi_ep; 1973c2462288SJayamohan Kallickal struct iscsi_endpoint *ep; 1974bfead3b2SJayamohan Kallickal struct beiscsi_hba *phba; 19756733b39aSJayamohan Kallickal 1976bfead3b2SJayamohan Kallickal cq = pbe_eq->cq; 19776733b39aSJayamohan Kallickal sol = queue_tail_node(cq); 1978bfead3b2SJayamohan Kallickal phba = pbe_eq->phba; 19796733b39aSJayamohan Kallickal 19806733b39aSJayamohan Kallickal while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] & 19816733b39aSJayamohan Kallickal CQE_VALID_MASK) { 19826733b39aSJayamohan Kallickal be_dws_le_to_cpu(sol, sizeof(struct sol_cqe)); 19836733b39aSJayamohan Kallickal 19840a513dd8SJohn Soni Jose cid = ((sol->dw[offsetof(struct amap_sol_cqe, cid)/32] & 19850a513dd8SJohn Soni Jose CQE_CID_MASK) >> 6); 19860a513dd8SJohn Soni Jose code = (sol->dw[offsetof(struct amap_sol_cqe, code)/32] & 19870a513dd8SJohn Soni Jose CQE_CODE_MASK); 19880a513dd8SJohn Soni Jose ep = phba->ep_array[cid - phba->fw_config.iscsi_cid_start]; 198932951dd8SJayamohan Kallickal 1990c2462288SJayamohan Kallickal beiscsi_ep = ep->dd_data; 1991c2462288SJayamohan Kallickal beiscsi_conn = beiscsi_ep->conn; 1992756d29c8SJayamohan Kallickal 19936733b39aSJayamohan Kallickal if (num_processed >= 32) { 1994bfead3b2SJayamohan Kallickal hwi_ring_cq_db(phba, cq->id, 19956733b39aSJayamohan Kallickal num_processed, 0, 0); 19966733b39aSJayamohan Kallickal tot_nump += num_processed; 19976733b39aSJayamohan Kallickal num_processed = 0; 19986733b39aSJayamohan Kallickal } 19996733b39aSJayamohan Kallickal 20000a513dd8SJohn Soni Jose switch (code) { 20016733b39aSJayamohan Kallickal case SOL_CMD_COMPLETE: 20026733b39aSJayamohan Kallickal hwi_complete_cmd(beiscsi_conn, phba, sol); 20036733b39aSJayamohan Kallickal break; 20046733b39aSJayamohan Kallickal case DRIVERMSG_NOTIFY: 200599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, 200699bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 20076763daaeSJohn Soni Jose "BM_%d : Received %s[%d] on CID : %d\n", 20086763daaeSJohn Soni Jose cqe_desc[code], code, cid); 200999bc5d55SJohn Soni Jose 20106733b39aSJayamohan Kallickal dmsg = (struct dmsg_cqe *)sol; 20116733b39aSJayamohan Kallickal hwi_complete_drvr_msgs(beiscsi_conn, phba, sol); 20126733b39aSJayamohan Kallickal break; 20136733b39aSJayamohan Kallickal case UNSOL_HDR_NOTIFY: 201499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, 201599bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 20166763daaeSJohn Soni Jose "BM_%d : Received %s[%d] on CID : %d\n", 20176763daaeSJohn Soni Jose cqe_desc[code], code, cid); 201899bc5d55SJohn Soni Jose 2019bfead3b2SJayamohan Kallickal hwi_process_default_pdu_ring(beiscsi_conn, phba, 2020bfead3b2SJayamohan Kallickal (struct i_t_dpdu_cqe *)sol); 2021bfead3b2SJayamohan Kallickal break; 20226733b39aSJayamohan Kallickal case UNSOL_DATA_NOTIFY: 202399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, 202499bc5d55SJohn Soni Jose BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 20256763daaeSJohn Soni Jose "BM_%d : Received %s[%d] on CID : %d\n", 20266763daaeSJohn Soni Jose cqe_desc[code], code, cid); 202799bc5d55SJohn Soni Jose 20286733b39aSJayamohan Kallickal hwi_process_default_pdu_ring(beiscsi_conn, phba, 20296733b39aSJayamohan Kallickal (struct i_t_dpdu_cqe *)sol); 20306733b39aSJayamohan Kallickal break; 20316733b39aSJayamohan Kallickal case CXN_INVALIDATE_INDEX_NOTIFY: 20326733b39aSJayamohan Kallickal case CMD_INVALIDATED_NOTIFY: 20336733b39aSJayamohan Kallickal case CXN_INVALIDATE_NOTIFY: 203499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 203599bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 20366763daaeSJohn Soni Jose "BM_%d : Ignoring %s[%d] on CID : %d\n", 20376763daaeSJohn Soni Jose cqe_desc[code], code, cid); 20386733b39aSJayamohan Kallickal break; 20396733b39aSJayamohan Kallickal case SOL_CMD_KILLED_DATA_DIGEST_ERR: 20406733b39aSJayamohan Kallickal case CMD_KILLED_INVALID_STATSN_RCVD: 20416733b39aSJayamohan Kallickal case CMD_KILLED_INVALID_R2T_RCVD: 20426733b39aSJayamohan Kallickal case CMD_CXN_KILLED_LUN_INVALID: 20436733b39aSJayamohan Kallickal case CMD_CXN_KILLED_ICD_INVALID: 20446733b39aSJayamohan Kallickal case CMD_CXN_KILLED_ITT_INVALID: 20456733b39aSJayamohan Kallickal case CMD_CXN_KILLED_SEQ_OUTOFORDER: 20466733b39aSJayamohan Kallickal case CMD_CXN_KILLED_INVALID_DATASN_RCVD: 204799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 204899bc5d55SJohn Soni Jose BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 20496763daaeSJohn Soni Jose "BM_%d : Cmd Notification %s[%d] on CID : %d\n", 20506763daaeSJohn Soni Jose cqe_desc[code], code, cid); 20516733b39aSJayamohan Kallickal break; 20526733b39aSJayamohan Kallickal case UNSOL_DATA_DIGEST_ERROR_NOTIFY: 205399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 205499bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 20556763daaeSJohn Soni Jose "BM_%d : Dropping %s[%d] on DPDU ring on CID : %d\n", 20566763daaeSJohn Soni Jose cqe_desc[code], code, cid); 20576733b39aSJayamohan Kallickal hwi_flush_default_pdu_buffer(phba, beiscsi_conn, 20586733b39aSJayamohan Kallickal (struct i_t_dpdu_cqe *) sol); 20596733b39aSJayamohan Kallickal break; 20606733b39aSJayamohan Kallickal case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL: 20616733b39aSJayamohan Kallickal case CXN_KILLED_BURST_LEN_MISMATCH: 20626733b39aSJayamohan Kallickal case CXN_KILLED_AHS_RCVD: 20636733b39aSJayamohan Kallickal case CXN_KILLED_HDR_DIGEST_ERR: 20646733b39aSJayamohan Kallickal case CXN_KILLED_UNKNOWN_HDR: 20656733b39aSJayamohan Kallickal case CXN_KILLED_STALE_ITT_TTT_RCVD: 20666733b39aSJayamohan Kallickal case CXN_KILLED_INVALID_ITT_TTT_RCVD: 20676733b39aSJayamohan Kallickal case CXN_KILLED_TIMED_OUT: 20686733b39aSJayamohan Kallickal case CXN_KILLED_FIN_RCVD: 20696763daaeSJohn Soni Jose case CXN_KILLED_RST_SENT: 20706763daaeSJohn Soni Jose case CXN_KILLED_RST_RCVD: 20716733b39aSJayamohan Kallickal case CXN_KILLED_BAD_UNSOL_PDU_RCVD: 20726733b39aSJayamohan Kallickal case CXN_KILLED_BAD_WRB_INDEX_ERROR: 20736733b39aSJayamohan Kallickal case CXN_KILLED_OVER_RUN_RESIDUAL: 20746733b39aSJayamohan Kallickal case CXN_KILLED_UNDER_RUN_RESIDUAL: 20756733b39aSJayamohan Kallickal case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN: 207699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 207799bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 20786763daaeSJohn Soni Jose "BM_%d : Event %s[%d] received on CID : %d\n", 20796763daaeSJohn Soni Jose cqe_desc[code], code, cid); 20800a513dd8SJohn Soni Jose if (beiscsi_conn) 20816733b39aSJayamohan Kallickal iscsi_conn_failure(beiscsi_conn->conn, 20826733b39aSJayamohan Kallickal ISCSI_ERR_CONN_FAILED); 20836733b39aSJayamohan Kallickal break; 20846733b39aSJayamohan Kallickal default: 208599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 208699bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 20876763daaeSJohn Soni Jose "BM_%d : Invalid CQE Event Received Code : %d" 20886763daaeSJohn Soni Jose "CID 0x%x...\n", 20890a513dd8SJohn Soni Jose code, cid); 20906733b39aSJayamohan Kallickal break; 20916733b39aSJayamohan Kallickal } 20926733b39aSJayamohan Kallickal 20936733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0); 20946733b39aSJayamohan Kallickal queue_tail_inc(cq); 20956733b39aSJayamohan Kallickal sol = queue_tail_node(cq); 20966733b39aSJayamohan Kallickal num_processed++; 20976733b39aSJayamohan Kallickal } 20986733b39aSJayamohan Kallickal 20996733b39aSJayamohan Kallickal if (num_processed > 0) { 21006733b39aSJayamohan Kallickal tot_nump += num_processed; 2101bfead3b2SJayamohan Kallickal hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0); 21026733b39aSJayamohan Kallickal } 21036733b39aSJayamohan Kallickal return tot_nump; 21046733b39aSJayamohan Kallickal } 21056733b39aSJayamohan Kallickal 2106756d29c8SJayamohan Kallickal void beiscsi_process_all_cqs(struct work_struct *work) 21076733b39aSJayamohan Kallickal { 21086733b39aSJayamohan Kallickal unsigned long flags; 2109bfead3b2SJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 2110bfead3b2SJayamohan Kallickal struct hwi_context_memory *phwi_context; 2111bfead3b2SJayamohan Kallickal struct be_eq_obj *pbe_eq; 21126733b39aSJayamohan Kallickal struct beiscsi_hba *phba = 21136733b39aSJayamohan Kallickal container_of(work, struct beiscsi_hba, work_cqs); 21146733b39aSJayamohan Kallickal 2115bfead3b2SJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 2116bfead3b2SJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 2117bfead3b2SJayamohan Kallickal if (phba->msix_enabled) 2118bfead3b2SJayamohan Kallickal pbe_eq = &phwi_context->be_eq[phba->num_cpus]; 2119bfead3b2SJayamohan Kallickal else 2120bfead3b2SJayamohan Kallickal pbe_eq = &phwi_context->be_eq[0]; 2121bfead3b2SJayamohan Kallickal 21226733b39aSJayamohan Kallickal if (phba->todo_mcc_cq) { 21236733b39aSJayamohan Kallickal spin_lock_irqsave(&phba->isr_lock, flags); 21246733b39aSJayamohan Kallickal phba->todo_mcc_cq = 0; 21256733b39aSJayamohan Kallickal spin_unlock_irqrestore(&phba->isr_lock, flags); 2126756d29c8SJayamohan Kallickal beiscsi_process_mcc_isr(phba); 21276733b39aSJayamohan Kallickal } 21286733b39aSJayamohan Kallickal 21296733b39aSJayamohan Kallickal if (phba->todo_cq) { 21306733b39aSJayamohan Kallickal spin_lock_irqsave(&phba->isr_lock, flags); 21316733b39aSJayamohan Kallickal phba->todo_cq = 0; 21326733b39aSJayamohan Kallickal spin_unlock_irqrestore(&phba->isr_lock, flags); 2133bfead3b2SJayamohan Kallickal beiscsi_process_cq(pbe_eq); 21346733b39aSJayamohan Kallickal } 21356733b39aSJayamohan Kallickal } 21366733b39aSJayamohan Kallickal 21376733b39aSJayamohan Kallickal static int be_iopoll(struct blk_iopoll *iop, int budget) 21386733b39aSJayamohan Kallickal { 21396733b39aSJayamohan Kallickal static unsigned int ret; 21406733b39aSJayamohan Kallickal struct beiscsi_hba *phba; 2141bfead3b2SJayamohan Kallickal struct be_eq_obj *pbe_eq; 21426733b39aSJayamohan Kallickal 2143bfead3b2SJayamohan Kallickal pbe_eq = container_of(iop, struct be_eq_obj, iopoll); 2144bfead3b2SJayamohan Kallickal ret = beiscsi_process_cq(pbe_eq); 21456733b39aSJayamohan Kallickal if (ret < budget) { 2146bfead3b2SJayamohan Kallickal phba = pbe_eq->phba; 21476733b39aSJayamohan Kallickal blk_iopoll_complete(iop); 214899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, 214999bc5d55SJohn Soni Jose BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 215099bc5d55SJohn Soni Jose "BM_%d : rearm pbe_eq->q.id =%d\n", 215199bc5d55SJohn Soni Jose pbe_eq->q.id); 2152bfead3b2SJayamohan Kallickal hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1); 21536733b39aSJayamohan Kallickal } 21546733b39aSJayamohan Kallickal return ret; 21556733b39aSJayamohan Kallickal } 21566733b39aSJayamohan Kallickal 21576733b39aSJayamohan Kallickal static void 21586733b39aSJayamohan Kallickal hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg, 21596733b39aSJayamohan Kallickal unsigned int num_sg, struct beiscsi_io_task *io_task) 21606733b39aSJayamohan Kallickal { 21616733b39aSJayamohan Kallickal struct iscsi_sge *psgl; 216258ff4bd0SJayamohan Kallickal unsigned int sg_len, index; 21636733b39aSJayamohan Kallickal unsigned int sge_len = 0; 21646733b39aSJayamohan Kallickal unsigned long long addr; 21656733b39aSJayamohan Kallickal struct scatterlist *l_sg; 21666733b39aSJayamohan Kallickal unsigned int offset; 21676733b39aSJayamohan Kallickal 21686733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb, 21696733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_lo); 21706733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb, 21716733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_hi); 21726733b39aSJayamohan Kallickal 21736733b39aSJayamohan Kallickal l_sg = sg; 217448bd86cfSJayamohan Kallickal for (index = 0; (index < num_sg) && (index < 2); index++, 217548bd86cfSJayamohan Kallickal sg = sg_next(sg)) { 21766733b39aSJayamohan Kallickal if (index == 0) { 21776733b39aSJayamohan Kallickal sg_len = sg_dma_len(sg); 21786733b39aSJayamohan Kallickal addr = (u64) sg_dma_address(sg); 21796733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb, 2180457ff3b7SJayamohan Kallickal ((u32)(addr & 0xFFFFFFFF))); 21816733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb, 2182457ff3b7SJayamohan Kallickal ((u32)(addr >> 32))); 21836733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb, 21846733b39aSJayamohan Kallickal sg_len); 21856733b39aSJayamohan Kallickal sge_len = sg_len; 21866733b39aSJayamohan Kallickal } else { 21876733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset, 21886733b39aSJayamohan Kallickal pwrb, sge_len); 21896733b39aSJayamohan Kallickal sg_len = sg_dma_len(sg); 21906733b39aSJayamohan Kallickal addr = (u64) sg_dma_address(sg); 21916733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb, 2192457ff3b7SJayamohan Kallickal ((u32)(addr & 0xFFFFFFFF))); 21936733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb, 2194457ff3b7SJayamohan Kallickal ((u32)(addr >> 32))); 21956733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb, 21966733b39aSJayamohan Kallickal sg_len); 21976733b39aSJayamohan Kallickal } 21986733b39aSJayamohan Kallickal } 21996733b39aSJayamohan Kallickal psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag; 22006733b39aSJayamohan Kallickal memset(psgl, 0, sizeof(*psgl) * BE2_SGE); 22016733b39aSJayamohan Kallickal 22026733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2); 22036733b39aSJayamohan Kallickal 22046733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 22056733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_hi); 22066733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 22076733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_lo); 22086733b39aSJayamohan Kallickal 2209caf818f1SJayamohan Kallickal if (num_sg == 1) { 2210caf818f1SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 2211caf818f1SJayamohan Kallickal 1); 2212caf818f1SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 2213caf818f1SJayamohan Kallickal 0); 2214caf818f1SJayamohan Kallickal } else if (num_sg == 2) { 2215caf818f1SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 2216caf818f1SJayamohan Kallickal 0); 2217caf818f1SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 2218caf818f1SJayamohan Kallickal 1); 2219caf818f1SJayamohan Kallickal } else { 2220caf818f1SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 2221caf818f1SJayamohan Kallickal 0); 2222caf818f1SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 2223caf818f1SJayamohan Kallickal 0); 2224caf818f1SJayamohan Kallickal } 22256733b39aSJayamohan Kallickal sg = l_sg; 22266733b39aSJayamohan Kallickal psgl++; 22276733b39aSJayamohan Kallickal psgl++; 22286733b39aSJayamohan Kallickal offset = 0; 222948bd86cfSJayamohan Kallickal for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) { 22306733b39aSJayamohan Kallickal sg_len = sg_dma_len(sg); 22316733b39aSJayamohan Kallickal addr = (u64) sg_dma_address(sg); 22326733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 22336733b39aSJayamohan Kallickal (addr & 0xFFFFFFFF)); 22346733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 22356733b39aSJayamohan Kallickal (addr >> 32)); 22366733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len); 22376733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset); 22386733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0); 22396733b39aSJayamohan Kallickal offset += sg_len; 22406733b39aSJayamohan Kallickal } 22416733b39aSJayamohan Kallickal psgl--; 22426733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1); 22436733b39aSJayamohan Kallickal } 22446733b39aSJayamohan Kallickal 2245d629c471SJohn Soni Jose /** 2246d629c471SJohn Soni Jose * hwi_write_buffer()- Populate the WRB with task info 2247d629c471SJohn Soni Jose * @pwrb: ptr to the WRB entry 2248d629c471SJohn Soni Jose * @task: iscsi task which is to be executed 2249d629c471SJohn Soni Jose **/ 22506733b39aSJayamohan Kallickal static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task) 22516733b39aSJayamohan Kallickal { 22526733b39aSJayamohan Kallickal struct iscsi_sge *psgl; 22536733b39aSJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 22546733b39aSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn = io_task->conn; 22556733b39aSJayamohan Kallickal struct beiscsi_hba *phba = beiscsi_conn->phba; 22566733b39aSJayamohan Kallickal 22576733b39aSJayamohan Kallickal io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2; 22586733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb, 22596733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_lo); 22606733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb, 22616733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_hi); 22626733b39aSJayamohan Kallickal 22636733b39aSJayamohan Kallickal if (task->data) { 22646733b39aSJayamohan Kallickal if (task->data_count) { 22656733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1); 2266d629c471SJohn Soni Jose io_task->mtask_addr = pci_map_single(phba->pcidev, 22676733b39aSJayamohan Kallickal task->data, 2268d629c471SJohn Soni Jose task->data_count, 2269d629c471SJohn Soni Jose PCI_DMA_TODEVICE); 2270d629c471SJohn Soni Jose 2271d629c471SJohn Soni Jose io_task->mtask_data_count = task->data_count; 22726733b39aSJayamohan Kallickal } else { 22736733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); 2274d629c471SJohn Soni Jose io_task->mtask_addr = 0; 22756733b39aSJayamohan Kallickal } 22766733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb, 2277d629c471SJohn Soni Jose lower_32_bits(io_task->mtask_addr)); 22786733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb, 2279d629c471SJohn Soni Jose upper_32_bits(io_task->mtask_addr)); 22806733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb, 22816733b39aSJayamohan Kallickal task->data_count); 22826733b39aSJayamohan Kallickal 22836733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1); 22846733b39aSJayamohan Kallickal } else { 22856733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); 2286d629c471SJohn Soni Jose io_task->mtask_addr = 0; 22876733b39aSJayamohan Kallickal } 22886733b39aSJayamohan Kallickal 22896733b39aSJayamohan Kallickal psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag; 22906733b39aSJayamohan Kallickal 22916733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len); 22926733b39aSJayamohan Kallickal 22936733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 22946733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_hi); 22956733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 22966733b39aSJayamohan Kallickal io_task->bhs_pa.u.a32.address_lo); 22976733b39aSJayamohan Kallickal if (task->data) { 22986733b39aSJayamohan Kallickal psgl++; 22996733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0); 23006733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0); 23016733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0); 23026733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0); 23036733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0); 23046733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0); 23056733b39aSJayamohan Kallickal 23066733b39aSJayamohan Kallickal psgl++; 23076733b39aSJayamohan Kallickal if (task->data) { 23086733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 2309d629c471SJohn Soni Jose lower_32_bits(io_task->mtask_addr)); 23106733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 2311d629c471SJohn Soni Jose upper_32_bits(io_task->mtask_addr)); 23126733b39aSJayamohan Kallickal } 23136733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106); 23146733b39aSJayamohan Kallickal } 23156733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1); 23166733b39aSJayamohan Kallickal } 23176733b39aSJayamohan Kallickal 23186733b39aSJayamohan Kallickal static void beiscsi_find_mem_req(struct beiscsi_hba *phba) 23196733b39aSJayamohan Kallickal { 2320bfead3b2SJayamohan Kallickal unsigned int num_cq_pages, num_async_pdu_buf_pages; 23216733b39aSJayamohan Kallickal unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn; 23226733b39aSJayamohan Kallickal unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages; 23236733b39aSJayamohan Kallickal 23246733b39aSJayamohan Kallickal num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \ 23256733b39aSJayamohan Kallickal sizeof(struct sol_cqe)); 23266733b39aSJayamohan Kallickal num_async_pdu_buf_pages = 23276733b39aSJayamohan Kallickal PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \ 23286733b39aSJayamohan Kallickal phba->params.defpdu_hdr_sz); 23296733b39aSJayamohan Kallickal num_async_pdu_buf_sgl_pages = 23306733b39aSJayamohan Kallickal PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \ 23316733b39aSJayamohan Kallickal sizeof(struct phys_addr)); 23326733b39aSJayamohan Kallickal num_async_pdu_data_pages = 23336733b39aSJayamohan Kallickal PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \ 23346733b39aSJayamohan Kallickal phba->params.defpdu_data_sz); 23356733b39aSJayamohan Kallickal num_async_pdu_data_sgl_pages = 23366733b39aSJayamohan Kallickal PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \ 23376733b39aSJayamohan Kallickal sizeof(struct phys_addr)); 23386733b39aSJayamohan Kallickal 23396733b39aSJayamohan Kallickal phba->params.hwi_ws_sz = sizeof(struct hwi_controller); 23406733b39aSJayamohan Kallickal 23416733b39aSJayamohan Kallickal phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 * 23426733b39aSJayamohan Kallickal BE_ISCSI_PDU_HEADER_SIZE; 23436733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_ADDN_CONTEXT] = 23446733b39aSJayamohan Kallickal sizeof(struct hwi_context_memory); 23456733b39aSJayamohan Kallickal 23466733b39aSJayamohan Kallickal 23476733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb) 23486733b39aSJayamohan Kallickal * (phba->params.wrbs_per_cxn) 23496733b39aSJayamohan Kallickal * phba->params.cxns_per_ctrl; 23506733b39aSJayamohan Kallickal wrb_sz_per_cxn = sizeof(struct wrb_handle) * 23516733b39aSJayamohan Kallickal (phba->params.wrbs_per_cxn); 23526733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) * 23536733b39aSJayamohan Kallickal phba->params.cxns_per_ctrl); 23546733b39aSJayamohan Kallickal 23556733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) * 23566733b39aSJayamohan Kallickal phba->params.icds_per_ctrl; 23576733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) * 23586733b39aSJayamohan Kallickal phba->params.num_sge_per_io * phba->params.icds_per_ctrl; 23596733b39aSJayamohan Kallickal 23606733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] = 23616733b39aSJayamohan Kallickal num_async_pdu_buf_pages * PAGE_SIZE; 23626733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] = 23636733b39aSJayamohan Kallickal num_async_pdu_data_pages * PAGE_SIZE; 23646733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] = 23656733b39aSJayamohan Kallickal num_async_pdu_buf_sgl_pages * PAGE_SIZE; 23666733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_ASYNC_DATA_RING] = 23676733b39aSJayamohan Kallickal num_async_pdu_data_sgl_pages * PAGE_SIZE; 23686733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] = 23696733b39aSJayamohan Kallickal phba->params.asyncpdus_per_ctrl * 23706733b39aSJayamohan Kallickal sizeof(struct async_pdu_handle); 23716733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] = 23726733b39aSJayamohan Kallickal phba->params.asyncpdus_per_ctrl * 23736733b39aSJayamohan Kallickal sizeof(struct async_pdu_handle); 23746733b39aSJayamohan Kallickal phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] = 23756733b39aSJayamohan Kallickal sizeof(struct hwi_async_pdu_context) + 23766733b39aSJayamohan Kallickal (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry)); 23776733b39aSJayamohan Kallickal } 23786733b39aSJayamohan Kallickal 23796733b39aSJayamohan Kallickal static int beiscsi_alloc_mem(struct beiscsi_hba *phba) 23806733b39aSJayamohan Kallickal { 23816733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 23826733b39aSJayamohan Kallickal dma_addr_t bus_add; 23836733b39aSJayamohan Kallickal struct mem_array *mem_arr, *mem_arr_orig; 23846733b39aSJayamohan Kallickal unsigned int i, j, alloc_size, curr_alloc_size; 23856733b39aSJayamohan Kallickal 23863ec78271SJayamohan Kallickal phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL); 23876733b39aSJayamohan Kallickal if (!phba->phwi_ctrlr) 23886733b39aSJayamohan Kallickal return -ENOMEM; 23896733b39aSJayamohan Kallickal 23906733b39aSJayamohan Kallickal phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr), 23916733b39aSJayamohan Kallickal GFP_KERNEL); 23926733b39aSJayamohan Kallickal if (!phba->init_mem) { 23936733b39aSJayamohan Kallickal kfree(phba->phwi_ctrlr); 23946733b39aSJayamohan Kallickal return -ENOMEM; 23956733b39aSJayamohan Kallickal } 23966733b39aSJayamohan Kallickal 23976733b39aSJayamohan Kallickal mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT, 23986733b39aSJayamohan Kallickal GFP_KERNEL); 23996733b39aSJayamohan Kallickal if (!mem_arr_orig) { 24006733b39aSJayamohan Kallickal kfree(phba->init_mem); 24016733b39aSJayamohan Kallickal kfree(phba->phwi_ctrlr); 24026733b39aSJayamohan Kallickal return -ENOMEM; 24036733b39aSJayamohan Kallickal } 24046733b39aSJayamohan Kallickal 24056733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 24066733b39aSJayamohan Kallickal for (i = 0; i < SE_MEM_MAX; i++) { 24076733b39aSJayamohan Kallickal j = 0; 24086733b39aSJayamohan Kallickal mem_arr = mem_arr_orig; 24096733b39aSJayamohan Kallickal alloc_size = phba->mem_req[i]; 24106733b39aSJayamohan Kallickal memset(mem_arr, 0, sizeof(struct mem_array) * 24116733b39aSJayamohan Kallickal BEISCSI_MAX_FRAGS_INIT); 24126733b39aSJayamohan Kallickal curr_alloc_size = min(be_max_phys_size * 1024, alloc_size); 24136733b39aSJayamohan Kallickal do { 24146733b39aSJayamohan Kallickal mem_arr->virtual_address = pci_alloc_consistent( 24156733b39aSJayamohan Kallickal phba->pcidev, 24166733b39aSJayamohan Kallickal curr_alloc_size, 24176733b39aSJayamohan Kallickal &bus_add); 24186733b39aSJayamohan Kallickal if (!mem_arr->virtual_address) { 24196733b39aSJayamohan Kallickal if (curr_alloc_size <= BE_MIN_MEM_SIZE) 24206733b39aSJayamohan Kallickal goto free_mem; 24216733b39aSJayamohan Kallickal if (curr_alloc_size - 24226733b39aSJayamohan Kallickal rounddown_pow_of_two(curr_alloc_size)) 24236733b39aSJayamohan Kallickal curr_alloc_size = rounddown_pow_of_two 24246733b39aSJayamohan Kallickal (curr_alloc_size); 24256733b39aSJayamohan Kallickal else 24266733b39aSJayamohan Kallickal curr_alloc_size = curr_alloc_size / 2; 24276733b39aSJayamohan Kallickal } else { 24286733b39aSJayamohan Kallickal mem_arr->bus_address.u. 24296733b39aSJayamohan Kallickal a64.address = (__u64) bus_add; 24306733b39aSJayamohan Kallickal mem_arr->size = curr_alloc_size; 24316733b39aSJayamohan Kallickal alloc_size -= curr_alloc_size; 24326733b39aSJayamohan Kallickal curr_alloc_size = min(be_max_phys_size * 24336733b39aSJayamohan Kallickal 1024, alloc_size); 24346733b39aSJayamohan Kallickal j++; 24356733b39aSJayamohan Kallickal mem_arr++; 24366733b39aSJayamohan Kallickal } 24376733b39aSJayamohan Kallickal } while (alloc_size); 24386733b39aSJayamohan Kallickal mem_descr->num_elements = j; 24396733b39aSJayamohan Kallickal mem_descr->size_in_bytes = phba->mem_req[i]; 24406733b39aSJayamohan Kallickal mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j, 24416733b39aSJayamohan Kallickal GFP_KERNEL); 24426733b39aSJayamohan Kallickal if (!mem_descr->mem_array) 24436733b39aSJayamohan Kallickal goto free_mem; 24446733b39aSJayamohan Kallickal 24456733b39aSJayamohan Kallickal memcpy(mem_descr->mem_array, mem_arr_orig, 24466733b39aSJayamohan Kallickal sizeof(struct mem_array) * j); 24476733b39aSJayamohan Kallickal mem_descr++; 24486733b39aSJayamohan Kallickal } 24496733b39aSJayamohan Kallickal kfree(mem_arr_orig); 24506733b39aSJayamohan Kallickal return 0; 24516733b39aSJayamohan Kallickal free_mem: 24526733b39aSJayamohan Kallickal mem_descr->num_elements = j; 24536733b39aSJayamohan Kallickal while ((i) || (j)) { 24546733b39aSJayamohan Kallickal for (j = mem_descr->num_elements; j > 0; j--) { 24556733b39aSJayamohan Kallickal pci_free_consistent(phba->pcidev, 24566733b39aSJayamohan Kallickal mem_descr->mem_array[j - 1].size, 24576733b39aSJayamohan Kallickal mem_descr->mem_array[j - 1]. 24586733b39aSJayamohan Kallickal virtual_address, 2459457ff3b7SJayamohan Kallickal (unsigned long)mem_descr-> 2460457ff3b7SJayamohan Kallickal mem_array[j - 1]. 24616733b39aSJayamohan Kallickal bus_address.u.a64.address); 24626733b39aSJayamohan Kallickal } 24636733b39aSJayamohan Kallickal if (i) { 24646733b39aSJayamohan Kallickal i--; 24656733b39aSJayamohan Kallickal kfree(mem_descr->mem_array); 24666733b39aSJayamohan Kallickal mem_descr--; 24676733b39aSJayamohan Kallickal } 24686733b39aSJayamohan Kallickal } 24696733b39aSJayamohan Kallickal kfree(mem_arr_orig); 24706733b39aSJayamohan Kallickal kfree(phba->init_mem); 24716733b39aSJayamohan Kallickal kfree(phba->phwi_ctrlr); 24726733b39aSJayamohan Kallickal return -ENOMEM; 24736733b39aSJayamohan Kallickal } 24746733b39aSJayamohan Kallickal 24756733b39aSJayamohan Kallickal static int beiscsi_get_memory(struct beiscsi_hba *phba) 24766733b39aSJayamohan Kallickal { 24776733b39aSJayamohan Kallickal beiscsi_find_mem_req(phba); 24786733b39aSJayamohan Kallickal return beiscsi_alloc_mem(phba); 24796733b39aSJayamohan Kallickal } 24806733b39aSJayamohan Kallickal 24816733b39aSJayamohan Kallickal static void iscsi_init_global_templates(struct beiscsi_hba *phba) 24826733b39aSJayamohan Kallickal { 24836733b39aSJayamohan Kallickal struct pdu_data_out *pdata_out; 24846733b39aSJayamohan Kallickal struct pdu_nop_out *pnop_out; 24856733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 24866733b39aSJayamohan Kallickal 24876733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 24886733b39aSJayamohan Kallickal mem_descr += ISCSI_MEM_GLOBAL_HEADER; 24896733b39aSJayamohan Kallickal pdata_out = 24906733b39aSJayamohan Kallickal (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address; 24916733b39aSJayamohan Kallickal memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE); 24926733b39aSJayamohan Kallickal 24936733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out, 24946733b39aSJayamohan Kallickal IIOC_SCSI_DATA); 24956733b39aSJayamohan Kallickal 24966733b39aSJayamohan Kallickal pnop_out = 24976733b39aSJayamohan Kallickal (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0]. 24986733b39aSJayamohan Kallickal virtual_address + BE_ISCSI_PDU_HEADER_SIZE); 24996733b39aSJayamohan Kallickal 25006733b39aSJayamohan Kallickal memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE); 25016733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF); 25026733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1); 25036733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0); 25046733b39aSJayamohan Kallickal } 25056733b39aSJayamohan Kallickal 25063ec78271SJayamohan Kallickal static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba) 25076733b39aSJayamohan Kallickal { 25086733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb; 25093ec78271SJayamohan Kallickal struct wrb_handle *pwrb_handle = NULL; 25106733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 25116733b39aSJayamohan Kallickal struct hwi_wrb_context *pwrb_context; 25123ec78271SJayamohan Kallickal struct iscsi_wrb *pwrb = NULL; 25133ec78271SJayamohan Kallickal unsigned int num_cxn_wrbh = 0; 25143ec78271SJayamohan Kallickal unsigned int num_cxn_wrb = 0, j, idx = 0, index; 25156733b39aSJayamohan Kallickal 25166733b39aSJayamohan Kallickal mem_descr_wrbh = phba->init_mem; 25176733b39aSJayamohan Kallickal mem_descr_wrbh += HWI_MEM_WRBH; 25186733b39aSJayamohan Kallickal 25196733b39aSJayamohan Kallickal mem_descr_wrb = phba->init_mem; 25206733b39aSJayamohan Kallickal mem_descr_wrb += HWI_MEM_WRB; 25216733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 25226733b39aSJayamohan Kallickal 25236733b39aSJayamohan Kallickal for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) { 25246733b39aSJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[index]; 25256733b39aSJayamohan Kallickal pwrb_context->pwrb_handle_base = 25266733b39aSJayamohan Kallickal kzalloc(sizeof(struct wrb_handle *) * 25276733b39aSJayamohan Kallickal phba->params.wrbs_per_cxn, GFP_KERNEL); 25283ec78271SJayamohan Kallickal if (!pwrb_context->pwrb_handle_base) { 252999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 253099bc5d55SJohn Soni Jose "BM_%d : Mem Alloc Failed. Failing to load\n"); 25313ec78271SJayamohan Kallickal goto init_wrb_hndl_failed; 25323ec78271SJayamohan Kallickal } 25336733b39aSJayamohan Kallickal pwrb_context->pwrb_handle_basestd = 25346733b39aSJayamohan Kallickal kzalloc(sizeof(struct wrb_handle *) * 25356733b39aSJayamohan Kallickal phba->params.wrbs_per_cxn, GFP_KERNEL); 25363ec78271SJayamohan Kallickal if (!pwrb_context->pwrb_handle_basestd) { 253799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 253899bc5d55SJohn Soni Jose "BM_%d : Mem Alloc Failed. Failing to load\n"); 25393ec78271SJayamohan Kallickal goto init_wrb_hndl_failed; 25406733b39aSJayamohan Kallickal } 25413ec78271SJayamohan Kallickal if (!num_cxn_wrbh) { 25426733b39aSJayamohan Kallickal pwrb_handle = 25436733b39aSJayamohan Kallickal mem_descr_wrbh->mem_array[idx].virtual_address; 25443ec78271SJayamohan Kallickal num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) / 25456733b39aSJayamohan Kallickal ((sizeof(struct wrb_handle)) * 25466733b39aSJayamohan Kallickal phba->params.wrbs_per_cxn)); 25473ec78271SJayamohan Kallickal idx++; 25483ec78271SJayamohan Kallickal } 25496733b39aSJayamohan Kallickal pwrb_context->alloc_index = 0; 25503ec78271SJayamohan Kallickal pwrb_context->wrb_handles_available = 0; 25513ec78271SJayamohan Kallickal pwrb_context->free_index = 0; 25523ec78271SJayamohan Kallickal 25533ec78271SJayamohan Kallickal if (num_cxn_wrbh) { 25546733b39aSJayamohan Kallickal for (j = 0; j < phba->params.wrbs_per_cxn; j++) { 25556733b39aSJayamohan Kallickal pwrb_context->pwrb_handle_base[j] = pwrb_handle; 25566733b39aSJayamohan Kallickal pwrb_context->pwrb_handle_basestd[j] = 25576733b39aSJayamohan Kallickal pwrb_handle; 25586733b39aSJayamohan Kallickal pwrb_context->wrb_handles_available++; 2559bfead3b2SJayamohan Kallickal pwrb_handle->wrb_index = j; 25606733b39aSJayamohan Kallickal pwrb_handle++; 25616733b39aSJayamohan Kallickal } 25626733b39aSJayamohan Kallickal num_cxn_wrbh--; 25636733b39aSJayamohan Kallickal } 25646733b39aSJayamohan Kallickal } 25656733b39aSJayamohan Kallickal idx = 0; 25663ec78271SJayamohan Kallickal for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) { 25673ec78271SJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[index]; 25683ec78271SJayamohan Kallickal if (!num_cxn_wrb) { 25696733b39aSJayamohan Kallickal pwrb = mem_descr_wrb->mem_array[idx].virtual_address; 25707c56533cSJayamohan Kallickal num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) / 25717c56533cSJayamohan Kallickal ((sizeof(struct iscsi_wrb) * 25727c56533cSJayamohan Kallickal phba->params.wrbs_per_cxn)); 25733ec78271SJayamohan Kallickal idx++; 25743ec78271SJayamohan Kallickal } 25753ec78271SJayamohan Kallickal 25766733b39aSJayamohan Kallickal if (num_cxn_wrb) { 25776733b39aSJayamohan Kallickal for (j = 0; j < phba->params.wrbs_per_cxn; j++) { 25786733b39aSJayamohan Kallickal pwrb_handle = pwrb_context->pwrb_handle_base[j]; 25796733b39aSJayamohan Kallickal pwrb_handle->pwrb = pwrb; 25806733b39aSJayamohan Kallickal pwrb++; 25816733b39aSJayamohan Kallickal } 25826733b39aSJayamohan Kallickal num_cxn_wrb--; 25836733b39aSJayamohan Kallickal } 25846733b39aSJayamohan Kallickal } 25853ec78271SJayamohan Kallickal return 0; 25863ec78271SJayamohan Kallickal init_wrb_hndl_failed: 25873ec78271SJayamohan Kallickal for (j = index; j > 0; j--) { 25883ec78271SJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[j]; 25893ec78271SJayamohan Kallickal kfree(pwrb_context->pwrb_handle_base); 25903ec78271SJayamohan Kallickal kfree(pwrb_context->pwrb_handle_basestd); 25913ec78271SJayamohan Kallickal } 25923ec78271SJayamohan Kallickal return -ENOMEM; 25936733b39aSJayamohan Kallickal } 25946733b39aSJayamohan Kallickal 25956733b39aSJayamohan Kallickal static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba) 25966733b39aSJayamohan Kallickal { 25976733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 25986733b39aSJayamohan Kallickal struct hba_parameters *p = &phba->params; 25996733b39aSJayamohan Kallickal struct hwi_async_pdu_context *pasync_ctx; 26006733b39aSJayamohan Kallickal struct async_pdu_handle *pasync_header_h, *pasync_data_h; 2601dc63aac6SJayamohan Kallickal unsigned int index, idx, num_per_mem, num_async_data; 26026733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 26036733b39aSJayamohan Kallickal 26046733b39aSJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 26056733b39aSJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT; 26066733b39aSJayamohan Kallickal 26076733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 26086733b39aSJayamohan Kallickal phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *) 26096733b39aSJayamohan Kallickal mem_descr->mem_array[0].virtual_address; 26106733b39aSJayamohan Kallickal pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx; 26116733b39aSJayamohan Kallickal memset(pasync_ctx, 0, sizeof(*pasync_ctx)); 26126733b39aSJayamohan Kallickal 2613dc63aac6SJayamohan Kallickal pasync_ctx->num_entries = p->asyncpdus_per_ctrl; 2614dc63aac6SJayamohan Kallickal pasync_ctx->buffer_size = p->defpdu_hdr_sz; 26156733b39aSJayamohan Kallickal 26166733b39aSJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 26176733b39aSJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_HEADER_BUF; 26186733b39aSJayamohan Kallickal if (mem_descr->mem_array[0].virtual_address) { 261999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 262099bc5d55SJohn Soni Jose "BM_%d : hwi_init_async_pdu_ctx" 262199bc5d55SJohn Soni Jose " HWI_MEM_ASYNC_HEADER_BUF va=%p\n", 262299bc5d55SJohn Soni Jose mem_descr->mem_array[0].virtual_address); 26236733b39aSJayamohan Kallickal } else 262499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 262599bc5d55SJohn Soni Jose "BM_%d : No Virtual address\n"); 26266733b39aSJayamohan Kallickal 26276733b39aSJayamohan Kallickal pasync_ctx->async_header.va_base = 26286733b39aSJayamohan Kallickal mem_descr->mem_array[0].virtual_address; 26296733b39aSJayamohan Kallickal 26306733b39aSJayamohan Kallickal pasync_ctx->async_header.pa_base.u.a64.address = 26316733b39aSJayamohan Kallickal mem_descr->mem_array[0].bus_address.u.a64.address; 26326733b39aSJayamohan Kallickal 26336733b39aSJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 26346733b39aSJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_HEADER_RING; 26356733b39aSJayamohan Kallickal if (mem_descr->mem_array[0].virtual_address) { 263699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 263799bc5d55SJohn Soni Jose "BM_%d : hwi_init_async_pdu_ctx" 263899bc5d55SJohn Soni Jose " HWI_MEM_ASYNC_HEADER_RING va=%p\n", 263999bc5d55SJohn Soni Jose mem_descr->mem_array[0].virtual_address); 26406733b39aSJayamohan Kallickal } else 264199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 264299bc5d55SJohn Soni Jose "BM_%d : No Virtual address\n"); 264399bc5d55SJohn Soni Jose 26446733b39aSJayamohan Kallickal pasync_ctx->async_header.ring_base = 26456733b39aSJayamohan Kallickal mem_descr->mem_array[0].virtual_address; 26466733b39aSJayamohan Kallickal 26476733b39aSJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 26486733b39aSJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE; 26496733b39aSJayamohan Kallickal if (mem_descr->mem_array[0].virtual_address) { 265099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 265199bc5d55SJohn Soni Jose "BM_%d : hwi_init_async_pdu_ctx" 265299bc5d55SJohn Soni Jose " HWI_MEM_ASYNC_HEADER_HANDLE va=%p\n", 265399bc5d55SJohn Soni Jose mem_descr->mem_array[0].virtual_address); 26546733b39aSJayamohan Kallickal } else 265599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 265699bc5d55SJohn Soni Jose "BM_%d : No Virtual address\n"); 26576733b39aSJayamohan Kallickal 26586733b39aSJayamohan Kallickal pasync_ctx->async_header.handle_base = 26596733b39aSJayamohan Kallickal mem_descr->mem_array[0].virtual_address; 26606733b39aSJayamohan Kallickal pasync_ctx->async_header.writables = 0; 26616733b39aSJayamohan Kallickal INIT_LIST_HEAD(&pasync_ctx->async_header.free_list); 26626733b39aSJayamohan Kallickal 26636733b39aSJayamohan Kallickal 26646733b39aSJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 26656733b39aSJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_DATA_RING; 26666733b39aSJayamohan Kallickal if (mem_descr->mem_array[0].virtual_address) { 266799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 266899bc5d55SJohn Soni Jose "BM_%d : hwi_init_async_pdu_ctx" 266999bc5d55SJohn Soni Jose " HWI_MEM_ASYNC_DATA_RING va=%p\n", 267099bc5d55SJohn Soni Jose mem_descr->mem_array[0].virtual_address); 26716733b39aSJayamohan Kallickal } else 267299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 267399bc5d55SJohn Soni Jose "BM_%d : No Virtual address\n"); 26746733b39aSJayamohan Kallickal 26756733b39aSJayamohan Kallickal pasync_ctx->async_data.ring_base = 26766733b39aSJayamohan Kallickal mem_descr->mem_array[0].virtual_address; 26776733b39aSJayamohan Kallickal 26786733b39aSJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 26796733b39aSJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_DATA_HANDLE; 26806733b39aSJayamohan Kallickal if (!mem_descr->mem_array[0].virtual_address) 268199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 268299bc5d55SJohn Soni Jose "BM_%d : No Virtual address\n"); 26836733b39aSJayamohan Kallickal 26846733b39aSJayamohan Kallickal pasync_ctx->async_data.handle_base = 26856733b39aSJayamohan Kallickal mem_descr->mem_array[0].virtual_address; 26866733b39aSJayamohan Kallickal pasync_ctx->async_data.writables = 0; 26876733b39aSJayamohan Kallickal INIT_LIST_HEAD(&pasync_ctx->async_data.free_list); 26886733b39aSJayamohan Kallickal 26896733b39aSJayamohan Kallickal pasync_header_h = 26906733b39aSJayamohan Kallickal (struct async_pdu_handle *)pasync_ctx->async_header.handle_base; 26916733b39aSJayamohan Kallickal pasync_data_h = 26926733b39aSJayamohan Kallickal (struct async_pdu_handle *)pasync_ctx->async_data.handle_base; 26936733b39aSJayamohan Kallickal 2694dc63aac6SJayamohan Kallickal mem_descr = (struct be_mem_descriptor *)phba->init_mem; 2695dc63aac6SJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_DATA_BUF; 2696dc63aac6SJayamohan Kallickal if (mem_descr->mem_array[0].virtual_address) { 269799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 269899bc5d55SJohn Soni Jose "BM_%d : hwi_init_async_pdu_ctx" 269999bc5d55SJohn Soni Jose " HWI_MEM_ASYNC_DATA_BUF va=%p\n", 270099bc5d55SJohn Soni Jose mem_descr->mem_array[0].virtual_address); 2701dc63aac6SJayamohan Kallickal } else 270299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 270399bc5d55SJohn Soni Jose "BM_%d : No Virtual address\n"); 270499bc5d55SJohn Soni Jose 2705dc63aac6SJayamohan Kallickal idx = 0; 2706dc63aac6SJayamohan Kallickal pasync_ctx->async_data.va_base = 2707dc63aac6SJayamohan Kallickal mem_descr->mem_array[idx].virtual_address; 2708dc63aac6SJayamohan Kallickal pasync_ctx->async_data.pa_base.u.a64.address = 2709dc63aac6SJayamohan Kallickal mem_descr->mem_array[idx].bus_address.u.a64.address; 2710dc63aac6SJayamohan Kallickal 2711dc63aac6SJayamohan Kallickal num_async_data = ((mem_descr->mem_array[idx].size) / 2712dc63aac6SJayamohan Kallickal phba->params.defpdu_data_sz); 2713dc63aac6SJayamohan Kallickal num_per_mem = 0; 2714dc63aac6SJayamohan Kallickal 27156733b39aSJayamohan Kallickal for (index = 0; index < p->asyncpdus_per_ctrl; index++) { 27166733b39aSJayamohan Kallickal pasync_header_h->cri = -1; 27176733b39aSJayamohan Kallickal pasync_header_h->index = (char)index; 27186733b39aSJayamohan Kallickal INIT_LIST_HEAD(&pasync_header_h->link); 27196733b39aSJayamohan Kallickal pasync_header_h->pbuffer = 27206733b39aSJayamohan Kallickal (void *)((unsigned long) 27216733b39aSJayamohan Kallickal (pasync_ctx->async_header.va_base) + 27226733b39aSJayamohan Kallickal (p->defpdu_hdr_sz * index)); 27236733b39aSJayamohan Kallickal 27246733b39aSJayamohan Kallickal pasync_header_h->pa.u.a64.address = 27256733b39aSJayamohan Kallickal pasync_ctx->async_header.pa_base.u.a64.address + 27266733b39aSJayamohan Kallickal (p->defpdu_hdr_sz * index); 27276733b39aSJayamohan Kallickal 27286733b39aSJayamohan Kallickal list_add_tail(&pasync_header_h->link, 27296733b39aSJayamohan Kallickal &pasync_ctx->async_header.free_list); 27306733b39aSJayamohan Kallickal pasync_header_h++; 27316733b39aSJayamohan Kallickal pasync_ctx->async_header.free_entries++; 27326733b39aSJayamohan Kallickal pasync_ctx->async_header.writables++; 27336733b39aSJayamohan Kallickal 27346733b39aSJayamohan Kallickal INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list); 27356733b39aSJayamohan Kallickal INIT_LIST_HEAD(&pasync_ctx->async_entry[index]. 27366733b39aSJayamohan Kallickal header_busy_list); 27376733b39aSJayamohan Kallickal pasync_data_h->cri = -1; 27386733b39aSJayamohan Kallickal pasync_data_h->index = (char)index; 27396733b39aSJayamohan Kallickal INIT_LIST_HEAD(&pasync_data_h->link); 2740dc63aac6SJayamohan Kallickal 2741dc63aac6SJayamohan Kallickal if (!num_async_data) { 2742dc63aac6SJayamohan Kallickal num_per_mem = 0; 2743dc63aac6SJayamohan Kallickal idx++; 2744dc63aac6SJayamohan Kallickal pasync_ctx->async_data.va_base = 2745dc63aac6SJayamohan Kallickal mem_descr->mem_array[idx].virtual_address; 2746dc63aac6SJayamohan Kallickal pasync_ctx->async_data.pa_base.u.a64.address = 2747dc63aac6SJayamohan Kallickal mem_descr->mem_array[idx]. 2748dc63aac6SJayamohan Kallickal bus_address.u.a64.address; 2749dc63aac6SJayamohan Kallickal 2750dc63aac6SJayamohan Kallickal num_async_data = ((mem_descr->mem_array[idx].size) / 2751dc63aac6SJayamohan Kallickal phba->params.defpdu_data_sz); 2752dc63aac6SJayamohan Kallickal } 27536733b39aSJayamohan Kallickal pasync_data_h->pbuffer = 27546733b39aSJayamohan Kallickal (void *)((unsigned long) 27556733b39aSJayamohan Kallickal (pasync_ctx->async_data.va_base) + 2756dc63aac6SJayamohan Kallickal (p->defpdu_data_sz * num_per_mem)); 27576733b39aSJayamohan Kallickal 27586733b39aSJayamohan Kallickal pasync_data_h->pa.u.a64.address = 27596733b39aSJayamohan Kallickal pasync_ctx->async_data.pa_base.u.a64.address + 2760dc63aac6SJayamohan Kallickal (p->defpdu_data_sz * num_per_mem); 2761dc63aac6SJayamohan Kallickal num_per_mem++; 2762dc63aac6SJayamohan Kallickal num_async_data--; 27636733b39aSJayamohan Kallickal 27646733b39aSJayamohan Kallickal list_add_tail(&pasync_data_h->link, 27656733b39aSJayamohan Kallickal &pasync_ctx->async_data.free_list); 27666733b39aSJayamohan Kallickal pasync_data_h++; 27676733b39aSJayamohan Kallickal pasync_ctx->async_data.free_entries++; 27686733b39aSJayamohan Kallickal pasync_ctx->async_data.writables++; 27696733b39aSJayamohan Kallickal 27706733b39aSJayamohan Kallickal INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list); 27716733b39aSJayamohan Kallickal } 27726733b39aSJayamohan Kallickal 27736733b39aSJayamohan Kallickal pasync_ctx->async_header.host_write_ptr = 0; 27746733b39aSJayamohan Kallickal pasync_ctx->async_header.ep_read_ptr = -1; 27756733b39aSJayamohan Kallickal pasync_ctx->async_data.host_write_ptr = 0; 27766733b39aSJayamohan Kallickal pasync_ctx->async_data.ep_read_ptr = -1; 27776733b39aSJayamohan Kallickal } 27786733b39aSJayamohan Kallickal 27796733b39aSJayamohan Kallickal static int 27806733b39aSJayamohan Kallickal be_sgl_create_contiguous(void *virtual_address, 27816733b39aSJayamohan Kallickal u64 physical_address, u32 length, 27826733b39aSJayamohan Kallickal struct be_dma_mem *sgl) 27836733b39aSJayamohan Kallickal { 27846733b39aSJayamohan Kallickal WARN_ON(!virtual_address); 27856733b39aSJayamohan Kallickal WARN_ON(!physical_address); 27866733b39aSJayamohan Kallickal WARN_ON(!length > 0); 27876733b39aSJayamohan Kallickal WARN_ON(!sgl); 27886733b39aSJayamohan Kallickal 27896733b39aSJayamohan Kallickal sgl->va = virtual_address; 2790457ff3b7SJayamohan Kallickal sgl->dma = (unsigned long)physical_address; 27916733b39aSJayamohan Kallickal sgl->size = length; 27926733b39aSJayamohan Kallickal 27936733b39aSJayamohan Kallickal return 0; 27946733b39aSJayamohan Kallickal } 27956733b39aSJayamohan Kallickal 27966733b39aSJayamohan Kallickal static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl) 27976733b39aSJayamohan Kallickal { 27986733b39aSJayamohan Kallickal memset(sgl, 0, sizeof(*sgl)); 27996733b39aSJayamohan Kallickal } 28006733b39aSJayamohan Kallickal 28016733b39aSJayamohan Kallickal static void 28026733b39aSJayamohan Kallickal hwi_build_be_sgl_arr(struct beiscsi_hba *phba, 28036733b39aSJayamohan Kallickal struct mem_array *pmem, struct be_dma_mem *sgl) 28046733b39aSJayamohan Kallickal { 28056733b39aSJayamohan Kallickal if (sgl->va) 28066733b39aSJayamohan Kallickal be_sgl_destroy_contiguous(sgl); 28076733b39aSJayamohan Kallickal 28086733b39aSJayamohan Kallickal be_sgl_create_contiguous(pmem->virtual_address, 28096733b39aSJayamohan Kallickal pmem->bus_address.u.a64.address, 28106733b39aSJayamohan Kallickal pmem->size, sgl); 28116733b39aSJayamohan Kallickal } 28126733b39aSJayamohan Kallickal 28136733b39aSJayamohan Kallickal static void 28146733b39aSJayamohan Kallickal hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba, 28156733b39aSJayamohan Kallickal struct mem_array *pmem, struct be_dma_mem *sgl) 28166733b39aSJayamohan Kallickal { 28176733b39aSJayamohan Kallickal if (sgl->va) 28186733b39aSJayamohan Kallickal be_sgl_destroy_contiguous(sgl); 28196733b39aSJayamohan Kallickal 28206733b39aSJayamohan Kallickal be_sgl_create_contiguous((unsigned char *)pmem->virtual_address, 28216733b39aSJayamohan Kallickal pmem->bus_address.u.a64.address, 28226733b39aSJayamohan Kallickal pmem->size, sgl); 28236733b39aSJayamohan Kallickal } 28246733b39aSJayamohan Kallickal 28256733b39aSJayamohan Kallickal static int be_fill_queue(struct be_queue_info *q, 28266733b39aSJayamohan Kallickal u16 len, u16 entry_size, void *vaddress) 28276733b39aSJayamohan Kallickal { 28286733b39aSJayamohan Kallickal struct be_dma_mem *mem = &q->dma_mem; 28296733b39aSJayamohan Kallickal 28306733b39aSJayamohan Kallickal memset(q, 0, sizeof(*q)); 28316733b39aSJayamohan Kallickal q->len = len; 28326733b39aSJayamohan Kallickal q->entry_size = entry_size; 28336733b39aSJayamohan Kallickal mem->size = len * entry_size; 28346733b39aSJayamohan Kallickal mem->va = vaddress; 28356733b39aSJayamohan Kallickal if (!mem->va) 28366733b39aSJayamohan Kallickal return -ENOMEM; 28376733b39aSJayamohan Kallickal memset(mem->va, 0, mem->size); 28386733b39aSJayamohan Kallickal return 0; 28396733b39aSJayamohan Kallickal } 28406733b39aSJayamohan Kallickal 2841bfead3b2SJayamohan Kallickal static int beiscsi_create_eqs(struct beiscsi_hba *phba, 28426733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context) 28436733b39aSJayamohan Kallickal { 2844bfead3b2SJayamohan Kallickal unsigned int i, num_eq_pages; 284599bc5d55SJohn Soni Jose int ret = 0, eq_for_mcc; 28466733b39aSJayamohan Kallickal struct be_queue_info *eq; 28476733b39aSJayamohan Kallickal struct be_dma_mem *mem; 28486733b39aSJayamohan Kallickal void *eq_vaddress; 2849bfead3b2SJayamohan Kallickal dma_addr_t paddr; 28506733b39aSJayamohan Kallickal 2851bfead3b2SJayamohan Kallickal num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \ 2852bfead3b2SJayamohan Kallickal sizeof(struct be_eq_entry)); 2853bfead3b2SJayamohan Kallickal 2854bfead3b2SJayamohan Kallickal if (phba->msix_enabled) 2855bfead3b2SJayamohan Kallickal eq_for_mcc = 1; 2856bfead3b2SJayamohan Kallickal else 2857bfead3b2SJayamohan Kallickal eq_for_mcc = 0; 2858bfead3b2SJayamohan Kallickal for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) { 2859bfead3b2SJayamohan Kallickal eq = &phwi_context->be_eq[i].q; 28606733b39aSJayamohan Kallickal mem = &eq->dma_mem; 2861bfead3b2SJayamohan Kallickal phwi_context->be_eq[i].phba = phba; 2862bfead3b2SJayamohan Kallickal eq_vaddress = pci_alloc_consistent(phba->pcidev, 2863bfead3b2SJayamohan Kallickal num_eq_pages * PAGE_SIZE, 2864bfead3b2SJayamohan Kallickal &paddr); 2865bfead3b2SJayamohan Kallickal if (!eq_vaddress) 2866bfead3b2SJayamohan Kallickal goto create_eq_error; 28676733b39aSJayamohan Kallickal 2868bfead3b2SJayamohan Kallickal mem->va = eq_vaddress; 28696733b39aSJayamohan Kallickal ret = be_fill_queue(eq, phba->params.num_eq_entries, 28706733b39aSJayamohan Kallickal sizeof(struct be_eq_entry), eq_vaddress); 28716733b39aSJayamohan Kallickal if (ret) { 287299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 287399bc5d55SJohn Soni Jose "BM_%d : be_fill_queue Failed for EQ\n"); 2874bfead3b2SJayamohan Kallickal goto create_eq_error; 28756733b39aSJayamohan Kallickal } 28766733b39aSJayamohan Kallickal 2877bfead3b2SJayamohan Kallickal mem->dma = paddr; 28786733b39aSJayamohan Kallickal ret = beiscsi_cmd_eq_create(&phba->ctrl, eq, 2879bfead3b2SJayamohan Kallickal phwi_context->cur_eqd); 28806733b39aSJayamohan Kallickal if (ret) { 288199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 288299bc5d55SJohn Soni Jose "BM_%d : beiscsi_cmd_eq_create" 28836733b39aSJayamohan Kallickal "Failed for EQ\n"); 2884bfead3b2SJayamohan Kallickal goto create_eq_error; 2885bfead3b2SJayamohan Kallickal } 288699bc5d55SJohn Soni Jose 288799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 288899bc5d55SJohn Soni Jose "BM_%d : eqid = %d\n", 288999bc5d55SJohn Soni Jose phwi_context->be_eq[i].q.id); 2890bfead3b2SJayamohan Kallickal } 2891bfead3b2SJayamohan Kallickal return 0; 2892bfead3b2SJayamohan Kallickal create_eq_error: 2893107dfcbaSJohn Soni Jose for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) { 2894bfead3b2SJayamohan Kallickal eq = &phwi_context->be_eq[i].q; 2895bfead3b2SJayamohan Kallickal mem = &eq->dma_mem; 2896bfead3b2SJayamohan Kallickal if (mem->va) 2897bfead3b2SJayamohan Kallickal pci_free_consistent(phba->pcidev, num_eq_pages 2898bfead3b2SJayamohan Kallickal * PAGE_SIZE, 2899bfead3b2SJayamohan Kallickal mem->va, mem->dma); 2900bfead3b2SJayamohan Kallickal } 29016733b39aSJayamohan Kallickal return ret; 29026733b39aSJayamohan Kallickal } 29036733b39aSJayamohan Kallickal 2904bfead3b2SJayamohan Kallickal static int beiscsi_create_cqs(struct beiscsi_hba *phba, 29056733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context) 29066733b39aSJayamohan Kallickal { 2907bfead3b2SJayamohan Kallickal unsigned int i, num_cq_pages; 290899bc5d55SJohn Soni Jose int ret = 0; 29096733b39aSJayamohan Kallickal struct be_queue_info *cq, *eq; 29106733b39aSJayamohan Kallickal struct be_dma_mem *mem; 2911bfead3b2SJayamohan Kallickal struct be_eq_obj *pbe_eq; 29126733b39aSJayamohan Kallickal void *cq_vaddress; 2913bfead3b2SJayamohan Kallickal dma_addr_t paddr; 29146733b39aSJayamohan Kallickal 2915bfead3b2SJayamohan Kallickal num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \ 2916bfead3b2SJayamohan Kallickal sizeof(struct sol_cqe)); 2917bfead3b2SJayamohan Kallickal 2918bfead3b2SJayamohan Kallickal for (i = 0; i < phba->num_cpus; i++) { 2919bfead3b2SJayamohan Kallickal cq = &phwi_context->be_cq[i]; 2920bfead3b2SJayamohan Kallickal eq = &phwi_context->be_eq[i].q; 2921bfead3b2SJayamohan Kallickal pbe_eq = &phwi_context->be_eq[i]; 2922bfead3b2SJayamohan Kallickal pbe_eq->cq = cq; 2923bfead3b2SJayamohan Kallickal pbe_eq->phba = phba; 29246733b39aSJayamohan Kallickal mem = &cq->dma_mem; 2925bfead3b2SJayamohan Kallickal cq_vaddress = pci_alloc_consistent(phba->pcidev, 2926bfead3b2SJayamohan Kallickal num_cq_pages * PAGE_SIZE, 2927bfead3b2SJayamohan Kallickal &paddr); 2928bfead3b2SJayamohan Kallickal if (!cq_vaddress) 2929bfead3b2SJayamohan Kallickal goto create_cq_error; 29307da50879SJayamohan Kallickal ret = be_fill_queue(cq, phba->params.num_cq_entries, 29316733b39aSJayamohan Kallickal sizeof(struct sol_cqe), cq_vaddress); 29326733b39aSJayamohan Kallickal if (ret) { 293399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 293499bc5d55SJohn Soni Jose "BM_%d : be_fill_queue Failed " 293599bc5d55SJohn Soni Jose "for ISCSI CQ\n"); 2936bfead3b2SJayamohan Kallickal goto create_cq_error; 29376733b39aSJayamohan Kallickal } 29386733b39aSJayamohan Kallickal 2939bfead3b2SJayamohan Kallickal mem->dma = paddr; 2940bfead3b2SJayamohan Kallickal ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false, 2941bfead3b2SJayamohan Kallickal false, 0); 29426733b39aSJayamohan Kallickal if (ret) { 294399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 294499bc5d55SJohn Soni Jose "BM_%d : beiscsi_cmd_eq_create" 2945bfead3b2SJayamohan Kallickal "Failed for ISCSI CQ\n"); 2946bfead3b2SJayamohan Kallickal goto create_cq_error; 29476733b39aSJayamohan Kallickal } 294899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 294999bc5d55SJohn Soni Jose "BM_%d : iscsi cq_id is %d for eq_id %d\n" 295099bc5d55SJohn Soni Jose "iSCSI CQ CREATED\n", cq->id, eq->id); 2951bfead3b2SJayamohan Kallickal } 29526733b39aSJayamohan Kallickal return 0; 2953bfead3b2SJayamohan Kallickal 2954bfead3b2SJayamohan Kallickal create_cq_error: 2955bfead3b2SJayamohan Kallickal for (i = 0; i < phba->num_cpus; i++) { 2956bfead3b2SJayamohan Kallickal cq = &phwi_context->be_cq[i]; 2957bfead3b2SJayamohan Kallickal mem = &cq->dma_mem; 2958bfead3b2SJayamohan Kallickal if (mem->va) 2959bfead3b2SJayamohan Kallickal pci_free_consistent(phba->pcidev, num_cq_pages 2960bfead3b2SJayamohan Kallickal * PAGE_SIZE, 2961bfead3b2SJayamohan Kallickal mem->va, mem->dma); 2962bfead3b2SJayamohan Kallickal } 2963bfead3b2SJayamohan Kallickal return ret; 2964bfead3b2SJayamohan Kallickal 29656733b39aSJayamohan Kallickal } 29666733b39aSJayamohan Kallickal 29676733b39aSJayamohan Kallickal static int 29686733b39aSJayamohan Kallickal beiscsi_create_def_hdr(struct beiscsi_hba *phba, 29696733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context, 29706733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr, 29716733b39aSJayamohan Kallickal unsigned int def_pdu_ring_sz) 29726733b39aSJayamohan Kallickal { 29736733b39aSJayamohan Kallickal unsigned int idx; 29746733b39aSJayamohan Kallickal int ret; 29756733b39aSJayamohan Kallickal struct be_queue_info *dq, *cq; 29766733b39aSJayamohan Kallickal struct be_dma_mem *mem; 29776733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 29786733b39aSJayamohan Kallickal void *dq_vaddress; 29796733b39aSJayamohan Kallickal 29806733b39aSJayamohan Kallickal idx = 0; 29816733b39aSJayamohan Kallickal dq = &phwi_context->be_def_hdrq; 2982bfead3b2SJayamohan Kallickal cq = &phwi_context->be_cq[0]; 29836733b39aSJayamohan Kallickal mem = &dq->dma_mem; 29846733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 29856733b39aSJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_HEADER_RING; 29866733b39aSJayamohan Kallickal dq_vaddress = mem_descr->mem_array[idx].virtual_address; 29876733b39aSJayamohan Kallickal ret = be_fill_queue(dq, mem_descr->mem_array[0].size / 29886733b39aSJayamohan Kallickal sizeof(struct phys_addr), 29896733b39aSJayamohan Kallickal sizeof(struct phys_addr), dq_vaddress); 29906733b39aSJayamohan Kallickal if (ret) { 299199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 299299bc5d55SJohn Soni Jose "BM_%d : be_fill_queue Failed for DEF PDU HDR\n"); 29936733b39aSJayamohan Kallickal return ret; 29946733b39aSJayamohan Kallickal } 2995457ff3b7SJayamohan Kallickal mem->dma = (unsigned long)mem_descr->mem_array[idx]. 2996457ff3b7SJayamohan Kallickal bus_address.u.a64.address; 29976733b39aSJayamohan Kallickal ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq, 29986733b39aSJayamohan Kallickal def_pdu_ring_sz, 29996733b39aSJayamohan Kallickal phba->params.defpdu_hdr_sz); 30006733b39aSJayamohan Kallickal if (ret) { 300199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 300299bc5d55SJohn Soni Jose "BM_%d : be_cmd_create_default_pdu_queue Failed DEFHDR\n"); 30036733b39aSJayamohan Kallickal return ret; 30046733b39aSJayamohan Kallickal } 30056733b39aSJayamohan Kallickal phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id; 300699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 300799bc5d55SJohn Soni Jose "BM_%d : iscsi def pdu id is %d\n", 30086733b39aSJayamohan Kallickal phwi_context->be_def_hdrq.id); 300999bc5d55SJohn Soni Jose 30106733b39aSJayamohan Kallickal hwi_post_async_buffers(phba, 1); 30116733b39aSJayamohan Kallickal return 0; 30126733b39aSJayamohan Kallickal } 30136733b39aSJayamohan Kallickal 30146733b39aSJayamohan Kallickal static int 30156733b39aSJayamohan Kallickal beiscsi_create_def_data(struct beiscsi_hba *phba, 30166733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context, 30176733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr, 30186733b39aSJayamohan Kallickal unsigned int def_pdu_ring_sz) 30196733b39aSJayamohan Kallickal { 30206733b39aSJayamohan Kallickal unsigned int idx; 30216733b39aSJayamohan Kallickal int ret; 30226733b39aSJayamohan Kallickal struct be_queue_info *dataq, *cq; 30236733b39aSJayamohan Kallickal struct be_dma_mem *mem; 30246733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 30256733b39aSJayamohan Kallickal void *dq_vaddress; 30266733b39aSJayamohan Kallickal 30276733b39aSJayamohan Kallickal idx = 0; 30286733b39aSJayamohan Kallickal dataq = &phwi_context->be_def_dataq; 3029bfead3b2SJayamohan Kallickal cq = &phwi_context->be_cq[0]; 30306733b39aSJayamohan Kallickal mem = &dataq->dma_mem; 30316733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 30326733b39aSJayamohan Kallickal mem_descr += HWI_MEM_ASYNC_DATA_RING; 30336733b39aSJayamohan Kallickal dq_vaddress = mem_descr->mem_array[idx].virtual_address; 30346733b39aSJayamohan Kallickal ret = be_fill_queue(dataq, mem_descr->mem_array[0].size / 30356733b39aSJayamohan Kallickal sizeof(struct phys_addr), 30366733b39aSJayamohan Kallickal sizeof(struct phys_addr), dq_vaddress); 30376733b39aSJayamohan Kallickal if (ret) { 303899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 303999bc5d55SJohn Soni Jose "BM_%d : be_fill_queue Failed for DEF PDU DATA\n"); 30406733b39aSJayamohan Kallickal return ret; 30416733b39aSJayamohan Kallickal } 3042457ff3b7SJayamohan Kallickal mem->dma = (unsigned long)mem_descr->mem_array[idx]. 3043457ff3b7SJayamohan Kallickal bus_address.u.a64.address; 30446733b39aSJayamohan Kallickal ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq, 30456733b39aSJayamohan Kallickal def_pdu_ring_sz, 30466733b39aSJayamohan Kallickal phba->params.defpdu_data_sz); 30476733b39aSJayamohan Kallickal if (ret) { 304899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 304999bc5d55SJohn Soni Jose "BM_%d be_cmd_create_default_pdu_queue" 305099bc5d55SJohn Soni Jose " Failed for DEF PDU DATA\n"); 30516733b39aSJayamohan Kallickal return ret; 30526733b39aSJayamohan Kallickal } 30536733b39aSJayamohan Kallickal phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id; 305499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 305599bc5d55SJohn Soni Jose "BM_%d : iscsi def data id is %d\n", 30566733b39aSJayamohan Kallickal phwi_context->be_def_dataq.id); 305799bc5d55SJohn Soni Jose 30586733b39aSJayamohan Kallickal hwi_post_async_buffers(phba, 0); 305999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 306099bc5d55SJohn Soni Jose "BM_%d : DEFAULT PDU DATA RING CREATED\n"); 306199bc5d55SJohn Soni Jose 30626733b39aSJayamohan Kallickal return 0; 30636733b39aSJayamohan Kallickal } 30646733b39aSJayamohan Kallickal 30656733b39aSJayamohan Kallickal static int 30666733b39aSJayamohan Kallickal beiscsi_post_pages(struct beiscsi_hba *phba) 30676733b39aSJayamohan Kallickal { 30686733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 30696733b39aSJayamohan Kallickal struct mem_array *pm_arr; 30706733b39aSJayamohan Kallickal unsigned int page_offset, i; 30716733b39aSJayamohan Kallickal struct be_dma_mem sgl; 30726733b39aSJayamohan Kallickal int status; 30736733b39aSJayamohan Kallickal 30746733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 30756733b39aSJayamohan Kallickal mem_descr += HWI_MEM_SGE; 30766733b39aSJayamohan Kallickal pm_arr = mem_descr->mem_array; 30776733b39aSJayamohan Kallickal 30786733b39aSJayamohan Kallickal page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io * 30796733b39aSJayamohan Kallickal phba->fw_config.iscsi_icd_start) / PAGE_SIZE; 30806733b39aSJayamohan Kallickal for (i = 0; i < mem_descr->num_elements; i++) { 30816733b39aSJayamohan Kallickal hwi_build_be_sgl_arr(phba, pm_arr, &sgl); 30826733b39aSJayamohan Kallickal status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl, 30836733b39aSJayamohan Kallickal page_offset, 30846733b39aSJayamohan Kallickal (pm_arr->size / PAGE_SIZE)); 30856733b39aSJayamohan Kallickal page_offset += pm_arr->size / PAGE_SIZE; 30866733b39aSJayamohan Kallickal if (status != 0) { 308799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 308899bc5d55SJohn Soni Jose "BM_%d : post sgl failed.\n"); 30896733b39aSJayamohan Kallickal return status; 30906733b39aSJayamohan Kallickal } 30916733b39aSJayamohan Kallickal pm_arr++; 30926733b39aSJayamohan Kallickal } 309399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 309499bc5d55SJohn Soni Jose "BM_%d : POSTED PAGES\n"); 30956733b39aSJayamohan Kallickal return 0; 30966733b39aSJayamohan Kallickal } 30976733b39aSJayamohan Kallickal 3098bfead3b2SJayamohan Kallickal static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q) 3099bfead3b2SJayamohan Kallickal { 3100bfead3b2SJayamohan Kallickal struct be_dma_mem *mem = &q->dma_mem; 3101c8b25598SJayamohan Kallickal if (mem->va) { 3102bfead3b2SJayamohan Kallickal pci_free_consistent(phba->pcidev, mem->size, 3103bfead3b2SJayamohan Kallickal mem->va, mem->dma); 3104c8b25598SJayamohan Kallickal mem->va = NULL; 3105c8b25598SJayamohan Kallickal } 3106bfead3b2SJayamohan Kallickal } 3107bfead3b2SJayamohan Kallickal 3108bfead3b2SJayamohan Kallickal static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q, 3109bfead3b2SJayamohan Kallickal u16 len, u16 entry_size) 3110bfead3b2SJayamohan Kallickal { 3111bfead3b2SJayamohan Kallickal struct be_dma_mem *mem = &q->dma_mem; 3112bfead3b2SJayamohan Kallickal 3113bfead3b2SJayamohan Kallickal memset(q, 0, sizeof(*q)); 3114bfead3b2SJayamohan Kallickal q->len = len; 3115bfead3b2SJayamohan Kallickal q->entry_size = entry_size; 3116bfead3b2SJayamohan Kallickal mem->size = len * entry_size; 3117bfead3b2SJayamohan Kallickal mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma); 3118bfead3b2SJayamohan Kallickal if (!mem->va) 3119d3ad2bb3SJayamohan Kallickal return -ENOMEM; 3120bfead3b2SJayamohan Kallickal memset(mem->va, 0, mem->size); 3121bfead3b2SJayamohan Kallickal return 0; 3122bfead3b2SJayamohan Kallickal } 3123bfead3b2SJayamohan Kallickal 31246733b39aSJayamohan Kallickal static int 31256733b39aSJayamohan Kallickal beiscsi_create_wrb_rings(struct beiscsi_hba *phba, 31266733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context, 31276733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr) 31286733b39aSJayamohan Kallickal { 31296733b39aSJayamohan Kallickal unsigned int wrb_mem_index, offset, size, num_wrb_rings; 31306733b39aSJayamohan Kallickal u64 pa_addr_lo; 31316733b39aSJayamohan Kallickal unsigned int idx, num, i; 31326733b39aSJayamohan Kallickal struct mem_array *pwrb_arr; 31336733b39aSJayamohan Kallickal void *wrb_vaddr; 31346733b39aSJayamohan Kallickal struct be_dma_mem sgl; 31356733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 31366733b39aSJayamohan Kallickal int status; 31376733b39aSJayamohan Kallickal 31386733b39aSJayamohan Kallickal idx = 0; 31396733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 31406733b39aSJayamohan Kallickal mem_descr += HWI_MEM_WRB; 31416733b39aSJayamohan Kallickal pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl, 31426733b39aSJayamohan Kallickal GFP_KERNEL); 31436733b39aSJayamohan Kallickal if (!pwrb_arr) { 314499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 314599bc5d55SJohn Soni Jose "BM_%d : Memory alloc failed in create wrb ring.\n"); 31466733b39aSJayamohan Kallickal return -ENOMEM; 31476733b39aSJayamohan Kallickal } 31486733b39aSJayamohan Kallickal wrb_vaddr = mem_descr->mem_array[idx].virtual_address; 31496733b39aSJayamohan Kallickal pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address; 31506733b39aSJayamohan Kallickal num_wrb_rings = mem_descr->mem_array[idx].size / 31516733b39aSJayamohan Kallickal (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb)); 31526733b39aSJayamohan Kallickal 31536733b39aSJayamohan Kallickal for (num = 0; num < phba->params.cxns_per_ctrl; num++) { 31546733b39aSJayamohan Kallickal if (num_wrb_rings) { 31556733b39aSJayamohan Kallickal pwrb_arr[num].virtual_address = wrb_vaddr; 31566733b39aSJayamohan Kallickal pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo; 31576733b39aSJayamohan Kallickal pwrb_arr[num].size = phba->params.wrbs_per_cxn * 31586733b39aSJayamohan Kallickal sizeof(struct iscsi_wrb); 31596733b39aSJayamohan Kallickal wrb_vaddr += pwrb_arr[num].size; 31606733b39aSJayamohan Kallickal pa_addr_lo += pwrb_arr[num].size; 31616733b39aSJayamohan Kallickal num_wrb_rings--; 31626733b39aSJayamohan Kallickal } else { 31636733b39aSJayamohan Kallickal idx++; 31646733b39aSJayamohan Kallickal wrb_vaddr = mem_descr->mem_array[idx].virtual_address; 31656733b39aSJayamohan Kallickal pa_addr_lo = mem_descr->mem_array[idx].\ 31666733b39aSJayamohan Kallickal bus_address.u.a64.address; 31676733b39aSJayamohan Kallickal num_wrb_rings = mem_descr->mem_array[idx].size / 31686733b39aSJayamohan Kallickal (phba->params.wrbs_per_cxn * 31696733b39aSJayamohan Kallickal sizeof(struct iscsi_wrb)); 31706733b39aSJayamohan Kallickal pwrb_arr[num].virtual_address = wrb_vaddr; 31716733b39aSJayamohan Kallickal pwrb_arr[num].bus_address.u.a64.address\ 31726733b39aSJayamohan Kallickal = pa_addr_lo; 31736733b39aSJayamohan Kallickal pwrb_arr[num].size = phba->params.wrbs_per_cxn * 31746733b39aSJayamohan Kallickal sizeof(struct iscsi_wrb); 31756733b39aSJayamohan Kallickal wrb_vaddr += pwrb_arr[num].size; 31766733b39aSJayamohan Kallickal pa_addr_lo += pwrb_arr[num].size; 31776733b39aSJayamohan Kallickal num_wrb_rings--; 31786733b39aSJayamohan Kallickal } 31796733b39aSJayamohan Kallickal } 31806733b39aSJayamohan Kallickal for (i = 0; i < phba->params.cxns_per_ctrl; i++) { 31816733b39aSJayamohan Kallickal wrb_mem_index = 0; 31826733b39aSJayamohan Kallickal offset = 0; 31836733b39aSJayamohan Kallickal size = 0; 31846733b39aSJayamohan Kallickal 31856733b39aSJayamohan Kallickal hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl); 31866733b39aSJayamohan Kallickal status = be_cmd_wrbq_create(&phba->ctrl, &sgl, 31876733b39aSJayamohan Kallickal &phwi_context->be_wrbq[i]); 31886733b39aSJayamohan Kallickal if (status != 0) { 318999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 319099bc5d55SJohn Soni Jose "BM_%d : wrbq create failed."); 31911462b8ffSDan Carpenter kfree(pwrb_arr); 31926733b39aSJayamohan Kallickal return status; 31936733b39aSJayamohan Kallickal } 31947da50879SJayamohan Kallickal phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i]. 31957da50879SJayamohan Kallickal id; 31966733b39aSJayamohan Kallickal } 31976733b39aSJayamohan Kallickal kfree(pwrb_arr); 31986733b39aSJayamohan Kallickal return 0; 31996733b39aSJayamohan Kallickal } 32006733b39aSJayamohan Kallickal 32016733b39aSJayamohan Kallickal static void free_wrb_handles(struct beiscsi_hba *phba) 32026733b39aSJayamohan Kallickal { 32036733b39aSJayamohan Kallickal unsigned int index; 32046733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 32056733b39aSJayamohan Kallickal struct hwi_wrb_context *pwrb_context; 32066733b39aSJayamohan Kallickal 32076733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 32086733b39aSJayamohan Kallickal for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) { 32096733b39aSJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[index]; 32106733b39aSJayamohan Kallickal kfree(pwrb_context->pwrb_handle_base); 32116733b39aSJayamohan Kallickal kfree(pwrb_context->pwrb_handle_basestd); 32126733b39aSJayamohan Kallickal } 32136733b39aSJayamohan Kallickal } 32146733b39aSJayamohan Kallickal 3215bfead3b2SJayamohan Kallickal static void be_mcc_queues_destroy(struct beiscsi_hba *phba) 3216bfead3b2SJayamohan Kallickal { 3217bfead3b2SJayamohan Kallickal struct be_queue_info *q; 3218bfead3b2SJayamohan Kallickal struct be_ctrl_info *ctrl = &phba->ctrl; 3219bfead3b2SJayamohan Kallickal 3220bfead3b2SJayamohan Kallickal q = &phba->ctrl.mcc_obj.q; 3221bfead3b2SJayamohan Kallickal if (q->created) 3222bfead3b2SJayamohan Kallickal beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ); 3223bfead3b2SJayamohan Kallickal be_queue_free(phba, q); 3224bfead3b2SJayamohan Kallickal 3225bfead3b2SJayamohan Kallickal q = &phba->ctrl.mcc_obj.cq; 3226bfead3b2SJayamohan Kallickal if (q->created) 3227bfead3b2SJayamohan Kallickal beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ); 3228bfead3b2SJayamohan Kallickal be_queue_free(phba, q); 3229bfead3b2SJayamohan Kallickal } 3230bfead3b2SJayamohan Kallickal 32316733b39aSJayamohan Kallickal static void hwi_cleanup(struct beiscsi_hba *phba) 32326733b39aSJayamohan Kallickal { 32336733b39aSJayamohan Kallickal struct be_queue_info *q; 32346733b39aSJayamohan Kallickal struct be_ctrl_info *ctrl = &phba->ctrl; 32356733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 32366733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context; 3237bfead3b2SJayamohan Kallickal int i, eq_num; 32386733b39aSJayamohan Kallickal 32396733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 32406733b39aSJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 32416733b39aSJayamohan Kallickal for (i = 0; i < phba->params.cxns_per_ctrl; i++) { 32426733b39aSJayamohan Kallickal q = &phwi_context->be_wrbq[i]; 32436733b39aSJayamohan Kallickal if (q->created) 32446733b39aSJayamohan Kallickal beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ); 32456733b39aSJayamohan Kallickal } 32466733b39aSJayamohan Kallickal free_wrb_handles(phba); 32476733b39aSJayamohan Kallickal 32486733b39aSJayamohan Kallickal q = &phwi_context->be_def_hdrq; 32496733b39aSJayamohan Kallickal if (q->created) 32506733b39aSJayamohan Kallickal beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ); 32516733b39aSJayamohan Kallickal 32526733b39aSJayamohan Kallickal q = &phwi_context->be_def_dataq; 32536733b39aSJayamohan Kallickal if (q->created) 32546733b39aSJayamohan Kallickal beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ); 32556733b39aSJayamohan Kallickal 32566733b39aSJayamohan Kallickal beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL); 32576733b39aSJayamohan Kallickal 3258bfead3b2SJayamohan Kallickal for (i = 0; i < (phba->num_cpus); i++) { 3259bfead3b2SJayamohan Kallickal q = &phwi_context->be_cq[i]; 32606733b39aSJayamohan Kallickal if (q->created) 32616733b39aSJayamohan Kallickal beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ); 3262bfead3b2SJayamohan Kallickal } 3263bfead3b2SJayamohan Kallickal if (phba->msix_enabled) 3264bfead3b2SJayamohan Kallickal eq_num = 1; 3265bfead3b2SJayamohan Kallickal else 3266bfead3b2SJayamohan Kallickal eq_num = 0; 3267bfead3b2SJayamohan Kallickal for (i = 0; i < (phba->num_cpus + eq_num); i++) { 3268bfead3b2SJayamohan Kallickal q = &phwi_context->be_eq[i].q; 32696733b39aSJayamohan Kallickal if (q->created) 32706733b39aSJayamohan Kallickal beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ); 32716733b39aSJayamohan Kallickal } 3272bfead3b2SJayamohan Kallickal be_mcc_queues_destroy(phba); 3273bfead3b2SJayamohan Kallickal } 3274bfead3b2SJayamohan Kallickal 3275bfead3b2SJayamohan Kallickal static int be_mcc_queues_create(struct beiscsi_hba *phba, 3276bfead3b2SJayamohan Kallickal struct hwi_context_memory *phwi_context) 3277bfead3b2SJayamohan Kallickal { 3278bfead3b2SJayamohan Kallickal struct be_queue_info *q, *cq; 3279bfead3b2SJayamohan Kallickal struct be_ctrl_info *ctrl = &phba->ctrl; 3280bfead3b2SJayamohan Kallickal 3281bfead3b2SJayamohan Kallickal /* Alloc MCC compl queue */ 3282bfead3b2SJayamohan Kallickal cq = &phba->ctrl.mcc_obj.cq; 3283bfead3b2SJayamohan Kallickal if (be_queue_alloc(phba, cq, MCC_CQ_LEN, 3284bfead3b2SJayamohan Kallickal sizeof(struct be_mcc_compl))) 3285bfead3b2SJayamohan Kallickal goto err; 3286bfead3b2SJayamohan Kallickal /* Ask BE to create MCC compl queue; */ 3287bfead3b2SJayamohan Kallickal if (phba->msix_enabled) { 3288bfead3b2SJayamohan Kallickal if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq 3289bfead3b2SJayamohan Kallickal [phba->num_cpus].q, false, true, 0)) 3290bfead3b2SJayamohan Kallickal goto mcc_cq_free; 3291bfead3b2SJayamohan Kallickal } else { 3292bfead3b2SJayamohan Kallickal if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q, 3293bfead3b2SJayamohan Kallickal false, true, 0)) 3294bfead3b2SJayamohan Kallickal goto mcc_cq_free; 3295bfead3b2SJayamohan Kallickal } 3296bfead3b2SJayamohan Kallickal 3297bfead3b2SJayamohan Kallickal /* Alloc MCC queue */ 3298bfead3b2SJayamohan Kallickal q = &phba->ctrl.mcc_obj.q; 3299bfead3b2SJayamohan Kallickal if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb))) 3300bfead3b2SJayamohan Kallickal goto mcc_cq_destroy; 3301bfead3b2SJayamohan Kallickal 3302bfead3b2SJayamohan Kallickal /* Ask BE to create MCC queue */ 330335e66019SJayamohan Kallickal if (beiscsi_cmd_mccq_create(phba, q, cq)) 3304bfead3b2SJayamohan Kallickal goto mcc_q_free; 3305bfead3b2SJayamohan Kallickal 3306bfead3b2SJayamohan Kallickal return 0; 3307bfead3b2SJayamohan Kallickal 3308bfead3b2SJayamohan Kallickal mcc_q_free: 3309bfead3b2SJayamohan Kallickal be_queue_free(phba, q); 3310bfead3b2SJayamohan Kallickal mcc_cq_destroy: 3311bfead3b2SJayamohan Kallickal beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ); 3312bfead3b2SJayamohan Kallickal mcc_cq_free: 3313bfead3b2SJayamohan Kallickal be_queue_free(phba, cq); 3314bfead3b2SJayamohan Kallickal err: 3315d3ad2bb3SJayamohan Kallickal return -ENOMEM; 3316bfead3b2SJayamohan Kallickal } 3317bfead3b2SJayamohan Kallickal 3318107dfcbaSJohn Soni Jose /** 3319107dfcbaSJohn Soni Jose * find_num_cpus()- Get the CPU online count 3320107dfcbaSJohn Soni Jose * @phba: ptr to priv structure 3321107dfcbaSJohn Soni Jose * 3322107dfcbaSJohn Soni Jose * CPU count is used for creating EQ. 3323107dfcbaSJohn Soni Jose **/ 3324107dfcbaSJohn Soni Jose static void find_num_cpus(struct beiscsi_hba *phba) 3325bfead3b2SJayamohan Kallickal { 3326bfead3b2SJayamohan Kallickal int num_cpus = 0; 3327bfead3b2SJayamohan Kallickal 3328bfead3b2SJayamohan Kallickal num_cpus = num_online_cpus(); 3329bfead3b2SJayamohan Kallickal 3330107dfcbaSJohn Soni Jose phba->num_cpus = (num_cpus >= BEISCSI_MAX_NUM_CPU) ? 3331107dfcbaSJohn Soni Jose (BEISCSI_MAX_NUM_CPU - 1) : num_cpus; 3332bfead3b2SJayamohan Kallickal } 33336733b39aSJayamohan Kallickal 33346733b39aSJayamohan Kallickal static int hwi_init_port(struct beiscsi_hba *phba) 33356733b39aSJayamohan Kallickal { 33366733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 33376733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context; 33386733b39aSJayamohan Kallickal unsigned int def_pdu_ring_sz; 33396733b39aSJayamohan Kallickal struct be_ctrl_info *ctrl = &phba->ctrl; 33406733b39aSJayamohan Kallickal int status; 33416733b39aSJayamohan Kallickal 33426733b39aSJayamohan Kallickal def_pdu_ring_sz = 33436733b39aSJayamohan Kallickal phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr); 33446733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 33456733b39aSJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 3346bfead3b2SJayamohan Kallickal phwi_context->max_eqd = 0; 3347bfead3b2SJayamohan Kallickal phwi_context->min_eqd = 0; 3348bfead3b2SJayamohan Kallickal phwi_context->cur_eqd = 64; 33496733b39aSJayamohan Kallickal be_cmd_fw_initialize(&phba->ctrl); 3350bfead3b2SJayamohan Kallickal 3351bfead3b2SJayamohan Kallickal status = beiscsi_create_eqs(phba, phwi_context); 33526733b39aSJayamohan Kallickal if (status != 0) { 335399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 335499bc5d55SJohn Soni Jose "BM_%d : EQ not created\n"); 33556733b39aSJayamohan Kallickal goto error; 33566733b39aSJayamohan Kallickal } 33576733b39aSJayamohan Kallickal 3358bfead3b2SJayamohan Kallickal status = be_mcc_queues_create(phba, phwi_context); 3359bfead3b2SJayamohan Kallickal if (status != 0) 3360bfead3b2SJayamohan Kallickal goto error; 3361bfead3b2SJayamohan Kallickal 3362bfead3b2SJayamohan Kallickal status = mgmt_check_supported_fw(ctrl, phba); 33636733b39aSJayamohan Kallickal if (status != 0) { 336499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 336599bc5d55SJohn Soni Jose "BM_%d : Unsupported fw version\n"); 33666733b39aSJayamohan Kallickal goto error; 33676733b39aSJayamohan Kallickal } 33686733b39aSJayamohan Kallickal 3369bfead3b2SJayamohan Kallickal status = beiscsi_create_cqs(phba, phwi_context); 33706733b39aSJayamohan Kallickal if (status != 0) { 337199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 337299bc5d55SJohn Soni Jose "BM_%d : CQ not created\n"); 33736733b39aSJayamohan Kallickal goto error; 33746733b39aSJayamohan Kallickal } 33756733b39aSJayamohan Kallickal 33766733b39aSJayamohan Kallickal status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr, 33776733b39aSJayamohan Kallickal def_pdu_ring_sz); 33786733b39aSJayamohan Kallickal if (status != 0) { 337999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 338099bc5d55SJohn Soni Jose "BM_%d : Default Header not created\n"); 33816733b39aSJayamohan Kallickal goto error; 33826733b39aSJayamohan Kallickal } 33836733b39aSJayamohan Kallickal 33846733b39aSJayamohan Kallickal status = beiscsi_create_def_data(phba, phwi_context, 33856733b39aSJayamohan Kallickal phwi_ctrlr, def_pdu_ring_sz); 33866733b39aSJayamohan Kallickal if (status != 0) { 338799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 338899bc5d55SJohn Soni Jose "BM_%d : Default Data not created\n"); 33896733b39aSJayamohan Kallickal goto error; 33906733b39aSJayamohan Kallickal } 33916733b39aSJayamohan Kallickal 33926733b39aSJayamohan Kallickal status = beiscsi_post_pages(phba); 33936733b39aSJayamohan Kallickal if (status != 0) { 339499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 339599bc5d55SJohn Soni Jose "BM_%d : Post SGL Pages Failed\n"); 33966733b39aSJayamohan Kallickal goto error; 33976733b39aSJayamohan Kallickal } 33986733b39aSJayamohan Kallickal 33996733b39aSJayamohan Kallickal status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr); 34006733b39aSJayamohan Kallickal if (status != 0) { 340199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 340299bc5d55SJohn Soni Jose "BM_%d : WRB Rings not created\n"); 34036733b39aSJayamohan Kallickal goto error; 34046733b39aSJayamohan Kallickal } 34056733b39aSJayamohan Kallickal 340699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 340799bc5d55SJohn Soni Jose "BM_%d : hwi_init_port success\n"); 34086733b39aSJayamohan Kallickal return 0; 34096733b39aSJayamohan Kallickal 34106733b39aSJayamohan Kallickal error: 341199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 341299bc5d55SJohn Soni Jose "BM_%d : hwi_init_port failed"); 34136733b39aSJayamohan Kallickal hwi_cleanup(phba); 3414a49e06d5SJayamohan Kallickal return status; 34156733b39aSJayamohan Kallickal } 34166733b39aSJayamohan Kallickal 34176733b39aSJayamohan Kallickal static int hwi_init_controller(struct beiscsi_hba *phba) 34186733b39aSJayamohan Kallickal { 34196733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 34206733b39aSJayamohan Kallickal 34216733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 34226733b39aSJayamohan Kallickal if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) { 34236733b39aSJayamohan Kallickal phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba-> 34246733b39aSJayamohan Kallickal init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address; 342599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 342699bc5d55SJohn Soni Jose "BM_%d : phwi_ctrlr->phwi_ctxt=%p\n", 34276733b39aSJayamohan Kallickal phwi_ctrlr->phwi_ctxt); 34286733b39aSJayamohan Kallickal } else { 342999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 343099bc5d55SJohn Soni Jose "BM_%d : HWI_MEM_ADDN_CONTEXT is more " 343199bc5d55SJohn Soni Jose "than one element.Failing to load\n"); 34326733b39aSJayamohan Kallickal return -ENOMEM; 34336733b39aSJayamohan Kallickal } 34346733b39aSJayamohan Kallickal 34356733b39aSJayamohan Kallickal iscsi_init_global_templates(phba); 34363ec78271SJayamohan Kallickal if (beiscsi_init_wrb_handle(phba)) 34373ec78271SJayamohan Kallickal return -ENOMEM; 34383ec78271SJayamohan Kallickal 34396733b39aSJayamohan Kallickal hwi_init_async_pdu_ctx(phba); 34406733b39aSJayamohan Kallickal if (hwi_init_port(phba) != 0) { 344199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 344299bc5d55SJohn Soni Jose "BM_%d : hwi_init_controller failed\n"); 344399bc5d55SJohn Soni Jose 34446733b39aSJayamohan Kallickal return -ENOMEM; 34456733b39aSJayamohan Kallickal } 34466733b39aSJayamohan Kallickal return 0; 34476733b39aSJayamohan Kallickal } 34486733b39aSJayamohan Kallickal 34496733b39aSJayamohan Kallickal static void beiscsi_free_mem(struct beiscsi_hba *phba) 34506733b39aSJayamohan Kallickal { 34516733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 34526733b39aSJayamohan Kallickal int i, j; 34536733b39aSJayamohan Kallickal 34546733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 34556733b39aSJayamohan Kallickal i = 0; 34566733b39aSJayamohan Kallickal j = 0; 34576733b39aSJayamohan Kallickal for (i = 0; i < SE_MEM_MAX; i++) { 34586733b39aSJayamohan Kallickal for (j = mem_descr->num_elements; j > 0; j--) { 34596733b39aSJayamohan Kallickal pci_free_consistent(phba->pcidev, 34606733b39aSJayamohan Kallickal mem_descr->mem_array[j - 1].size, 34616733b39aSJayamohan Kallickal mem_descr->mem_array[j - 1].virtual_address, 3462457ff3b7SJayamohan Kallickal (unsigned long)mem_descr->mem_array[j - 1]. 3463457ff3b7SJayamohan Kallickal bus_address.u.a64.address); 34646733b39aSJayamohan Kallickal } 34656733b39aSJayamohan Kallickal kfree(mem_descr->mem_array); 34666733b39aSJayamohan Kallickal mem_descr++; 34676733b39aSJayamohan Kallickal } 34686733b39aSJayamohan Kallickal kfree(phba->init_mem); 34696733b39aSJayamohan Kallickal kfree(phba->phwi_ctrlr); 34706733b39aSJayamohan Kallickal } 34716733b39aSJayamohan Kallickal 34726733b39aSJayamohan Kallickal static int beiscsi_init_controller(struct beiscsi_hba *phba) 34736733b39aSJayamohan Kallickal { 34746733b39aSJayamohan Kallickal int ret = -ENOMEM; 34756733b39aSJayamohan Kallickal 34766733b39aSJayamohan Kallickal ret = beiscsi_get_memory(phba); 34776733b39aSJayamohan Kallickal if (ret < 0) { 347899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 347999bc5d55SJohn Soni Jose "BM_%d : beiscsi_dev_probe -" 34806733b39aSJayamohan Kallickal "Failed in beiscsi_alloc_memory\n"); 34816733b39aSJayamohan Kallickal return ret; 34826733b39aSJayamohan Kallickal } 34836733b39aSJayamohan Kallickal 34846733b39aSJayamohan Kallickal ret = hwi_init_controller(phba); 34856733b39aSJayamohan Kallickal if (ret) 34866733b39aSJayamohan Kallickal goto free_init; 348799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 348899bc5d55SJohn Soni Jose "BM_%d : Return success from beiscsi_init_controller"); 348999bc5d55SJohn Soni Jose 34906733b39aSJayamohan Kallickal return 0; 34916733b39aSJayamohan Kallickal 34926733b39aSJayamohan Kallickal free_init: 34936733b39aSJayamohan Kallickal beiscsi_free_mem(phba); 3494a49e06d5SJayamohan Kallickal return ret; 34956733b39aSJayamohan Kallickal } 34966733b39aSJayamohan Kallickal 34976733b39aSJayamohan Kallickal static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba) 34986733b39aSJayamohan Kallickal { 34996733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg; 35006733b39aSJayamohan Kallickal struct sgl_handle *psgl_handle; 35016733b39aSJayamohan Kallickal struct iscsi_sge *pfrag; 35026733b39aSJayamohan Kallickal unsigned int arr_index, i, idx; 35036733b39aSJayamohan Kallickal 35046733b39aSJayamohan Kallickal phba->io_sgl_hndl_avbl = 0; 35056733b39aSJayamohan Kallickal phba->eh_sgl_hndl_avbl = 0; 3506bfead3b2SJayamohan Kallickal 35076733b39aSJayamohan Kallickal mem_descr_sglh = phba->init_mem; 35086733b39aSJayamohan Kallickal mem_descr_sglh += HWI_MEM_SGLH; 35096733b39aSJayamohan Kallickal if (1 == mem_descr_sglh->num_elements) { 35106733b39aSJayamohan Kallickal phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) * 35116733b39aSJayamohan Kallickal phba->params.ios_per_ctrl, 35126733b39aSJayamohan Kallickal GFP_KERNEL); 35136733b39aSJayamohan Kallickal if (!phba->io_sgl_hndl_base) { 351499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 351599bc5d55SJohn Soni Jose "BM_%d : Mem Alloc Failed. Failing to load\n"); 35166733b39aSJayamohan Kallickal return -ENOMEM; 35176733b39aSJayamohan Kallickal } 35186733b39aSJayamohan Kallickal phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) * 35196733b39aSJayamohan Kallickal (phba->params.icds_per_ctrl - 35206733b39aSJayamohan Kallickal phba->params.ios_per_ctrl), 35216733b39aSJayamohan Kallickal GFP_KERNEL); 35226733b39aSJayamohan Kallickal if (!phba->eh_sgl_hndl_base) { 35236733b39aSJayamohan Kallickal kfree(phba->io_sgl_hndl_base); 352499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 352599bc5d55SJohn Soni Jose "BM_%d : Mem Alloc Failed. Failing to load\n"); 35266733b39aSJayamohan Kallickal return -ENOMEM; 35276733b39aSJayamohan Kallickal } 35286733b39aSJayamohan Kallickal } else { 352999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 353099bc5d55SJohn Soni Jose "BM_%d : HWI_MEM_SGLH is more than one element." 35316733b39aSJayamohan Kallickal "Failing to load\n"); 35326733b39aSJayamohan Kallickal return -ENOMEM; 35336733b39aSJayamohan Kallickal } 35346733b39aSJayamohan Kallickal 35356733b39aSJayamohan Kallickal arr_index = 0; 35366733b39aSJayamohan Kallickal idx = 0; 35376733b39aSJayamohan Kallickal while (idx < mem_descr_sglh->num_elements) { 35386733b39aSJayamohan Kallickal psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address; 35396733b39aSJayamohan Kallickal 35406733b39aSJayamohan Kallickal for (i = 0; i < (mem_descr_sglh->mem_array[idx].size / 35416733b39aSJayamohan Kallickal sizeof(struct sgl_handle)); i++) { 35426733b39aSJayamohan Kallickal if (arr_index < phba->params.ios_per_ctrl) { 35436733b39aSJayamohan Kallickal phba->io_sgl_hndl_base[arr_index] = psgl_handle; 35446733b39aSJayamohan Kallickal phba->io_sgl_hndl_avbl++; 35456733b39aSJayamohan Kallickal arr_index++; 35466733b39aSJayamohan Kallickal } else { 35476733b39aSJayamohan Kallickal phba->eh_sgl_hndl_base[arr_index - 35486733b39aSJayamohan Kallickal phba->params.ios_per_ctrl] = 35496733b39aSJayamohan Kallickal psgl_handle; 35506733b39aSJayamohan Kallickal arr_index++; 35516733b39aSJayamohan Kallickal phba->eh_sgl_hndl_avbl++; 35526733b39aSJayamohan Kallickal } 35536733b39aSJayamohan Kallickal psgl_handle++; 35546733b39aSJayamohan Kallickal } 35556733b39aSJayamohan Kallickal idx++; 35566733b39aSJayamohan Kallickal } 355799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 355899bc5d55SJohn Soni Jose "BM_%d : phba->io_sgl_hndl_avbl=%d" 35596733b39aSJayamohan Kallickal "phba->eh_sgl_hndl_avbl=%d\n", 35606733b39aSJayamohan Kallickal phba->io_sgl_hndl_avbl, 35616733b39aSJayamohan Kallickal phba->eh_sgl_hndl_avbl); 356299bc5d55SJohn Soni Jose 35636733b39aSJayamohan Kallickal mem_descr_sg = phba->init_mem; 35646733b39aSJayamohan Kallickal mem_descr_sg += HWI_MEM_SGE; 356599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 356699bc5d55SJohn Soni Jose "\n BM_%d : mem_descr_sg->num_elements=%d\n", 35676733b39aSJayamohan Kallickal mem_descr_sg->num_elements); 356899bc5d55SJohn Soni Jose 35696733b39aSJayamohan Kallickal arr_index = 0; 35706733b39aSJayamohan Kallickal idx = 0; 35716733b39aSJayamohan Kallickal while (idx < mem_descr_sg->num_elements) { 35726733b39aSJayamohan Kallickal pfrag = mem_descr_sg->mem_array[idx].virtual_address; 35736733b39aSJayamohan Kallickal 35746733b39aSJayamohan Kallickal for (i = 0; 35756733b39aSJayamohan Kallickal i < (mem_descr_sg->mem_array[idx].size) / 35766733b39aSJayamohan Kallickal (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io); 35776733b39aSJayamohan Kallickal i++) { 35786733b39aSJayamohan Kallickal if (arr_index < phba->params.ios_per_ctrl) 35796733b39aSJayamohan Kallickal psgl_handle = phba->io_sgl_hndl_base[arr_index]; 35806733b39aSJayamohan Kallickal else 35816733b39aSJayamohan Kallickal psgl_handle = phba->eh_sgl_hndl_base[arr_index - 35826733b39aSJayamohan Kallickal phba->params.ios_per_ctrl]; 35836733b39aSJayamohan Kallickal psgl_handle->pfrag = pfrag; 35846733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0); 35856733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0); 35866733b39aSJayamohan Kallickal pfrag += phba->params.num_sge_per_io; 35876733b39aSJayamohan Kallickal psgl_handle->sgl_index = 35887da50879SJayamohan Kallickal phba->fw_config.iscsi_icd_start + arr_index++; 35896733b39aSJayamohan Kallickal } 35906733b39aSJayamohan Kallickal idx++; 35916733b39aSJayamohan Kallickal } 35926733b39aSJayamohan Kallickal phba->io_sgl_free_index = 0; 35936733b39aSJayamohan Kallickal phba->io_sgl_alloc_index = 0; 35946733b39aSJayamohan Kallickal phba->eh_sgl_free_index = 0; 35956733b39aSJayamohan Kallickal phba->eh_sgl_alloc_index = 0; 35966733b39aSJayamohan Kallickal return 0; 35976733b39aSJayamohan Kallickal } 35986733b39aSJayamohan Kallickal 35996733b39aSJayamohan Kallickal static int hba_setup_cid_tbls(struct beiscsi_hba *phba) 36006733b39aSJayamohan Kallickal { 36016733b39aSJayamohan Kallickal int i, new_cid; 36026733b39aSJayamohan Kallickal 3603c2462288SJayamohan Kallickal phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl, 36046733b39aSJayamohan Kallickal GFP_KERNEL); 36056733b39aSJayamohan Kallickal if (!phba->cid_array) { 360699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 360799bc5d55SJohn Soni Jose "BM_%d : Failed to allocate memory in " 36086733b39aSJayamohan Kallickal "hba_setup_cid_tbls\n"); 36096733b39aSJayamohan Kallickal return -ENOMEM; 36106733b39aSJayamohan Kallickal } 3611c2462288SJayamohan Kallickal phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) * 36126733b39aSJayamohan Kallickal phba->params.cxns_per_ctrl * 2, GFP_KERNEL); 36136733b39aSJayamohan Kallickal if (!phba->ep_array) { 361499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 361599bc5d55SJohn Soni Jose "BM_%d : Failed to allocate memory in " 36166733b39aSJayamohan Kallickal "hba_setup_cid_tbls\n"); 36176733b39aSJayamohan Kallickal kfree(phba->cid_array); 36186733b39aSJayamohan Kallickal return -ENOMEM; 36196733b39aSJayamohan Kallickal } 36207da50879SJayamohan Kallickal new_cid = phba->fw_config.iscsi_cid_start; 36216733b39aSJayamohan Kallickal for (i = 0; i < phba->params.cxns_per_ctrl; i++) { 36226733b39aSJayamohan Kallickal phba->cid_array[i] = new_cid; 36236733b39aSJayamohan Kallickal new_cid += 2; 36246733b39aSJayamohan Kallickal } 36256733b39aSJayamohan Kallickal phba->avlbl_cids = phba->params.cxns_per_ctrl; 36266733b39aSJayamohan Kallickal return 0; 36276733b39aSJayamohan Kallickal } 36286733b39aSJayamohan Kallickal 3629238f6b72SJayamohan Kallickal static void hwi_enable_intr(struct beiscsi_hba *phba) 36306733b39aSJayamohan Kallickal { 36316733b39aSJayamohan Kallickal struct be_ctrl_info *ctrl = &phba->ctrl; 36326733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 36336733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context; 36346733b39aSJayamohan Kallickal struct be_queue_info *eq; 36356733b39aSJayamohan Kallickal u8 __iomem *addr; 3636bfead3b2SJayamohan Kallickal u32 reg, i; 36376733b39aSJayamohan Kallickal u32 enabled; 36386733b39aSJayamohan Kallickal 36396733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 36406733b39aSJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 36416733b39aSJayamohan Kallickal 36426733b39aSJayamohan Kallickal addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg + 36436733b39aSJayamohan Kallickal PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET); 36446733b39aSJayamohan Kallickal reg = ioread32(addr); 36456733b39aSJayamohan Kallickal 36466733b39aSJayamohan Kallickal enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 36476733b39aSJayamohan Kallickal if (!enabled) { 36486733b39aSJayamohan Kallickal reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 364999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 365099bc5d55SJohn Soni Jose "BM_%d : reg =x%08x addr=%p\n", reg, addr); 36516733b39aSJayamohan Kallickal iowrite32(reg, addr); 3652665d6d94SJayamohan Kallickal } 3653665d6d94SJayamohan Kallickal 3654c03af1aeSJayamohan Kallickal if (!phba->msix_enabled) { 3655c03af1aeSJayamohan Kallickal eq = &phwi_context->be_eq[0].q; 365699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 365799bc5d55SJohn Soni Jose "BM_%d : eq->id=%d\n", eq->id); 365899bc5d55SJohn Soni Jose 3659c03af1aeSJayamohan Kallickal hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1); 3660c03af1aeSJayamohan Kallickal } else { 3661bfead3b2SJayamohan Kallickal for (i = 0; i <= phba->num_cpus; i++) { 3662bfead3b2SJayamohan Kallickal eq = &phwi_context->be_eq[i].q; 366399bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 366499bc5d55SJohn Soni Jose "BM_%d : eq->id=%d\n", eq->id); 36656733b39aSJayamohan Kallickal hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1); 3666bfead3b2SJayamohan Kallickal } 3667c03af1aeSJayamohan Kallickal } 3668c03af1aeSJayamohan Kallickal } 36696733b39aSJayamohan Kallickal 36706733b39aSJayamohan Kallickal static void hwi_disable_intr(struct beiscsi_hba *phba) 36716733b39aSJayamohan Kallickal { 36726733b39aSJayamohan Kallickal struct be_ctrl_info *ctrl = &phba->ctrl; 36736733b39aSJayamohan Kallickal 36746733b39aSJayamohan Kallickal u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET; 36756733b39aSJayamohan Kallickal u32 reg = ioread32(addr); 36766733b39aSJayamohan Kallickal 36776733b39aSJayamohan Kallickal u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 36786733b39aSJayamohan Kallickal if (enabled) { 36796733b39aSJayamohan Kallickal reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 36806733b39aSJayamohan Kallickal iowrite32(reg, addr); 36816733b39aSJayamohan Kallickal } else 368299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 368399bc5d55SJohn Soni Jose "BM_%d : In hwi_disable_intr, Already Disabled\n"); 36846733b39aSJayamohan Kallickal } 36856733b39aSJayamohan Kallickal 36869aef4200SJohn Soni Jose /** 36879aef4200SJohn Soni Jose * beiscsi_get_boot_info()- Get the boot session info 36889aef4200SJohn Soni Jose * @phba: The device priv structure instance 36899aef4200SJohn Soni Jose * 36909aef4200SJohn Soni Jose * Get the boot target info and store in driver priv structure 36919aef4200SJohn Soni Jose * 36929aef4200SJohn Soni Jose * return values 36939aef4200SJohn Soni Jose * Success: 0 36949aef4200SJohn Soni Jose * Failure: Non-Zero Value 36959aef4200SJohn Soni Jose **/ 3696c7acc5b8SJayamohan Kallickal static int beiscsi_get_boot_info(struct beiscsi_hba *phba) 3697c7acc5b8SJayamohan Kallickal { 36980e43895eSMike Christie struct be_cmd_get_session_resp *session_resp; 3699c7acc5b8SJayamohan Kallickal struct be_mcc_wrb *wrb; 3700c7acc5b8SJayamohan Kallickal struct be_dma_mem nonemb_cmd; 3701c7acc5b8SJayamohan Kallickal unsigned int tag, wrb_num; 3702c7acc5b8SJayamohan Kallickal unsigned short status, extd_status; 37039aef4200SJohn Soni Jose unsigned int s_handle; 3704c7acc5b8SJayamohan Kallickal struct be_queue_info *mccq = &phba->ctrl.mcc_obj.q; 3705f457a46fSMike Christie int ret = -ENOMEM; 3706c7acc5b8SJayamohan Kallickal 37079aef4200SJohn Soni Jose /* Get the session handle of the boot target */ 37089aef4200SJohn Soni Jose ret = be_mgmt_get_boot_shandle(phba, &s_handle); 37099aef4200SJohn Soni Jose if (ret) { 371099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 371199bc5d55SJohn Soni Jose BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG, 371299bc5d55SJohn Soni Jose "BM_%d : No boot session\n"); 37139aef4200SJohn Soni Jose return ret; 3714c7acc5b8SJayamohan Kallickal } 3715c7acc5b8SJayamohan Kallickal nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev, 3716c7acc5b8SJayamohan Kallickal sizeof(*session_resp), 3717c7acc5b8SJayamohan Kallickal &nonemb_cmd.dma); 3718c7acc5b8SJayamohan Kallickal if (nonemb_cmd.va == NULL) { 371999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 372099bc5d55SJohn Soni Jose BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG, 372199bc5d55SJohn Soni Jose "BM_%d : Failed to allocate memory for" 3722c7acc5b8SJayamohan Kallickal "beiscsi_get_session_info\n"); 372399bc5d55SJohn Soni Jose 3724c7acc5b8SJayamohan Kallickal return -ENOMEM; 3725c7acc5b8SJayamohan Kallickal } 3726c7acc5b8SJayamohan Kallickal 3727c7acc5b8SJayamohan Kallickal memset(nonemb_cmd.va, 0, sizeof(*session_resp)); 37289aef4200SJohn Soni Jose tag = mgmt_get_session_info(phba, s_handle, 37290e43895eSMike Christie &nonemb_cmd); 3730c7acc5b8SJayamohan Kallickal if (!tag) { 373199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 373299bc5d55SJohn Soni Jose BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG, 373399bc5d55SJohn Soni Jose "BM_%d : beiscsi_get_session_info" 3734c7acc5b8SJayamohan Kallickal " Failed\n"); 373599bc5d55SJohn Soni Jose 3736c7acc5b8SJayamohan Kallickal goto boot_freemem; 3737c7acc5b8SJayamohan Kallickal } else 3738c7acc5b8SJayamohan Kallickal wait_event_interruptible(phba->ctrl.mcc_wait[tag], 3739c7acc5b8SJayamohan Kallickal phba->ctrl.mcc_numtag[tag]); 3740c7acc5b8SJayamohan Kallickal 3741c7acc5b8SJayamohan Kallickal wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16; 3742c7acc5b8SJayamohan Kallickal extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8; 3743c7acc5b8SJayamohan Kallickal status = phba->ctrl.mcc_numtag[tag] & 0x000000FF; 3744c7acc5b8SJayamohan Kallickal if (status || extd_status) { 374599bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 374699bc5d55SJohn Soni Jose BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG, 374799bc5d55SJohn Soni Jose "BM_%d : beiscsi_get_session_info Failed" 3748c7acc5b8SJayamohan Kallickal " status = %d extd_status = %d\n", 3749c7acc5b8SJayamohan Kallickal status, extd_status); 375099bc5d55SJohn Soni Jose 3751c7acc5b8SJayamohan Kallickal free_mcc_tag(&phba->ctrl, tag); 3752c7acc5b8SJayamohan Kallickal goto boot_freemem; 3753c7acc5b8SJayamohan Kallickal } 3754c7acc5b8SJayamohan Kallickal wrb = queue_get_wrb(mccq, wrb_num); 3755c7acc5b8SJayamohan Kallickal free_mcc_tag(&phba->ctrl, tag); 3756c7acc5b8SJayamohan Kallickal session_resp = nonemb_cmd.va ; 3757f457a46fSMike Christie 3758c7acc5b8SJayamohan Kallickal memcpy(&phba->boot_sess, &session_resp->session_info, 3759c7acc5b8SJayamohan Kallickal sizeof(struct mgmt_session_info)); 3760f457a46fSMike Christie ret = 0; 3761f457a46fSMike Christie 3762c7acc5b8SJayamohan Kallickal boot_freemem: 3763c7acc5b8SJayamohan Kallickal pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 3764c7acc5b8SJayamohan Kallickal nonemb_cmd.va, nonemb_cmd.dma); 3765f457a46fSMike Christie return ret; 3766f457a46fSMike Christie } 3767f457a46fSMike Christie 3768f457a46fSMike Christie static void beiscsi_boot_release(void *data) 3769f457a46fSMike Christie { 3770f457a46fSMike Christie struct beiscsi_hba *phba = data; 3771f457a46fSMike Christie 3772f457a46fSMike Christie scsi_host_put(phba->shost); 3773f457a46fSMike Christie } 3774f457a46fSMike Christie 3775f457a46fSMike Christie static int beiscsi_setup_boot_info(struct beiscsi_hba *phba) 3776f457a46fSMike Christie { 3777f457a46fSMike Christie struct iscsi_boot_kobj *boot_kobj; 3778f457a46fSMike Christie 3779f457a46fSMike Christie /* get boot info using mgmt cmd */ 3780f457a46fSMike Christie if (beiscsi_get_boot_info(phba)) 3781f457a46fSMike Christie /* Try to see if we can carry on without this */ 3782f457a46fSMike Christie return 0; 3783f457a46fSMike Christie 3784f457a46fSMike Christie phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no); 3785f457a46fSMike Christie if (!phba->boot_kset) 3786f457a46fSMike Christie return -ENOMEM; 3787f457a46fSMike Christie 3788f457a46fSMike Christie /* get a ref because the show function will ref the phba */ 3789f457a46fSMike Christie if (!scsi_host_get(phba->shost)) 3790f457a46fSMike Christie goto free_kset; 3791f457a46fSMike Christie boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba, 3792f457a46fSMike Christie beiscsi_show_boot_tgt_info, 3793f457a46fSMike Christie beiscsi_tgt_get_attr_visibility, 3794f457a46fSMike Christie beiscsi_boot_release); 3795f457a46fSMike Christie if (!boot_kobj) 3796f457a46fSMike Christie goto put_shost; 3797f457a46fSMike Christie 3798f457a46fSMike Christie if (!scsi_host_get(phba->shost)) 3799f457a46fSMike Christie goto free_kset; 3800f457a46fSMike Christie boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba, 3801f457a46fSMike Christie beiscsi_show_boot_ini_info, 3802f457a46fSMike Christie beiscsi_ini_get_attr_visibility, 3803f457a46fSMike Christie beiscsi_boot_release); 3804f457a46fSMike Christie if (!boot_kobj) 3805f457a46fSMike Christie goto put_shost; 3806f457a46fSMike Christie 3807f457a46fSMike Christie if (!scsi_host_get(phba->shost)) 3808f457a46fSMike Christie goto free_kset; 3809f457a46fSMike Christie boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba, 3810f457a46fSMike Christie beiscsi_show_boot_eth_info, 3811f457a46fSMike Christie beiscsi_eth_get_attr_visibility, 3812f457a46fSMike Christie beiscsi_boot_release); 3813f457a46fSMike Christie if (!boot_kobj) 3814f457a46fSMike Christie goto put_shost; 3815f457a46fSMike Christie return 0; 3816f457a46fSMike Christie 3817f457a46fSMike Christie put_shost: 3818f457a46fSMike Christie scsi_host_put(phba->shost); 3819f457a46fSMike Christie free_kset: 3820f457a46fSMike Christie iscsi_boot_destroy_kset(phba->boot_kset); 3821c7acc5b8SJayamohan Kallickal return -ENOMEM; 3822c7acc5b8SJayamohan Kallickal } 3823c7acc5b8SJayamohan Kallickal 38246733b39aSJayamohan Kallickal static int beiscsi_init_port(struct beiscsi_hba *phba) 38256733b39aSJayamohan Kallickal { 38266733b39aSJayamohan Kallickal int ret; 38276733b39aSJayamohan Kallickal 38286733b39aSJayamohan Kallickal ret = beiscsi_init_controller(phba); 38296733b39aSJayamohan Kallickal if (ret < 0) { 383099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 383199bc5d55SJohn Soni Jose "BM_%d : beiscsi_dev_probe - Failed in" 38326733b39aSJayamohan Kallickal "beiscsi_init_controller\n"); 38336733b39aSJayamohan Kallickal return ret; 38346733b39aSJayamohan Kallickal } 38356733b39aSJayamohan Kallickal ret = beiscsi_init_sgl_handle(phba); 38366733b39aSJayamohan Kallickal if (ret < 0) { 383799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 383899bc5d55SJohn Soni Jose "BM_%d : beiscsi_dev_probe - Failed in" 38396733b39aSJayamohan Kallickal "beiscsi_init_sgl_handle\n"); 38406733b39aSJayamohan Kallickal goto do_cleanup_ctrlr; 38416733b39aSJayamohan Kallickal } 38426733b39aSJayamohan Kallickal 38436733b39aSJayamohan Kallickal if (hba_setup_cid_tbls(phba)) { 384499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 384599bc5d55SJohn Soni Jose "BM_%d : Failed in hba_setup_cid_tbls\n"); 38466733b39aSJayamohan Kallickal kfree(phba->io_sgl_hndl_base); 38476733b39aSJayamohan Kallickal kfree(phba->eh_sgl_hndl_base); 38486733b39aSJayamohan Kallickal goto do_cleanup_ctrlr; 38496733b39aSJayamohan Kallickal } 38506733b39aSJayamohan Kallickal 38516733b39aSJayamohan Kallickal return ret; 38526733b39aSJayamohan Kallickal 38536733b39aSJayamohan Kallickal do_cleanup_ctrlr: 38546733b39aSJayamohan Kallickal hwi_cleanup(phba); 38556733b39aSJayamohan Kallickal return ret; 38566733b39aSJayamohan Kallickal } 38576733b39aSJayamohan Kallickal 38586733b39aSJayamohan Kallickal static void hwi_purge_eq(struct beiscsi_hba *phba) 38596733b39aSJayamohan Kallickal { 38606733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 38616733b39aSJayamohan Kallickal struct hwi_context_memory *phwi_context; 38626733b39aSJayamohan Kallickal struct be_queue_info *eq; 38636733b39aSJayamohan Kallickal struct be_eq_entry *eqe = NULL; 3864bfead3b2SJayamohan Kallickal int i, eq_msix; 3865756d29c8SJayamohan Kallickal unsigned int num_processed; 38666733b39aSJayamohan Kallickal 38676733b39aSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 38686733b39aSJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 3869bfead3b2SJayamohan Kallickal if (phba->msix_enabled) 3870bfead3b2SJayamohan Kallickal eq_msix = 1; 3871bfead3b2SJayamohan Kallickal else 3872bfead3b2SJayamohan Kallickal eq_msix = 0; 3873bfead3b2SJayamohan Kallickal 3874bfead3b2SJayamohan Kallickal for (i = 0; i < (phba->num_cpus + eq_msix); i++) { 3875bfead3b2SJayamohan Kallickal eq = &phwi_context->be_eq[i].q; 38766733b39aSJayamohan Kallickal eqe = queue_tail_node(eq); 3877756d29c8SJayamohan Kallickal num_processed = 0; 38786733b39aSJayamohan Kallickal while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 38796733b39aSJayamohan Kallickal & EQE_VALID_MASK) { 38806733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 38816733b39aSJayamohan Kallickal queue_tail_inc(eq); 38826733b39aSJayamohan Kallickal eqe = queue_tail_node(eq); 3883756d29c8SJayamohan Kallickal num_processed++; 38846733b39aSJayamohan Kallickal } 3885756d29c8SJayamohan Kallickal 3886756d29c8SJayamohan Kallickal if (num_processed) 3887756d29c8SJayamohan Kallickal hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1); 38886733b39aSJayamohan Kallickal } 3889bfead3b2SJayamohan Kallickal } 38906733b39aSJayamohan Kallickal 38916733b39aSJayamohan Kallickal static void beiscsi_clean_port(struct beiscsi_hba *phba) 38926733b39aSJayamohan Kallickal { 389303a12310SJayamohan Kallickal int mgmt_status; 38946733b39aSJayamohan Kallickal 38956733b39aSJayamohan Kallickal mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0); 38966733b39aSJayamohan Kallickal if (mgmt_status) 389799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 389899bc5d55SJohn Soni Jose "BM_%d : mgmt_epfw_cleanup FAILED\n"); 3899756d29c8SJayamohan Kallickal 39006733b39aSJayamohan Kallickal hwi_purge_eq(phba); 3901756d29c8SJayamohan Kallickal hwi_cleanup(phba); 39026733b39aSJayamohan Kallickal kfree(phba->io_sgl_hndl_base); 39036733b39aSJayamohan Kallickal kfree(phba->eh_sgl_hndl_base); 39046733b39aSJayamohan Kallickal kfree(phba->cid_array); 39056733b39aSJayamohan Kallickal kfree(phba->ep_array); 39066733b39aSJayamohan Kallickal } 39076733b39aSJayamohan Kallickal 3908d629c471SJohn Soni Jose /** 3909d629c471SJohn Soni Jose * beiscsi_cleanup_task()- Free driver resources of the task 3910d629c471SJohn Soni Jose * @task: ptr to the iscsi task 3911d629c471SJohn Soni Jose * 3912d629c471SJohn Soni Jose **/ 39131282ab76SMike Christie static void beiscsi_cleanup_task(struct iscsi_task *task) 39141282ab76SMike Christie { 39151282ab76SMike Christie struct beiscsi_io_task *io_task = task->dd_data; 39161282ab76SMike Christie struct iscsi_conn *conn = task->conn; 39171282ab76SMike Christie struct beiscsi_conn *beiscsi_conn = conn->dd_data; 39181282ab76SMike Christie struct beiscsi_hba *phba = beiscsi_conn->phba; 39191282ab76SMike Christie struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess; 39201282ab76SMike Christie struct hwi_wrb_context *pwrb_context; 39211282ab76SMike Christie struct hwi_controller *phwi_ctrlr; 39221282ab76SMike Christie 39231282ab76SMike Christie phwi_ctrlr = phba->phwi_ctrlr; 39241282ab76SMike Christie pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid 39251282ab76SMike Christie - phba->fw_config.iscsi_cid_start]; 39261282ab76SMike Christie 39271282ab76SMike Christie if (io_task->cmd_bhs) { 39281282ab76SMike Christie pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs, 39291282ab76SMike Christie io_task->bhs_pa.u.a64.address); 39301282ab76SMike Christie io_task->cmd_bhs = NULL; 39311282ab76SMike Christie } 39321282ab76SMike Christie 39331282ab76SMike Christie if (task->sc) { 39341282ab76SMike Christie if (io_task->pwrb_handle) { 39351282ab76SMike Christie free_wrb_handle(phba, pwrb_context, 39361282ab76SMike Christie io_task->pwrb_handle); 39371282ab76SMike Christie io_task->pwrb_handle = NULL; 39381282ab76SMike Christie } 39391282ab76SMike Christie 39401282ab76SMike Christie if (io_task->psgl_handle) { 39411282ab76SMike Christie spin_lock(&phba->io_sgl_lock); 39421282ab76SMike Christie free_io_sgl_handle(phba, io_task->psgl_handle); 39431282ab76SMike Christie spin_unlock(&phba->io_sgl_lock); 39441282ab76SMike Christie io_task->psgl_handle = NULL; 39451282ab76SMike Christie } 39461282ab76SMike Christie } else { 39471282ab76SMike Christie if (!beiscsi_conn->login_in_progress) { 39481282ab76SMike Christie if (io_task->pwrb_handle) { 39491282ab76SMike Christie free_wrb_handle(phba, pwrb_context, 39501282ab76SMike Christie io_task->pwrb_handle); 39511282ab76SMike Christie io_task->pwrb_handle = NULL; 39521282ab76SMike Christie } 39531282ab76SMike Christie if (io_task->psgl_handle) { 39541282ab76SMike Christie spin_lock(&phba->mgmt_sgl_lock); 39551282ab76SMike Christie free_mgmt_sgl_handle(phba, 39561282ab76SMike Christie io_task->psgl_handle); 39571282ab76SMike Christie spin_unlock(&phba->mgmt_sgl_lock); 39581282ab76SMike Christie io_task->psgl_handle = NULL; 39591282ab76SMike Christie } 3960d629c471SJohn Soni Jose if (io_task->mtask_addr) { 3961d629c471SJohn Soni Jose pci_unmap_single(phba->pcidev, 3962d629c471SJohn Soni Jose io_task->mtask_addr, 3963d629c471SJohn Soni Jose io_task->mtask_data_count, 3964d629c471SJohn Soni Jose PCI_DMA_TODEVICE); 3965d629c471SJohn Soni Jose io_task->mtask_addr = 0; 3966d629c471SJohn Soni Jose } 39671282ab76SMike Christie } 39681282ab76SMike Christie } 39691282ab76SMike Christie } 39701282ab76SMike Christie 39716733b39aSJayamohan Kallickal void 39726733b39aSJayamohan Kallickal beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn, 39736733b39aSJayamohan Kallickal struct beiscsi_offload_params *params) 39746733b39aSJayamohan Kallickal { 39756733b39aSJayamohan Kallickal struct wrb_handle *pwrb_handle; 39766733b39aSJayamohan Kallickal struct iscsi_target_context_update_wrb *pwrb = NULL; 39776733b39aSJayamohan Kallickal struct be_mem_descriptor *mem_descr; 39786733b39aSJayamohan Kallickal struct beiscsi_hba *phba = beiscsi_conn->phba; 39791282ab76SMike Christie struct iscsi_task *task = beiscsi_conn->task; 39801282ab76SMike Christie struct iscsi_session *session = task->conn->session; 39816733b39aSJayamohan Kallickal u32 doorbell = 0; 39826733b39aSJayamohan Kallickal 39836733b39aSJayamohan Kallickal /* 39846733b39aSJayamohan Kallickal * We can always use 0 here because it is reserved by libiscsi for 39856733b39aSJayamohan Kallickal * login/startup related tasks. 39866733b39aSJayamohan Kallickal */ 39871282ab76SMike Christie beiscsi_conn->login_in_progress = 0; 39881282ab76SMike Christie spin_lock_bh(&session->lock); 39891282ab76SMike Christie beiscsi_cleanup_task(task); 39901282ab76SMike Christie spin_unlock_bh(&session->lock); 39911282ab76SMike Christie 39927da50879SJayamohan Kallickal pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid - 3993d5431488SJayamohan Kallickal phba->fw_config.iscsi_cid_start)); 39946733b39aSJayamohan Kallickal pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb; 39956733b39aSJayamohan Kallickal memset(pwrb, 0, sizeof(*pwrb)); 39966733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, 39976733b39aSJayamohan Kallickal max_burst_length, pwrb, params->dw[offsetof 39986733b39aSJayamohan Kallickal (struct amap_beiscsi_offload_params, 39996733b39aSJayamohan Kallickal max_burst_length) / 32]); 40006733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, 40016733b39aSJayamohan Kallickal max_send_data_segment_length, pwrb, 40026733b39aSJayamohan Kallickal params->dw[offsetof(struct amap_beiscsi_offload_params, 40036733b39aSJayamohan Kallickal max_send_data_segment_length) / 32]); 40046733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, 40056733b39aSJayamohan Kallickal first_burst_length, 40066733b39aSJayamohan Kallickal pwrb, 40076733b39aSJayamohan Kallickal params->dw[offsetof(struct amap_beiscsi_offload_params, 40086733b39aSJayamohan Kallickal first_burst_length) / 32]); 40096733b39aSJayamohan Kallickal 40106733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb, 40116733b39aSJayamohan Kallickal (params->dw[offsetof(struct amap_beiscsi_offload_params, 40126733b39aSJayamohan Kallickal erl) / 32] & OFFLD_PARAMS_ERL)); 40136733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb, 40146733b39aSJayamohan Kallickal (params->dw[offsetof(struct amap_beiscsi_offload_params, 40156733b39aSJayamohan Kallickal dde) / 32] & OFFLD_PARAMS_DDE) >> 2); 40166733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb, 40176733b39aSJayamohan Kallickal (params->dw[offsetof(struct amap_beiscsi_offload_params, 40186733b39aSJayamohan Kallickal hde) / 32] & OFFLD_PARAMS_HDE) >> 3); 40196733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb, 40206733b39aSJayamohan Kallickal (params->dw[offsetof(struct amap_beiscsi_offload_params, 40216733b39aSJayamohan Kallickal ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4); 40226733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb, 40236733b39aSJayamohan Kallickal (params->dw[offsetof(struct amap_beiscsi_offload_params, 40246733b39aSJayamohan Kallickal imd) / 32] & OFFLD_PARAMS_IMD) >> 5); 40256733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn, 40266733b39aSJayamohan Kallickal pwrb, 40276733b39aSJayamohan Kallickal (params->dw[offsetof(struct amap_beiscsi_offload_params, 40286733b39aSJayamohan Kallickal exp_statsn) / 32] + 1)); 40296733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb, 40306733b39aSJayamohan Kallickal 0x7); 40316733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx, 40326733b39aSJayamohan Kallickal pwrb, pwrb_handle->wrb_index); 40336733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb, 40346733b39aSJayamohan Kallickal pwrb, pwrb_handle->nxt_wrb_index); 40356733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, 40366733b39aSJayamohan Kallickal session_state, pwrb, 0); 40376733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack, 40386733b39aSJayamohan Kallickal pwrb, 1); 40396733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq, 40406733b39aSJayamohan Kallickal pwrb, 0); 40416733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb, 40426733b39aSJayamohan Kallickal 0); 40436733b39aSJayamohan Kallickal 40446733b39aSJayamohan Kallickal mem_descr = phba->init_mem; 40456733b39aSJayamohan Kallickal mem_descr += ISCSI_MEM_GLOBAL_HEADER; 40466733b39aSJayamohan Kallickal 40476733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, 40486733b39aSJayamohan Kallickal pad_buffer_addr_hi, pwrb, 40496733b39aSJayamohan Kallickal mem_descr->mem_array[0].bus_address.u.a32.address_hi); 40506733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, 40516733b39aSJayamohan Kallickal pad_buffer_addr_lo, pwrb, 40526733b39aSJayamohan Kallickal mem_descr->mem_array[0].bus_address.u.a32.address_lo); 40536733b39aSJayamohan Kallickal 40546733b39aSJayamohan Kallickal be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb)); 40556733b39aSJayamohan Kallickal 40566733b39aSJayamohan Kallickal doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK; 4057bfead3b2SJayamohan Kallickal doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK) 4058bfead3b2SJayamohan Kallickal << DB_DEF_PDU_WRB_INDEX_SHIFT; 40596733b39aSJayamohan Kallickal doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 40606733b39aSJayamohan Kallickal 40616733b39aSJayamohan Kallickal iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET); 40626733b39aSJayamohan Kallickal } 40636733b39aSJayamohan Kallickal 40646733b39aSJayamohan Kallickal static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt, 40656733b39aSJayamohan Kallickal int *index, int *age) 40666733b39aSJayamohan Kallickal { 4067bfead3b2SJayamohan Kallickal *index = (int)itt; 40686733b39aSJayamohan Kallickal if (age) 40696733b39aSJayamohan Kallickal *age = conn->session->age; 40706733b39aSJayamohan Kallickal } 40716733b39aSJayamohan Kallickal 40726733b39aSJayamohan Kallickal /** 40736733b39aSJayamohan Kallickal * beiscsi_alloc_pdu - allocates pdu and related resources 40746733b39aSJayamohan Kallickal * @task: libiscsi task 40756733b39aSJayamohan Kallickal * @opcode: opcode of pdu for task 40766733b39aSJayamohan Kallickal * 40776733b39aSJayamohan Kallickal * This is called with the session lock held. It will allocate 40786733b39aSJayamohan Kallickal * the wrb and sgl if needed for the command. And it will prep 40796733b39aSJayamohan Kallickal * the pdu's itt. beiscsi_parse_pdu will later translate 40806733b39aSJayamohan Kallickal * the pdu itt to the libiscsi task itt. 40816733b39aSJayamohan Kallickal */ 40826733b39aSJayamohan Kallickal static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode) 40836733b39aSJayamohan Kallickal { 40846733b39aSJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 40856733b39aSJayamohan Kallickal struct iscsi_conn *conn = task->conn; 40866733b39aSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn = conn->dd_data; 40876733b39aSJayamohan Kallickal struct beiscsi_hba *phba = beiscsi_conn->phba; 40886733b39aSJayamohan Kallickal struct hwi_wrb_context *pwrb_context; 40896733b39aSJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 40906733b39aSJayamohan Kallickal itt_t itt; 40912afc95bfSJayamohan Kallickal struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess; 40922afc95bfSJayamohan Kallickal dma_addr_t paddr; 40936733b39aSJayamohan Kallickal 40942afc95bfSJayamohan Kallickal io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool, 4095bc7accecSMike Christie GFP_ATOMIC, &paddr); 40962afc95bfSJayamohan Kallickal if (!io_task->cmd_bhs) 40972afc95bfSJayamohan Kallickal return -ENOMEM; 40982afc95bfSJayamohan Kallickal io_task->bhs_pa.u.a64.address = paddr; 4099bfead3b2SJayamohan Kallickal io_task->libiscsi_itt = (itt_t)task->itt; 41006733b39aSJayamohan Kallickal io_task->conn = beiscsi_conn; 41016733b39aSJayamohan Kallickal 41026733b39aSJayamohan Kallickal task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr; 41036733b39aSJayamohan Kallickal task->hdr_max = sizeof(struct be_cmd_bhs); 4104d2cecf0dSJayamohan Kallickal io_task->psgl_handle = NULL; 41053ec78271SJayamohan Kallickal io_task->pwrb_handle = NULL; 41066733b39aSJayamohan Kallickal 41076733b39aSJayamohan Kallickal if (task->sc) { 41086733b39aSJayamohan Kallickal spin_lock(&phba->io_sgl_lock); 41096733b39aSJayamohan Kallickal io_task->psgl_handle = alloc_io_sgl_handle(phba); 41106733b39aSJayamohan Kallickal spin_unlock(&phba->io_sgl_lock); 41112afc95bfSJayamohan Kallickal if (!io_task->psgl_handle) 41122afc95bfSJayamohan Kallickal goto free_hndls; 4113d2cecf0dSJayamohan Kallickal io_task->pwrb_handle = alloc_wrb_handle(phba, 4114d2cecf0dSJayamohan Kallickal beiscsi_conn->beiscsi_conn_cid - 4115d2cecf0dSJayamohan Kallickal phba->fw_config.iscsi_cid_start); 4116d2cecf0dSJayamohan Kallickal if (!io_task->pwrb_handle) 4117d2cecf0dSJayamohan Kallickal goto free_io_hndls; 41186733b39aSJayamohan Kallickal } else { 41196733b39aSJayamohan Kallickal io_task->scsi_cmnd = NULL; 4120d7aea67bSJayamohan Kallickal if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) { 41216733b39aSJayamohan Kallickal if (!beiscsi_conn->login_in_progress) { 41226733b39aSJayamohan Kallickal spin_lock(&phba->mgmt_sgl_lock); 41236733b39aSJayamohan Kallickal io_task->psgl_handle = (struct sgl_handle *) 41246733b39aSJayamohan Kallickal alloc_mgmt_sgl_handle(phba); 41256733b39aSJayamohan Kallickal spin_unlock(&phba->mgmt_sgl_lock); 41262afc95bfSJayamohan Kallickal if (!io_task->psgl_handle) 41272afc95bfSJayamohan Kallickal goto free_hndls; 41282afc95bfSJayamohan Kallickal 41296733b39aSJayamohan Kallickal beiscsi_conn->login_in_progress = 1; 41306733b39aSJayamohan Kallickal beiscsi_conn->plogin_sgl_handle = 41316733b39aSJayamohan Kallickal io_task->psgl_handle; 4132d2cecf0dSJayamohan Kallickal io_task->pwrb_handle = 4133d2cecf0dSJayamohan Kallickal alloc_wrb_handle(phba, 4134d2cecf0dSJayamohan Kallickal beiscsi_conn->beiscsi_conn_cid - 4135d2cecf0dSJayamohan Kallickal phba->fw_config.iscsi_cid_start); 4136d2cecf0dSJayamohan Kallickal if (!io_task->pwrb_handle) 4137d2cecf0dSJayamohan Kallickal goto free_io_hndls; 4138d2cecf0dSJayamohan Kallickal beiscsi_conn->plogin_wrb_handle = 4139d2cecf0dSJayamohan Kallickal io_task->pwrb_handle; 4140d2cecf0dSJayamohan Kallickal 41416733b39aSJayamohan Kallickal } else { 41426733b39aSJayamohan Kallickal io_task->psgl_handle = 41436733b39aSJayamohan Kallickal beiscsi_conn->plogin_sgl_handle; 4144d2cecf0dSJayamohan Kallickal io_task->pwrb_handle = 4145d2cecf0dSJayamohan Kallickal beiscsi_conn->plogin_wrb_handle; 41466733b39aSJayamohan Kallickal } 41471282ab76SMike Christie beiscsi_conn->task = task; 41486733b39aSJayamohan Kallickal } else { 41496733b39aSJayamohan Kallickal spin_lock(&phba->mgmt_sgl_lock); 41506733b39aSJayamohan Kallickal io_task->psgl_handle = alloc_mgmt_sgl_handle(phba); 41516733b39aSJayamohan Kallickal spin_unlock(&phba->mgmt_sgl_lock); 41522afc95bfSJayamohan Kallickal if (!io_task->psgl_handle) 41532afc95bfSJayamohan Kallickal goto free_hndls; 4154d2cecf0dSJayamohan Kallickal io_task->pwrb_handle = 4155d2cecf0dSJayamohan Kallickal alloc_wrb_handle(phba, 4156d2cecf0dSJayamohan Kallickal beiscsi_conn->beiscsi_conn_cid - 4157d2cecf0dSJayamohan Kallickal phba->fw_config.iscsi_cid_start); 4158d2cecf0dSJayamohan Kallickal if (!io_task->pwrb_handle) 4159d2cecf0dSJayamohan Kallickal goto free_mgmt_hndls; 4160d2cecf0dSJayamohan Kallickal 41616733b39aSJayamohan Kallickal } 41626733b39aSJayamohan Kallickal } 4163bfead3b2SJayamohan Kallickal itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle-> 4164bfead3b2SJayamohan Kallickal wrb_index << 16) | (unsigned int) 4165bfead3b2SJayamohan Kallickal (io_task->psgl_handle->sgl_index)); 4166bfead3b2SJayamohan Kallickal io_task->pwrb_handle->pio_handle = task; 4167bfead3b2SJayamohan Kallickal 41686733b39aSJayamohan Kallickal io_task->cmd_bhs->iscsi_hdr.itt = itt; 41696733b39aSJayamohan Kallickal return 0; 41702afc95bfSJayamohan Kallickal 4171d2cecf0dSJayamohan Kallickal free_io_hndls: 4172d2cecf0dSJayamohan Kallickal spin_lock(&phba->io_sgl_lock); 4173d2cecf0dSJayamohan Kallickal free_io_sgl_handle(phba, io_task->psgl_handle); 4174d2cecf0dSJayamohan Kallickal spin_unlock(&phba->io_sgl_lock); 4175d2cecf0dSJayamohan Kallickal goto free_hndls; 4176d2cecf0dSJayamohan Kallickal free_mgmt_hndls: 4177d2cecf0dSJayamohan Kallickal spin_lock(&phba->mgmt_sgl_lock); 4178d2cecf0dSJayamohan Kallickal free_mgmt_sgl_handle(phba, io_task->psgl_handle); 4179d2cecf0dSJayamohan Kallickal spin_unlock(&phba->mgmt_sgl_lock); 41802afc95bfSJayamohan Kallickal free_hndls: 41812afc95bfSJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 41827da50879SJayamohan Kallickal pwrb_context = &phwi_ctrlr->wrb_context[ 41837da50879SJayamohan Kallickal beiscsi_conn->beiscsi_conn_cid - 41847da50879SJayamohan Kallickal phba->fw_config.iscsi_cid_start]; 4185d2cecf0dSJayamohan Kallickal if (io_task->pwrb_handle) 41862afc95bfSJayamohan Kallickal free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle); 41872afc95bfSJayamohan Kallickal io_task->pwrb_handle = NULL; 41882afc95bfSJayamohan Kallickal pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs, 41892afc95bfSJayamohan Kallickal io_task->bhs_pa.u.a64.address); 41901282ab76SMike Christie io_task->cmd_bhs = NULL; 419199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, 419299bc5d55SJohn Soni Jose BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 419399bc5d55SJohn Soni Jose "BM_%d : Alloc of SGL_ICD Failed\n"); 41942afc95bfSJayamohan Kallickal return -ENOMEM; 41956733b39aSJayamohan Kallickal } 41966733b39aSJayamohan Kallickal 41976733b39aSJayamohan Kallickal static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg, 41986733b39aSJayamohan Kallickal unsigned int num_sg, unsigned int xferlen, 41996733b39aSJayamohan Kallickal unsigned int writedir) 42006733b39aSJayamohan Kallickal { 42016733b39aSJayamohan Kallickal 42026733b39aSJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 42036733b39aSJayamohan Kallickal struct iscsi_conn *conn = task->conn; 42046733b39aSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn = conn->dd_data; 42056733b39aSJayamohan Kallickal struct beiscsi_hba *phba = beiscsi_conn->phba; 42066733b39aSJayamohan Kallickal struct iscsi_wrb *pwrb = NULL; 42076733b39aSJayamohan Kallickal unsigned int doorbell = 0; 42086733b39aSJayamohan Kallickal 42096733b39aSJayamohan Kallickal pwrb = io_task->pwrb_handle->pwrb; 42106733b39aSJayamohan Kallickal io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0; 42116733b39aSJayamohan Kallickal io_task->bhs_len = sizeof(struct be_cmd_bhs); 42126733b39aSJayamohan Kallickal 42136733b39aSJayamohan Kallickal if (writedir) { 4214bfead3b2SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 4215bfead3b2SJayamohan Kallickal INI_WR_CMD); 42166733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1); 42176733b39aSJayamohan Kallickal } else { 4218bfead3b2SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 4219bfead3b2SJayamohan Kallickal INI_RD_CMD); 42206733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); 42216733b39aSJayamohan Kallickal } 42226733b39aSJayamohan Kallickal 42236733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb, 4224dc63aac6SJayamohan Kallickal cpu_to_be16(*(unsigned short *) 4225dc63aac6SJayamohan Kallickal &io_task->cmd_bhs->iscsi_hdr.lun)); 42266733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen); 42276733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb, 42286733b39aSJayamohan Kallickal io_task->pwrb_handle->wrb_index); 42296733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 42306733b39aSJayamohan Kallickal be32_to_cpu(task->cmdsn)); 42316733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb, 42326733b39aSJayamohan Kallickal io_task->psgl_handle->sgl_index); 42336733b39aSJayamohan Kallickal 42346733b39aSJayamohan Kallickal hwi_write_sgl(pwrb, sg, num_sg, io_task); 42356733b39aSJayamohan Kallickal 42366733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb, 42376733b39aSJayamohan Kallickal io_task->pwrb_handle->nxt_wrb_index); 42386733b39aSJayamohan Kallickal be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb)); 42396733b39aSJayamohan Kallickal 42406733b39aSJayamohan Kallickal doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK; 42416733b39aSJayamohan Kallickal doorbell |= (io_task->pwrb_handle->wrb_index & 42426733b39aSJayamohan Kallickal DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT; 42436733b39aSJayamohan Kallickal doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 42446733b39aSJayamohan Kallickal 42456733b39aSJayamohan Kallickal iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET); 42466733b39aSJayamohan Kallickal return 0; 42476733b39aSJayamohan Kallickal } 42486733b39aSJayamohan Kallickal 42496733b39aSJayamohan Kallickal static int beiscsi_mtask(struct iscsi_task *task) 42506733b39aSJayamohan Kallickal { 4251dafab8e0SJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 42526733b39aSJayamohan Kallickal struct iscsi_conn *conn = task->conn; 42536733b39aSJayamohan Kallickal struct beiscsi_conn *beiscsi_conn = conn->dd_data; 42546733b39aSJayamohan Kallickal struct beiscsi_hba *phba = beiscsi_conn->phba; 42556733b39aSJayamohan Kallickal struct iscsi_wrb *pwrb = NULL; 42566733b39aSJayamohan Kallickal unsigned int doorbell = 0; 4257dafab8e0SJayamohan Kallickal unsigned int cid; 42586733b39aSJayamohan Kallickal 4259bfead3b2SJayamohan Kallickal cid = beiscsi_conn->beiscsi_conn_cid; 42606733b39aSJayamohan Kallickal pwrb = io_task->pwrb_handle->pwrb; 4261caf818f1SJayamohan Kallickal memset(pwrb, 0, sizeof(*pwrb)); 42626733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 42636733b39aSJayamohan Kallickal be32_to_cpu(task->cmdsn)); 42646733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb, 42656733b39aSJayamohan Kallickal io_task->pwrb_handle->wrb_index); 42666733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb, 42676733b39aSJayamohan Kallickal io_task->psgl_handle->sgl_index); 4268dafab8e0SJayamohan Kallickal 42696733b39aSJayamohan Kallickal switch (task->hdr->opcode & ISCSI_OPCODE_MASK) { 42706733b39aSJayamohan Kallickal case ISCSI_OP_LOGIN: 4271bfead3b2SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 4272bfead3b2SJayamohan Kallickal TGT_DM_CMD); 42736733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); 42746733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1); 42756733b39aSJayamohan Kallickal hwi_write_buffer(pwrb, task); 42766733b39aSJayamohan Kallickal break; 42776733b39aSJayamohan Kallickal case ISCSI_OP_NOOP_OUT: 42781390b01bSJayamohan Kallickal if (task->hdr->ttt != ISCSI_RESERVED_TAG) { 42791390b01bSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 42801390b01bSJayamohan Kallickal TGT_DM_CMD); 42811390b01bSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, 42821390b01bSJayamohan Kallickal pwrb, 0); 4283685e16fdSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 1); 42841390b01bSJayamohan Kallickal } else { 4285bfead3b2SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 4286bfead3b2SJayamohan Kallickal INI_RD_CMD); 4287685e16fdSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); 42881390b01bSJayamohan Kallickal } 42896733b39aSJayamohan Kallickal hwi_write_buffer(pwrb, task); 42906733b39aSJayamohan Kallickal break; 42916733b39aSJayamohan Kallickal case ISCSI_OP_TEXT: 4292bfead3b2SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 4293b30c6dabSJayamohan Kallickal TGT_DM_CMD); 42940ecb0b45SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); 42956733b39aSJayamohan Kallickal hwi_write_buffer(pwrb, task); 42966733b39aSJayamohan Kallickal break; 42976733b39aSJayamohan Kallickal case ISCSI_OP_SCSI_TMFUNC: 4298bfead3b2SJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 4299bfead3b2SJayamohan Kallickal INI_TMF_CMD); 43006733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); 43016733b39aSJayamohan Kallickal hwi_write_buffer(pwrb, task); 43026733b39aSJayamohan Kallickal break; 43036733b39aSJayamohan Kallickal case ISCSI_OP_LOGOUT: 43046733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); 43056733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 43066733b39aSJayamohan Kallickal HWH_TYPE_LOGOUT); 43076733b39aSJayamohan Kallickal hwi_write_buffer(pwrb, task); 43086733b39aSJayamohan Kallickal break; 43096733b39aSJayamohan Kallickal 43106733b39aSJayamohan Kallickal default: 431199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 431299bc5d55SJohn Soni Jose "BM_%d : opcode =%d Not supported\n", 43136733b39aSJayamohan Kallickal task->hdr->opcode & ISCSI_OPCODE_MASK); 431499bc5d55SJohn Soni Jose 43156733b39aSJayamohan Kallickal return -EINVAL; 43166733b39aSJayamohan Kallickal } 43176733b39aSJayamohan Kallickal 43186733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, 431951a46250SJayamohan Kallickal task->data_count); 43206733b39aSJayamohan Kallickal AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb, 43216733b39aSJayamohan Kallickal io_task->pwrb_handle->nxt_wrb_index); 43226733b39aSJayamohan Kallickal be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb)); 43236733b39aSJayamohan Kallickal 4324bfead3b2SJayamohan Kallickal doorbell |= cid & DB_WRB_POST_CID_MASK; 43256733b39aSJayamohan Kallickal doorbell |= (io_task->pwrb_handle->wrb_index & 43266733b39aSJayamohan Kallickal DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT; 43276733b39aSJayamohan Kallickal doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 43286733b39aSJayamohan Kallickal iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET); 43296733b39aSJayamohan Kallickal return 0; 43306733b39aSJayamohan Kallickal } 43316733b39aSJayamohan Kallickal 43326733b39aSJayamohan Kallickal static int beiscsi_task_xmit(struct iscsi_task *task) 43336733b39aSJayamohan Kallickal { 43346733b39aSJayamohan Kallickal struct beiscsi_io_task *io_task = task->dd_data; 43356733b39aSJayamohan Kallickal struct scsi_cmnd *sc = task->sc; 43366733b39aSJayamohan Kallickal struct scatterlist *sg; 43376733b39aSJayamohan Kallickal int num_sg; 43386733b39aSJayamohan Kallickal unsigned int writedir = 0, xferlen = 0; 43396733b39aSJayamohan Kallickal 43406733b39aSJayamohan Kallickal if (!sc) 43416733b39aSJayamohan Kallickal return beiscsi_mtask(task); 43426733b39aSJayamohan Kallickal 43436733b39aSJayamohan Kallickal io_task->scsi_cmnd = sc; 43446733b39aSJayamohan Kallickal num_sg = scsi_dma_map(sc); 43456733b39aSJayamohan Kallickal if (num_sg < 0) { 434699bc5d55SJohn Soni Jose struct iscsi_conn *conn = task->conn; 434799bc5d55SJohn Soni Jose struct beiscsi_hba *phba = NULL; 434899bc5d55SJohn Soni Jose 434999bc5d55SJohn Soni Jose phba = ((struct beiscsi_conn *)conn->dd_data)->phba; 435099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_IO, 435199bc5d55SJohn Soni Jose "BM_%d : scsi_dma_map Failed\n"); 435299bc5d55SJohn Soni Jose 43536733b39aSJayamohan Kallickal return num_sg; 43546733b39aSJayamohan Kallickal } 43556733b39aSJayamohan Kallickal xferlen = scsi_bufflen(sc); 43566733b39aSJayamohan Kallickal sg = scsi_sglist(sc); 435799bc5d55SJohn Soni Jose if (sc->sc_data_direction == DMA_TO_DEVICE) 43586733b39aSJayamohan Kallickal writedir = 1; 435999bc5d55SJohn Soni Jose else 43606733b39aSJayamohan Kallickal writedir = 0; 436199bc5d55SJohn Soni Jose 43626733b39aSJayamohan Kallickal return beiscsi_iotask(task, sg, num_sg, xferlen, writedir); 43636733b39aSJayamohan Kallickal } 43646733b39aSJayamohan Kallickal 4365ffce3e2eSJayamohan Kallickal /** 4366ffce3e2eSJayamohan Kallickal * beiscsi_bsg_request - handle bsg request from ISCSI transport 4367ffce3e2eSJayamohan Kallickal * @job: job to handle 4368ffce3e2eSJayamohan Kallickal */ 4369ffce3e2eSJayamohan Kallickal static int beiscsi_bsg_request(struct bsg_job *job) 4370ffce3e2eSJayamohan Kallickal { 4371ffce3e2eSJayamohan Kallickal struct Scsi_Host *shost; 4372ffce3e2eSJayamohan Kallickal struct beiscsi_hba *phba; 4373ffce3e2eSJayamohan Kallickal struct iscsi_bsg_request *bsg_req = job->request; 4374ffce3e2eSJayamohan Kallickal int rc = -EINVAL; 4375ffce3e2eSJayamohan Kallickal unsigned int tag; 4376ffce3e2eSJayamohan Kallickal struct be_dma_mem nonemb_cmd; 4377ffce3e2eSJayamohan Kallickal struct be_cmd_resp_hdr *resp; 4378ffce3e2eSJayamohan Kallickal struct iscsi_bsg_reply *bsg_reply = job->reply; 4379ffce3e2eSJayamohan Kallickal unsigned short status, extd_status; 4380ffce3e2eSJayamohan Kallickal 4381ffce3e2eSJayamohan Kallickal shost = iscsi_job_to_shost(job); 4382ffce3e2eSJayamohan Kallickal phba = iscsi_host_priv(shost); 4383ffce3e2eSJayamohan Kallickal 4384ffce3e2eSJayamohan Kallickal switch (bsg_req->msgcode) { 4385ffce3e2eSJayamohan Kallickal case ISCSI_BSG_HST_VENDOR: 4386ffce3e2eSJayamohan Kallickal nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev, 4387ffce3e2eSJayamohan Kallickal job->request_payload.payload_len, 4388ffce3e2eSJayamohan Kallickal &nonemb_cmd.dma); 4389ffce3e2eSJayamohan Kallickal if (nonemb_cmd.va == NULL) { 439099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 439199bc5d55SJohn Soni Jose "BM_%d : Failed to allocate memory for " 4392ffce3e2eSJayamohan Kallickal "beiscsi_bsg_request\n"); 4393ffce3e2eSJayamohan Kallickal return -EIO; 4394ffce3e2eSJayamohan Kallickal } 4395ffce3e2eSJayamohan Kallickal tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job, 4396ffce3e2eSJayamohan Kallickal &nonemb_cmd); 4397ffce3e2eSJayamohan Kallickal if (!tag) { 439899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 439999bc5d55SJohn Soni Jose "BM_%d : be_cmd_get_mac_addr Failed\n"); 440099bc5d55SJohn Soni Jose 4401ffce3e2eSJayamohan Kallickal pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 4402ffce3e2eSJayamohan Kallickal nonemb_cmd.va, nonemb_cmd.dma); 4403ffce3e2eSJayamohan Kallickal return -EAGAIN; 4404ffce3e2eSJayamohan Kallickal } else 4405ffce3e2eSJayamohan Kallickal wait_event_interruptible(phba->ctrl.mcc_wait[tag], 4406ffce3e2eSJayamohan Kallickal phba->ctrl.mcc_numtag[tag]); 4407ffce3e2eSJayamohan Kallickal extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8; 4408ffce3e2eSJayamohan Kallickal status = phba->ctrl.mcc_numtag[tag] & 0x000000FF; 4409ffce3e2eSJayamohan Kallickal free_mcc_tag(&phba->ctrl, tag); 4410ffce3e2eSJayamohan Kallickal resp = (struct be_cmd_resp_hdr *)nonemb_cmd.va; 4411ffce3e2eSJayamohan Kallickal sg_copy_from_buffer(job->reply_payload.sg_list, 4412ffce3e2eSJayamohan Kallickal job->reply_payload.sg_cnt, 4413ffce3e2eSJayamohan Kallickal nonemb_cmd.va, (resp->response_length 4414ffce3e2eSJayamohan Kallickal + sizeof(*resp))); 4415ffce3e2eSJayamohan Kallickal bsg_reply->reply_payload_rcv_len = resp->response_length; 4416ffce3e2eSJayamohan Kallickal bsg_reply->result = status; 4417ffce3e2eSJayamohan Kallickal bsg_job_done(job, bsg_reply->result, 4418ffce3e2eSJayamohan Kallickal bsg_reply->reply_payload_rcv_len); 4419ffce3e2eSJayamohan Kallickal pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 4420ffce3e2eSJayamohan Kallickal nonemb_cmd.va, nonemb_cmd.dma); 4421ffce3e2eSJayamohan Kallickal if (status || extd_status) { 442299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 442399bc5d55SJohn Soni Jose "BM_%d : be_cmd_get_mac_addr Failed" 4424ffce3e2eSJayamohan Kallickal " status = %d extd_status = %d\n", 4425ffce3e2eSJayamohan Kallickal status, extd_status); 442699bc5d55SJohn Soni Jose 4427ffce3e2eSJayamohan Kallickal return -EIO; 4428ffce3e2eSJayamohan Kallickal } 4429ffce3e2eSJayamohan Kallickal break; 4430ffce3e2eSJayamohan Kallickal 4431ffce3e2eSJayamohan Kallickal default: 443299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 443399bc5d55SJohn Soni Jose "BM_%d : Unsupported bsg command: 0x%x\n", 4434ffce3e2eSJayamohan Kallickal bsg_req->msgcode); 4435ffce3e2eSJayamohan Kallickal break; 4436ffce3e2eSJayamohan Kallickal } 4437ffce3e2eSJayamohan Kallickal 4438ffce3e2eSJayamohan Kallickal return rc; 4439ffce3e2eSJayamohan Kallickal } 4440ffce3e2eSJayamohan Kallickal 444199bc5d55SJohn Soni Jose void beiscsi_hba_attrs_init(struct beiscsi_hba *phba) 444299bc5d55SJohn Soni Jose { 444399bc5d55SJohn Soni Jose /* Set the logging parameter */ 444499bc5d55SJohn Soni Jose beiscsi_log_enable_init(phba, beiscsi_log_enable); 444599bc5d55SJohn Soni Jose } 444699bc5d55SJohn Soni Jose 444725602c97SJayamohan Kallickal static void beiscsi_quiesce(struct beiscsi_hba *phba) 44486733b39aSJayamohan Kallickal { 4449bfead3b2SJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 4450bfead3b2SJayamohan Kallickal struct hwi_context_memory *phwi_context; 4451bfead3b2SJayamohan Kallickal struct be_eq_obj *pbe_eq; 4452bfead3b2SJayamohan Kallickal unsigned int i, msix_vec; 4453e9b91193SJayamohan Kallickal u8 *real_offset = 0; 4454e9b91193SJayamohan Kallickal u32 value = 0; 44556733b39aSJayamohan Kallickal 4456bfead3b2SJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 4457bfead3b2SJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 44586733b39aSJayamohan Kallickal hwi_disable_intr(phba); 4459bfead3b2SJayamohan Kallickal if (phba->msix_enabled) { 4460bfead3b2SJayamohan Kallickal for (i = 0; i <= phba->num_cpus; i++) { 4461bfead3b2SJayamohan Kallickal msix_vec = phba->msix_entries[i].vector; 4462bfead3b2SJayamohan Kallickal free_irq(msix_vec, &phwi_context->be_eq[i]); 44638fcfb210SJayamohan Kallickal kfree(phba->msi_name[i]); 4464bfead3b2SJayamohan Kallickal } 4465bfead3b2SJayamohan Kallickal } else 44666733b39aSJayamohan Kallickal if (phba->pcidev->irq) 44676733b39aSJayamohan Kallickal free_irq(phba->pcidev->irq, phba); 4468bfead3b2SJayamohan Kallickal pci_disable_msix(phba->pcidev); 44696733b39aSJayamohan Kallickal destroy_workqueue(phba->wq); 44706733b39aSJayamohan Kallickal if (blk_iopoll_enabled) 4471bfead3b2SJayamohan Kallickal for (i = 0; i < phba->num_cpus; i++) { 4472bfead3b2SJayamohan Kallickal pbe_eq = &phwi_context->be_eq[i]; 4473bfead3b2SJayamohan Kallickal blk_iopoll_disable(&pbe_eq->iopoll); 4474bfead3b2SJayamohan Kallickal } 44756733b39aSJayamohan Kallickal 44766733b39aSJayamohan Kallickal beiscsi_clean_port(phba); 44776733b39aSJayamohan Kallickal beiscsi_free_mem(phba); 4478e9b91193SJayamohan Kallickal real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE; 4479e9b91193SJayamohan Kallickal 4480e9b91193SJayamohan Kallickal value = readl((void *)real_offset); 4481e9b91193SJayamohan Kallickal 4482e9b91193SJayamohan Kallickal if (value & 0x00010000) { 4483e9b91193SJayamohan Kallickal value &= 0xfffeffff; 4484e9b91193SJayamohan Kallickal writel(value, (void *)real_offset); 4485e9b91193SJayamohan Kallickal } 44866733b39aSJayamohan Kallickal beiscsi_unmap_pci_function(phba); 44876733b39aSJayamohan Kallickal pci_free_consistent(phba->pcidev, 44886733b39aSJayamohan Kallickal phba->ctrl.mbox_mem_alloced.size, 44896733b39aSJayamohan Kallickal phba->ctrl.mbox_mem_alloced.va, 44906733b39aSJayamohan Kallickal phba->ctrl.mbox_mem_alloced.dma); 449125602c97SJayamohan Kallickal } 449225602c97SJayamohan Kallickal 449325602c97SJayamohan Kallickal static void beiscsi_remove(struct pci_dev *pcidev) 449425602c97SJayamohan Kallickal { 449525602c97SJayamohan Kallickal 449625602c97SJayamohan Kallickal struct beiscsi_hba *phba = NULL; 449725602c97SJayamohan Kallickal 449825602c97SJayamohan Kallickal phba = pci_get_drvdata(pcidev); 449925602c97SJayamohan Kallickal if (!phba) { 450025602c97SJayamohan Kallickal dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n"); 450125602c97SJayamohan Kallickal return; 450225602c97SJayamohan Kallickal } 450325602c97SJayamohan Kallickal 45040e43895eSMike Christie beiscsi_destroy_def_ifaces(phba); 450525602c97SJayamohan Kallickal beiscsi_quiesce(phba); 45060b1d3cbfSJayamohan Kallickal iscsi_boot_destroy_kset(phba->boot_kset); 45076733b39aSJayamohan Kallickal iscsi_host_remove(phba->shost); 45086733b39aSJayamohan Kallickal pci_dev_put(phba->pcidev); 45096733b39aSJayamohan Kallickal iscsi_host_free(phba->shost); 45108dce69ffSJayamohan Kallickal pci_disable_device(pcidev); 45116733b39aSJayamohan Kallickal } 45126733b39aSJayamohan Kallickal 451325602c97SJayamohan Kallickal static void beiscsi_shutdown(struct pci_dev *pcidev) 451425602c97SJayamohan Kallickal { 451525602c97SJayamohan Kallickal 451625602c97SJayamohan Kallickal struct beiscsi_hba *phba = NULL; 451725602c97SJayamohan Kallickal 451825602c97SJayamohan Kallickal phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev); 451925602c97SJayamohan Kallickal if (!phba) { 452025602c97SJayamohan Kallickal dev_err(&pcidev->dev, "beiscsi_shutdown called with no phba\n"); 452125602c97SJayamohan Kallickal return; 452225602c97SJayamohan Kallickal } 452325602c97SJayamohan Kallickal 452425602c97SJayamohan Kallickal beiscsi_quiesce(phba); 45258dce69ffSJayamohan Kallickal pci_disable_device(pcidev); 452625602c97SJayamohan Kallickal } 452725602c97SJayamohan Kallickal 4528bfead3b2SJayamohan Kallickal static void beiscsi_msix_enable(struct beiscsi_hba *phba) 4529bfead3b2SJayamohan Kallickal { 4530bfead3b2SJayamohan Kallickal int i, status; 4531bfead3b2SJayamohan Kallickal 4532bfead3b2SJayamohan Kallickal for (i = 0; i <= phba->num_cpus; i++) 4533bfead3b2SJayamohan Kallickal phba->msix_entries[i].entry = i; 4534bfead3b2SJayamohan Kallickal 4535bfead3b2SJayamohan Kallickal status = pci_enable_msix(phba->pcidev, phba->msix_entries, 4536bfead3b2SJayamohan Kallickal (phba->num_cpus + 1)); 4537bfead3b2SJayamohan Kallickal if (!status) 4538bfead3b2SJayamohan Kallickal phba->msix_enabled = true; 4539bfead3b2SJayamohan Kallickal 4540bfead3b2SJayamohan Kallickal return; 4541bfead3b2SJayamohan Kallickal } 4542bfead3b2SJayamohan Kallickal 45436733b39aSJayamohan Kallickal static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev, 45446733b39aSJayamohan Kallickal const struct pci_device_id *id) 45456733b39aSJayamohan Kallickal { 45466733b39aSJayamohan Kallickal struct beiscsi_hba *phba = NULL; 4547bfead3b2SJayamohan Kallickal struct hwi_controller *phwi_ctrlr; 4548bfead3b2SJayamohan Kallickal struct hwi_context_memory *phwi_context; 4549bfead3b2SJayamohan Kallickal struct be_eq_obj *pbe_eq; 4550107dfcbaSJohn Soni Jose int ret, i; 4551e9b91193SJayamohan Kallickal u8 *real_offset = 0; 4552e9b91193SJayamohan Kallickal u32 value = 0; 45536733b39aSJayamohan Kallickal 45546733b39aSJayamohan Kallickal ret = beiscsi_enable_pci(pcidev); 45556733b39aSJayamohan Kallickal if (ret < 0) { 455699bc5d55SJohn Soni Jose dev_err(&pcidev->dev, 455799bc5d55SJohn Soni Jose "beiscsi_dev_probe - Failed to enable pci device\n"); 45586733b39aSJayamohan Kallickal return ret; 45596733b39aSJayamohan Kallickal } 45606733b39aSJayamohan Kallickal 45616733b39aSJayamohan Kallickal phba = beiscsi_hba_alloc(pcidev); 45626733b39aSJayamohan Kallickal if (!phba) { 456399bc5d55SJohn Soni Jose dev_err(&pcidev->dev, 456499bc5d55SJohn Soni Jose "beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n"); 45656733b39aSJayamohan Kallickal goto disable_pci; 45666733b39aSJayamohan Kallickal } 45676733b39aSJayamohan Kallickal 456899bc5d55SJohn Soni Jose /* Initialize Driver configuration Paramters */ 456999bc5d55SJohn Soni Jose beiscsi_hba_attrs_init(phba); 457099bc5d55SJohn Soni Jose 4571f98c96b0SJayamohan Kallickal switch (pcidev->device) { 4572f98c96b0SJayamohan Kallickal case BE_DEVICE_ID1: 4573f98c96b0SJayamohan Kallickal case OC_DEVICE_ID1: 4574f98c96b0SJayamohan Kallickal case OC_DEVICE_ID2: 4575f98c96b0SJayamohan Kallickal phba->generation = BE_GEN2; 4576f98c96b0SJayamohan Kallickal break; 4577f98c96b0SJayamohan Kallickal case BE_DEVICE_ID2: 4578f98c96b0SJayamohan Kallickal case OC_DEVICE_ID3: 4579f98c96b0SJayamohan Kallickal phba->generation = BE_GEN3; 4580f98c96b0SJayamohan Kallickal break; 4581f98c96b0SJayamohan Kallickal default: 4582f98c96b0SJayamohan Kallickal phba->generation = 0; 4583f98c96b0SJayamohan Kallickal } 4584f98c96b0SJayamohan Kallickal 4585bfead3b2SJayamohan Kallickal if (enable_msix) 4586107dfcbaSJohn Soni Jose find_num_cpus(phba); 4587bfead3b2SJayamohan Kallickal else 4588107dfcbaSJohn Soni Jose phba->num_cpus = 1; 4589107dfcbaSJohn Soni Jose 459099bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 459199bc5d55SJohn Soni Jose "BM_%d : num_cpus = %d\n", 459299bc5d55SJohn Soni Jose phba->num_cpus); 4593bfead3b2SJayamohan Kallickal 4594b547f2d6SJayamohan Kallickal if (enable_msix) { 4595bfead3b2SJayamohan Kallickal beiscsi_msix_enable(phba); 4596b547f2d6SJayamohan Kallickal if (!phba->msix_enabled) 4597b547f2d6SJayamohan Kallickal phba->num_cpus = 1; 4598b547f2d6SJayamohan Kallickal } 45996733b39aSJayamohan Kallickal ret = be_ctrl_init(phba, pcidev); 46006733b39aSJayamohan Kallickal if (ret) { 460199bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 460299bc5d55SJohn Soni Jose "BM_%d : beiscsi_dev_probe-" 46036733b39aSJayamohan Kallickal "Failed in be_ctrl_init\n"); 46046733b39aSJayamohan Kallickal goto hba_free; 46056733b39aSJayamohan Kallickal } 46066733b39aSJayamohan Kallickal 4607e9b91193SJayamohan Kallickal if (!num_hba) { 4608e9b91193SJayamohan Kallickal real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE; 4609e9b91193SJayamohan Kallickal value = readl((void *)real_offset); 4610e9b91193SJayamohan Kallickal if (value & 0x00010000) { 4611e9b91193SJayamohan Kallickal gcrashmode++; 461299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 461399bc5d55SJohn Soni Jose "BM_%d : Loading Driver in crashdump mode\n"); 4614e5285860SJayamohan Kallickal ret = beiscsi_cmd_reset_function(phba); 4615e9b91193SJayamohan Kallickal if (ret) { 461699bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 461799bc5d55SJohn Soni Jose "BM_%d : Reset Failed. Aborting Crashdump\n"); 4618e9b91193SJayamohan Kallickal goto hba_free; 4619e9b91193SJayamohan Kallickal } 4620e9b91193SJayamohan Kallickal ret = be_chk_reset_complete(phba); 4621e9b91193SJayamohan Kallickal if (ret) { 462299bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 462399bc5d55SJohn Soni Jose "BM_%d : Failed to get out of reset." 4624e9b91193SJayamohan Kallickal "Aborting Crashdump\n"); 4625e9b91193SJayamohan Kallickal goto hba_free; 4626e9b91193SJayamohan Kallickal } 4627e9b91193SJayamohan Kallickal } else { 4628e9b91193SJayamohan Kallickal value |= 0x00010000; 4629e9b91193SJayamohan Kallickal writel(value, (void *)real_offset); 4630e9b91193SJayamohan Kallickal num_hba++; 4631e9b91193SJayamohan Kallickal } 4632e9b91193SJayamohan Kallickal } 4633e9b91193SJayamohan Kallickal 46346733b39aSJayamohan Kallickal spin_lock_init(&phba->io_sgl_lock); 46356733b39aSJayamohan Kallickal spin_lock_init(&phba->mgmt_sgl_lock); 46366733b39aSJayamohan Kallickal spin_lock_init(&phba->isr_lock); 46377da50879SJayamohan Kallickal ret = mgmt_get_fw_config(&phba->ctrl, phba); 46387da50879SJayamohan Kallickal if (ret != 0) { 463999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 464099bc5d55SJohn Soni Jose "BM_%d : Error getting fw config\n"); 46417da50879SJayamohan Kallickal goto free_port; 46427da50879SJayamohan Kallickal } 46437da50879SJayamohan Kallickal phba->shost->max_id = phba->fw_config.iscsi_cid_count; 46446733b39aSJayamohan Kallickal beiscsi_get_params(phba); 4645aa874f07SJayamohan Kallickal phba->shost->can_queue = phba->params.ios_per_ctrl; 46466733b39aSJayamohan Kallickal ret = beiscsi_init_port(phba); 46476733b39aSJayamohan Kallickal if (ret < 0) { 464899bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 464999bc5d55SJohn Soni Jose "BM_%d : beiscsi_dev_probe-" 46506733b39aSJayamohan Kallickal "Failed in beiscsi_init_port\n"); 46516733b39aSJayamohan Kallickal goto free_port; 46526733b39aSJayamohan Kallickal } 46536733b39aSJayamohan Kallickal 4654756d29c8SJayamohan Kallickal for (i = 0; i < MAX_MCC_CMD ; i++) { 4655756d29c8SJayamohan Kallickal init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]); 4656756d29c8SJayamohan Kallickal phba->ctrl.mcc_tag[i] = i + 1; 4657756d29c8SJayamohan Kallickal phba->ctrl.mcc_numtag[i + 1] = 0; 4658756d29c8SJayamohan Kallickal phba->ctrl.mcc_tag_available++; 4659756d29c8SJayamohan Kallickal } 4660756d29c8SJayamohan Kallickal 4661756d29c8SJayamohan Kallickal phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0; 4662756d29c8SJayamohan Kallickal 46636733b39aSJayamohan Kallickal snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u", 46646733b39aSJayamohan Kallickal phba->shost->host_no); 4665278274d5STejun Heo phba->wq = alloc_workqueue(phba->wq_name, WQ_MEM_RECLAIM, 1); 46666733b39aSJayamohan Kallickal if (!phba->wq) { 466799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 466899bc5d55SJohn Soni Jose "BM_%d : beiscsi_dev_probe-" 46696733b39aSJayamohan Kallickal "Failed to allocate work queue\n"); 46706733b39aSJayamohan Kallickal goto free_twq; 46716733b39aSJayamohan Kallickal } 46726733b39aSJayamohan Kallickal 46736733b39aSJayamohan Kallickal INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs); 46746733b39aSJayamohan Kallickal 4675bfead3b2SJayamohan Kallickal phwi_ctrlr = phba->phwi_ctrlr; 4676bfead3b2SJayamohan Kallickal phwi_context = phwi_ctrlr->phwi_ctxt; 46776733b39aSJayamohan Kallickal if (blk_iopoll_enabled) { 4678bfead3b2SJayamohan Kallickal for (i = 0; i < phba->num_cpus; i++) { 4679bfead3b2SJayamohan Kallickal pbe_eq = &phwi_context->be_eq[i]; 4680bfead3b2SJayamohan Kallickal blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget, 4681bfead3b2SJayamohan Kallickal be_iopoll); 4682bfead3b2SJayamohan Kallickal blk_iopoll_enable(&pbe_eq->iopoll); 46836733b39aSJayamohan Kallickal } 4684bfead3b2SJayamohan Kallickal } 46856733b39aSJayamohan Kallickal ret = beiscsi_init_irqs(phba); 46866733b39aSJayamohan Kallickal if (ret < 0) { 468799bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 468899bc5d55SJohn Soni Jose "BM_%d : beiscsi_dev_probe-" 46896733b39aSJayamohan Kallickal "Failed to beiscsi_init_irqs\n"); 46906733b39aSJayamohan Kallickal goto free_blkenbld; 46916733b39aSJayamohan Kallickal } 4692238f6b72SJayamohan Kallickal hwi_enable_intr(phba); 4693f457a46fSMike Christie 4694f457a46fSMike Christie if (beiscsi_setup_boot_info(phba)) 4695f457a46fSMike Christie /* 4696f457a46fSMike Christie * log error but continue, because we may not be using 4697f457a46fSMike Christie * iscsi boot. 4698f457a46fSMike Christie */ 469999bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 470099bc5d55SJohn Soni Jose "BM_%d : Could not set up " 47010e43895eSMike Christie "iSCSI boot info.\n"); 4702f457a46fSMike Christie 47030e43895eSMike Christie beiscsi_create_def_ifaces(phba); 470499bc5d55SJohn Soni Jose beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 470599bc5d55SJohn Soni Jose "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n"); 47066733b39aSJayamohan Kallickal return 0; 47076733b39aSJayamohan Kallickal 47086733b39aSJayamohan Kallickal free_blkenbld: 47096733b39aSJayamohan Kallickal destroy_workqueue(phba->wq); 47106733b39aSJayamohan Kallickal if (blk_iopoll_enabled) 4711bfead3b2SJayamohan Kallickal for (i = 0; i < phba->num_cpus; i++) { 4712bfead3b2SJayamohan Kallickal pbe_eq = &phwi_context->be_eq[i]; 4713bfead3b2SJayamohan Kallickal blk_iopoll_disable(&pbe_eq->iopoll); 4714bfead3b2SJayamohan Kallickal } 47156733b39aSJayamohan Kallickal free_twq: 47166733b39aSJayamohan Kallickal beiscsi_clean_port(phba); 47176733b39aSJayamohan Kallickal beiscsi_free_mem(phba); 47186733b39aSJayamohan Kallickal free_port: 4719e9b91193SJayamohan Kallickal real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE; 4720e9b91193SJayamohan Kallickal 4721e9b91193SJayamohan Kallickal value = readl((void *)real_offset); 4722e9b91193SJayamohan Kallickal 4723e9b91193SJayamohan Kallickal if (value & 0x00010000) { 4724e9b91193SJayamohan Kallickal value &= 0xfffeffff; 4725e9b91193SJayamohan Kallickal writel(value, (void *)real_offset); 4726e9b91193SJayamohan Kallickal } 4727e9b91193SJayamohan Kallickal 47286733b39aSJayamohan Kallickal pci_free_consistent(phba->pcidev, 47296733b39aSJayamohan Kallickal phba->ctrl.mbox_mem_alloced.size, 47306733b39aSJayamohan Kallickal phba->ctrl.mbox_mem_alloced.va, 47316733b39aSJayamohan Kallickal phba->ctrl.mbox_mem_alloced.dma); 47326733b39aSJayamohan Kallickal beiscsi_unmap_pci_function(phba); 47336733b39aSJayamohan Kallickal hba_free: 4734238f6b72SJayamohan Kallickal if (phba->msix_enabled) 4735238f6b72SJayamohan Kallickal pci_disable_msix(phba->pcidev); 47366733b39aSJayamohan Kallickal iscsi_host_remove(phba->shost); 47376733b39aSJayamohan Kallickal pci_dev_put(phba->pcidev); 47386733b39aSJayamohan Kallickal iscsi_host_free(phba->shost); 47396733b39aSJayamohan Kallickal disable_pci: 47406733b39aSJayamohan Kallickal pci_disable_device(pcidev); 47416733b39aSJayamohan Kallickal return ret; 47426733b39aSJayamohan Kallickal } 47436733b39aSJayamohan Kallickal 47446733b39aSJayamohan Kallickal struct iscsi_transport beiscsi_iscsi_transport = { 47456733b39aSJayamohan Kallickal .owner = THIS_MODULE, 47466733b39aSJayamohan Kallickal .name = DRV_NAME, 47479db0fb3aSJayamohan Kallickal .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO | 47486733b39aSJayamohan Kallickal CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD, 47496733b39aSJayamohan Kallickal .create_session = beiscsi_session_create, 47506733b39aSJayamohan Kallickal .destroy_session = beiscsi_session_destroy, 47516733b39aSJayamohan Kallickal .create_conn = beiscsi_conn_create, 47526733b39aSJayamohan Kallickal .bind_conn = beiscsi_conn_bind, 47536733b39aSJayamohan Kallickal .destroy_conn = iscsi_conn_teardown, 47543128c6c7SMike Christie .attr_is_visible = be2iscsi_attr_is_visible, 47550e43895eSMike Christie .set_iface_param = be2iscsi_iface_set_param, 47560e43895eSMike Christie .get_iface_param = be2iscsi_iface_get_param, 47576733b39aSJayamohan Kallickal .set_param = beiscsi_set_param, 4758c7f7fd5bSMike Christie .get_conn_param = iscsi_conn_get_param, 47596733b39aSJayamohan Kallickal .get_session_param = iscsi_session_get_param, 47606733b39aSJayamohan Kallickal .get_host_param = beiscsi_get_host_param, 47616733b39aSJayamohan Kallickal .start_conn = beiscsi_conn_start, 4762fa95d206SMike Christie .stop_conn = iscsi_conn_stop, 47636733b39aSJayamohan Kallickal .send_pdu = iscsi_conn_send_pdu, 47646733b39aSJayamohan Kallickal .xmit_task = beiscsi_task_xmit, 47656733b39aSJayamohan Kallickal .cleanup_task = beiscsi_cleanup_task, 47666733b39aSJayamohan Kallickal .alloc_pdu = beiscsi_alloc_pdu, 47676733b39aSJayamohan Kallickal .parse_pdu_itt = beiscsi_parse_pdu, 47686733b39aSJayamohan Kallickal .get_stats = beiscsi_conn_get_stats, 4769c7f7fd5bSMike Christie .get_ep_param = beiscsi_ep_get_param, 47706733b39aSJayamohan Kallickal .ep_connect = beiscsi_ep_connect, 47716733b39aSJayamohan Kallickal .ep_poll = beiscsi_ep_poll, 47726733b39aSJayamohan Kallickal .ep_disconnect = beiscsi_ep_disconnect, 47736733b39aSJayamohan Kallickal .session_recovery_timedout = iscsi_session_recovery_timedout, 4774ffce3e2eSJayamohan Kallickal .bsg_request = beiscsi_bsg_request, 47756733b39aSJayamohan Kallickal }; 47766733b39aSJayamohan Kallickal 47776733b39aSJayamohan Kallickal static struct pci_driver beiscsi_pci_driver = { 47786733b39aSJayamohan Kallickal .name = DRV_NAME, 47796733b39aSJayamohan Kallickal .probe = beiscsi_dev_probe, 47806733b39aSJayamohan Kallickal .remove = beiscsi_remove, 478125602c97SJayamohan Kallickal .shutdown = beiscsi_shutdown, 47826733b39aSJayamohan Kallickal .id_table = beiscsi_pci_id_table 47836733b39aSJayamohan Kallickal }; 47846733b39aSJayamohan Kallickal 4785bfead3b2SJayamohan Kallickal 47866733b39aSJayamohan Kallickal static int __init beiscsi_module_init(void) 47876733b39aSJayamohan Kallickal { 47886733b39aSJayamohan Kallickal int ret; 47896733b39aSJayamohan Kallickal 47906733b39aSJayamohan Kallickal beiscsi_scsi_transport = 47916733b39aSJayamohan Kallickal iscsi_register_transport(&beiscsi_iscsi_transport); 47926733b39aSJayamohan Kallickal if (!beiscsi_scsi_transport) { 479399bc5d55SJohn Soni Jose printk(KERN_ERR 479499bc5d55SJohn Soni Jose "beiscsi_module_init - Unable to register beiscsi transport.\n"); 4795f55a24f2SJayamohan Kallickal return -ENOMEM; 47966733b39aSJayamohan Kallickal } 479799bc5d55SJohn Soni Jose printk(KERN_INFO "In beiscsi_module_init, tt=%p\n", 47986733b39aSJayamohan Kallickal &beiscsi_iscsi_transport); 47996733b39aSJayamohan Kallickal 48006733b39aSJayamohan Kallickal ret = pci_register_driver(&beiscsi_pci_driver); 48016733b39aSJayamohan Kallickal if (ret) { 480299bc5d55SJohn Soni Jose printk(KERN_ERR 480399bc5d55SJohn Soni Jose "beiscsi_module_init - Unable to register beiscsi pci driver.\n"); 48046733b39aSJayamohan Kallickal goto unregister_iscsi_transport; 48056733b39aSJayamohan Kallickal } 48066733b39aSJayamohan Kallickal return 0; 48076733b39aSJayamohan Kallickal 48086733b39aSJayamohan Kallickal unregister_iscsi_transport: 48096733b39aSJayamohan Kallickal iscsi_unregister_transport(&beiscsi_iscsi_transport); 48106733b39aSJayamohan Kallickal return ret; 48116733b39aSJayamohan Kallickal } 48126733b39aSJayamohan Kallickal 48136733b39aSJayamohan Kallickal static void __exit beiscsi_module_exit(void) 48146733b39aSJayamohan Kallickal { 48156733b39aSJayamohan Kallickal pci_unregister_driver(&beiscsi_pci_driver); 48166733b39aSJayamohan Kallickal iscsi_unregister_transport(&beiscsi_iscsi_transport); 48176733b39aSJayamohan Kallickal } 48186733b39aSJayamohan Kallickal 48196733b39aSJayamohan Kallickal module_init(beiscsi_module_init); 48206733b39aSJayamohan Kallickal module_exit(beiscsi_module_exit); 4821