1cfdda9d7SSteve Wise /* 2cfdda9d7SSteve Wise * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. 3cfdda9d7SSteve Wise * 4cfdda9d7SSteve Wise * This software is available to you under a choice of one of two 5cfdda9d7SSteve Wise * licenses. You may choose to be licensed under the terms of the GNU 6cfdda9d7SSteve Wise * General Public License (GPL) Version 2, available from the file 7cfdda9d7SSteve Wise * COPYING in the main directory of this source tree, or the 8cfdda9d7SSteve Wise * OpenIB.org BSD license below: 9cfdda9d7SSteve Wise * 10cfdda9d7SSteve Wise * Redistribution and use in source and binary forms, with or 11cfdda9d7SSteve Wise * without modification, are permitted provided that the following 12cfdda9d7SSteve Wise * conditions are met: 13cfdda9d7SSteve Wise * 14cfdda9d7SSteve Wise * - Redistributions of source code must retain the above 15cfdda9d7SSteve Wise * copyright notice, this list of conditions and the following 16cfdda9d7SSteve Wise * disclaimer. 17cfdda9d7SSteve Wise * 18cfdda9d7SSteve Wise * - Redistributions in binary form must reproduce the above 19cfdda9d7SSteve Wise * copyright notice, this list of conditions and the following 20cfdda9d7SSteve Wise * disclaimer in the documentation and/or other materials 21cfdda9d7SSteve Wise * provided with the distribution. 22cfdda9d7SSteve Wise * 23cfdda9d7SSteve Wise * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24cfdda9d7SSteve Wise * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25cfdda9d7SSteve Wise * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26cfdda9d7SSteve Wise * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27cfdda9d7SSteve Wise * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28cfdda9d7SSteve Wise * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29cfdda9d7SSteve Wise * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30cfdda9d7SSteve Wise * SOFTWARE. 31cfdda9d7SSteve Wise */ 32cfdda9d7SSteve Wise #include <linux/module.h> 33cfdda9d7SSteve Wise #include <linux/moduleparam.h> 34cfdda9d7SSteve Wise #include <linux/debugfs.h> 35e572568fSVipul Pandya #include <linux/vmalloc.h> 36da388973SHariprasad Shenai #include <linux/math64.h> 37cfdda9d7SSteve Wise 38cfdda9d7SSteve Wise #include <rdma/ib_verbs.h> 39cfdda9d7SSteve Wise 40cfdda9d7SSteve Wise #include "iw_cxgb4.h" 41cfdda9d7SSteve Wise 42cfdda9d7SSteve Wise #define DRV_VERSION "0.1" 43cfdda9d7SSteve Wise 44cfdda9d7SSteve Wise MODULE_AUTHOR("Steve Wise"); 45f079af7aSVipul Pandya MODULE_DESCRIPTION("Chelsio T4/T5 RDMA Driver"); 46cfdda9d7SSteve Wise MODULE_LICENSE("Dual BSD/GPL"); 47cfdda9d7SSteve Wise MODULE_VERSION(DRV_VERSION); 48cfdda9d7SSteve Wise 4980ccdd60SVipul Pandya static int allow_db_fc_on_t5; 5080ccdd60SVipul Pandya module_param(allow_db_fc_on_t5, int, 0644); 5180ccdd60SVipul Pandya MODULE_PARM_DESC(allow_db_fc_on_t5, 5280ccdd60SVipul Pandya "Allow DB Flow Control on T5 (default = 0)"); 5380ccdd60SVipul Pandya 5480ccdd60SVipul Pandya static int allow_db_coalescing_on_t5; 5580ccdd60SVipul Pandya module_param(allow_db_coalescing_on_t5, int, 0644); 5680ccdd60SVipul Pandya MODULE_PARM_DESC(allow_db_coalescing_on_t5, 5780ccdd60SVipul Pandya "Allow DB Coalescing on T5 (default = 0)"); 5880ccdd60SVipul Pandya 597730b4c7SHariprasad Shenai int c4iw_wr_log = 0; 607730b4c7SHariprasad Shenai module_param(c4iw_wr_log, int, 0444); 617730b4c7SHariprasad Shenai MODULE_PARM_DESC(c4iw_wr_log, "Enables logging of work request timing data."); 627730b4c7SHariprasad Shenai 6365d4c01aSSteve Wise static int c4iw_wr_log_size_order = 12; 647730b4c7SHariprasad Shenai module_param(c4iw_wr_log_size_order, int, 0444); 657730b4c7SHariprasad Shenai MODULE_PARM_DESC(c4iw_wr_log_size_order, 667730b4c7SHariprasad Shenai "Number of entries (log2) in the work request timing log."); 677730b4c7SHariprasad Shenai 682c974781SVipul Pandya struct uld_ctx { 692c974781SVipul Pandya struct list_head entry; 702c974781SVipul Pandya struct cxgb4_lld_info lldi; 712c974781SVipul Pandya struct c4iw_dev *dev; 722c974781SVipul Pandya }; 732c974781SVipul Pandya 742f25e9a5SSteve Wise static LIST_HEAD(uld_ctx_list); 75cfdda9d7SSteve Wise static DEFINE_MUTEX(dev_mutex); 76cfdda9d7SSteve Wise 7705eb2389SSteve Wise #define DB_FC_RESUME_SIZE 64 7805eb2389SSteve Wise #define DB_FC_RESUME_DELAY 1 7905eb2389SSteve Wise #define DB_FC_DRAIN_THRESH 0 8005eb2389SSteve Wise 81cfdda9d7SSteve Wise static struct dentry *c4iw_debugfs_root; 82cfdda9d7SSteve Wise 839e8d1fa3SSteve Wise struct c4iw_debugfs_data { 84cfdda9d7SSteve Wise struct c4iw_dev *devp; 85cfdda9d7SSteve Wise char *buf; 86cfdda9d7SSteve Wise int bufsize; 87cfdda9d7SSteve Wise int pos; 88cfdda9d7SSteve Wise }; 89cfdda9d7SSteve Wise 909eccfe10SSteve Wise /* registered cxgb4 netlink callbacks */ 919eccfe10SSteve Wise static struct ibnl_client_cbs c4iw_nl_cb_table[] = { 929eccfe10SSteve Wise [RDMA_NL_IWPM_REG_PID] = {.dump = iwpm_register_pid_cb}, 939eccfe10SSteve Wise [RDMA_NL_IWPM_ADD_MAPPING] = {.dump = iwpm_add_mapping_cb}, 949eccfe10SSteve Wise [RDMA_NL_IWPM_QUERY_MAPPING] = {.dump = iwpm_add_and_query_mapping_cb}, 959eccfe10SSteve Wise [RDMA_NL_IWPM_HANDLE_ERR] = {.dump = iwpm_mapping_error_cb}, 969eccfe10SSteve Wise [RDMA_NL_IWPM_MAPINFO] = {.dump = iwpm_mapping_info_cb}, 979eccfe10SSteve Wise [RDMA_NL_IWPM_MAPINFO_NUM] = {.dump = iwpm_ack_mapping_info_cb} 989eccfe10SSteve Wise }; 999eccfe10SSteve Wise 1009e8d1fa3SSteve Wise static int count_idrs(int id, void *p, void *data) 101cfdda9d7SSteve Wise { 102cfdda9d7SSteve Wise int *countp = data; 103cfdda9d7SSteve Wise 104cfdda9d7SSteve Wise *countp = *countp + 1; 105cfdda9d7SSteve Wise return 0; 106cfdda9d7SSteve Wise } 107cfdda9d7SSteve Wise 1089e8d1fa3SSteve Wise static ssize_t debugfs_read(struct file *file, char __user *buf, size_t count, 1099e8d1fa3SSteve Wise loff_t *ppos) 1109e8d1fa3SSteve Wise { 1119e8d1fa3SSteve Wise struct c4iw_debugfs_data *d = file->private_data; 1129e8d1fa3SSteve Wise 1133160977aSSteve Wise return simple_read_from_buffer(buf, count, ppos, d->buf, d->pos); 1149e8d1fa3SSteve Wise } 1159e8d1fa3SSteve Wise 1167730b4c7SHariprasad Shenai void c4iw_log_wr_stats(struct t4_wq *wq, struct t4_cqe *cqe) 1177730b4c7SHariprasad Shenai { 1187730b4c7SHariprasad Shenai struct wr_log_entry le; 1197730b4c7SHariprasad Shenai int idx; 1207730b4c7SHariprasad Shenai 1217730b4c7SHariprasad Shenai if (!wq->rdev->wr_log) 1227730b4c7SHariprasad Shenai return; 1237730b4c7SHariprasad Shenai 1247730b4c7SHariprasad Shenai idx = (atomic_inc_return(&wq->rdev->wr_log_idx) - 1) & 1257730b4c7SHariprasad Shenai (wq->rdev->wr_log_size - 1); 1267730b4c7SHariprasad Shenai le.poll_sge_ts = cxgb4_read_sge_timestamp(wq->rdev->lldi.ports[0]); 1277730b4c7SHariprasad Shenai getnstimeofday(&le.poll_host_ts); 1287730b4c7SHariprasad Shenai le.valid = 1; 1297730b4c7SHariprasad Shenai le.cqe_sge_ts = CQE_TS(cqe); 1307730b4c7SHariprasad Shenai if (SQ_TYPE(cqe)) { 1317730b4c7SHariprasad Shenai le.qid = wq->sq.qid; 1327730b4c7SHariprasad Shenai le.opcode = CQE_OPCODE(cqe); 1337730b4c7SHariprasad Shenai le.post_host_ts = wq->sq.sw_sq[wq->sq.cidx].host_ts; 1347730b4c7SHariprasad Shenai le.post_sge_ts = wq->sq.sw_sq[wq->sq.cidx].sge_ts; 1357730b4c7SHariprasad Shenai le.wr_id = CQE_WRID_SQ_IDX(cqe); 1367730b4c7SHariprasad Shenai } else { 1377730b4c7SHariprasad Shenai le.qid = wq->rq.qid; 1387730b4c7SHariprasad Shenai le.opcode = FW_RI_RECEIVE; 1397730b4c7SHariprasad Shenai le.post_host_ts = wq->rq.sw_rq[wq->rq.cidx].host_ts; 1407730b4c7SHariprasad Shenai le.post_sge_ts = wq->rq.sw_rq[wq->rq.cidx].sge_ts; 1417730b4c7SHariprasad Shenai le.wr_id = CQE_WRID_MSN(cqe); 1427730b4c7SHariprasad Shenai } 1437730b4c7SHariprasad Shenai wq->rdev->wr_log[idx] = le; 1447730b4c7SHariprasad Shenai } 1457730b4c7SHariprasad Shenai 1467730b4c7SHariprasad Shenai static int wr_log_show(struct seq_file *seq, void *v) 1477730b4c7SHariprasad Shenai { 1487730b4c7SHariprasad Shenai struct c4iw_dev *dev = seq->private; 1497730b4c7SHariprasad Shenai struct timespec prev_ts = {0, 0}; 1507730b4c7SHariprasad Shenai struct wr_log_entry *lep; 1517730b4c7SHariprasad Shenai int prev_ts_set = 0; 1527730b4c7SHariprasad Shenai int idx, end; 1537730b4c7SHariprasad Shenai 1546198dd8dSHariprasad S #define ts2ns(ts) div64_u64((ts) * dev->rdev.lldi.cclk_ps, 1000) 1557730b4c7SHariprasad Shenai 1567730b4c7SHariprasad Shenai idx = atomic_read(&dev->rdev.wr_log_idx) & 1577730b4c7SHariprasad Shenai (dev->rdev.wr_log_size - 1); 1587730b4c7SHariprasad Shenai end = idx - 1; 1597730b4c7SHariprasad Shenai if (end < 0) 1607730b4c7SHariprasad Shenai end = dev->rdev.wr_log_size - 1; 1617730b4c7SHariprasad Shenai lep = &dev->rdev.wr_log[idx]; 1627730b4c7SHariprasad Shenai while (idx != end) { 1637730b4c7SHariprasad Shenai if (lep->valid) { 1647730b4c7SHariprasad Shenai if (!prev_ts_set) { 1657730b4c7SHariprasad Shenai prev_ts_set = 1; 1667730b4c7SHariprasad Shenai prev_ts = lep->poll_host_ts; 1677730b4c7SHariprasad Shenai } 1687730b4c7SHariprasad Shenai seq_printf(seq, "%04u: sec %lu nsec %lu qid %u opcode " 1697730b4c7SHariprasad Shenai "%u %s 0x%x host_wr_delta sec %lu nsec %lu " 1707730b4c7SHariprasad Shenai "post_sge_ts 0x%llx cqe_sge_ts 0x%llx " 1717730b4c7SHariprasad Shenai "poll_sge_ts 0x%llx post_poll_delta_ns %llu " 1727730b4c7SHariprasad Shenai "cqe_poll_delta_ns %llu\n", 1737730b4c7SHariprasad Shenai idx, 1747730b4c7SHariprasad Shenai timespec_sub(lep->poll_host_ts, 1757730b4c7SHariprasad Shenai prev_ts).tv_sec, 1767730b4c7SHariprasad Shenai timespec_sub(lep->poll_host_ts, 1777730b4c7SHariprasad Shenai prev_ts).tv_nsec, 1787730b4c7SHariprasad Shenai lep->qid, lep->opcode, 1797730b4c7SHariprasad Shenai lep->opcode == FW_RI_RECEIVE ? 1807730b4c7SHariprasad Shenai "msn" : "wrid", 1817730b4c7SHariprasad Shenai lep->wr_id, 1827730b4c7SHariprasad Shenai timespec_sub(lep->poll_host_ts, 1837730b4c7SHariprasad Shenai lep->post_host_ts).tv_sec, 1847730b4c7SHariprasad Shenai timespec_sub(lep->poll_host_ts, 1857730b4c7SHariprasad Shenai lep->post_host_ts).tv_nsec, 1867730b4c7SHariprasad Shenai lep->post_sge_ts, lep->cqe_sge_ts, 1877730b4c7SHariprasad Shenai lep->poll_sge_ts, 1887730b4c7SHariprasad Shenai ts2ns(lep->poll_sge_ts - lep->post_sge_ts), 1897730b4c7SHariprasad Shenai ts2ns(lep->poll_sge_ts - lep->cqe_sge_ts)); 1907730b4c7SHariprasad Shenai prev_ts = lep->poll_host_ts; 1917730b4c7SHariprasad Shenai } 1927730b4c7SHariprasad Shenai idx++; 1937730b4c7SHariprasad Shenai if (idx > (dev->rdev.wr_log_size - 1)) 1947730b4c7SHariprasad Shenai idx = 0; 1957730b4c7SHariprasad Shenai lep = &dev->rdev.wr_log[idx]; 1967730b4c7SHariprasad Shenai } 1977730b4c7SHariprasad Shenai #undef ts2ns 1987730b4c7SHariprasad Shenai return 0; 1997730b4c7SHariprasad Shenai } 2007730b4c7SHariprasad Shenai 2017730b4c7SHariprasad Shenai static int wr_log_open(struct inode *inode, struct file *file) 2027730b4c7SHariprasad Shenai { 2037730b4c7SHariprasad Shenai return single_open(file, wr_log_show, inode->i_private); 2047730b4c7SHariprasad Shenai } 2057730b4c7SHariprasad Shenai 2067730b4c7SHariprasad Shenai static ssize_t wr_log_clear(struct file *file, const char __user *buf, 2077730b4c7SHariprasad Shenai size_t count, loff_t *pos) 2087730b4c7SHariprasad Shenai { 2097730b4c7SHariprasad Shenai struct c4iw_dev *dev = ((struct seq_file *)file->private_data)->private; 2107730b4c7SHariprasad Shenai int i; 2117730b4c7SHariprasad Shenai 2127730b4c7SHariprasad Shenai if (dev->rdev.wr_log) 2137730b4c7SHariprasad Shenai for (i = 0; i < dev->rdev.wr_log_size; i++) 2147730b4c7SHariprasad Shenai dev->rdev.wr_log[i].valid = 0; 2157730b4c7SHariprasad Shenai return count; 2167730b4c7SHariprasad Shenai } 2177730b4c7SHariprasad Shenai 2187730b4c7SHariprasad Shenai static const struct file_operations wr_log_debugfs_fops = { 2197730b4c7SHariprasad Shenai .owner = THIS_MODULE, 2207730b4c7SHariprasad Shenai .open = wr_log_open, 2217730b4c7SHariprasad Shenai .release = single_release, 2227730b4c7SHariprasad Shenai .read = seq_read, 2237730b4c7SHariprasad Shenai .llseek = seq_lseek, 2247730b4c7SHariprasad Shenai .write = wr_log_clear, 2257730b4c7SHariprasad Shenai }; 2267730b4c7SHariprasad Shenai 2279e8d1fa3SSteve Wise static int dump_qp(int id, void *p, void *data) 228cfdda9d7SSteve Wise { 229cfdda9d7SSteve Wise struct c4iw_qp *qp = p; 2309e8d1fa3SSteve Wise struct c4iw_debugfs_data *qpd = data; 231cfdda9d7SSteve Wise int space; 232cfdda9d7SSteve Wise int cc; 233cfdda9d7SSteve Wise 234cfdda9d7SSteve Wise if (id != qp->wq.sq.qid) 235cfdda9d7SSteve Wise return 0; 236cfdda9d7SSteve Wise 237cfdda9d7SSteve Wise space = qpd->bufsize - qpd->pos - 1; 238cfdda9d7SSteve Wise if (space == 0) 239cfdda9d7SSteve Wise return 1; 240cfdda9d7SSteve Wise 241830662f6SVipul Pandya if (qp->ep) { 242830662f6SVipul Pandya if (qp->ep->com.local_addr.ss_family == AF_INET) { 243830662f6SVipul Pandya struct sockaddr_in *lsin = (struct sockaddr_in *) 244830662f6SVipul Pandya &qp->ep->com.local_addr; 245830662f6SVipul Pandya struct sockaddr_in *rsin = (struct sockaddr_in *) 246830662f6SVipul Pandya &qp->ep->com.remote_addr; 2479eccfe10SSteve Wise struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) 2489eccfe10SSteve Wise &qp->ep->com.mapped_local_addr; 2499eccfe10SSteve Wise struct sockaddr_in *mapped_rsin = (struct sockaddr_in *) 2509eccfe10SSteve Wise &qp->ep->com.mapped_remote_addr; 251830662f6SVipul Pandya 252db5d040dSSteve Wise cc = snprintf(qpd->buf + qpd->pos, space, 253830662f6SVipul Pandya "rc qp sq id %u rq id %u state %u " 254830662f6SVipul Pandya "onchip %u ep tid %u state %u " 2559eccfe10SSteve Wise "%pI4:%u/%u->%pI4:%u/%u\n", 256830662f6SVipul Pandya qp->wq.sq.qid, qp->wq.rq.qid, 257830662f6SVipul Pandya (int)qp->attr.state, 258db5d040dSSteve Wise qp->wq.sq.flags & T4_SQ_ONCHIP, 259cfdda9d7SSteve Wise qp->ep->hwtid, (int)qp->ep->com.state, 260830662f6SVipul Pandya &lsin->sin_addr, ntohs(lsin->sin_port), 2619eccfe10SSteve Wise ntohs(mapped_lsin->sin_port), 2629eccfe10SSteve Wise &rsin->sin_addr, ntohs(rsin->sin_port), 2639eccfe10SSteve Wise ntohs(mapped_rsin->sin_port)); 264830662f6SVipul Pandya } else { 265830662f6SVipul Pandya struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) 266830662f6SVipul Pandya &qp->ep->com.local_addr; 267830662f6SVipul Pandya struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *) 268830662f6SVipul Pandya &qp->ep->com.remote_addr; 2699eccfe10SSteve Wise struct sockaddr_in6 *mapped_lsin6 = 2709eccfe10SSteve Wise (struct sockaddr_in6 *) 2719eccfe10SSteve Wise &qp->ep->com.mapped_local_addr; 2729eccfe10SSteve Wise struct sockaddr_in6 *mapped_rsin6 = 2739eccfe10SSteve Wise (struct sockaddr_in6 *) 2749eccfe10SSteve Wise &qp->ep->com.mapped_remote_addr; 275830662f6SVipul Pandya 276830662f6SVipul Pandya cc = snprintf(qpd->buf + qpd->pos, space, 277830662f6SVipul Pandya "rc qp sq id %u rq id %u state %u " 278830662f6SVipul Pandya "onchip %u ep tid %u state %u " 2799eccfe10SSteve Wise "%pI6:%u/%u->%pI6:%u/%u\n", 280830662f6SVipul Pandya qp->wq.sq.qid, qp->wq.rq.qid, 281830662f6SVipul Pandya (int)qp->attr.state, 282830662f6SVipul Pandya qp->wq.sq.flags & T4_SQ_ONCHIP, 283830662f6SVipul Pandya qp->ep->hwtid, (int)qp->ep->com.state, 284830662f6SVipul Pandya &lsin6->sin6_addr, 285830662f6SVipul Pandya ntohs(lsin6->sin6_port), 2869eccfe10SSteve Wise ntohs(mapped_lsin6->sin6_port), 287830662f6SVipul Pandya &rsin6->sin6_addr, 2889eccfe10SSteve Wise ntohs(rsin6->sin6_port), 2899eccfe10SSteve Wise ntohs(mapped_rsin6->sin6_port)); 290830662f6SVipul Pandya } 291830662f6SVipul Pandya } else 292db5d040dSSteve Wise cc = snprintf(qpd->buf + qpd->pos, space, 293db5d040dSSteve Wise "qp sq id %u rq id %u state %u onchip %u\n", 294db5d040dSSteve Wise qp->wq.sq.qid, qp->wq.rq.qid, 295db5d040dSSteve Wise (int)qp->attr.state, 296db5d040dSSteve Wise qp->wq.sq.flags & T4_SQ_ONCHIP); 297cfdda9d7SSteve Wise if (cc < space) 298cfdda9d7SSteve Wise qpd->pos += cc; 299cfdda9d7SSteve Wise return 0; 300cfdda9d7SSteve Wise } 301cfdda9d7SSteve Wise 302cfdda9d7SSteve Wise static int qp_release(struct inode *inode, struct file *file) 303cfdda9d7SSteve Wise { 3049e8d1fa3SSteve Wise struct c4iw_debugfs_data *qpd = file->private_data; 305cfdda9d7SSteve Wise if (!qpd) { 306cfdda9d7SSteve Wise printk(KERN_INFO "%s null qpd?\n", __func__); 307cfdda9d7SSteve Wise return 0; 308cfdda9d7SSteve Wise } 309d716a2a0SVipul Pandya vfree(qpd->buf); 310cfdda9d7SSteve Wise kfree(qpd); 311cfdda9d7SSteve Wise return 0; 312cfdda9d7SSteve Wise } 313cfdda9d7SSteve Wise 314cfdda9d7SSteve Wise static int qp_open(struct inode *inode, struct file *file) 315cfdda9d7SSteve Wise { 3169e8d1fa3SSteve Wise struct c4iw_debugfs_data *qpd; 317cfdda9d7SSteve Wise int ret = 0; 318cfdda9d7SSteve Wise int count = 1; 319cfdda9d7SSteve Wise 320cfdda9d7SSteve Wise qpd = kmalloc(sizeof *qpd, GFP_KERNEL); 321cfdda9d7SSteve Wise if (!qpd) { 322cfdda9d7SSteve Wise ret = -ENOMEM; 323cfdda9d7SSteve Wise goto out; 324cfdda9d7SSteve Wise } 325cfdda9d7SSteve Wise qpd->devp = inode->i_private; 326cfdda9d7SSteve Wise qpd->pos = 0; 327cfdda9d7SSteve Wise 328cfdda9d7SSteve Wise spin_lock_irq(&qpd->devp->lock); 3299e8d1fa3SSteve Wise idr_for_each(&qpd->devp->qpidr, count_idrs, &count); 330cfdda9d7SSteve Wise spin_unlock_irq(&qpd->devp->lock); 331cfdda9d7SSteve Wise 332cfdda9d7SSteve Wise qpd->bufsize = count * 128; 333d716a2a0SVipul Pandya qpd->buf = vmalloc(qpd->bufsize); 334cfdda9d7SSteve Wise if (!qpd->buf) { 335cfdda9d7SSteve Wise ret = -ENOMEM; 336cfdda9d7SSteve Wise goto err1; 337cfdda9d7SSteve Wise } 338cfdda9d7SSteve Wise 339cfdda9d7SSteve Wise spin_lock_irq(&qpd->devp->lock); 3409e8d1fa3SSteve Wise idr_for_each(&qpd->devp->qpidr, dump_qp, qpd); 341cfdda9d7SSteve Wise spin_unlock_irq(&qpd->devp->lock); 342cfdda9d7SSteve Wise 343cfdda9d7SSteve Wise qpd->buf[qpd->pos++] = 0; 344cfdda9d7SSteve Wise file->private_data = qpd; 345cfdda9d7SSteve Wise goto out; 346cfdda9d7SSteve Wise err1: 347cfdda9d7SSteve Wise kfree(qpd); 348cfdda9d7SSteve Wise out: 349cfdda9d7SSteve Wise return ret; 350cfdda9d7SSteve Wise } 351cfdda9d7SSteve Wise 352cfdda9d7SSteve Wise static const struct file_operations qp_debugfs_fops = { 353cfdda9d7SSteve Wise .owner = THIS_MODULE, 354cfdda9d7SSteve Wise .open = qp_open, 355cfdda9d7SSteve Wise .release = qp_release, 3569e8d1fa3SSteve Wise .read = debugfs_read, 3578bbac892SSteve Wise .llseek = default_llseek, 3589e8d1fa3SSteve Wise }; 3599e8d1fa3SSteve Wise 3609e8d1fa3SSteve Wise static int dump_stag(int id, void *p, void *data) 3619e8d1fa3SSteve Wise { 3629e8d1fa3SSteve Wise struct c4iw_debugfs_data *stagd = data; 3639e8d1fa3SSteve Wise int space; 3649e8d1fa3SSteve Wise int cc; 365031cf476SHariprasad Shenai struct fw_ri_tpte tpte; 366031cf476SHariprasad Shenai int ret; 3679e8d1fa3SSteve Wise 3689e8d1fa3SSteve Wise space = stagd->bufsize - stagd->pos - 1; 3699e8d1fa3SSteve Wise if (space == 0) 3709e8d1fa3SSteve Wise return 1; 3719e8d1fa3SSteve Wise 372031cf476SHariprasad Shenai ret = cxgb4_read_tpte(stagd->devp->rdev.lldi.ports[0], (u32)id<<8, 373031cf476SHariprasad Shenai (__be32 *)&tpte); 374031cf476SHariprasad Shenai if (ret) { 375031cf476SHariprasad Shenai dev_err(&stagd->devp->rdev.lldi.pdev->dev, 376031cf476SHariprasad Shenai "%s cxgb4_read_tpte err %d\n", __func__, ret); 377031cf476SHariprasad Shenai return ret; 378031cf476SHariprasad Shenai } 379031cf476SHariprasad Shenai cc = snprintf(stagd->buf + stagd->pos, space, 380031cf476SHariprasad Shenai "stag: idx 0x%x valid %d key 0x%x state %d pdid %d " 381031cf476SHariprasad Shenai "perm 0x%x ps %d len 0x%llx va 0x%llx\n", 382031cf476SHariprasad Shenai (u32)id<<8, 383cf7fe64aSHariprasad Shenai FW_RI_TPTE_VALID_G(ntohl(tpte.valid_to_pdid)), 384cf7fe64aSHariprasad Shenai FW_RI_TPTE_STAGKEY_G(ntohl(tpte.valid_to_pdid)), 385cf7fe64aSHariprasad Shenai FW_RI_TPTE_STAGSTATE_G(ntohl(tpte.valid_to_pdid)), 386cf7fe64aSHariprasad Shenai FW_RI_TPTE_PDID_G(ntohl(tpte.valid_to_pdid)), 387cf7fe64aSHariprasad Shenai FW_RI_TPTE_PERM_G(ntohl(tpte.locread_to_qpid)), 388cf7fe64aSHariprasad Shenai FW_RI_TPTE_PS_G(ntohl(tpte.locread_to_qpid)), 389031cf476SHariprasad Shenai ((u64)ntohl(tpte.len_hi) << 32) | ntohl(tpte.len_lo), 390031cf476SHariprasad Shenai ((u64)ntohl(tpte.va_hi) << 32) | ntohl(tpte.va_lo_fbo)); 3919e8d1fa3SSteve Wise if (cc < space) 3929e8d1fa3SSteve Wise stagd->pos += cc; 3939e8d1fa3SSteve Wise return 0; 3949e8d1fa3SSteve Wise } 3959e8d1fa3SSteve Wise 3969e8d1fa3SSteve Wise static int stag_release(struct inode *inode, struct file *file) 3979e8d1fa3SSteve Wise { 3989e8d1fa3SSteve Wise struct c4iw_debugfs_data *stagd = file->private_data; 3999e8d1fa3SSteve Wise if (!stagd) { 4009e8d1fa3SSteve Wise printk(KERN_INFO "%s null stagd?\n", __func__); 4019e8d1fa3SSteve Wise return 0; 4029e8d1fa3SSteve Wise } 403031cf476SHariprasad Shenai vfree(stagd->buf); 4049e8d1fa3SSteve Wise kfree(stagd); 4059e8d1fa3SSteve Wise return 0; 4069e8d1fa3SSteve Wise } 4079e8d1fa3SSteve Wise 4089e8d1fa3SSteve Wise static int stag_open(struct inode *inode, struct file *file) 4099e8d1fa3SSteve Wise { 4109e8d1fa3SSteve Wise struct c4iw_debugfs_data *stagd; 4119e8d1fa3SSteve Wise int ret = 0; 4129e8d1fa3SSteve Wise int count = 1; 4139e8d1fa3SSteve Wise 4149e8d1fa3SSteve Wise stagd = kmalloc(sizeof *stagd, GFP_KERNEL); 4159e8d1fa3SSteve Wise if (!stagd) { 4169e8d1fa3SSteve Wise ret = -ENOMEM; 4179e8d1fa3SSteve Wise goto out; 4189e8d1fa3SSteve Wise } 4199e8d1fa3SSteve Wise stagd->devp = inode->i_private; 4209e8d1fa3SSteve Wise stagd->pos = 0; 4219e8d1fa3SSteve Wise 4229e8d1fa3SSteve Wise spin_lock_irq(&stagd->devp->lock); 4239e8d1fa3SSteve Wise idr_for_each(&stagd->devp->mmidr, count_idrs, &count); 4249e8d1fa3SSteve Wise spin_unlock_irq(&stagd->devp->lock); 4259e8d1fa3SSteve Wise 426031cf476SHariprasad Shenai stagd->bufsize = count * 256; 427031cf476SHariprasad Shenai stagd->buf = vmalloc(stagd->bufsize); 4289e8d1fa3SSteve Wise if (!stagd->buf) { 4299e8d1fa3SSteve Wise ret = -ENOMEM; 4309e8d1fa3SSteve Wise goto err1; 4319e8d1fa3SSteve Wise } 4329e8d1fa3SSteve Wise 4339e8d1fa3SSteve Wise spin_lock_irq(&stagd->devp->lock); 4349e8d1fa3SSteve Wise idr_for_each(&stagd->devp->mmidr, dump_stag, stagd); 4359e8d1fa3SSteve Wise spin_unlock_irq(&stagd->devp->lock); 4369e8d1fa3SSteve Wise 4379e8d1fa3SSteve Wise stagd->buf[stagd->pos++] = 0; 4389e8d1fa3SSteve Wise file->private_data = stagd; 4399e8d1fa3SSteve Wise goto out; 4409e8d1fa3SSteve Wise err1: 4419e8d1fa3SSteve Wise kfree(stagd); 4429e8d1fa3SSteve Wise out: 4439e8d1fa3SSteve Wise return ret; 4449e8d1fa3SSteve Wise } 4459e8d1fa3SSteve Wise 4469e8d1fa3SSteve Wise static const struct file_operations stag_debugfs_fops = { 4479e8d1fa3SSteve Wise .owner = THIS_MODULE, 4489e8d1fa3SSteve Wise .open = stag_open, 4499e8d1fa3SSteve Wise .release = stag_release, 4509e8d1fa3SSteve Wise .read = debugfs_read, 4518bbac892SSteve Wise .llseek = default_llseek, 452cfdda9d7SSteve Wise }; 453cfdda9d7SSteve Wise 45405eb2389SSteve Wise static char *db_state_str[] = {"NORMAL", "FLOW_CONTROL", "RECOVERY", "STOPPED"}; 455422eea0aSVipul Pandya 4568d81ef34SVipul Pandya static int stats_show(struct seq_file *seq, void *v) 4578d81ef34SVipul Pandya { 4588d81ef34SVipul Pandya struct c4iw_dev *dev = seq->private; 4598d81ef34SVipul Pandya 460ec3eead2SVipul Pandya seq_printf(seq, " Object: %10s %10s %10s %10s\n", "Total", "Current", 461ec3eead2SVipul Pandya "Max", "Fail"); 462ec3eead2SVipul Pandya seq_printf(seq, " PDID: %10llu %10llu %10llu %10llu\n", 4638d81ef34SVipul Pandya dev->rdev.stats.pd.total, dev->rdev.stats.pd.cur, 464ec3eead2SVipul Pandya dev->rdev.stats.pd.max, dev->rdev.stats.pd.fail); 465ec3eead2SVipul Pandya seq_printf(seq, " QID: %10llu %10llu %10llu %10llu\n", 4668d81ef34SVipul Pandya dev->rdev.stats.qid.total, dev->rdev.stats.qid.cur, 467ec3eead2SVipul Pandya dev->rdev.stats.qid.max, dev->rdev.stats.qid.fail); 468ec3eead2SVipul Pandya seq_printf(seq, " TPTMEM: %10llu %10llu %10llu %10llu\n", 4698d81ef34SVipul Pandya dev->rdev.stats.stag.total, dev->rdev.stats.stag.cur, 470ec3eead2SVipul Pandya dev->rdev.stats.stag.max, dev->rdev.stats.stag.fail); 471ec3eead2SVipul Pandya seq_printf(seq, " PBLMEM: %10llu %10llu %10llu %10llu\n", 4728d81ef34SVipul Pandya dev->rdev.stats.pbl.total, dev->rdev.stats.pbl.cur, 473ec3eead2SVipul Pandya dev->rdev.stats.pbl.max, dev->rdev.stats.pbl.fail); 474ec3eead2SVipul Pandya seq_printf(seq, " RQTMEM: %10llu %10llu %10llu %10llu\n", 4758d81ef34SVipul Pandya dev->rdev.stats.rqt.total, dev->rdev.stats.rqt.cur, 476ec3eead2SVipul Pandya dev->rdev.stats.rqt.max, dev->rdev.stats.rqt.fail); 477ec3eead2SVipul Pandya seq_printf(seq, " OCQPMEM: %10llu %10llu %10llu %10llu\n", 4788d81ef34SVipul Pandya dev->rdev.stats.ocqp.total, dev->rdev.stats.ocqp.cur, 479ec3eead2SVipul Pandya dev->rdev.stats.ocqp.max, dev->rdev.stats.ocqp.fail); 4802c974781SVipul Pandya seq_printf(seq, " DB FULL: %10llu\n", dev->rdev.stats.db_full); 4812c974781SVipul Pandya seq_printf(seq, " DB EMPTY: %10llu\n", dev->rdev.stats.db_empty); 4822c974781SVipul Pandya seq_printf(seq, " DB DROP: %10llu\n", dev->rdev.stats.db_drop); 48305eb2389SSteve Wise seq_printf(seq, " DB State: %s Transitions %llu FC Interruptions %llu\n", 484422eea0aSVipul Pandya db_state_str[dev->db_state], 48505eb2389SSteve Wise dev->rdev.stats.db_state_transitions, 48605eb2389SSteve Wise dev->rdev.stats.db_fc_interruptions); 4871cab775cSVipul Pandya seq_printf(seq, "TCAM_FULL: %10llu\n", dev->rdev.stats.tcam_full); 488793dad94SVipul Pandya seq_printf(seq, "ACT_OFLD_CONN_FAILS: %10llu\n", 489793dad94SVipul Pandya dev->rdev.stats.act_ofld_conn_fails); 490793dad94SVipul Pandya seq_printf(seq, "PAS_OFLD_CONN_FAILS: %10llu\n", 491793dad94SVipul Pandya dev->rdev.stats.pas_ofld_conn_fails); 4924c2c5763SHariprasad Shenai seq_printf(seq, "AVAILABLE IRD: %10u\n", dev->avail_ird); 4938d81ef34SVipul Pandya return 0; 4948d81ef34SVipul Pandya } 4958d81ef34SVipul Pandya 4968d81ef34SVipul Pandya static int stats_open(struct inode *inode, struct file *file) 4978d81ef34SVipul Pandya { 4988d81ef34SVipul Pandya return single_open(file, stats_show, inode->i_private); 4998d81ef34SVipul Pandya } 5008d81ef34SVipul Pandya 5018d81ef34SVipul Pandya static ssize_t stats_clear(struct file *file, const char __user *buf, 5028d81ef34SVipul Pandya size_t count, loff_t *pos) 5038d81ef34SVipul Pandya { 5048d81ef34SVipul Pandya struct c4iw_dev *dev = ((struct seq_file *)file->private_data)->private; 5058d81ef34SVipul Pandya 5068d81ef34SVipul Pandya mutex_lock(&dev->rdev.stats.lock); 5078d81ef34SVipul Pandya dev->rdev.stats.pd.max = 0; 508ec3eead2SVipul Pandya dev->rdev.stats.pd.fail = 0; 5098d81ef34SVipul Pandya dev->rdev.stats.qid.max = 0; 510ec3eead2SVipul Pandya dev->rdev.stats.qid.fail = 0; 5118d81ef34SVipul Pandya dev->rdev.stats.stag.max = 0; 512ec3eead2SVipul Pandya dev->rdev.stats.stag.fail = 0; 5138d81ef34SVipul Pandya dev->rdev.stats.pbl.max = 0; 514ec3eead2SVipul Pandya dev->rdev.stats.pbl.fail = 0; 5158d81ef34SVipul Pandya dev->rdev.stats.rqt.max = 0; 516ec3eead2SVipul Pandya dev->rdev.stats.rqt.fail = 0; 5178d81ef34SVipul Pandya dev->rdev.stats.ocqp.max = 0; 518ec3eead2SVipul Pandya dev->rdev.stats.ocqp.fail = 0; 5192c974781SVipul Pandya dev->rdev.stats.db_full = 0; 5202c974781SVipul Pandya dev->rdev.stats.db_empty = 0; 5212c974781SVipul Pandya dev->rdev.stats.db_drop = 0; 522422eea0aSVipul Pandya dev->rdev.stats.db_state_transitions = 0; 523793dad94SVipul Pandya dev->rdev.stats.tcam_full = 0; 524793dad94SVipul Pandya dev->rdev.stats.act_ofld_conn_fails = 0; 525793dad94SVipul Pandya dev->rdev.stats.pas_ofld_conn_fails = 0; 5268d81ef34SVipul Pandya mutex_unlock(&dev->rdev.stats.lock); 5278d81ef34SVipul Pandya return count; 5288d81ef34SVipul Pandya } 5298d81ef34SVipul Pandya 5308d81ef34SVipul Pandya static const struct file_operations stats_debugfs_fops = { 5318d81ef34SVipul Pandya .owner = THIS_MODULE, 5328d81ef34SVipul Pandya .open = stats_open, 5338d81ef34SVipul Pandya .release = single_release, 5348d81ef34SVipul Pandya .read = seq_read, 5358d81ef34SVipul Pandya .llseek = seq_lseek, 5368d81ef34SVipul Pandya .write = stats_clear, 5378d81ef34SVipul Pandya }; 5388d81ef34SVipul Pandya 539793dad94SVipul Pandya static int dump_ep(int id, void *p, void *data) 540793dad94SVipul Pandya { 541793dad94SVipul Pandya struct c4iw_ep *ep = p; 542793dad94SVipul Pandya struct c4iw_debugfs_data *epd = data; 543793dad94SVipul Pandya int space; 544793dad94SVipul Pandya int cc; 545793dad94SVipul Pandya 546793dad94SVipul Pandya space = epd->bufsize - epd->pos - 1; 547793dad94SVipul Pandya if (space == 0) 548793dad94SVipul Pandya return 1; 549793dad94SVipul Pandya 550830662f6SVipul Pandya if (ep->com.local_addr.ss_family == AF_INET) { 551830662f6SVipul Pandya struct sockaddr_in *lsin = (struct sockaddr_in *) 552830662f6SVipul Pandya &ep->com.local_addr; 553830662f6SVipul Pandya struct sockaddr_in *rsin = (struct sockaddr_in *) 554830662f6SVipul Pandya &ep->com.remote_addr; 5559eccfe10SSteve Wise struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) 5569eccfe10SSteve Wise &ep->com.mapped_local_addr; 5579eccfe10SSteve Wise struct sockaddr_in *mapped_rsin = (struct sockaddr_in *) 5589eccfe10SSteve Wise &ep->com.mapped_remote_addr; 559830662f6SVipul Pandya 560793dad94SVipul Pandya cc = snprintf(epd->buf + epd->pos, space, 561830662f6SVipul Pandya "ep %p cm_id %p qp %p state %d flags 0x%lx " 562830662f6SVipul Pandya "history 0x%lx hwtid %d atid %d " 5639eccfe10SSteve Wise "%pI4:%d/%d <-> %pI4:%d/%d\n", 564830662f6SVipul Pandya ep, ep->com.cm_id, ep->com.qp, 565830662f6SVipul Pandya (int)ep->com.state, ep->com.flags, 566830662f6SVipul Pandya ep->com.history, ep->hwtid, ep->atid, 567830662f6SVipul Pandya &lsin->sin_addr, ntohs(lsin->sin_port), 5689eccfe10SSteve Wise ntohs(mapped_lsin->sin_port), 5699eccfe10SSteve Wise &rsin->sin_addr, ntohs(rsin->sin_port), 5709eccfe10SSteve Wise ntohs(mapped_rsin->sin_port)); 571830662f6SVipul Pandya } else { 572830662f6SVipul Pandya struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) 573830662f6SVipul Pandya &ep->com.local_addr; 574830662f6SVipul Pandya struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *) 575830662f6SVipul Pandya &ep->com.remote_addr; 5769eccfe10SSteve Wise struct sockaddr_in6 *mapped_lsin6 = (struct sockaddr_in6 *) 5779eccfe10SSteve Wise &ep->com.mapped_local_addr; 5789eccfe10SSteve Wise struct sockaddr_in6 *mapped_rsin6 = (struct sockaddr_in6 *) 5799eccfe10SSteve Wise &ep->com.mapped_remote_addr; 580830662f6SVipul Pandya 581830662f6SVipul Pandya cc = snprintf(epd->buf + epd->pos, space, 582830662f6SVipul Pandya "ep %p cm_id %p qp %p state %d flags 0x%lx " 583830662f6SVipul Pandya "history 0x%lx hwtid %d atid %d " 5849eccfe10SSteve Wise "%pI6:%d/%d <-> %pI6:%d/%d\n", 585830662f6SVipul Pandya ep, ep->com.cm_id, ep->com.qp, 586830662f6SVipul Pandya (int)ep->com.state, ep->com.flags, 587830662f6SVipul Pandya ep->com.history, ep->hwtid, ep->atid, 588830662f6SVipul Pandya &lsin6->sin6_addr, ntohs(lsin6->sin6_port), 5899eccfe10SSteve Wise ntohs(mapped_lsin6->sin6_port), 5909eccfe10SSteve Wise &rsin6->sin6_addr, ntohs(rsin6->sin6_port), 5919eccfe10SSteve Wise ntohs(mapped_rsin6->sin6_port)); 592830662f6SVipul Pandya } 593793dad94SVipul Pandya if (cc < space) 594793dad94SVipul Pandya epd->pos += cc; 595793dad94SVipul Pandya return 0; 596793dad94SVipul Pandya } 597793dad94SVipul Pandya 598793dad94SVipul Pandya static int dump_listen_ep(int id, void *p, void *data) 599793dad94SVipul Pandya { 600793dad94SVipul Pandya struct c4iw_listen_ep *ep = p; 601793dad94SVipul Pandya struct c4iw_debugfs_data *epd = data; 602793dad94SVipul Pandya int space; 603793dad94SVipul Pandya int cc; 604793dad94SVipul Pandya 605793dad94SVipul Pandya space = epd->bufsize - epd->pos - 1; 606793dad94SVipul Pandya if (space == 0) 607793dad94SVipul Pandya return 1; 608793dad94SVipul Pandya 609830662f6SVipul Pandya if (ep->com.local_addr.ss_family == AF_INET) { 610830662f6SVipul Pandya struct sockaddr_in *lsin = (struct sockaddr_in *) 611830662f6SVipul Pandya &ep->com.local_addr; 6129eccfe10SSteve Wise struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) 6139eccfe10SSteve Wise &ep->com.mapped_local_addr; 614830662f6SVipul Pandya 615793dad94SVipul Pandya cc = snprintf(epd->buf + epd->pos, space, 616830662f6SVipul Pandya "ep %p cm_id %p state %d flags 0x%lx stid %d " 6179eccfe10SSteve Wise "backlog %d %pI4:%d/%d\n", 618830662f6SVipul Pandya ep, ep->com.cm_id, (int)ep->com.state, 619793dad94SVipul Pandya ep->com.flags, ep->stid, ep->backlog, 6209eccfe10SSteve Wise &lsin->sin_addr, ntohs(lsin->sin_port), 6219eccfe10SSteve Wise ntohs(mapped_lsin->sin_port)); 622830662f6SVipul Pandya } else { 623830662f6SVipul Pandya struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) 624830662f6SVipul Pandya &ep->com.local_addr; 6259eccfe10SSteve Wise struct sockaddr_in6 *mapped_lsin6 = (struct sockaddr_in6 *) 6269eccfe10SSteve Wise &ep->com.mapped_local_addr; 627830662f6SVipul Pandya 628830662f6SVipul Pandya cc = snprintf(epd->buf + epd->pos, space, 629830662f6SVipul Pandya "ep %p cm_id %p state %d flags 0x%lx stid %d " 6309eccfe10SSteve Wise "backlog %d %pI6:%d/%d\n", 631830662f6SVipul Pandya ep, ep->com.cm_id, (int)ep->com.state, 632830662f6SVipul Pandya ep->com.flags, ep->stid, ep->backlog, 6339eccfe10SSteve Wise &lsin6->sin6_addr, ntohs(lsin6->sin6_port), 6349eccfe10SSteve Wise ntohs(mapped_lsin6->sin6_port)); 635830662f6SVipul Pandya } 636793dad94SVipul Pandya if (cc < space) 637793dad94SVipul Pandya epd->pos += cc; 638793dad94SVipul Pandya return 0; 639793dad94SVipul Pandya } 640793dad94SVipul Pandya 641793dad94SVipul Pandya static int ep_release(struct inode *inode, struct file *file) 642793dad94SVipul Pandya { 643793dad94SVipul Pandya struct c4iw_debugfs_data *epd = file->private_data; 644793dad94SVipul Pandya if (!epd) { 645793dad94SVipul Pandya pr_info("%s null qpd?\n", __func__); 646793dad94SVipul Pandya return 0; 647793dad94SVipul Pandya } 648793dad94SVipul Pandya vfree(epd->buf); 649793dad94SVipul Pandya kfree(epd); 650793dad94SVipul Pandya return 0; 651793dad94SVipul Pandya } 652793dad94SVipul Pandya 653793dad94SVipul Pandya static int ep_open(struct inode *inode, struct file *file) 654793dad94SVipul Pandya { 655793dad94SVipul Pandya struct c4iw_debugfs_data *epd; 656793dad94SVipul Pandya int ret = 0; 657793dad94SVipul Pandya int count = 1; 658793dad94SVipul Pandya 659793dad94SVipul Pandya epd = kmalloc(sizeof(*epd), GFP_KERNEL); 660793dad94SVipul Pandya if (!epd) { 661793dad94SVipul Pandya ret = -ENOMEM; 662793dad94SVipul Pandya goto out; 663793dad94SVipul Pandya } 664793dad94SVipul Pandya epd->devp = inode->i_private; 665793dad94SVipul Pandya epd->pos = 0; 666793dad94SVipul Pandya 667793dad94SVipul Pandya spin_lock_irq(&epd->devp->lock); 668793dad94SVipul Pandya idr_for_each(&epd->devp->hwtid_idr, count_idrs, &count); 669793dad94SVipul Pandya idr_for_each(&epd->devp->atid_idr, count_idrs, &count); 670793dad94SVipul Pandya idr_for_each(&epd->devp->stid_idr, count_idrs, &count); 671793dad94SVipul Pandya spin_unlock_irq(&epd->devp->lock); 672793dad94SVipul Pandya 67363a71ba6SPramod Kumar epd->bufsize = count * 240; 674793dad94SVipul Pandya epd->buf = vmalloc(epd->bufsize); 675793dad94SVipul Pandya if (!epd->buf) { 676793dad94SVipul Pandya ret = -ENOMEM; 677793dad94SVipul Pandya goto err1; 678793dad94SVipul Pandya } 679793dad94SVipul Pandya 680793dad94SVipul Pandya spin_lock_irq(&epd->devp->lock); 681793dad94SVipul Pandya idr_for_each(&epd->devp->hwtid_idr, dump_ep, epd); 682793dad94SVipul Pandya idr_for_each(&epd->devp->atid_idr, dump_ep, epd); 683793dad94SVipul Pandya idr_for_each(&epd->devp->stid_idr, dump_listen_ep, epd); 684793dad94SVipul Pandya spin_unlock_irq(&epd->devp->lock); 685793dad94SVipul Pandya 686793dad94SVipul Pandya file->private_data = epd; 687793dad94SVipul Pandya goto out; 688793dad94SVipul Pandya err1: 689793dad94SVipul Pandya kfree(epd); 690793dad94SVipul Pandya out: 691793dad94SVipul Pandya return ret; 692793dad94SVipul Pandya } 693793dad94SVipul Pandya 694793dad94SVipul Pandya static const struct file_operations ep_debugfs_fops = { 695793dad94SVipul Pandya .owner = THIS_MODULE, 696793dad94SVipul Pandya .open = ep_open, 697793dad94SVipul Pandya .release = ep_release, 698793dad94SVipul Pandya .read = debugfs_read, 699793dad94SVipul Pandya }; 700793dad94SVipul Pandya 701cfdda9d7SSteve Wise static int setup_debugfs(struct c4iw_dev *devp) 702cfdda9d7SSteve Wise { 703cfdda9d7SSteve Wise if (!devp->debugfs_root) 704cfdda9d7SSteve Wise return -1; 705cfdda9d7SSteve Wise 706e59b4e91SDavid Howells debugfs_create_file_size("qps", S_IWUSR, devp->debugfs_root, 707e59b4e91SDavid Howells (void *)devp, &qp_debugfs_fops, 4096); 7089e8d1fa3SSteve Wise 709e59b4e91SDavid Howells debugfs_create_file_size("stags", S_IWUSR, devp->debugfs_root, 710e59b4e91SDavid Howells (void *)devp, &stag_debugfs_fops, 4096); 7118d81ef34SVipul Pandya 712e59b4e91SDavid Howells debugfs_create_file_size("stats", S_IWUSR, devp->debugfs_root, 713e59b4e91SDavid Howells (void *)devp, &stats_debugfs_fops, 4096); 7148d81ef34SVipul Pandya 715e59b4e91SDavid Howells debugfs_create_file_size("eps", S_IWUSR, devp->debugfs_root, 716e59b4e91SDavid Howells (void *)devp, &ep_debugfs_fops, 4096); 717793dad94SVipul Pandya 718e59b4e91SDavid Howells if (c4iw_wr_log) 719e59b4e91SDavid Howells debugfs_create_file_size("wr_log", S_IWUSR, devp->debugfs_root, 720e59b4e91SDavid Howells (void *)devp, &wr_log_debugfs_fops, 4096); 721cfdda9d7SSteve Wise return 0; 722cfdda9d7SSteve Wise } 723cfdda9d7SSteve Wise 724cfdda9d7SSteve Wise void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev, 725cfdda9d7SSteve Wise struct c4iw_dev_ucontext *uctx) 726cfdda9d7SSteve Wise { 727cfdda9d7SSteve Wise struct list_head *pos, *nxt; 728cfdda9d7SSteve Wise struct c4iw_qid_list *entry; 729cfdda9d7SSteve Wise 730cfdda9d7SSteve Wise mutex_lock(&uctx->lock); 731cfdda9d7SSteve Wise list_for_each_safe(pos, nxt, &uctx->qpids) { 732cfdda9d7SSteve Wise entry = list_entry(pos, struct c4iw_qid_list, entry); 733cfdda9d7SSteve Wise list_del_init(&entry->entry); 7348d81ef34SVipul Pandya if (!(entry->qid & rdev->qpmask)) { 735ec3eead2SVipul Pandya c4iw_put_resource(&rdev->resource.qid_table, 736ec3eead2SVipul Pandya entry->qid); 7378d81ef34SVipul Pandya mutex_lock(&rdev->stats.lock); 7388d81ef34SVipul Pandya rdev->stats.qid.cur -= rdev->qpmask + 1; 7398d81ef34SVipul Pandya mutex_unlock(&rdev->stats.lock); 7408d81ef34SVipul Pandya } 741cfdda9d7SSteve Wise kfree(entry); 742cfdda9d7SSteve Wise } 743cfdda9d7SSteve Wise 744cfdda9d7SSteve Wise list_for_each_safe(pos, nxt, &uctx->qpids) { 745cfdda9d7SSteve Wise entry = list_entry(pos, struct c4iw_qid_list, entry); 746cfdda9d7SSteve Wise list_del_init(&entry->entry); 747cfdda9d7SSteve Wise kfree(entry); 748cfdda9d7SSteve Wise } 749cfdda9d7SSteve Wise mutex_unlock(&uctx->lock); 750cfdda9d7SSteve Wise } 751cfdda9d7SSteve Wise 752cfdda9d7SSteve Wise void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev, 753cfdda9d7SSteve Wise struct c4iw_dev_ucontext *uctx) 754cfdda9d7SSteve Wise { 755cfdda9d7SSteve Wise INIT_LIST_HEAD(&uctx->qpids); 756cfdda9d7SSteve Wise INIT_LIST_HEAD(&uctx->cqids); 757cfdda9d7SSteve Wise mutex_init(&uctx->lock); 758cfdda9d7SSteve Wise } 759cfdda9d7SSteve Wise 760cfdda9d7SSteve Wise /* Caller takes care of locking if needed */ 761cfdda9d7SSteve Wise static int c4iw_rdev_open(struct c4iw_rdev *rdev) 762cfdda9d7SSteve Wise { 763cfdda9d7SSteve Wise int err; 764cfdda9d7SSteve Wise 765cfdda9d7SSteve Wise c4iw_init_dev_ucontext(rdev, &rdev->uctx); 766cfdda9d7SSteve Wise 767cfdda9d7SSteve Wise /* 7684a75a86cSHariprasad S * This implementation assumes udb_density == ucq_density! Eventually 7694a75a86cSHariprasad S * we might need to support this but for now fail the open. Also the 7704a75a86cSHariprasad S * cqid and qpid range must match for now. 7714a75a86cSHariprasad S */ 7724a75a86cSHariprasad S if (rdev->lldi.udb_density != rdev->lldi.ucq_density) { 7734a75a86cSHariprasad S pr_err(MOD "%s: unsupported udb/ucq densities %u/%u\n", 7744a75a86cSHariprasad S pci_name(rdev->lldi.pdev), rdev->lldi.udb_density, 7754a75a86cSHariprasad S rdev->lldi.ucq_density); 7764a75a86cSHariprasad S err = -EINVAL; 7774a75a86cSHariprasad S goto err1; 7784a75a86cSHariprasad S } 7794a75a86cSHariprasad S if (rdev->lldi.vr->qp.start != rdev->lldi.vr->cq.start || 7804a75a86cSHariprasad S rdev->lldi.vr->qp.size != rdev->lldi.vr->cq.size) { 7814a75a86cSHariprasad S pr_err(MOD "%s: unsupported qp and cq id ranges " 7824a75a86cSHariprasad S "qp start %u size %u cq start %u size %u\n", 7834a75a86cSHariprasad S pci_name(rdev->lldi.pdev), rdev->lldi.vr->qp.start, 7844a75a86cSHariprasad S rdev->lldi.vr->qp.size, rdev->lldi.vr->cq.size, 7854a75a86cSHariprasad S rdev->lldi.vr->cq.size); 7864a75a86cSHariprasad S err = -EINVAL; 7874a75a86cSHariprasad S goto err1; 7884a75a86cSHariprasad S } 7894a75a86cSHariprasad S 7904a75a86cSHariprasad S /* 791cfdda9d7SSteve Wise * qpshift is the number of bits to shift the qpid left in order 792cfdda9d7SSteve Wise * to get the correct address of the doorbell for that qp. 793cfdda9d7SSteve Wise */ 794cfdda9d7SSteve Wise rdev->qpshift = PAGE_SHIFT - ilog2(rdev->lldi.udb_density); 795cfdda9d7SSteve Wise rdev->qpmask = rdev->lldi.udb_density - 1; 796cfdda9d7SSteve Wise rdev->cqshift = PAGE_SHIFT - ilog2(rdev->lldi.ucq_density); 797cfdda9d7SSteve Wise rdev->cqmask = rdev->lldi.ucq_density - 1; 798cfdda9d7SSteve Wise PDBG("%s dev %s stag start 0x%0x size 0x%0x num stags %d " 79993fb72e4SSteve Wise "pbl start 0x%0x size 0x%0x rq start 0x%0x size 0x%0x " 80093fb72e4SSteve Wise "qp qid start %u size %u cq qid start %u size %u\n", 801cfdda9d7SSteve Wise __func__, pci_name(rdev->lldi.pdev), rdev->lldi.vr->stag.start, 802cfdda9d7SSteve Wise rdev->lldi.vr->stag.size, c4iw_num_stags(rdev), 803cfdda9d7SSteve Wise rdev->lldi.vr->pbl.start, 804cfdda9d7SSteve Wise rdev->lldi.vr->pbl.size, rdev->lldi.vr->rq.start, 80593fb72e4SSteve Wise rdev->lldi.vr->rq.size, 80693fb72e4SSteve Wise rdev->lldi.vr->qp.start, 80793fb72e4SSteve Wise rdev->lldi.vr->qp.size, 80893fb72e4SSteve Wise rdev->lldi.vr->cq.start, 80993fb72e4SSteve Wise rdev->lldi.vr->cq.size); 8106198dd8dSHariprasad S PDBG("udb len 0x%x udb base %p db_reg %p gts_reg %p qpshift %lu " 811cfdda9d7SSteve Wise "qpmask 0x%x cqshift %lu cqmask 0x%x\n", 812cfdda9d7SSteve Wise (unsigned)pci_resource_len(rdev->lldi.pdev, 2), 8136198dd8dSHariprasad S (void *)pci_resource_start(rdev->lldi.pdev, 2), 814cfdda9d7SSteve Wise rdev->lldi.db_reg, 815cfdda9d7SSteve Wise rdev->lldi.gts_reg, 816cfdda9d7SSteve Wise rdev->qpshift, rdev->qpmask, 817cfdda9d7SSteve Wise rdev->cqshift, rdev->cqmask); 818cfdda9d7SSteve Wise 819cfdda9d7SSteve Wise if (c4iw_num_stags(rdev) == 0) { 820cfdda9d7SSteve Wise err = -EINVAL; 821cfdda9d7SSteve Wise goto err1; 822cfdda9d7SSteve Wise } 823cfdda9d7SSteve Wise 8248d81ef34SVipul Pandya rdev->stats.pd.total = T4_MAX_NUM_PD; 8258d81ef34SVipul Pandya rdev->stats.stag.total = rdev->lldi.vr->stag.size; 8268d81ef34SVipul Pandya rdev->stats.pbl.total = rdev->lldi.vr->pbl.size; 8278d81ef34SVipul Pandya rdev->stats.rqt.total = rdev->lldi.vr->rq.size; 8288d81ef34SVipul Pandya rdev->stats.ocqp.total = rdev->lldi.vr->ocq.size; 8298d81ef34SVipul Pandya rdev->stats.qid.total = rdev->lldi.vr->qp.size; 8308d81ef34SVipul Pandya 831cfdda9d7SSteve Wise err = c4iw_init_resource(rdev, c4iw_num_stags(rdev), T4_MAX_NUM_PD); 832cfdda9d7SSteve Wise if (err) { 833cfdda9d7SSteve Wise printk(KERN_ERR MOD "error %d initializing resources\n", err); 834cfdda9d7SSteve Wise goto err1; 835cfdda9d7SSteve Wise } 836cfdda9d7SSteve Wise err = c4iw_pblpool_create(rdev); 837cfdda9d7SSteve Wise if (err) { 838cfdda9d7SSteve Wise printk(KERN_ERR MOD "error %d initializing pbl pool\n", err); 839cfdda9d7SSteve Wise goto err2; 840cfdda9d7SSteve Wise } 841cfdda9d7SSteve Wise err = c4iw_rqtpool_create(rdev); 842cfdda9d7SSteve Wise if (err) { 843cfdda9d7SSteve Wise printk(KERN_ERR MOD "error %d initializing rqt pool\n", err); 844cfdda9d7SSteve Wise goto err3; 845cfdda9d7SSteve Wise } 846c6d7b267SSteve Wise err = c4iw_ocqp_pool_create(rdev); 847c6d7b267SSteve Wise if (err) { 848c6d7b267SSteve Wise printk(KERN_ERR MOD "error %d initializing ocqp pool\n", err); 849c6d7b267SSteve Wise goto err4; 850c6d7b267SSteve Wise } 85105eb2389SSteve Wise rdev->status_page = (struct t4_dev_status_page *) 85205eb2389SSteve Wise __get_free_page(GFP_KERNEL); 85305eb2389SSteve Wise if (!rdev->status_page) { 85405eb2389SSteve Wise pr_err(MOD "error allocating status page\n"); 85505eb2389SSteve Wise goto err4; 85605eb2389SSteve Wise } 8578fd90bb8SDavid S. Miller 8587730b4c7SHariprasad Shenai if (c4iw_wr_log) { 8597730b4c7SHariprasad Shenai rdev->wr_log = kzalloc((1 << c4iw_wr_log_size_order) * 8607730b4c7SHariprasad Shenai sizeof(*rdev->wr_log), GFP_KERNEL); 8617730b4c7SHariprasad Shenai if (rdev->wr_log) { 8627730b4c7SHariprasad Shenai rdev->wr_log_size = 1 << c4iw_wr_log_size_order; 8637730b4c7SHariprasad Shenai atomic_set(&rdev->wr_log_idx, 0); 8647730b4c7SHariprasad Shenai } else { 8657730b4c7SHariprasad Shenai pr_err(MOD "error allocating wr_log. Logging disabled\n"); 8667730b4c7SHariprasad Shenai } 8677730b4c7SHariprasad Shenai } 8688fd90bb8SDavid S. Miller 8696b54d54dSSteve Wise rdev->status_page->db_off = 0; 8708fd90bb8SDavid S. Miller 871cfdda9d7SSteve Wise return 0; 872c6d7b267SSteve Wise err4: 873c6d7b267SSteve Wise c4iw_rqtpool_destroy(rdev); 874cfdda9d7SSteve Wise err3: 875cfdda9d7SSteve Wise c4iw_pblpool_destroy(rdev); 876cfdda9d7SSteve Wise err2: 877cfdda9d7SSteve Wise c4iw_destroy_resource(&rdev->resource); 878cfdda9d7SSteve Wise err1: 879cfdda9d7SSteve Wise return err; 880cfdda9d7SSteve Wise } 881cfdda9d7SSteve Wise 882cfdda9d7SSteve Wise static void c4iw_rdev_close(struct c4iw_rdev *rdev) 883cfdda9d7SSteve Wise { 8847730b4c7SHariprasad Shenai kfree(rdev->wr_log); 88505eb2389SSteve Wise free_page((unsigned long)rdev->status_page); 886cfdda9d7SSteve Wise c4iw_pblpool_destroy(rdev); 887cfdda9d7SSteve Wise c4iw_rqtpool_destroy(rdev); 888cfdda9d7SSteve Wise c4iw_destroy_resource(&rdev->resource); 889cfdda9d7SSteve Wise } 890cfdda9d7SSteve Wise 8919efe10a1SSteve Wise static void c4iw_dealloc(struct uld_ctx *ctx) 892cfdda9d7SSteve Wise { 8932f25e9a5SSteve Wise c4iw_rdev_close(&ctx->dev->rdev); 8942f25e9a5SSteve Wise idr_destroy(&ctx->dev->cqidr); 8952f25e9a5SSteve Wise idr_destroy(&ctx->dev->qpidr); 8962f25e9a5SSteve Wise idr_destroy(&ctx->dev->mmidr); 897793dad94SVipul Pandya idr_destroy(&ctx->dev->hwtid_idr); 898793dad94SVipul Pandya idr_destroy(&ctx->dev->stid_idr); 899793dad94SVipul Pandya idr_destroy(&ctx->dev->atid_idr); 900fa658a98SSteve Wise if (ctx->dev->rdev.bar2_kva) 901fa658a98SSteve Wise iounmap(ctx->dev->rdev.bar2_kva); 902fa658a98SSteve Wise if (ctx->dev->rdev.oc_mw_kva) 9032f25e9a5SSteve Wise iounmap(ctx->dev->rdev.oc_mw_kva); 9042f25e9a5SSteve Wise ib_dealloc_device(&ctx->dev->ibdev); 9052f25e9a5SSteve Wise ctx->dev = NULL; 906cfdda9d7SSteve Wise } 907cfdda9d7SSteve Wise 9089efe10a1SSteve Wise static void c4iw_remove(struct uld_ctx *ctx) 9099efe10a1SSteve Wise { 9109efe10a1SSteve Wise PDBG("%s c4iw_dev %p\n", __func__, ctx->dev); 9119efe10a1SSteve Wise c4iw_unregister_device(ctx->dev); 9129efe10a1SSteve Wise c4iw_dealloc(ctx); 9139efe10a1SSteve Wise } 9149efe10a1SSteve Wise 9159efe10a1SSteve Wise static int rdma_supported(const struct cxgb4_lld_info *infop) 9169efe10a1SSteve Wise { 9179efe10a1SSteve Wise return infop->vr->stag.size > 0 && infop->vr->pbl.size > 0 && 9189efe10a1SSteve Wise infop->vr->rq.size > 0 && infop->vr->qp.size > 0 && 919f079af7aSVipul Pandya infop->vr->cq.size > 0; 9209efe10a1SSteve Wise } 9219efe10a1SSteve Wise 922cfdda9d7SSteve Wise static struct c4iw_dev *c4iw_alloc(const struct cxgb4_lld_info *infop) 923cfdda9d7SSteve Wise { 924cfdda9d7SSteve Wise struct c4iw_dev *devp; 925cfdda9d7SSteve Wise int ret; 926cfdda9d7SSteve Wise 9279efe10a1SSteve Wise if (!rdma_supported(infop)) { 9289efe10a1SSteve Wise printk(KERN_INFO MOD "%s: RDMA not supported on this device.\n", 9299efe10a1SSteve Wise pci_name(infop->pdev)); 9309efe10a1SSteve Wise return ERR_PTR(-ENOSYS); 9319efe10a1SSteve Wise } 932f079af7aSVipul Pandya if (!ocqp_supported(infop)) 933f079af7aSVipul Pandya pr_info("%s: On-Chip Queues not supported on this device.\n", 934f079af7aSVipul Pandya pci_name(infop->pdev)); 93580ccdd60SVipul Pandya 936cfdda9d7SSteve Wise devp = (struct c4iw_dev *)ib_alloc_device(sizeof(*devp)); 937cfdda9d7SSteve Wise if (!devp) { 938cfdda9d7SSteve Wise printk(KERN_ERR MOD "Cannot allocate ib device\n"); 939bbe9a0a2SSteve Wise return ERR_PTR(-ENOMEM); 940cfdda9d7SSteve Wise } 941cfdda9d7SSteve Wise devp->rdev.lldi = *infop; 942cfdda9d7SSteve Wise 94304e10e21SHariprasad Shenai /* init various hw-queue params based on lld info */ 94404e10e21SHariprasad Shenai PDBG("%s: Ing. padding boundary is %d, egrsstatuspagesize = %d\n", 94504e10e21SHariprasad Shenai __func__, devp->rdev.lldi.sge_ingpadboundary, 94604e10e21SHariprasad Shenai devp->rdev.lldi.sge_egrstatuspagesize); 94704e10e21SHariprasad Shenai 94804e10e21SHariprasad Shenai devp->rdev.hw_queue.t4_eq_status_entries = 94904e10e21SHariprasad Shenai devp->rdev.lldi.sge_ingpadboundary > 64 ? 2 : 1; 95066eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_max_eq_size = 65520; 95166eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_max_iq_size = 65520; 95266eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_max_rq_size = 8192 - 95366eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_eq_status_entries - 1; 95404e10e21SHariprasad Shenai devp->rdev.hw_queue.t4_max_sq_size = 95566eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_max_eq_size - 95666eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_eq_status_entries - 1; 95704e10e21SHariprasad Shenai devp->rdev.hw_queue.t4_max_qp_depth = 95866eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_max_rq_size; 95904e10e21SHariprasad Shenai devp->rdev.hw_queue.t4_max_cq_depth = 96066eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_max_iq_size - 2; 96104e10e21SHariprasad Shenai devp->rdev.hw_queue.t4_stat_len = 96204e10e21SHariprasad Shenai devp->rdev.lldi.sge_egrstatuspagesize; 96304e10e21SHariprasad Shenai 964fa658a98SSteve Wise /* 965fa658a98SSteve Wise * For T5 devices, we map all of BAR2 with WC. 966fa658a98SSteve Wise * For T4 devices with onchip qp mem, we map only that part 967fa658a98SSteve Wise * of BAR2 with WC. 968fa658a98SSteve Wise */ 969fa658a98SSteve Wise devp->rdev.bar2_pa = pci_resource_start(devp->rdev.lldi.pdev, 2); 970fa658a98SSteve Wise if (is_t5(devp->rdev.lldi.adapter_type)) { 971fa658a98SSteve Wise devp->rdev.bar2_kva = ioremap_wc(devp->rdev.bar2_pa, 972fa658a98SSteve Wise pci_resource_len(devp->rdev.lldi.pdev, 2)); 973fa658a98SSteve Wise if (!devp->rdev.bar2_kva) { 974fa658a98SSteve Wise pr_err(MOD "Unable to ioremap BAR2\n"); 97565b302adSChristoph Jaeger ib_dealloc_device(&devp->ibdev); 976fa658a98SSteve Wise return ERR_PTR(-EINVAL); 977fa658a98SSteve Wise } 978fa658a98SSteve Wise } else if (ocqp_supported(infop)) { 979fa658a98SSteve Wise devp->rdev.oc_mw_pa = 980fa658a98SSteve Wise pci_resource_start(devp->rdev.lldi.pdev, 2) + 981fa658a98SSteve Wise pci_resource_len(devp->rdev.lldi.pdev, 2) - 982fa658a98SSteve Wise roundup_pow_of_two(devp->rdev.lldi.vr->ocq.size); 983c6d7b267SSteve Wise devp->rdev.oc_mw_kva = ioremap_wc(devp->rdev.oc_mw_pa, 984c6d7b267SSteve Wise devp->rdev.lldi.vr->ocq.size); 985fa658a98SSteve Wise if (!devp->rdev.oc_mw_kva) { 986fa658a98SSteve Wise pr_err(MOD "Unable to ioremap onchip mem\n"); 98765b302adSChristoph Jaeger ib_dealloc_device(&devp->ibdev); 988fa658a98SSteve Wise return ERR_PTR(-EINVAL); 989fa658a98SSteve Wise } 990fa658a98SSteve Wise } 991c6d7b267SSteve Wise 9922f25e9a5SSteve Wise PDBG(KERN_INFO MOD "ocq memory: " 993c6d7b267SSteve Wise "hw_start 0x%x size %u mw_pa 0x%lx mw_kva %p\n", 994c6d7b267SSteve Wise devp->rdev.lldi.vr->ocq.start, devp->rdev.lldi.vr->ocq.size, 995c6d7b267SSteve Wise devp->rdev.oc_mw_pa, devp->rdev.oc_mw_kva); 996c6d7b267SSteve Wise 997cfdda9d7SSteve Wise ret = c4iw_rdev_open(&devp->rdev); 998cfdda9d7SSteve Wise if (ret) { 999cfdda9d7SSteve Wise printk(KERN_ERR MOD "Unable to open CXIO rdev err %d\n", ret); 1000cfdda9d7SSteve Wise ib_dealloc_device(&devp->ibdev); 1001bbe9a0a2SSteve Wise return ERR_PTR(ret); 1002cfdda9d7SSteve Wise } 1003cfdda9d7SSteve Wise 1004cfdda9d7SSteve Wise idr_init(&devp->cqidr); 1005cfdda9d7SSteve Wise idr_init(&devp->qpidr); 1006cfdda9d7SSteve Wise idr_init(&devp->mmidr); 1007793dad94SVipul Pandya idr_init(&devp->hwtid_idr); 1008793dad94SVipul Pandya idr_init(&devp->stid_idr); 1009793dad94SVipul Pandya idr_init(&devp->atid_idr); 1010cfdda9d7SSteve Wise spin_lock_init(&devp->lock); 10118d81ef34SVipul Pandya mutex_init(&devp->rdev.stats.lock); 10122c974781SVipul Pandya mutex_init(&devp->db_mutex); 101305eb2389SSteve Wise INIT_LIST_HEAD(&devp->db_fc_list); 10144c2c5763SHariprasad Shenai devp->avail_ird = devp->rdev.lldi.max_ird_adapter; 1015cfdda9d7SSteve Wise 1016cfdda9d7SSteve Wise if (c4iw_debugfs_root) { 1017cfdda9d7SSteve Wise devp->debugfs_root = debugfs_create_dir( 1018cfdda9d7SSteve Wise pci_name(devp->rdev.lldi.pdev), 1019cfdda9d7SSteve Wise c4iw_debugfs_root); 1020cfdda9d7SSteve Wise setup_debugfs(devp); 1021cfdda9d7SSteve Wise } 10229eccfe10SSteve Wise 10239eccfe10SSteve Wise 1024cfdda9d7SSteve Wise return devp; 1025cfdda9d7SSteve Wise } 1026cfdda9d7SSteve Wise 1027cfdda9d7SSteve Wise static void *c4iw_uld_add(const struct cxgb4_lld_info *infop) 1028cfdda9d7SSteve Wise { 10292f25e9a5SSteve Wise struct uld_ctx *ctx; 1030cfdda9d7SSteve Wise static int vers_printed; 1031cfdda9d7SSteve Wise int i; 1032cfdda9d7SSteve Wise 1033cfdda9d7SSteve Wise if (!vers_printed++) 1034f079af7aSVipul Pandya pr_info("Chelsio T4/T5 RDMA Driver - version %s\n", 1035cfdda9d7SSteve Wise DRV_VERSION); 1036cfdda9d7SSteve Wise 10372f25e9a5SSteve Wise ctx = kzalloc(sizeof *ctx, GFP_KERNEL); 10382f25e9a5SSteve Wise if (!ctx) { 10392f25e9a5SSteve Wise ctx = ERR_PTR(-ENOMEM); 1040cfdda9d7SSteve Wise goto out; 10412f25e9a5SSteve Wise } 10422f25e9a5SSteve Wise ctx->lldi = *infop; 1043cfdda9d7SSteve Wise 1044cfdda9d7SSteve Wise PDBG("%s found device %s nchan %u nrxq %u ntxq %u nports %u\n", 10452f25e9a5SSteve Wise __func__, pci_name(ctx->lldi.pdev), 10462f25e9a5SSteve Wise ctx->lldi.nchan, ctx->lldi.nrxq, 10472f25e9a5SSteve Wise ctx->lldi.ntxq, ctx->lldi.nports); 1048cfdda9d7SSteve Wise 10492f25e9a5SSteve Wise mutex_lock(&dev_mutex); 10502f25e9a5SSteve Wise list_add_tail(&ctx->entry, &uld_ctx_list); 10512f25e9a5SSteve Wise mutex_unlock(&dev_mutex); 10522f25e9a5SSteve Wise 10532f25e9a5SSteve Wise for (i = 0; i < ctx->lldi.nrxq; i++) 10542f25e9a5SSteve Wise PDBG("rxqid[%u] %u\n", i, ctx->lldi.rxq_ids[i]); 1055cfdda9d7SSteve Wise out: 10562f25e9a5SSteve Wise return ctx; 1057cfdda9d7SSteve Wise } 1058cfdda9d7SSteve Wise 10591cab775cSVipul Pandya static inline struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl, 10601cab775cSVipul Pandya const __be64 *rsp, 10611cab775cSVipul Pandya u32 pktshift) 10621cab775cSVipul Pandya { 10631cab775cSVipul Pandya struct sk_buff *skb; 10641cab775cSVipul Pandya 10651cab775cSVipul Pandya /* 10661cab775cSVipul Pandya * Allocate space for cpl_pass_accept_req which will be synthesized by 10671cab775cSVipul Pandya * driver. Once the driver synthesizes the request the skb will go 10681cab775cSVipul Pandya * through the regular cpl_pass_accept_req processing. 10691cab775cSVipul Pandya * The math here assumes sizeof cpl_pass_accept_req >= sizeof 10701cab775cSVipul Pandya * cpl_rx_pkt. 10711cab775cSVipul Pandya */ 10721cab775cSVipul Pandya skb = alloc_skb(gl->tot_len + sizeof(struct cpl_pass_accept_req) + 10731cab775cSVipul Pandya sizeof(struct rss_header) - pktshift, GFP_ATOMIC); 10741cab775cSVipul Pandya if (unlikely(!skb)) 10751cab775cSVipul Pandya return NULL; 10761cab775cSVipul Pandya 10771cab775cSVipul Pandya __skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req) + 10781cab775cSVipul Pandya sizeof(struct rss_header) - pktshift); 10791cab775cSVipul Pandya 10801cab775cSVipul Pandya /* 10811cab775cSVipul Pandya * This skb will contain: 10821cab775cSVipul Pandya * rss_header from the rspq descriptor (1 flit) 10831cab775cSVipul Pandya * cpl_rx_pkt struct from the rspq descriptor (2 flits) 10841cab775cSVipul Pandya * space for the difference between the size of an 10851cab775cSVipul Pandya * rx_pkt and pass_accept_req cpl (1 flit) 10861cab775cSVipul Pandya * the packet data from the gl 10871cab775cSVipul Pandya */ 10881cab775cSVipul Pandya skb_copy_to_linear_data(skb, rsp, sizeof(struct cpl_pass_accept_req) + 10891cab775cSVipul Pandya sizeof(struct rss_header)); 10901cab775cSVipul Pandya skb_copy_to_linear_data_offset(skb, sizeof(struct rss_header) + 10911cab775cSVipul Pandya sizeof(struct cpl_pass_accept_req), 10921cab775cSVipul Pandya gl->va + pktshift, 10931cab775cSVipul Pandya gl->tot_len - pktshift); 10941cab775cSVipul Pandya return skb; 10951cab775cSVipul Pandya } 10961cab775cSVipul Pandya 10971cab775cSVipul Pandya static inline int recv_rx_pkt(struct c4iw_dev *dev, const struct pkt_gl *gl, 10981cab775cSVipul Pandya const __be64 *rsp) 10991cab775cSVipul Pandya { 11001cab775cSVipul Pandya unsigned int opcode = *(u8 *)rsp; 11011cab775cSVipul Pandya struct sk_buff *skb; 11021cab775cSVipul Pandya 11031cab775cSVipul Pandya if (opcode != CPL_RX_PKT) 11041cab775cSVipul Pandya goto out; 11051cab775cSVipul Pandya 11061cab775cSVipul Pandya skb = copy_gl_to_skb_pkt(gl , rsp, dev->rdev.lldi.sge_pktshift); 11071cab775cSVipul Pandya if (skb == NULL) 11081cab775cSVipul Pandya goto out; 11091cab775cSVipul Pandya 11101cab775cSVipul Pandya if (c4iw_handlers[opcode] == NULL) { 11111cab775cSVipul Pandya pr_info("%s no handler opcode 0x%x...\n", __func__, 11121cab775cSVipul Pandya opcode); 11131cab775cSVipul Pandya kfree_skb(skb); 11141cab775cSVipul Pandya goto out; 11151cab775cSVipul Pandya } 11161cab775cSVipul Pandya c4iw_handlers[opcode](dev, skb); 11171cab775cSVipul Pandya return 1; 11181cab775cSVipul Pandya out: 11191cab775cSVipul Pandya return 0; 11201cab775cSVipul Pandya } 11211cab775cSVipul Pandya 1122cfdda9d7SSteve Wise static int c4iw_uld_rx_handler(void *handle, const __be64 *rsp, 1123cfdda9d7SSteve Wise const struct pkt_gl *gl) 1124cfdda9d7SSteve Wise { 11252f25e9a5SSteve Wise struct uld_ctx *ctx = handle; 11262f25e9a5SSteve Wise struct c4iw_dev *dev = ctx->dev; 1127cfdda9d7SSteve Wise struct sk_buff *skb; 11281cab775cSVipul Pandya u8 opcode; 1129cfdda9d7SSteve Wise 1130cfdda9d7SSteve Wise if (gl == NULL) { 1131cfdda9d7SSteve Wise /* omit RSS and rsp_ctrl at end of descriptor */ 1132cfdda9d7SSteve Wise unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8; 1133cfdda9d7SSteve Wise 1134cfdda9d7SSteve Wise skb = alloc_skb(256, GFP_ATOMIC); 1135cfdda9d7SSteve Wise if (!skb) 1136cfdda9d7SSteve Wise goto nomem; 1137cfdda9d7SSteve Wise __skb_put(skb, len); 1138cfdda9d7SSteve Wise skb_copy_to_linear_data(skb, &rsp[1], len); 1139cfdda9d7SSteve Wise } else if (gl == CXGB4_MSG_AN) { 1140cfdda9d7SSteve Wise const struct rsp_ctrl *rc = (void *)rsp; 1141cfdda9d7SSteve Wise 1142cfdda9d7SSteve Wise u32 qid = be32_to_cpu(rc->pldbuflen_qid); 1143cfdda9d7SSteve Wise c4iw_ev_handler(dev, qid); 1144cfdda9d7SSteve Wise return 0; 11451cab775cSVipul Pandya } else if (unlikely(*(u8 *)rsp != *(u8 *)gl->va)) { 11461cab775cSVipul Pandya if (recv_rx_pkt(dev, gl, rsp)) 11471cab775cSVipul Pandya return 0; 11481cab775cSVipul Pandya 11491cab775cSVipul Pandya pr_info("%s: unexpected FL contents at %p, " \ 11501cab775cSVipul Pandya "RSS %#llx, FL %#llx, len %u\n", 11511cab775cSVipul Pandya pci_name(ctx->lldi.pdev), gl->va, 11521cab775cSVipul Pandya (unsigned long long)be64_to_cpu(*rsp), 1153ef5d6355SVipul Pandya (unsigned long long)be64_to_cpu( 1154ef5d6355SVipul Pandya *(__force __be64 *)gl->va), 11551cab775cSVipul Pandya gl->tot_len); 11561cab775cSVipul Pandya 11571cab775cSVipul Pandya return 0; 1158cfdda9d7SSteve Wise } else { 1159da411ba1SSteve Wise skb = cxgb4_pktgl_to_skb(gl, 128, 128); 1160cfdda9d7SSteve Wise if (unlikely(!skb)) 1161cfdda9d7SSteve Wise goto nomem; 1162cfdda9d7SSteve Wise } 1163cfdda9d7SSteve Wise 11641cab775cSVipul Pandya opcode = *(u8 *)rsp; 1165dbb084ccSSteve Wise if (c4iw_handlers[opcode]) { 1166cfdda9d7SSteve Wise c4iw_handlers[opcode](dev, skb); 1167dbb084ccSSteve Wise } else { 11681cab775cSVipul Pandya pr_info("%s no handler opcode 0x%x...\n", __func__, 1169cfdda9d7SSteve Wise opcode); 1170dbb084ccSSteve Wise kfree_skb(skb); 1171dbb084ccSSteve Wise } 1172cfdda9d7SSteve Wise 1173cfdda9d7SSteve Wise return 0; 1174cfdda9d7SSteve Wise nomem: 1175cfdda9d7SSteve Wise return -1; 1176cfdda9d7SSteve Wise } 1177cfdda9d7SSteve Wise 1178cfdda9d7SSteve Wise static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state) 1179cfdda9d7SSteve Wise { 11802f25e9a5SSteve Wise struct uld_ctx *ctx = handle; 11811c01c538SSteve Wise 1182cfdda9d7SSteve Wise PDBG("%s new_state %u\n", __func__, new_state); 11831c01c538SSteve Wise switch (new_state) { 11841c01c538SSteve Wise case CXGB4_STATE_UP: 11852f25e9a5SSteve Wise printk(KERN_INFO MOD "%s: Up\n", pci_name(ctx->lldi.pdev)); 11862f25e9a5SSteve Wise if (!ctx->dev) { 11879efe10a1SSteve Wise int ret; 11882f25e9a5SSteve Wise 11892f25e9a5SSteve Wise ctx->dev = c4iw_alloc(&ctx->lldi); 11909efe10a1SSteve Wise if (IS_ERR(ctx->dev)) { 11919efe10a1SSteve Wise printk(KERN_ERR MOD 11929efe10a1SSteve Wise "%s: initialization failed: %ld\n", 11939efe10a1SSteve Wise pci_name(ctx->lldi.pdev), 11949efe10a1SSteve Wise PTR_ERR(ctx->dev)); 11959efe10a1SSteve Wise ctx->dev = NULL; 11969efe10a1SSteve Wise break; 11979efe10a1SSteve Wise } 11982f25e9a5SSteve Wise ret = c4iw_register_device(ctx->dev); 11999efe10a1SSteve Wise if (ret) { 12001c01c538SSteve Wise printk(KERN_ERR MOD 12011c01c538SSteve Wise "%s: RDMA registration failed: %d\n", 12022f25e9a5SSteve Wise pci_name(ctx->lldi.pdev), ret); 12039efe10a1SSteve Wise c4iw_dealloc(ctx); 12049efe10a1SSteve Wise } 12051c01c538SSteve Wise } 12061c01c538SSteve Wise break; 12071c01c538SSteve Wise case CXGB4_STATE_DOWN: 12081c01c538SSteve Wise printk(KERN_INFO MOD "%s: Down\n", 12092f25e9a5SSteve Wise pci_name(ctx->lldi.pdev)); 12102f25e9a5SSteve Wise if (ctx->dev) 12112f25e9a5SSteve Wise c4iw_remove(ctx); 12121c01c538SSteve Wise break; 12131c01c538SSteve Wise case CXGB4_STATE_START_RECOVERY: 12141c01c538SSteve Wise printk(KERN_INFO MOD "%s: Fatal Error\n", 12152f25e9a5SSteve Wise pci_name(ctx->lldi.pdev)); 12162f25e9a5SSteve Wise if (ctx->dev) { 1217767fbe81SSteve Wise struct ib_event event; 1218767fbe81SSteve Wise 12192f25e9a5SSteve Wise ctx->dev->rdev.flags |= T4_FATAL_ERROR; 1220767fbe81SSteve Wise memset(&event, 0, sizeof event); 1221767fbe81SSteve Wise event.event = IB_EVENT_DEVICE_FATAL; 12222f25e9a5SSteve Wise event.device = &ctx->dev->ibdev; 1223767fbe81SSteve Wise ib_dispatch_event(&event); 12242f25e9a5SSteve Wise c4iw_remove(ctx); 1225767fbe81SSteve Wise } 12261c01c538SSteve Wise break; 12271c01c538SSteve Wise case CXGB4_STATE_DETACH: 12281c01c538SSteve Wise printk(KERN_INFO MOD "%s: Detach\n", 12292f25e9a5SSteve Wise pci_name(ctx->lldi.pdev)); 12302f25e9a5SSteve Wise if (ctx->dev) 12312f25e9a5SSteve Wise c4iw_remove(ctx); 12321c01c538SSteve Wise break; 12331c01c538SSteve Wise } 1234cfdda9d7SSteve Wise return 0; 1235cfdda9d7SSteve Wise } 1236cfdda9d7SSteve Wise 12372c974781SVipul Pandya static int disable_qp_db(int id, void *p, void *data) 12382c974781SVipul Pandya { 12392c974781SVipul Pandya struct c4iw_qp *qp = p; 12402c974781SVipul Pandya 12412c974781SVipul Pandya t4_disable_wq_db(&qp->wq); 12422c974781SVipul Pandya return 0; 12432c974781SVipul Pandya } 12442c974781SVipul Pandya 12452c974781SVipul Pandya static void stop_queues(struct uld_ctx *ctx) 12462c974781SVipul Pandya { 124705eb2389SSteve Wise unsigned long flags; 124805eb2389SSteve Wise 124905eb2389SSteve Wise spin_lock_irqsave(&ctx->dev->lock, flags); 1250422eea0aSVipul Pandya ctx->dev->rdev.stats.db_state_transitions++; 125105eb2389SSteve Wise ctx->dev->db_state = STOPPED; 125205eb2389SSteve Wise if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED) 12532c974781SVipul Pandya idr_for_each(&ctx->dev->qpidr, disable_qp_db, NULL); 125405eb2389SSteve Wise else 125505eb2389SSteve Wise ctx->dev->rdev.status_page->db_off = 1; 125605eb2389SSteve Wise spin_unlock_irqrestore(&ctx->dev->lock, flags); 12572c974781SVipul Pandya } 12582c974781SVipul Pandya 12592c974781SVipul Pandya static int enable_qp_db(int id, void *p, void *data) 12602c974781SVipul Pandya { 12612c974781SVipul Pandya struct c4iw_qp *qp = p; 12622c974781SVipul Pandya 12632c974781SVipul Pandya t4_enable_wq_db(&qp->wq); 12642c974781SVipul Pandya return 0; 12652c974781SVipul Pandya } 12662c974781SVipul Pandya 126705eb2389SSteve Wise static void resume_rc_qp(struct c4iw_qp *qp) 126805eb2389SSteve Wise { 126905eb2389SSteve Wise spin_lock(&qp->lock); 1270fa658a98SSteve Wise t4_ring_sq_db(&qp->wq, qp->wq.sq.wq_pidx_inc, 1271fa658a98SSteve Wise is_t5(qp->rhp->rdev.lldi.adapter_type), NULL); 127205eb2389SSteve Wise qp->wq.sq.wq_pidx_inc = 0; 1273fa658a98SSteve Wise t4_ring_rq_db(&qp->wq, qp->wq.rq.wq_pidx_inc, 1274fa658a98SSteve Wise is_t5(qp->rhp->rdev.lldi.adapter_type), NULL); 127505eb2389SSteve Wise qp->wq.rq.wq_pidx_inc = 0; 127605eb2389SSteve Wise spin_unlock(&qp->lock); 127705eb2389SSteve Wise } 127805eb2389SSteve Wise 127905eb2389SSteve Wise static void resume_a_chunk(struct uld_ctx *ctx) 128005eb2389SSteve Wise { 128105eb2389SSteve Wise int i; 128205eb2389SSteve Wise struct c4iw_qp *qp; 128305eb2389SSteve Wise 128405eb2389SSteve Wise for (i = 0; i < DB_FC_RESUME_SIZE; i++) { 128505eb2389SSteve Wise qp = list_first_entry(&ctx->dev->db_fc_list, struct c4iw_qp, 128605eb2389SSteve Wise db_fc_entry); 128705eb2389SSteve Wise list_del_init(&qp->db_fc_entry); 128805eb2389SSteve Wise resume_rc_qp(qp); 128905eb2389SSteve Wise if (list_empty(&ctx->dev->db_fc_list)) 129005eb2389SSteve Wise break; 129105eb2389SSteve Wise } 129205eb2389SSteve Wise } 129305eb2389SSteve Wise 12942c974781SVipul Pandya static void resume_queues(struct uld_ctx *ctx) 12952c974781SVipul Pandya { 12962c974781SVipul Pandya spin_lock_irq(&ctx->dev->lock); 129705eb2389SSteve Wise if (ctx->dev->db_state != STOPPED) 129805eb2389SSteve Wise goto out; 129905eb2389SSteve Wise ctx->dev->db_state = FLOW_CONTROL; 130005eb2389SSteve Wise while (1) { 130105eb2389SSteve Wise if (list_empty(&ctx->dev->db_fc_list)) { 130205eb2389SSteve Wise WARN_ON(ctx->dev->db_state != FLOW_CONTROL); 1303422eea0aSVipul Pandya ctx->dev->db_state = NORMAL; 1304422eea0aSVipul Pandya ctx->dev->rdev.stats.db_state_transitions++; 130505eb2389SSteve Wise if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED) { 130605eb2389SSteve Wise idr_for_each(&ctx->dev->qpidr, enable_qp_db, 130705eb2389SSteve Wise NULL); 130805eb2389SSteve Wise } else { 130905eb2389SSteve Wise ctx->dev->rdev.status_page->db_off = 0; 1310422eea0aSVipul Pandya } 131105eb2389SSteve Wise break; 131205eb2389SSteve Wise } else { 131305eb2389SSteve Wise if (cxgb4_dbfifo_count(ctx->dev->rdev.lldi.ports[0], 1) 131405eb2389SSteve Wise < (ctx->dev->rdev.lldi.dbfifo_int_thresh << 131505eb2389SSteve Wise DB_FC_DRAIN_THRESH)) { 131605eb2389SSteve Wise resume_a_chunk(ctx); 131705eb2389SSteve Wise } 131805eb2389SSteve Wise if (!list_empty(&ctx->dev->db_fc_list)) { 131905eb2389SSteve Wise spin_unlock_irq(&ctx->dev->lock); 132005eb2389SSteve Wise if (DB_FC_RESUME_DELAY) { 132105eb2389SSteve Wise set_current_state(TASK_UNINTERRUPTIBLE); 132205eb2389SSteve Wise schedule_timeout(DB_FC_RESUME_DELAY); 132305eb2389SSteve Wise } 132405eb2389SSteve Wise spin_lock_irq(&ctx->dev->lock); 132505eb2389SSteve Wise if (ctx->dev->db_state != FLOW_CONTROL) 132605eb2389SSteve Wise break; 132705eb2389SSteve Wise } 132805eb2389SSteve Wise } 132905eb2389SSteve Wise } 133005eb2389SSteve Wise out: 133105eb2389SSteve Wise if (ctx->dev->db_state != NORMAL) 133205eb2389SSteve Wise ctx->dev->rdev.stats.db_fc_interruptions++; 1333422eea0aSVipul Pandya spin_unlock_irq(&ctx->dev->lock); 1334422eea0aSVipul Pandya } 1335422eea0aSVipul Pandya 1336422eea0aSVipul Pandya struct qp_list { 1337422eea0aSVipul Pandya unsigned idx; 1338422eea0aSVipul Pandya struct c4iw_qp **qps; 1339422eea0aSVipul Pandya }; 1340422eea0aSVipul Pandya 1341422eea0aSVipul Pandya static int add_and_ref_qp(int id, void *p, void *data) 1342422eea0aSVipul Pandya { 1343422eea0aSVipul Pandya struct qp_list *qp_listp = data; 1344422eea0aSVipul Pandya struct c4iw_qp *qp = p; 1345422eea0aSVipul Pandya 1346422eea0aSVipul Pandya c4iw_qp_add_ref(&qp->ibqp); 1347422eea0aSVipul Pandya qp_listp->qps[qp_listp->idx++] = qp; 1348422eea0aSVipul Pandya return 0; 1349422eea0aSVipul Pandya } 1350422eea0aSVipul Pandya 1351422eea0aSVipul Pandya static int count_qps(int id, void *p, void *data) 1352422eea0aSVipul Pandya { 1353422eea0aSVipul Pandya unsigned *countp = data; 1354422eea0aSVipul Pandya (*countp)++; 1355422eea0aSVipul Pandya return 0; 1356422eea0aSVipul Pandya } 1357422eea0aSVipul Pandya 135805eb2389SSteve Wise static void deref_qps(struct qp_list *qp_list) 1359422eea0aSVipul Pandya { 1360422eea0aSVipul Pandya int idx; 1361422eea0aSVipul Pandya 136205eb2389SSteve Wise for (idx = 0; idx < qp_list->idx; idx++) 136305eb2389SSteve Wise c4iw_qp_rem_ref(&qp_list->qps[idx]->ibqp); 1364422eea0aSVipul Pandya } 1365422eea0aSVipul Pandya 1366422eea0aSVipul Pandya static void recover_lost_dbs(struct uld_ctx *ctx, struct qp_list *qp_list) 1367422eea0aSVipul Pandya { 1368422eea0aSVipul Pandya int idx; 1369422eea0aSVipul Pandya int ret; 1370422eea0aSVipul Pandya 1371422eea0aSVipul Pandya for (idx = 0; idx < qp_list->idx; idx++) { 1372422eea0aSVipul Pandya struct c4iw_qp *qp = qp_list->qps[idx]; 1373422eea0aSVipul Pandya 137405eb2389SSteve Wise spin_lock_irq(&qp->rhp->lock); 137505eb2389SSteve Wise spin_lock(&qp->lock); 1376422eea0aSVipul Pandya ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0], 1377422eea0aSVipul Pandya qp->wq.sq.qid, 1378422eea0aSVipul Pandya t4_sq_host_wq_pidx(&qp->wq), 1379422eea0aSVipul Pandya t4_sq_wq_size(&qp->wq)); 1380422eea0aSVipul Pandya if (ret) { 138105eb2389SSteve Wise pr_err(KERN_ERR MOD "%s: Fatal error - " 1382422eea0aSVipul Pandya "DB overflow recovery failed - " 1383422eea0aSVipul Pandya "error syncing SQ qid %u\n", 1384422eea0aSVipul Pandya pci_name(ctx->lldi.pdev), qp->wq.sq.qid); 138505eb2389SSteve Wise spin_unlock(&qp->lock); 138605eb2389SSteve Wise spin_unlock_irq(&qp->rhp->lock); 1387422eea0aSVipul Pandya return; 1388422eea0aSVipul Pandya } 138905eb2389SSteve Wise qp->wq.sq.wq_pidx_inc = 0; 1390422eea0aSVipul Pandya 1391422eea0aSVipul Pandya ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0], 1392422eea0aSVipul Pandya qp->wq.rq.qid, 1393422eea0aSVipul Pandya t4_rq_host_wq_pidx(&qp->wq), 1394422eea0aSVipul Pandya t4_rq_wq_size(&qp->wq)); 1395422eea0aSVipul Pandya 1396422eea0aSVipul Pandya if (ret) { 139705eb2389SSteve Wise pr_err(KERN_ERR MOD "%s: Fatal error - " 1398422eea0aSVipul Pandya "DB overflow recovery failed - " 1399422eea0aSVipul Pandya "error syncing RQ qid %u\n", 1400422eea0aSVipul Pandya pci_name(ctx->lldi.pdev), qp->wq.rq.qid); 140105eb2389SSteve Wise spin_unlock(&qp->lock); 140205eb2389SSteve Wise spin_unlock_irq(&qp->rhp->lock); 1403422eea0aSVipul Pandya return; 1404422eea0aSVipul Pandya } 140505eb2389SSteve Wise qp->wq.rq.wq_pidx_inc = 0; 140605eb2389SSteve Wise spin_unlock(&qp->lock); 140705eb2389SSteve Wise spin_unlock_irq(&qp->rhp->lock); 1408422eea0aSVipul Pandya 1409422eea0aSVipul Pandya /* Wait for the dbfifo to drain */ 1410422eea0aSVipul Pandya while (cxgb4_dbfifo_count(qp->rhp->rdev.lldi.ports[0], 1) > 0) { 1411422eea0aSVipul Pandya set_current_state(TASK_UNINTERRUPTIBLE); 1412422eea0aSVipul Pandya schedule_timeout(usecs_to_jiffies(10)); 1413422eea0aSVipul Pandya } 1414422eea0aSVipul Pandya } 1415422eea0aSVipul Pandya } 1416422eea0aSVipul Pandya 1417422eea0aSVipul Pandya static void recover_queues(struct uld_ctx *ctx) 1418422eea0aSVipul Pandya { 1419422eea0aSVipul Pandya int count = 0; 1420422eea0aSVipul Pandya struct qp_list qp_list; 1421422eea0aSVipul Pandya int ret; 1422422eea0aSVipul Pandya 1423422eea0aSVipul Pandya /* slow everybody down */ 1424422eea0aSVipul Pandya set_current_state(TASK_UNINTERRUPTIBLE); 1425422eea0aSVipul Pandya schedule_timeout(usecs_to_jiffies(1000)); 1426422eea0aSVipul Pandya 1427422eea0aSVipul Pandya /* flush the SGE contexts */ 1428422eea0aSVipul Pandya ret = cxgb4_flush_eq_cache(ctx->dev->rdev.lldi.ports[0]); 1429422eea0aSVipul Pandya if (ret) { 1430422eea0aSVipul Pandya printk(KERN_ERR MOD "%s: Fatal error - DB overflow recovery failed\n", 1431422eea0aSVipul Pandya pci_name(ctx->lldi.pdev)); 143205eb2389SSteve Wise return; 1433422eea0aSVipul Pandya } 1434422eea0aSVipul Pandya 1435422eea0aSVipul Pandya /* Count active queues so we can build a list of queues to recover */ 1436422eea0aSVipul Pandya spin_lock_irq(&ctx->dev->lock); 143705eb2389SSteve Wise WARN_ON(ctx->dev->db_state != STOPPED); 143805eb2389SSteve Wise ctx->dev->db_state = RECOVERY; 1439422eea0aSVipul Pandya idr_for_each(&ctx->dev->qpidr, count_qps, &count); 1440422eea0aSVipul Pandya 1441422eea0aSVipul Pandya qp_list.qps = kzalloc(count * sizeof *qp_list.qps, GFP_ATOMIC); 1442422eea0aSVipul Pandya if (!qp_list.qps) { 1443422eea0aSVipul Pandya printk(KERN_ERR MOD "%s: Fatal error - DB overflow recovery failed\n", 1444422eea0aSVipul Pandya pci_name(ctx->lldi.pdev)); 1445422eea0aSVipul Pandya spin_unlock_irq(&ctx->dev->lock); 144605eb2389SSteve Wise return; 1447422eea0aSVipul Pandya } 1448422eea0aSVipul Pandya qp_list.idx = 0; 1449422eea0aSVipul Pandya 1450422eea0aSVipul Pandya /* add and ref each qp so it doesn't get freed */ 1451422eea0aSVipul Pandya idr_for_each(&ctx->dev->qpidr, add_and_ref_qp, &qp_list); 1452422eea0aSVipul Pandya 1453422eea0aSVipul Pandya spin_unlock_irq(&ctx->dev->lock); 1454422eea0aSVipul Pandya 1455422eea0aSVipul Pandya /* now traverse the list in a safe context to recover the db state*/ 1456422eea0aSVipul Pandya recover_lost_dbs(ctx, &qp_list); 1457422eea0aSVipul Pandya 1458422eea0aSVipul Pandya /* we're almost done! deref the qps and clean up */ 145905eb2389SSteve Wise deref_qps(&qp_list); 1460422eea0aSVipul Pandya kfree(qp_list.qps); 1461422eea0aSVipul Pandya 1462422eea0aSVipul Pandya spin_lock_irq(&ctx->dev->lock); 146305eb2389SSteve Wise WARN_ON(ctx->dev->db_state != RECOVERY); 146405eb2389SSteve Wise ctx->dev->db_state = STOPPED; 14652c974781SVipul Pandya spin_unlock_irq(&ctx->dev->lock); 14662c974781SVipul Pandya } 14672c974781SVipul Pandya 14682c974781SVipul Pandya static int c4iw_uld_control(void *handle, enum cxgb4_control control, ...) 14692c974781SVipul Pandya { 14702c974781SVipul Pandya struct uld_ctx *ctx = handle; 14712c974781SVipul Pandya 14722c974781SVipul Pandya switch (control) { 14732c974781SVipul Pandya case CXGB4_CONTROL_DB_FULL: 14742c974781SVipul Pandya stop_queues(ctx); 14752c974781SVipul Pandya ctx->dev->rdev.stats.db_full++; 14762c974781SVipul Pandya break; 14772c974781SVipul Pandya case CXGB4_CONTROL_DB_EMPTY: 14782c974781SVipul Pandya resume_queues(ctx); 14792c974781SVipul Pandya mutex_lock(&ctx->dev->rdev.stats.lock); 14802c974781SVipul Pandya ctx->dev->rdev.stats.db_empty++; 14812c974781SVipul Pandya mutex_unlock(&ctx->dev->rdev.stats.lock); 14822c974781SVipul Pandya break; 14832c974781SVipul Pandya case CXGB4_CONTROL_DB_DROP: 1484422eea0aSVipul Pandya recover_queues(ctx); 14852c974781SVipul Pandya mutex_lock(&ctx->dev->rdev.stats.lock); 14862c974781SVipul Pandya ctx->dev->rdev.stats.db_drop++; 14872c974781SVipul Pandya mutex_unlock(&ctx->dev->rdev.stats.lock); 14882c974781SVipul Pandya break; 14892c974781SVipul Pandya default: 14902c974781SVipul Pandya printk(KERN_WARNING MOD "%s: unknown control cmd %u\n", 14912c974781SVipul Pandya pci_name(ctx->lldi.pdev), control); 14922c974781SVipul Pandya break; 14932c974781SVipul Pandya } 14942c974781SVipul Pandya return 0; 14952c974781SVipul Pandya } 14962c974781SVipul Pandya 1497cfdda9d7SSteve Wise static struct cxgb4_uld_info c4iw_uld_info = { 1498cfdda9d7SSteve Wise .name = DRV_NAME, 1499cfdda9d7SSteve Wise .add = c4iw_uld_add, 1500cfdda9d7SSteve Wise .rx_handler = c4iw_uld_rx_handler, 1501cfdda9d7SSteve Wise .state_change = c4iw_uld_state_change, 15022c974781SVipul Pandya .control = c4iw_uld_control, 1503cfdda9d7SSteve Wise }; 1504cfdda9d7SSteve Wise 1505cfdda9d7SSteve Wise static int __init c4iw_init_module(void) 1506cfdda9d7SSteve Wise { 1507cfdda9d7SSteve Wise int err; 1508cfdda9d7SSteve Wise 1509cfdda9d7SSteve Wise err = c4iw_cm_init(); 1510cfdda9d7SSteve Wise if (err) 1511cfdda9d7SSteve Wise return err; 1512cfdda9d7SSteve Wise 1513cfdda9d7SSteve Wise c4iw_debugfs_root = debugfs_create_dir(DRV_NAME, NULL); 1514cfdda9d7SSteve Wise if (!c4iw_debugfs_root) 1515cfdda9d7SSteve Wise printk(KERN_WARNING MOD 1516cfdda9d7SSteve Wise "could not create debugfs entry, continuing\n"); 1517cfdda9d7SSteve Wise 15189eccfe10SSteve Wise if (ibnl_add_client(RDMA_NL_C4IW, RDMA_NL_IWPM_NUM_OPS, 15199eccfe10SSteve Wise c4iw_nl_cb_table)) 15209eccfe10SSteve Wise pr_err("%s[%u]: Failed to add netlink callback\n" 15219eccfe10SSteve Wise , __func__, __LINE__); 15229eccfe10SSteve Wise 152346c1376dSSteve Wise err = iwpm_init(RDMA_NL_C4IW); 152446c1376dSSteve Wise if (err) { 152546c1376dSSteve Wise pr_err("port mapper initialization failed with %d\n", err); 152646c1376dSSteve Wise ibnl_remove_client(RDMA_NL_C4IW); 152746c1376dSSteve Wise c4iw_cm_term(); 152846c1376dSSteve Wise debugfs_remove_recursive(c4iw_debugfs_root); 152946c1376dSSteve Wise return err; 153046c1376dSSteve Wise } 153146c1376dSSteve Wise 1532cfdda9d7SSteve Wise cxgb4_register_uld(CXGB4_ULD_RDMA, &c4iw_uld_info); 1533cfdda9d7SSteve Wise 1534cfdda9d7SSteve Wise return 0; 1535cfdda9d7SSteve Wise } 1536cfdda9d7SSteve Wise 1537cfdda9d7SSteve Wise static void __exit c4iw_exit_module(void) 1538cfdda9d7SSteve Wise { 15392f25e9a5SSteve Wise struct uld_ctx *ctx, *tmp; 1540cfdda9d7SSteve Wise 1541cfdda9d7SSteve Wise mutex_lock(&dev_mutex); 15422f25e9a5SSteve Wise list_for_each_entry_safe(ctx, tmp, &uld_ctx_list, entry) { 15432f25e9a5SSteve Wise if (ctx->dev) 15442f25e9a5SSteve Wise c4iw_remove(ctx); 15452f25e9a5SSteve Wise kfree(ctx); 1546cfdda9d7SSteve Wise } 1547cfdda9d7SSteve Wise mutex_unlock(&dev_mutex); 1548fd388ce6SSteve Wise cxgb4_unregister_uld(CXGB4_ULD_RDMA); 154946c1376dSSteve Wise iwpm_exit(RDMA_NL_C4IW); 15509eccfe10SSteve Wise ibnl_remove_client(RDMA_NL_C4IW); 1551cfdda9d7SSteve Wise c4iw_cm_term(); 1552cfdda9d7SSteve Wise debugfs_remove_recursive(c4iw_debugfs_root); 1553cfdda9d7SSteve Wise } 1554cfdda9d7SSteve Wise 1555cfdda9d7SSteve Wise module_init(c4iw_init_module); 1556cfdda9d7SSteve Wise module_exit(c4iw_exit_module); 1557