1cfdda9d7SSteve Wise /* 2cfdda9d7SSteve Wise * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. 3cfdda9d7SSteve Wise * 4cfdda9d7SSteve Wise * This software is available to you under a choice of one of two 5cfdda9d7SSteve Wise * licenses. You may choose to be licensed under the terms of the GNU 6cfdda9d7SSteve Wise * General Public License (GPL) Version 2, available from the file 7cfdda9d7SSteve Wise * COPYING in the main directory of this source tree, or the 8cfdda9d7SSteve Wise * OpenIB.org BSD license below: 9cfdda9d7SSteve Wise * 10cfdda9d7SSteve Wise * Redistribution and use in source and binary forms, with or 11cfdda9d7SSteve Wise * without modification, are permitted provided that the following 12cfdda9d7SSteve Wise * conditions are met: 13cfdda9d7SSteve Wise * 14cfdda9d7SSteve Wise * - Redistributions of source code must retain the above 15cfdda9d7SSteve Wise * copyright notice, this list of conditions and the following 16cfdda9d7SSteve Wise * disclaimer. 17cfdda9d7SSteve Wise * 18cfdda9d7SSteve Wise * - Redistributions in binary form must reproduce the above 19cfdda9d7SSteve Wise * copyright notice, this list of conditions and the following 20cfdda9d7SSteve Wise * disclaimer in the documentation and/or other materials 21cfdda9d7SSteve Wise * provided with the distribution. 22cfdda9d7SSteve Wise * 23cfdda9d7SSteve Wise * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24cfdda9d7SSteve Wise * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25cfdda9d7SSteve Wise * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26cfdda9d7SSteve Wise * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27cfdda9d7SSteve Wise * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28cfdda9d7SSteve Wise * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29cfdda9d7SSteve Wise * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30cfdda9d7SSteve Wise * SOFTWARE. 31cfdda9d7SSteve Wise */ 32cfdda9d7SSteve Wise #include <linux/module.h> 33cfdda9d7SSteve Wise #include <linux/moduleparam.h> 34cfdda9d7SSteve Wise #include <linux/debugfs.h> 35e572568fSVipul Pandya #include <linux/vmalloc.h> 36da388973SHariprasad Shenai #include <linux/math64.h> 37cfdda9d7SSteve Wise 38cfdda9d7SSteve Wise #include <rdma/ib_verbs.h> 39cfdda9d7SSteve Wise 40cfdda9d7SSteve Wise #include "iw_cxgb4.h" 41cfdda9d7SSteve Wise 42cfdda9d7SSteve Wise #define DRV_VERSION "0.1" 43cfdda9d7SSteve Wise 44cfdda9d7SSteve Wise MODULE_AUTHOR("Steve Wise"); 45f079af7aSVipul Pandya MODULE_DESCRIPTION("Chelsio T4/T5 RDMA Driver"); 46cfdda9d7SSteve Wise MODULE_LICENSE("Dual BSD/GPL"); 47cfdda9d7SSteve Wise 4880ccdd60SVipul Pandya static int allow_db_fc_on_t5; 4980ccdd60SVipul Pandya module_param(allow_db_fc_on_t5, int, 0644); 5080ccdd60SVipul Pandya MODULE_PARM_DESC(allow_db_fc_on_t5, 5180ccdd60SVipul Pandya "Allow DB Flow Control on T5 (default = 0)"); 5280ccdd60SVipul Pandya 5380ccdd60SVipul Pandya static int allow_db_coalescing_on_t5; 5480ccdd60SVipul Pandya module_param(allow_db_coalescing_on_t5, int, 0644); 5580ccdd60SVipul Pandya MODULE_PARM_DESC(allow_db_coalescing_on_t5, 5680ccdd60SVipul Pandya "Allow DB Coalescing on T5 (default = 0)"); 5780ccdd60SVipul Pandya 587730b4c7SHariprasad Shenai int c4iw_wr_log = 0; 597730b4c7SHariprasad Shenai module_param(c4iw_wr_log, int, 0444); 607730b4c7SHariprasad Shenai MODULE_PARM_DESC(c4iw_wr_log, "Enables logging of work request timing data."); 617730b4c7SHariprasad Shenai 6265d4c01aSSteve Wise static int c4iw_wr_log_size_order = 12; 637730b4c7SHariprasad Shenai module_param(c4iw_wr_log_size_order, int, 0444); 647730b4c7SHariprasad Shenai MODULE_PARM_DESC(c4iw_wr_log_size_order, 657730b4c7SHariprasad Shenai "Number of entries (log2) in the work request timing log."); 667730b4c7SHariprasad Shenai 672f25e9a5SSteve Wise static LIST_HEAD(uld_ctx_list); 68cfdda9d7SSteve Wise static DEFINE_MUTEX(dev_mutex); 690cb65d42SColin Ian King static struct workqueue_struct *reg_workq; 70cfdda9d7SSteve Wise 7105eb2389SSteve Wise #define DB_FC_RESUME_SIZE 64 7205eb2389SSteve Wise #define DB_FC_RESUME_DELAY 1 7305eb2389SSteve Wise #define DB_FC_DRAIN_THRESH 0 7405eb2389SSteve Wise 75cfdda9d7SSteve Wise static struct dentry *c4iw_debugfs_root; 76cfdda9d7SSteve Wise 779e8d1fa3SSteve Wise struct c4iw_debugfs_data { 78cfdda9d7SSteve Wise struct c4iw_dev *devp; 79cfdda9d7SSteve Wise char *buf; 80cfdda9d7SSteve Wise int bufsize; 81cfdda9d7SSteve Wise int pos; 82cfdda9d7SSteve Wise }; 83cfdda9d7SSteve Wise 849e8d1fa3SSteve Wise static int count_idrs(int id, void *p, void *data) 85cfdda9d7SSteve Wise { 86cfdda9d7SSteve Wise int *countp = data; 87cfdda9d7SSteve Wise 88cfdda9d7SSteve Wise *countp = *countp + 1; 89cfdda9d7SSteve Wise return 0; 90cfdda9d7SSteve Wise } 91cfdda9d7SSteve Wise 929e8d1fa3SSteve Wise static ssize_t debugfs_read(struct file *file, char __user *buf, size_t count, 939e8d1fa3SSteve Wise loff_t *ppos) 949e8d1fa3SSteve Wise { 959e8d1fa3SSteve Wise struct c4iw_debugfs_data *d = file->private_data; 969e8d1fa3SSteve Wise 973160977aSSteve Wise return simple_read_from_buffer(buf, count, ppos, d->buf, d->pos); 989e8d1fa3SSteve Wise } 999e8d1fa3SSteve Wise 1007730b4c7SHariprasad Shenai void c4iw_log_wr_stats(struct t4_wq *wq, struct t4_cqe *cqe) 1017730b4c7SHariprasad Shenai { 1027730b4c7SHariprasad Shenai struct wr_log_entry le; 1037730b4c7SHariprasad Shenai int idx; 1047730b4c7SHariprasad Shenai 1057730b4c7SHariprasad Shenai if (!wq->rdev->wr_log) 1067730b4c7SHariprasad Shenai return; 1077730b4c7SHariprasad Shenai 1087730b4c7SHariprasad Shenai idx = (atomic_inc_return(&wq->rdev->wr_log_idx) - 1) & 1097730b4c7SHariprasad Shenai (wq->rdev->wr_log_size - 1); 1107730b4c7SHariprasad Shenai le.poll_sge_ts = cxgb4_read_sge_timestamp(wq->rdev->lldi.ports[0]); 111f8109d9eSArnd Bergmann le.poll_host_time = ktime_get(); 1127730b4c7SHariprasad Shenai le.valid = 1; 1137730b4c7SHariprasad Shenai le.cqe_sge_ts = CQE_TS(cqe); 1147730b4c7SHariprasad Shenai if (SQ_TYPE(cqe)) { 1157730b4c7SHariprasad Shenai le.qid = wq->sq.qid; 1167730b4c7SHariprasad Shenai le.opcode = CQE_OPCODE(cqe); 117f8109d9eSArnd Bergmann le.post_host_time = wq->sq.sw_sq[wq->sq.cidx].host_time; 1187730b4c7SHariprasad Shenai le.post_sge_ts = wq->sq.sw_sq[wq->sq.cidx].sge_ts; 1197730b4c7SHariprasad Shenai le.wr_id = CQE_WRID_SQ_IDX(cqe); 1207730b4c7SHariprasad Shenai } else { 1217730b4c7SHariprasad Shenai le.qid = wq->rq.qid; 1227730b4c7SHariprasad Shenai le.opcode = FW_RI_RECEIVE; 123f8109d9eSArnd Bergmann le.post_host_time = wq->rq.sw_rq[wq->rq.cidx].host_time; 1247730b4c7SHariprasad Shenai le.post_sge_ts = wq->rq.sw_rq[wq->rq.cidx].sge_ts; 1257730b4c7SHariprasad Shenai le.wr_id = CQE_WRID_MSN(cqe); 1267730b4c7SHariprasad Shenai } 1277730b4c7SHariprasad Shenai wq->rdev->wr_log[idx] = le; 1287730b4c7SHariprasad Shenai } 1297730b4c7SHariprasad Shenai 1307730b4c7SHariprasad Shenai static int wr_log_show(struct seq_file *seq, void *v) 1317730b4c7SHariprasad Shenai { 1327730b4c7SHariprasad Shenai struct c4iw_dev *dev = seq->private; 133f8109d9eSArnd Bergmann ktime_t prev_time; 1347730b4c7SHariprasad Shenai struct wr_log_entry *lep; 135f8109d9eSArnd Bergmann int prev_time_set = 0; 1367730b4c7SHariprasad Shenai int idx, end; 1377730b4c7SHariprasad Shenai 1386198dd8dSHariprasad S #define ts2ns(ts) div64_u64((ts) * dev->rdev.lldi.cclk_ps, 1000) 1397730b4c7SHariprasad Shenai 1407730b4c7SHariprasad Shenai idx = atomic_read(&dev->rdev.wr_log_idx) & 1417730b4c7SHariprasad Shenai (dev->rdev.wr_log_size - 1); 1427730b4c7SHariprasad Shenai end = idx - 1; 1437730b4c7SHariprasad Shenai if (end < 0) 1447730b4c7SHariprasad Shenai end = dev->rdev.wr_log_size - 1; 1457730b4c7SHariprasad Shenai lep = &dev->rdev.wr_log[idx]; 1467730b4c7SHariprasad Shenai while (idx != end) { 1477730b4c7SHariprasad Shenai if (lep->valid) { 148f8109d9eSArnd Bergmann if (!prev_time_set) { 149f8109d9eSArnd Bergmann prev_time_set = 1; 150f8109d9eSArnd Bergmann prev_time = lep->poll_host_time; 1517730b4c7SHariprasad Shenai } 152f8109d9eSArnd Bergmann seq_printf(seq, "%04u: nsec %llu qid %u opcode " 153f8109d9eSArnd Bergmann "%u %s 0x%x host_wr_delta nsec %llu " 1547730b4c7SHariprasad Shenai "post_sge_ts 0x%llx cqe_sge_ts 0x%llx " 1557730b4c7SHariprasad Shenai "poll_sge_ts 0x%llx post_poll_delta_ns %llu " 1567730b4c7SHariprasad Shenai "cqe_poll_delta_ns %llu\n", 1577730b4c7SHariprasad Shenai idx, 158f8109d9eSArnd Bergmann ktime_to_ns(ktime_sub(lep->poll_host_time, 159f8109d9eSArnd Bergmann prev_time)), 1607730b4c7SHariprasad Shenai lep->qid, lep->opcode, 1617730b4c7SHariprasad Shenai lep->opcode == FW_RI_RECEIVE ? 1627730b4c7SHariprasad Shenai "msn" : "wrid", 1637730b4c7SHariprasad Shenai lep->wr_id, 164f8109d9eSArnd Bergmann ktime_to_ns(ktime_sub(lep->poll_host_time, 165f8109d9eSArnd Bergmann lep->post_host_time)), 1667730b4c7SHariprasad Shenai lep->post_sge_ts, lep->cqe_sge_ts, 1677730b4c7SHariprasad Shenai lep->poll_sge_ts, 1687730b4c7SHariprasad Shenai ts2ns(lep->poll_sge_ts - lep->post_sge_ts), 1697730b4c7SHariprasad Shenai ts2ns(lep->poll_sge_ts - lep->cqe_sge_ts)); 170f8109d9eSArnd Bergmann prev_time = lep->poll_host_time; 1717730b4c7SHariprasad Shenai } 1727730b4c7SHariprasad Shenai idx++; 1737730b4c7SHariprasad Shenai if (idx > (dev->rdev.wr_log_size - 1)) 1747730b4c7SHariprasad Shenai idx = 0; 1757730b4c7SHariprasad Shenai lep = &dev->rdev.wr_log[idx]; 1767730b4c7SHariprasad Shenai } 1777730b4c7SHariprasad Shenai #undef ts2ns 1787730b4c7SHariprasad Shenai return 0; 1797730b4c7SHariprasad Shenai } 1807730b4c7SHariprasad Shenai 1817730b4c7SHariprasad Shenai static int wr_log_open(struct inode *inode, struct file *file) 1827730b4c7SHariprasad Shenai { 1837730b4c7SHariprasad Shenai return single_open(file, wr_log_show, inode->i_private); 1847730b4c7SHariprasad Shenai } 1857730b4c7SHariprasad Shenai 1867730b4c7SHariprasad Shenai static ssize_t wr_log_clear(struct file *file, const char __user *buf, 1877730b4c7SHariprasad Shenai size_t count, loff_t *pos) 1887730b4c7SHariprasad Shenai { 1897730b4c7SHariprasad Shenai struct c4iw_dev *dev = ((struct seq_file *)file->private_data)->private; 1907730b4c7SHariprasad Shenai int i; 1917730b4c7SHariprasad Shenai 1927730b4c7SHariprasad Shenai if (dev->rdev.wr_log) 1937730b4c7SHariprasad Shenai for (i = 0; i < dev->rdev.wr_log_size; i++) 1947730b4c7SHariprasad Shenai dev->rdev.wr_log[i].valid = 0; 1957730b4c7SHariprasad Shenai return count; 1967730b4c7SHariprasad Shenai } 1977730b4c7SHariprasad Shenai 1987730b4c7SHariprasad Shenai static const struct file_operations wr_log_debugfs_fops = { 1997730b4c7SHariprasad Shenai .owner = THIS_MODULE, 2007730b4c7SHariprasad Shenai .open = wr_log_open, 2017730b4c7SHariprasad Shenai .release = single_release, 2027730b4c7SHariprasad Shenai .read = seq_read, 2037730b4c7SHariprasad Shenai .llseek = seq_lseek, 2047730b4c7SHariprasad Shenai .write = wr_log_clear, 2057730b4c7SHariprasad Shenai }; 2067730b4c7SHariprasad Shenai 207bab572f1SGanesh Goudar static struct sockaddr_in zero_sin = { 208bab572f1SGanesh Goudar .sin_family = AF_INET, 209bab572f1SGanesh Goudar }; 210bab572f1SGanesh Goudar 211bab572f1SGanesh Goudar static struct sockaddr_in6 zero_sin6 = { 212bab572f1SGanesh Goudar .sin6_family = AF_INET6, 213bab572f1SGanesh Goudar }; 214bab572f1SGanesh Goudar 215bab572f1SGanesh Goudar static void set_ep_sin_addrs(struct c4iw_ep *ep, 216bab572f1SGanesh Goudar struct sockaddr_in **lsin, 217bab572f1SGanesh Goudar struct sockaddr_in **rsin, 218bab572f1SGanesh Goudar struct sockaddr_in **m_lsin, 219bab572f1SGanesh Goudar struct sockaddr_in **m_rsin) 220bab572f1SGanesh Goudar { 221bab572f1SGanesh Goudar struct iw_cm_id *id = ep->com.cm_id; 222bab572f1SGanesh Goudar 22344016b34SBharat Potnuri *m_lsin = (struct sockaddr_in *)&ep->com.local_addr; 22444016b34SBharat Potnuri *m_rsin = (struct sockaddr_in *)&ep->com.remote_addr; 225bab572f1SGanesh Goudar if (id) { 22644016b34SBharat Potnuri *lsin = (struct sockaddr_in *)&id->local_addr; 22744016b34SBharat Potnuri *rsin = (struct sockaddr_in *)&id->remote_addr; 228bab572f1SGanesh Goudar } else { 22944016b34SBharat Potnuri *lsin = &zero_sin; 23044016b34SBharat Potnuri *rsin = &zero_sin; 231bab572f1SGanesh Goudar } 232bab572f1SGanesh Goudar } 233bab572f1SGanesh Goudar 234bab572f1SGanesh Goudar static void set_ep_sin6_addrs(struct c4iw_ep *ep, 235bab572f1SGanesh Goudar struct sockaddr_in6 **lsin6, 236bab572f1SGanesh Goudar struct sockaddr_in6 **rsin6, 237bab572f1SGanesh Goudar struct sockaddr_in6 **m_lsin6, 238bab572f1SGanesh Goudar struct sockaddr_in6 **m_rsin6) 239bab572f1SGanesh Goudar { 240bab572f1SGanesh Goudar struct iw_cm_id *id = ep->com.cm_id; 241bab572f1SGanesh Goudar 24244016b34SBharat Potnuri *m_lsin6 = (struct sockaddr_in6 *)&ep->com.local_addr; 24344016b34SBharat Potnuri *m_rsin6 = (struct sockaddr_in6 *)&ep->com.remote_addr; 244bab572f1SGanesh Goudar if (id) { 24544016b34SBharat Potnuri *lsin6 = (struct sockaddr_in6 *)&id->local_addr; 24644016b34SBharat Potnuri *rsin6 = (struct sockaddr_in6 *)&id->remote_addr; 247bab572f1SGanesh Goudar } else { 24844016b34SBharat Potnuri *lsin6 = &zero_sin6; 24944016b34SBharat Potnuri *rsin6 = &zero_sin6; 250bab572f1SGanesh Goudar } 251bab572f1SGanesh Goudar } 252bab572f1SGanesh Goudar 2539e8d1fa3SSteve Wise static int dump_qp(int id, void *p, void *data) 254cfdda9d7SSteve Wise { 255cfdda9d7SSteve Wise struct c4iw_qp *qp = p; 2569e8d1fa3SSteve Wise struct c4iw_debugfs_data *qpd = data; 257cfdda9d7SSteve Wise int space; 258cfdda9d7SSteve Wise int cc; 259cfdda9d7SSteve Wise 260cfdda9d7SSteve Wise if (id != qp->wq.sq.qid) 261cfdda9d7SSteve Wise return 0; 262cfdda9d7SSteve Wise 263cfdda9d7SSteve Wise space = qpd->bufsize - qpd->pos - 1; 264cfdda9d7SSteve Wise if (space == 0) 265cfdda9d7SSteve Wise return 1; 266cfdda9d7SSteve Wise 267830662f6SVipul Pandya if (qp->ep) { 268bab572f1SGanesh Goudar struct c4iw_ep *ep = qp->ep; 269830662f6SVipul Pandya 270bab572f1SGanesh Goudar if (ep->com.local_addr.ss_family == AF_INET) { 271bab572f1SGanesh Goudar struct sockaddr_in *lsin; 272bab572f1SGanesh Goudar struct sockaddr_in *rsin; 273bab572f1SGanesh Goudar struct sockaddr_in *m_lsin; 274bab572f1SGanesh Goudar struct sockaddr_in *m_rsin; 275bab572f1SGanesh Goudar 276bab572f1SGanesh Goudar set_ep_sin_addrs(ep, &lsin, &rsin, &m_lsin, &m_rsin); 277db5d040dSSteve Wise cc = snprintf(qpd->buf + qpd->pos, space, 2786a0b6174SRaju Rangoju "rc qp sq id %u %s id %u state %u " 279830662f6SVipul Pandya "onchip %u ep tid %u state %u " 2809eccfe10SSteve Wise "%pI4:%u/%u->%pI4:%u/%u\n", 2816a0b6174SRaju Rangoju qp->wq.sq.qid, qp->srq ? "srq" : "rq", 2826a0b6174SRaju Rangoju qp->srq ? qp->srq->idx : qp->wq.rq.qid, 283830662f6SVipul Pandya (int)qp->attr.state, 284db5d040dSSteve Wise qp->wq.sq.flags & T4_SQ_ONCHIP, 285bab572f1SGanesh Goudar ep->hwtid, (int)ep->com.state, 286830662f6SVipul Pandya &lsin->sin_addr, ntohs(lsin->sin_port), 287bab572f1SGanesh Goudar ntohs(m_lsin->sin_port), 2889eccfe10SSteve Wise &rsin->sin_addr, ntohs(rsin->sin_port), 289bab572f1SGanesh Goudar ntohs(m_rsin->sin_port)); 290830662f6SVipul Pandya } else { 291bab572f1SGanesh Goudar struct sockaddr_in6 *lsin6; 292bab572f1SGanesh Goudar struct sockaddr_in6 *rsin6; 293bab572f1SGanesh Goudar struct sockaddr_in6 *m_lsin6; 294bab572f1SGanesh Goudar struct sockaddr_in6 *m_rsin6; 295830662f6SVipul Pandya 296bab572f1SGanesh Goudar set_ep_sin6_addrs(ep, &lsin6, &rsin6, &m_lsin6, 297bab572f1SGanesh Goudar &m_rsin6); 298830662f6SVipul Pandya cc = snprintf(qpd->buf + qpd->pos, space, 299830662f6SVipul Pandya "rc qp sq id %u rq id %u state %u " 300830662f6SVipul Pandya "onchip %u ep tid %u state %u " 3019eccfe10SSteve Wise "%pI6:%u/%u->%pI6:%u/%u\n", 302830662f6SVipul Pandya qp->wq.sq.qid, qp->wq.rq.qid, 303830662f6SVipul Pandya (int)qp->attr.state, 304830662f6SVipul Pandya qp->wq.sq.flags & T4_SQ_ONCHIP, 305bab572f1SGanesh Goudar ep->hwtid, (int)ep->com.state, 306830662f6SVipul Pandya &lsin6->sin6_addr, 307830662f6SVipul Pandya ntohs(lsin6->sin6_port), 308bab572f1SGanesh Goudar ntohs(m_lsin6->sin6_port), 309830662f6SVipul Pandya &rsin6->sin6_addr, 3109eccfe10SSteve Wise ntohs(rsin6->sin6_port), 311bab572f1SGanesh Goudar ntohs(m_rsin6->sin6_port)); 312830662f6SVipul Pandya } 313830662f6SVipul Pandya } else 314db5d040dSSteve Wise cc = snprintf(qpd->buf + qpd->pos, space, 315db5d040dSSteve Wise "qp sq id %u rq id %u state %u onchip %u\n", 316db5d040dSSteve Wise qp->wq.sq.qid, qp->wq.rq.qid, 317db5d040dSSteve Wise (int)qp->attr.state, 318db5d040dSSteve Wise qp->wq.sq.flags & T4_SQ_ONCHIP); 319cfdda9d7SSteve Wise if (cc < space) 320cfdda9d7SSteve Wise qpd->pos += cc; 321cfdda9d7SSteve Wise return 0; 322cfdda9d7SSteve Wise } 323cfdda9d7SSteve Wise 324cfdda9d7SSteve Wise static int qp_release(struct inode *inode, struct file *file) 325cfdda9d7SSteve Wise { 3269e8d1fa3SSteve Wise struct c4iw_debugfs_data *qpd = file->private_data; 327cfdda9d7SSteve Wise if (!qpd) { 328700456bdSJoe Perches pr_info("%s null qpd?\n", __func__); 329cfdda9d7SSteve Wise return 0; 330cfdda9d7SSteve Wise } 331d716a2a0SVipul Pandya vfree(qpd->buf); 332cfdda9d7SSteve Wise kfree(qpd); 333cfdda9d7SSteve Wise return 0; 334cfdda9d7SSteve Wise } 335cfdda9d7SSteve Wise 336cfdda9d7SSteve Wise static int qp_open(struct inode *inode, struct file *file) 337cfdda9d7SSteve Wise { 3389e8d1fa3SSteve Wise struct c4iw_debugfs_data *qpd; 339cfdda9d7SSteve Wise int count = 1; 340cfdda9d7SSteve Wise 341cfdda9d7SSteve Wise qpd = kmalloc(sizeof *qpd, GFP_KERNEL); 3424275a5b2SHariprasad S if (!qpd) 3434275a5b2SHariprasad S return -ENOMEM; 3444275a5b2SHariprasad S 345cfdda9d7SSteve Wise qpd->devp = inode->i_private; 346cfdda9d7SSteve Wise qpd->pos = 0; 347cfdda9d7SSteve Wise 348cfdda9d7SSteve Wise spin_lock_irq(&qpd->devp->lock); 3499e8d1fa3SSteve Wise idr_for_each(&qpd->devp->qpidr, count_idrs, &count); 350cfdda9d7SSteve Wise spin_unlock_irq(&qpd->devp->lock); 351cfdda9d7SSteve Wise 35268cebcabSHariprasad S qpd->bufsize = count * 180; 353d716a2a0SVipul Pandya qpd->buf = vmalloc(qpd->bufsize); 354cfdda9d7SSteve Wise if (!qpd->buf) { 3554275a5b2SHariprasad S kfree(qpd); 3564275a5b2SHariprasad S return -ENOMEM; 357cfdda9d7SSteve Wise } 358cfdda9d7SSteve Wise 359cfdda9d7SSteve Wise spin_lock_irq(&qpd->devp->lock); 3609e8d1fa3SSteve Wise idr_for_each(&qpd->devp->qpidr, dump_qp, qpd); 361cfdda9d7SSteve Wise spin_unlock_irq(&qpd->devp->lock); 362cfdda9d7SSteve Wise 363cfdda9d7SSteve Wise qpd->buf[qpd->pos++] = 0; 364cfdda9d7SSteve Wise file->private_data = qpd; 3654275a5b2SHariprasad S return 0; 366cfdda9d7SSteve Wise } 367cfdda9d7SSteve Wise 368cfdda9d7SSteve Wise static const struct file_operations qp_debugfs_fops = { 369cfdda9d7SSteve Wise .owner = THIS_MODULE, 370cfdda9d7SSteve Wise .open = qp_open, 371cfdda9d7SSteve Wise .release = qp_release, 3729e8d1fa3SSteve Wise .read = debugfs_read, 3738bbac892SSteve Wise .llseek = default_llseek, 3749e8d1fa3SSteve Wise }; 3759e8d1fa3SSteve Wise 3769e8d1fa3SSteve Wise static int dump_stag(int id, void *p, void *data) 3779e8d1fa3SSteve Wise { 3789e8d1fa3SSteve Wise struct c4iw_debugfs_data *stagd = data; 3799e8d1fa3SSteve Wise int space; 3809e8d1fa3SSteve Wise int cc; 381031cf476SHariprasad Shenai struct fw_ri_tpte tpte; 382031cf476SHariprasad Shenai int ret; 3839e8d1fa3SSteve Wise 3849e8d1fa3SSteve Wise space = stagd->bufsize - stagd->pos - 1; 3859e8d1fa3SSteve Wise if (space == 0) 3869e8d1fa3SSteve Wise return 1; 3879e8d1fa3SSteve Wise 388031cf476SHariprasad Shenai ret = cxgb4_read_tpte(stagd->devp->rdev.lldi.ports[0], (u32)id<<8, 389031cf476SHariprasad Shenai (__be32 *)&tpte); 390031cf476SHariprasad Shenai if (ret) { 391031cf476SHariprasad Shenai dev_err(&stagd->devp->rdev.lldi.pdev->dev, 392031cf476SHariprasad Shenai "%s cxgb4_read_tpte err %d\n", __func__, ret); 393031cf476SHariprasad Shenai return ret; 394031cf476SHariprasad Shenai } 395031cf476SHariprasad Shenai cc = snprintf(stagd->buf + stagd->pos, space, 396031cf476SHariprasad Shenai "stag: idx 0x%x valid %d key 0x%x state %d pdid %d " 397031cf476SHariprasad Shenai "perm 0x%x ps %d len 0x%llx va 0x%llx\n", 398031cf476SHariprasad Shenai (u32)id<<8, 399cf7fe64aSHariprasad Shenai FW_RI_TPTE_VALID_G(ntohl(tpte.valid_to_pdid)), 400cf7fe64aSHariprasad Shenai FW_RI_TPTE_STAGKEY_G(ntohl(tpte.valid_to_pdid)), 401cf7fe64aSHariprasad Shenai FW_RI_TPTE_STAGSTATE_G(ntohl(tpte.valid_to_pdid)), 402cf7fe64aSHariprasad Shenai FW_RI_TPTE_PDID_G(ntohl(tpte.valid_to_pdid)), 403cf7fe64aSHariprasad Shenai FW_RI_TPTE_PERM_G(ntohl(tpte.locread_to_qpid)), 404cf7fe64aSHariprasad Shenai FW_RI_TPTE_PS_G(ntohl(tpte.locread_to_qpid)), 405031cf476SHariprasad Shenai ((u64)ntohl(tpte.len_hi) << 32) | ntohl(tpte.len_lo), 406031cf476SHariprasad Shenai ((u64)ntohl(tpte.va_hi) << 32) | ntohl(tpte.va_lo_fbo)); 4079e8d1fa3SSteve Wise if (cc < space) 4089e8d1fa3SSteve Wise stagd->pos += cc; 4099e8d1fa3SSteve Wise return 0; 4109e8d1fa3SSteve Wise } 4119e8d1fa3SSteve Wise 4129e8d1fa3SSteve Wise static int stag_release(struct inode *inode, struct file *file) 4139e8d1fa3SSteve Wise { 4149e8d1fa3SSteve Wise struct c4iw_debugfs_data *stagd = file->private_data; 4159e8d1fa3SSteve Wise if (!stagd) { 416700456bdSJoe Perches pr_info("%s null stagd?\n", __func__); 4179e8d1fa3SSteve Wise return 0; 4189e8d1fa3SSteve Wise } 419031cf476SHariprasad Shenai vfree(stagd->buf); 4209e8d1fa3SSteve Wise kfree(stagd); 4219e8d1fa3SSteve Wise return 0; 4229e8d1fa3SSteve Wise } 4239e8d1fa3SSteve Wise 4249e8d1fa3SSteve Wise static int stag_open(struct inode *inode, struct file *file) 4259e8d1fa3SSteve Wise { 4269e8d1fa3SSteve Wise struct c4iw_debugfs_data *stagd; 4279e8d1fa3SSteve Wise int ret = 0; 4289e8d1fa3SSteve Wise int count = 1; 4299e8d1fa3SSteve Wise 4309e8d1fa3SSteve Wise stagd = kmalloc(sizeof *stagd, GFP_KERNEL); 4319e8d1fa3SSteve Wise if (!stagd) { 4329e8d1fa3SSteve Wise ret = -ENOMEM; 4339e8d1fa3SSteve Wise goto out; 4349e8d1fa3SSteve Wise } 4359e8d1fa3SSteve Wise stagd->devp = inode->i_private; 4369e8d1fa3SSteve Wise stagd->pos = 0; 4379e8d1fa3SSteve Wise 4389e8d1fa3SSteve Wise spin_lock_irq(&stagd->devp->lock); 4399e8d1fa3SSteve Wise idr_for_each(&stagd->devp->mmidr, count_idrs, &count); 4409e8d1fa3SSteve Wise spin_unlock_irq(&stagd->devp->lock); 4419e8d1fa3SSteve Wise 442031cf476SHariprasad Shenai stagd->bufsize = count * 256; 443031cf476SHariprasad Shenai stagd->buf = vmalloc(stagd->bufsize); 4449e8d1fa3SSteve Wise if (!stagd->buf) { 4459e8d1fa3SSteve Wise ret = -ENOMEM; 4469e8d1fa3SSteve Wise goto err1; 4479e8d1fa3SSteve Wise } 4489e8d1fa3SSteve Wise 4499e8d1fa3SSteve Wise spin_lock_irq(&stagd->devp->lock); 4509e8d1fa3SSteve Wise idr_for_each(&stagd->devp->mmidr, dump_stag, stagd); 4519e8d1fa3SSteve Wise spin_unlock_irq(&stagd->devp->lock); 4529e8d1fa3SSteve Wise 4539e8d1fa3SSteve Wise stagd->buf[stagd->pos++] = 0; 4549e8d1fa3SSteve Wise file->private_data = stagd; 4559e8d1fa3SSteve Wise goto out; 4569e8d1fa3SSteve Wise err1: 4579e8d1fa3SSteve Wise kfree(stagd); 4589e8d1fa3SSteve Wise out: 4599e8d1fa3SSteve Wise return ret; 4609e8d1fa3SSteve Wise } 4619e8d1fa3SSteve Wise 4629e8d1fa3SSteve Wise static const struct file_operations stag_debugfs_fops = { 4639e8d1fa3SSteve Wise .owner = THIS_MODULE, 4649e8d1fa3SSteve Wise .open = stag_open, 4659e8d1fa3SSteve Wise .release = stag_release, 4669e8d1fa3SSteve Wise .read = debugfs_read, 4678bbac892SSteve Wise .llseek = default_llseek, 468cfdda9d7SSteve Wise }; 469cfdda9d7SSteve Wise 47005eb2389SSteve Wise static char *db_state_str[] = {"NORMAL", "FLOW_CONTROL", "RECOVERY", "STOPPED"}; 471422eea0aSVipul Pandya 4728d81ef34SVipul Pandya static int stats_show(struct seq_file *seq, void *v) 4738d81ef34SVipul Pandya { 4748d81ef34SVipul Pandya struct c4iw_dev *dev = seq->private; 4758d81ef34SVipul Pandya 476ec3eead2SVipul Pandya seq_printf(seq, " Object: %10s %10s %10s %10s\n", "Total", "Current", 477ec3eead2SVipul Pandya "Max", "Fail"); 478ec3eead2SVipul Pandya seq_printf(seq, " PDID: %10llu %10llu %10llu %10llu\n", 4798d81ef34SVipul Pandya dev->rdev.stats.pd.total, dev->rdev.stats.pd.cur, 480ec3eead2SVipul Pandya dev->rdev.stats.pd.max, dev->rdev.stats.pd.fail); 481ec3eead2SVipul Pandya seq_printf(seq, " QID: %10llu %10llu %10llu %10llu\n", 4828d81ef34SVipul Pandya dev->rdev.stats.qid.total, dev->rdev.stats.qid.cur, 483ec3eead2SVipul Pandya dev->rdev.stats.qid.max, dev->rdev.stats.qid.fail); 4846a0b6174SRaju Rangoju seq_printf(seq, " SRQS: %10llu %10llu %10llu %10llu\n", 4856a0b6174SRaju Rangoju dev->rdev.stats.srqt.total, dev->rdev.stats.srqt.cur, 4866a0b6174SRaju Rangoju dev->rdev.stats.srqt.max, dev->rdev.stats.srqt.fail); 487ec3eead2SVipul Pandya seq_printf(seq, " TPTMEM: %10llu %10llu %10llu %10llu\n", 4888d81ef34SVipul Pandya dev->rdev.stats.stag.total, dev->rdev.stats.stag.cur, 489ec3eead2SVipul Pandya dev->rdev.stats.stag.max, dev->rdev.stats.stag.fail); 490ec3eead2SVipul Pandya seq_printf(seq, " PBLMEM: %10llu %10llu %10llu %10llu\n", 4918d81ef34SVipul Pandya dev->rdev.stats.pbl.total, dev->rdev.stats.pbl.cur, 492ec3eead2SVipul Pandya dev->rdev.stats.pbl.max, dev->rdev.stats.pbl.fail); 493ec3eead2SVipul Pandya seq_printf(seq, " RQTMEM: %10llu %10llu %10llu %10llu\n", 4948d81ef34SVipul Pandya dev->rdev.stats.rqt.total, dev->rdev.stats.rqt.cur, 495ec3eead2SVipul Pandya dev->rdev.stats.rqt.max, dev->rdev.stats.rqt.fail); 496ec3eead2SVipul Pandya seq_printf(seq, " OCQPMEM: %10llu %10llu %10llu %10llu\n", 4978d81ef34SVipul Pandya dev->rdev.stats.ocqp.total, dev->rdev.stats.ocqp.cur, 498ec3eead2SVipul Pandya dev->rdev.stats.ocqp.max, dev->rdev.stats.ocqp.fail); 4992c974781SVipul Pandya seq_printf(seq, " DB FULL: %10llu\n", dev->rdev.stats.db_full); 5002c974781SVipul Pandya seq_printf(seq, " DB EMPTY: %10llu\n", dev->rdev.stats.db_empty); 5012c974781SVipul Pandya seq_printf(seq, " DB DROP: %10llu\n", dev->rdev.stats.db_drop); 50205eb2389SSteve Wise seq_printf(seq, " DB State: %s Transitions %llu FC Interruptions %llu\n", 503422eea0aSVipul Pandya db_state_str[dev->db_state], 50405eb2389SSteve Wise dev->rdev.stats.db_state_transitions, 50505eb2389SSteve Wise dev->rdev.stats.db_fc_interruptions); 5061cab775cSVipul Pandya seq_printf(seq, "TCAM_FULL: %10llu\n", dev->rdev.stats.tcam_full); 507793dad94SVipul Pandya seq_printf(seq, "ACT_OFLD_CONN_FAILS: %10llu\n", 508793dad94SVipul Pandya dev->rdev.stats.act_ofld_conn_fails); 509793dad94SVipul Pandya seq_printf(seq, "PAS_OFLD_CONN_FAILS: %10llu\n", 510793dad94SVipul Pandya dev->rdev.stats.pas_ofld_conn_fails); 511179d03bbSHariprasad S seq_printf(seq, "NEG_ADV_RCVD: %10llu\n", dev->rdev.stats.neg_adv); 5124c2c5763SHariprasad Shenai seq_printf(seq, "AVAILABLE IRD: %10u\n", dev->avail_ird); 5138d81ef34SVipul Pandya return 0; 5148d81ef34SVipul Pandya } 5158d81ef34SVipul Pandya 5168d81ef34SVipul Pandya static int stats_open(struct inode *inode, struct file *file) 5178d81ef34SVipul Pandya { 5188d81ef34SVipul Pandya return single_open(file, stats_show, inode->i_private); 5198d81ef34SVipul Pandya } 5208d81ef34SVipul Pandya 5218d81ef34SVipul Pandya static ssize_t stats_clear(struct file *file, const char __user *buf, 5228d81ef34SVipul Pandya size_t count, loff_t *pos) 5238d81ef34SVipul Pandya { 5248d81ef34SVipul Pandya struct c4iw_dev *dev = ((struct seq_file *)file->private_data)->private; 5258d81ef34SVipul Pandya 5268d81ef34SVipul Pandya mutex_lock(&dev->rdev.stats.lock); 5278d81ef34SVipul Pandya dev->rdev.stats.pd.max = 0; 528ec3eead2SVipul Pandya dev->rdev.stats.pd.fail = 0; 5298d81ef34SVipul Pandya dev->rdev.stats.qid.max = 0; 530ec3eead2SVipul Pandya dev->rdev.stats.qid.fail = 0; 5318d81ef34SVipul Pandya dev->rdev.stats.stag.max = 0; 532ec3eead2SVipul Pandya dev->rdev.stats.stag.fail = 0; 5338d81ef34SVipul Pandya dev->rdev.stats.pbl.max = 0; 534ec3eead2SVipul Pandya dev->rdev.stats.pbl.fail = 0; 5358d81ef34SVipul Pandya dev->rdev.stats.rqt.max = 0; 536ec3eead2SVipul Pandya dev->rdev.stats.rqt.fail = 0; 5376a0b6174SRaju Rangoju dev->rdev.stats.rqt.max = 0; 5386a0b6174SRaju Rangoju dev->rdev.stats.rqt.fail = 0; 5398d81ef34SVipul Pandya dev->rdev.stats.ocqp.max = 0; 540ec3eead2SVipul Pandya dev->rdev.stats.ocqp.fail = 0; 5412c974781SVipul Pandya dev->rdev.stats.db_full = 0; 5422c974781SVipul Pandya dev->rdev.stats.db_empty = 0; 5432c974781SVipul Pandya dev->rdev.stats.db_drop = 0; 544422eea0aSVipul Pandya dev->rdev.stats.db_state_transitions = 0; 545793dad94SVipul Pandya dev->rdev.stats.tcam_full = 0; 546793dad94SVipul Pandya dev->rdev.stats.act_ofld_conn_fails = 0; 547793dad94SVipul Pandya dev->rdev.stats.pas_ofld_conn_fails = 0; 5488d81ef34SVipul Pandya mutex_unlock(&dev->rdev.stats.lock); 5498d81ef34SVipul Pandya return count; 5508d81ef34SVipul Pandya } 5518d81ef34SVipul Pandya 5528d81ef34SVipul Pandya static const struct file_operations stats_debugfs_fops = { 5538d81ef34SVipul Pandya .owner = THIS_MODULE, 5548d81ef34SVipul Pandya .open = stats_open, 5558d81ef34SVipul Pandya .release = single_release, 5568d81ef34SVipul Pandya .read = seq_read, 5578d81ef34SVipul Pandya .llseek = seq_lseek, 5588d81ef34SVipul Pandya .write = stats_clear, 5598d81ef34SVipul Pandya }; 5608d81ef34SVipul Pandya 561793dad94SVipul Pandya static int dump_ep(int id, void *p, void *data) 562793dad94SVipul Pandya { 563793dad94SVipul Pandya struct c4iw_ep *ep = p; 564793dad94SVipul Pandya struct c4iw_debugfs_data *epd = data; 565793dad94SVipul Pandya int space; 566793dad94SVipul Pandya int cc; 567793dad94SVipul Pandya 568793dad94SVipul Pandya space = epd->bufsize - epd->pos - 1; 569793dad94SVipul Pandya if (space == 0) 570793dad94SVipul Pandya return 1; 571793dad94SVipul Pandya 572830662f6SVipul Pandya if (ep->com.local_addr.ss_family == AF_INET) { 573bab572f1SGanesh Goudar struct sockaddr_in *lsin; 574bab572f1SGanesh Goudar struct sockaddr_in *rsin; 575bab572f1SGanesh Goudar struct sockaddr_in *m_lsin; 576bab572f1SGanesh Goudar struct sockaddr_in *m_rsin; 577830662f6SVipul Pandya 578bab572f1SGanesh Goudar set_ep_sin_addrs(ep, &lsin, &rsin, &m_lsin, &m_rsin); 579793dad94SVipul Pandya cc = snprintf(epd->buf + epd->pos, space, 580830662f6SVipul Pandya "ep %p cm_id %p qp %p state %d flags 0x%lx " 581830662f6SVipul Pandya "history 0x%lx hwtid %d atid %d " 582179d03bbSHariprasad S "conn_na %u abort_na %u " 5839eccfe10SSteve Wise "%pI4:%d/%d <-> %pI4:%d/%d\n", 584830662f6SVipul Pandya ep, ep->com.cm_id, ep->com.qp, 585830662f6SVipul Pandya (int)ep->com.state, ep->com.flags, 586830662f6SVipul Pandya ep->com.history, ep->hwtid, ep->atid, 587179d03bbSHariprasad S ep->stats.connect_neg_adv, 588179d03bbSHariprasad S ep->stats.abort_neg_adv, 589830662f6SVipul Pandya &lsin->sin_addr, ntohs(lsin->sin_port), 590bab572f1SGanesh Goudar ntohs(m_lsin->sin_port), 5919eccfe10SSteve Wise &rsin->sin_addr, ntohs(rsin->sin_port), 592bab572f1SGanesh Goudar ntohs(m_rsin->sin_port)); 593830662f6SVipul Pandya } else { 594bab572f1SGanesh Goudar struct sockaddr_in6 *lsin6; 595bab572f1SGanesh Goudar struct sockaddr_in6 *rsin6; 596bab572f1SGanesh Goudar struct sockaddr_in6 *m_lsin6; 597bab572f1SGanesh Goudar struct sockaddr_in6 *m_rsin6; 598830662f6SVipul Pandya 599bab572f1SGanesh Goudar set_ep_sin6_addrs(ep, &lsin6, &rsin6, &m_lsin6, &m_rsin6); 600830662f6SVipul Pandya cc = snprintf(epd->buf + epd->pos, space, 601830662f6SVipul Pandya "ep %p cm_id %p qp %p state %d flags 0x%lx " 602830662f6SVipul Pandya "history 0x%lx hwtid %d atid %d " 603179d03bbSHariprasad S "conn_na %u abort_na %u " 6049eccfe10SSteve Wise "%pI6:%d/%d <-> %pI6:%d/%d\n", 605830662f6SVipul Pandya ep, ep->com.cm_id, ep->com.qp, 606830662f6SVipul Pandya (int)ep->com.state, ep->com.flags, 607830662f6SVipul Pandya ep->com.history, ep->hwtid, ep->atid, 608179d03bbSHariprasad S ep->stats.connect_neg_adv, 609179d03bbSHariprasad S ep->stats.abort_neg_adv, 610830662f6SVipul Pandya &lsin6->sin6_addr, ntohs(lsin6->sin6_port), 611bab572f1SGanesh Goudar ntohs(m_lsin6->sin6_port), 6129eccfe10SSteve Wise &rsin6->sin6_addr, ntohs(rsin6->sin6_port), 613bab572f1SGanesh Goudar ntohs(m_rsin6->sin6_port)); 614830662f6SVipul Pandya } 615793dad94SVipul Pandya if (cc < space) 616793dad94SVipul Pandya epd->pos += cc; 617793dad94SVipul Pandya return 0; 618793dad94SVipul Pandya } 619793dad94SVipul Pandya 620793dad94SVipul Pandya static int dump_listen_ep(int id, void *p, void *data) 621793dad94SVipul Pandya { 622793dad94SVipul Pandya struct c4iw_listen_ep *ep = p; 623793dad94SVipul Pandya struct c4iw_debugfs_data *epd = data; 624793dad94SVipul Pandya int space; 625793dad94SVipul Pandya int cc; 626793dad94SVipul Pandya 627793dad94SVipul Pandya space = epd->bufsize - epd->pos - 1; 628793dad94SVipul Pandya if (space == 0) 629793dad94SVipul Pandya return 1; 630793dad94SVipul Pandya 631830662f6SVipul Pandya if (ep->com.local_addr.ss_family == AF_INET) { 632830662f6SVipul Pandya struct sockaddr_in *lsin = (struct sockaddr_in *) 633170003c8SSteve Wise &ep->com.cm_id->local_addr; 634bab572f1SGanesh Goudar struct sockaddr_in *m_lsin = (struct sockaddr_in *) 635170003c8SSteve Wise &ep->com.cm_id->m_local_addr; 636830662f6SVipul Pandya 637793dad94SVipul Pandya cc = snprintf(epd->buf + epd->pos, space, 638830662f6SVipul Pandya "ep %p cm_id %p state %d flags 0x%lx stid %d " 6399eccfe10SSteve Wise "backlog %d %pI4:%d/%d\n", 640830662f6SVipul Pandya ep, ep->com.cm_id, (int)ep->com.state, 641793dad94SVipul Pandya ep->com.flags, ep->stid, ep->backlog, 6429eccfe10SSteve Wise &lsin->sin_addr, ntohs(lsin->sin_port), 643bab572f1SGanesh Goudar ntohs(m_lsin->sin_port)); 644830662f6SVipul Pandya } else { 645830662f6SVipul Pandya struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) 646170003c8SSteve Wise &ep->com.cm_id->local_addr; 647bab572f1SGanesh Goudar struct sockaddr_in6 *m_lsin6 = (struct sockaddr_in6 *) 648170003c8SSteve Wise &ep->com.cm_id->m_local_addr; 649830662f6SVipul Pandya 650830662f6SVipul Pandya cc = snprintf(epd->buf + epd->pos, space, 651830662f6SVipul Pandya "ep %p cm_id %p state %d flags 0x%lx stid %d " 6529eccfe10SSteve Wise "backlog %d %pI6:%d/%d\n", 653830662f6SVipul Pandya ep, ep->com.cm_id, (int)ep->com.state, 654830662f6SVipul Pandya ep->com.flags, ep->stid, ep->backlog, 6559eccfe10SSteve Wise &lsin6->sin6_addr, ntohs(lsin6->sin6_port), 656bab572f1SGanesh Goudar ntohs(m_lsin6->sin6_port)); 657830662f6SVipul Pandya } 658793dad94SVipul Pandya if (cc < space) 659793dad94SVipul Pandya epd->pos += cc; 660793dad94SVipul Pandya return 0; 661793dad94SVipul Pandya } 662793dad94SVipul Pandya 663793dad94SVipul Pandya static int ep_release(struct inode *inode, struct file *file) 664793dad94SVipul Pandya { 665793dad94SVipul Pandya struct c4iw_debugfs_data *epd = file->private_data; 666793dad94SVipul Pandya if (!epd) { 667793dad94SVipul Pandya pr_info("%s null qpd?\n", __func__); 668793dad94SVipul Pandya return 0; 669793dad94SVipul Pandya } 670793dad94SVipul Pandya vfree(epd->buf); 671793dad94SVipul Pandya kfree(epd); 672793dad94SVipul Pandya return 0; 673793dad94SVipul Pandya } 674793dad94SVipul Pandya 675793dad94SVipul Pandya static int ep_open(struct inode *inode, struct file *file) 676793dad94SVipul Pandya { 677793dad94SVipul Pandya struct c4iw_debugfs_data *epd; 678793dad94SVipul Pandya int ret = 0; 679793dad94SVipul Pandya int count = 1; 680793dad94SVipul Pandya 681793dad94SVipul Pandya epd = kmalloc(sizeof(*epd), GFP_KERNEL); 682793dad94SVipul Pandya if (!epd) { 683793dad94SVipul Pandya ret = -ENOMEM; 684793dad94SVipul Pandya goto out; 685793dad94SVipul Pandya } 686793dad94SVipul Pandya epd->devp = inode->i_private; 687793dad94SVipul Pandya epd->pos = 0; 688793dad94SVipul Pandya 689793dad94SVipul Pandya spin_lock_irq(&epd->devp->lock); 690793dad94SVipul Pandya idr_for_each(&epd->devp->hwtid_idr, count_idrs, &count); 691793dad94SVipul Pandya idr_for_each(&epd->devp->atid_idr, count_idrs, &count); 692793dad94SVipul Pandya idr_for_each(&epd->devp->stid_idr, count_idrs, &count); 693793dad94SVipul Pandya spin_unlock_irq(&epd->devp->lock); 694793dad94SVipul Pandya 69563a71ba6SPramod Kumar epd->bufsize = count * 240; 696793dad94SVipul Pandya epd->buf = vmalloc(epd->bufsize); 697793dad94SVipul Pandya if (!epd->buf) { 698793dad94SVipul Pandya ret = -ENOMEM; 699793dad94SVipul Pandya goto err1; 700793dad94SVipul Pandya } 701793dad94SVipul Pandya 702793dad94SVipul Pandya spin_lock_irq(&epd->devp->lock); 703793dad94SVipul Pandya idr_for_each(&epd->devp->hwtid_idr, dump_ep, epd); 704793dad94SVipul Pandya idr_for_each(&epd->devp->atid_idr, dump_ep, epd); 705793dad94SVipul Pandya idr_for_each(&epd->devp->stid_idr, dump_listen_ep, epd); 706793dad94SVipul Pandya spin_unlock_irq(&epd->devp->lock); 707793dad94SVipul Pandya 708793dad94SVipul Pandya file->private_data = epd; 709793dad94SVipul Pandya goto out; 710793dad94SVipul Pandya err1: 711793dad94SVipul Pandya kfree(epd); 712793dad94SVipul Pandya out: 713793dad94SVipul Pandya return ret; 714793dad94SVipul Pandya } 715793dad94SVipul Pandya 716793dad94SVipul Pandya static const struct file_operations ep_debugfs_fops = { 717793dad94SVipul Pandya .owner = THIS_MODULE, 718793dad94SVipul Pandya .open = ep_open, 719793dad94SVipul Pandya .release = ep_release, 720793dad94SVipul Pandya .read = debugfs_read, 721793dad94SVipul Pandya }; 722793dad94SVipul Pandya 723cfdda9d7SSteve Wise static int setup_debugfs(struct c4iw_dev *devp) 724cfdda9d7SSteve Wise { 725cfdda9d7SSteve Wise if (!devp->debugfs_root) 726cfdda9d7SSteve Wise return -1; 727cfdda9d7SSteve Wise 728e59b4e91SDavid Howells debugfs_create_file_size("qps", S_IWUSR, devp->debugfs_root, 729e59b4e91SDavid Howells (void *)devp, &qp_debugfs_fops, 4096); 7309e8d1fa3SSteve Wise 731e59b4e91SDavid Howells debugfs_create_file_size("stags", S_IWUSR, devp->debugfs_root, 732e59b4e91SDavid Howells (void *)devp, &stag_debugfs_fops, 4096); 7338d81ef34SVipul Pandya 734e59b4e91SDavid Howells debugfs_create_file_size("stats", S_IWUSR, devp->debugfs_root, 735e59b4e91SDavid Howells (void *)devp, &stats_debugfs_fops, 4096); 7368d81ef34SVipul Pandya 737e59b4e91SDavid Howells debugfs_create_file_size("eps", S_IWUSR, devp->debugfs_root, 738e59b4e91SDavid Howells (void *)devp, &ep_debugfs_fops, 4096); 739793dad94SVipul Pandya 740e59b4e91SDavid Howells if (c4iw_wr_log) 741e59b4e91SDavid Howells debugfs_create_file_size("wr_log", S_IWUSR, devp->debugfs_root, 742e59b4e91SDavid Howells (void *)devp, &wr_log_debugfs_fops, 4096); 743cfdda9d7SSteve Wise return 0; 744cfdda9d7SSteve Wise } 745cfdda9d7SSteve Wise 746cfdda9d7SSteve Wise void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev, 747cfdda9d7SSteve Wise struct c4iw_dev_ucontext *uctx) 748cfdda9d7SSteve Wise { 749cfdda9d7SSteve Wise struct list_head *pos, *nxt; 750cfdda9d7SSteve Wise struct c4iw_qid_list *entry; 751cfdda9d7SSteve Wise 752cfdda9d7SSteve Wise mutex_lock(&uctx->lock); 753cfdda9d7SSteve Wise list_for_each_safe(pos, nxt, &uctx->qpids) { 754cfdda9d7SSteve Wise entry = list_entry(pos, struct c4iw_qid_list, entry); 755cfdda9d7SSteve Wise list_del_init(&entry->entry); 7568d81ef34SVipul Pandya if (!(entry->qid & rdev->qpmask)) { 757ec3eead2SVipul Pandya c4iw_put_resource(&rdev->resource.qid_table, 758ec3eead2SVipul Pandya entry->qid); 7598d81ef34SVipul Pandya mutex_lock(&rdev->stats.lock); 7608d81ef34SVipul Pandya rdev->stats.qid.cur -= rdev->qpmask + 1; 7618d81ef34SVipul Pandya mutex_unlock(&rdev->stats.lock); 7628d81ef34SVipul Pandya } 763cfdda9d7SSteve Wise kfree(entry); 764cfdda9d7SSteve Wise } 765cfdda9d7SSteve Wise 766d4702645SRaju Rangoju list_for_each_safe(pos, nxt, &uctx->cqids) { 767cfdda9d7SSteve Wise entry = list_entry(pos, struct c4iw_qid_list, entry); 768cfdda9d7SSteve Wise list_del_init(&entry->entry); 769cfdda9d7SSteve Wise kfree(entry); 770cfdda9d7SSteve Wise } 771cfdda9d7SSteve Wise mutex_unlock(&uctx->lock); 772cfdda9d7SSteve Wise } 773cfdda9d7SSteve Wise 774cfdda9d7SSteve Wise void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev, 775cfdda9d7SSteve Wise struct c4iw_dev_ucontext *uctx) 776cfdda9d7SSteve Wise { 777cfdda9d7SSteve Wise INIT_LIST_HEAD(&uctx->qpids); 778cfdda9d7SSteve Wise INIT_LIST_HEAD(&uctx->cqids); 779cfdda9d7SSteve Wise mutex_init(&uctx->lock); 780cfdda9d7SSteve Wise } 781cfdda9d7SSteve Wise 782cfdda9d7SSteve Wise /* Caller takes care of locking if needed */ 783cfdda9d7SSteve Wise static int c4iw_rdev_open(struct c4iw_rdev *rdev) 784cfdda9d7SSteve Wise { 785cfdda9d7SSteve Wise int err; 786cfdda9d7SSteve Wise 787cfdda9d7SSteve Wise c4iw_init_dev_ucontext(rdev, &rdev->uctx); 788cfdda9d7SSteve Wise 789cfdda9d7SSteve Wise /* 7904a75a86cSHariprasad S * This implementation assumes udb_density == ucq_density! Eventually 7914a75a86cSHariprasad S * we might need to support this but for now fail the open. Also the 7924a75a86cSHariprasad S * cqid and qpid range must match for now. 7934a75a86cSHariprasad S */ 7944a75a86cSHariprasad S if (rdev->lldi.udb_density != rdev->lldi.ucq_density) { 795700456bdSJoe Perches pr_err("%s: unsupported udb/ucq densities %u/%u\n", 7964a75a86cSHariprasad S pci_name(rdev->lldi.pdev), rdev->lldi.udb_density, 7974a75a86cSHariprasad S rdev->lldi.ucq_density); 7984275a5b2SHariprasad S return -EINVAL; 7994a75a86cSHariprasad S } 8004a75a86cSHariprasad S if (rdev->lldi.vr->qp.start != rdev->lldi.vr->cq.start || 8014a75a86cSHariprasad S rdev->lldi.vr->qp.size != rdev->lldi.vr->cq.size) { 802700456bdSJoe Perches pr_err("%s: unsupported qp and cq id ranges qp start %u size %u cq start %u size %u\n", 8034a75a86cSHariprasad S pci_name(rdev->lldi.pdev), rdev->lldi.vr->qp.start, 8044a75a86cSHariprasad S rdev->lldi.vr->qp.size, rdev->lldi.vr->cq.size, 8054a75a86cSHariprasad S rdev->lldi.vr->cq.size); 8064275a5b2SHariprasad S return -EINVAL; 8074a75a86cSHariprasad S } 8084a75a86cSHariprasad S 809cfdda9d7SSteve Wise rdev->qpmask = rdev->lldi.udb_density - 1; 810cfdda9d7SSteve Wise rdev->cqmask = rdev->lldi.ucq_density - 1; 8116a0b6174SRaju Rangoju pr_debug("dev %s stag start 0x%0x size 0x%0x num stags %d pbl start 0x%0x size 0x%0x rq start 0x%0x size 0x%0x qp qid start %u size %u cq qid start %u size %u srq size %u\n", 812548ddb19SBharat Potnuri pci_name(rdev->lldi.pdev), rdev->lldi.vr->stag.start, 813cfdda9d7SSteve Wise rdev->lldi.vr->stag.size, c4iw_num_stags(rdev), 814cfdda9d7SSteve Wise rdev->lldi.vr->pbl.start, 815cfdda9d7SSteve Wise rdev->lldi.vr->pbl.size, rdev->lldi.vr->rq.start, 81693fb72e4SSteve Wise rdev->lldi.vr->rq.size, 81793fb72e4SSteve Wise rdev->lldi.vr->qp.start, 81893fb72e4SSteve Wise rdev->lldi.vr->qp.size, 81993fb72e4SSteve Wise rdev->lldi.vr->cq.start, 8206a0b6174SRaju Rangoju rdev->lldi.vr->cq.size, 8216a0b6174SRaju Rangoju rdev->lldi.vr->srq.size); 822a9a42886SJoe Perches pr_debug("udb %pR db_reg %p gts_reg %p qpmask 0x%x cqmask 0x%x\n", 8233021376dSArnd Bergmann &rdev->lldi.pdev->resource[2], 82474217d4cSHariprasad S rdev->lldi.db_reg, rdev->lldi.gts_reg, 82574217d4cSHariprasad S rdev->qpmask, rdev->cqmask); 826cfdda9d7SSteve Wise 8274275a5b2SHariprasad S if (c4iw_num_stags(rdev) == 0) 8284275a5b2SHariprasad S return -EINVAL; 829cfdda9d7SSteve Wise 8308d81ef34SVipul Pandya rdev->stats.pd.total = T4_MAX_NUM_PD; 8318d81ef34SVipul Pandya rdev->stats.stag.total = rdev->lldi.vr->stag.size; 8328d81ef34SVipul Pandya rdev->stats.pbl.total = rdev->lldi.vr->pbl.size; 8338d81ef34SVipul Pandya rdev->stats.rqt.total = rdev->lldi.vr->rq.size; 8346a0b6174SRaju Rangoju rdev->stats.srqt.total = rdev->lldi.vr->srq.size; 8358d81ef34SVipul Pandya rdev->stats.ocqp.total = rdev->lldi.vr->ocq.size; 8368d81ef34SVipul Pandya rdev->stats.qid.total = rdev->lldi.vr->qp.size; 8378d81ef34SVipul Pandya 8386a0b6174SRaju Rangoju err = c4iw_init_resource(rdev, c4iw_num_stags(rdev), 8396a0b6174SRaju Rangoju T4_MAX_NUM_PD, rdev->lldi.vr->srq.size); 840cfdda9d7SSteve Wise if (err) { 841700456bdSJoe Perches pr_err("error %d initializing resources\n", err); 8424275a5b2SHariprasad S return err; 843cfdda9d7SSteve Wise } 844cfdda9d7SSteve Wise err = c4iw_pblpool_create(rdev); 845cfdda9d7SSteve Wise if (err) { 846700456bdSJoe Perches pr_err("error %d initializing pbl pool\n", err); 8474275a5b2SHariprasad S goto destroy_resource; 848cfdda9d7SSteve Wise } 849cfdda9d7SSteve Wise err = c4iw_rqtpool_create(rdev); 850cfdda9d7SSteve Wise if (err) { 851700456bdSJoe Perches pr_err("error %d initializing rqt pool\n", err); 8524275a5b2SHariprasad S goto destroy_pblpool; 853cfdda9d7SSteve Wise } 854c6d7b267SSteve Wise err = c4iw_ocqp_pool_create(rdev); 855c6d7b267SSteve Wise if (err) { 856700456bdSJoe Perches pr_err("error %d initializing ocqp pool\n", err); 8574275a5b2SHariprasad S goto destroy_rqtpool; 858c6d7b267SSteve Wise } 85905eb2389SSteve Wise rdev->status_page = (struct t4_dev_status_page *) 86005eb2389SSteve Wise __get_free_page(GFP_KERNEL); 86115f7e3c2SWei Yongjun if (!rdev->status_page) { 86215f7e3c2SWei Yongjun err = -ENOMEM; 86382b1df1bSHariprasad S goto destroy_ocqp_pool; 86415f7e3c2SWei Yongjun } 865c5dfb000SHariprasad S rdev->status_page->qp_start = rdev->lldi.vr->qp.start; 866c5dfb000SHariprasad S rdev->status_page->qp_size = rdev->lldi.vr->qp.size; 867c5dfb000SHariprasad S rdev->status_page->cq_start = rdev->lldi.vr->cq.start; 868c5dfb000SHariprasad S rdev->status_page->cq_size = rdev->lldi.vr->cq.size; 86994245f4aSPotnuri Bharat Teja rdev->status_page->write_cmpl_supported = rdev->lldi.write_cmpl_support; 8708fd90bb8SDavid S. Miller 8717730b4c7SHariprasad Shenai if (c4iw_wr_log) { 8726396bb22SKees Cook rdev->wr_log = kcalloc(1 << c4iw_wr_log_size_order, 8736396bb22SKees Cook sizeof(*rdev->wr_log), 8746396bb22SKees Cook GFP_KERNEL); 8757730b4c7SHariprasad Shenai if (rdev->wr_log) { 8767730b4c7SHariprasad Shenai rdev->wr_log_size = 1 << c4iw_wr_log_size_order; 8777730b4c7SHariprasad Shenai atomic_set(&rdev->wr_log_idx, 0); 8787730b4c7SHariprasad Shenai } 8797730b4c7SHariprasad Shenai } 8808fd90bb8SDavid S. Miller 881c12a67feSSteve Wise rdev->free_workq = create_singlethread_workqueue("iw_cxgb4_free"); 882c12a67feSSteve Wise if (!rdev->free_workq) { 883c12a67feSSteve Wise err = -ENOMEM; 884d4702645SRaju Rangoju goto err_free_status_page_and_wr_log; 885c12a67feSSteve Wise } 886c12a67feSSteve Wise 8876b54d54dSSteve Wise rdev->status_page->db_off = 0; 8888fd90bb8SDavid S. Miller 88926bff1bdSRaju Rangoju init_completion(&rdev->rqt_compl); 89026bff1bdSRaju Rangoju init_completion(&rdev->pbl_compl); 89126bff1bdSRaju Rangoju kref_init(&rdev->rqt_kref); 89226bff1bdSRaju Rangoju kref_init(&rdev->pbl_kref); 89326bff1bdSRaju Rangoju 894cfdda9d7SSteve Wise return 0; 895d4702645SRaju Rangoju err_free_status_page_and_wr_log: 896d4702645SRaju Rangoju if (c4iw_wr_log && rdev->wr_log) 897d4702645SRaju Rangoju kfree(rdev->wr_log); 898c12a67feSSteve Wise free_page((unsigned long)rdev->status_page); 89982b1df1bSHariprasad S destroy_ocqp_pool: 90082b1df1bSHariprasad S c4iw_ocqp_pool_destroy(rdev); 9014275a5b2SHariprasad S destroy_rqtpool: 902c6d7b267SSteve Wise c4iw_rqtpool_destroy(rdev); 9034275a5b2SHariprasad S destroy_pblpool: 904cfdda9d7SSteve Wise c4iw_pblpool_destroy(rdev); 9054275a5b2SHariprasad S destroy_resource: 906cfdda9d7SSteve Wise c4iw_destroy_resource(&rdev->resource); 907cfdda9d7SSteve Wise return err; 908cfdda9d7SSteve Wise } 909cfdda9d7SSteve Wise 910cfdda9d7SSteve Wise static void c4iw_rdev_close(struct c4iw_rdev *rdev) 911cfdda9d7SSteve Wise { 9127730b4c7SHariprasad Shenai kfree(rdev->wr_log); 913d4702645SRaju Rangoju c4iw_release_dev_ucontext(rdev, &rdev->uctx); 91405eb2389SSteve Wise free_page((unsigned long)rdev->status_page); 915cfdda9d7SSteve Wise c4iw_pblpool_destroy(rdev); 916cfdda9d7SSteve Wise c4iw_rqtpool_destroy(rdev); 91726bff1bdSRaju Rangoju wait_for_completion(&rdev->pbl_compl); 91826bff1bdSRaju Rangoju wait_for_completion(&rdev->rqt_compl); 919d4702645SRaju Rangoju c4iw_ocqp_pool_destroy(rdev); 92026bff1bdSRaju Rangoju destroy_workqueue(rdev->free_workq); 921cfdda9d7SSteve Wise c4iw_destroy_resource(&rdev->resource); 922cfdda9d7SSteve Wise } 923cfdda9d7SSteve Wise 9241c8f1da5SBharat Potnuri void c4iw_dealloc(struct uld_ctx *ctx) 925cfdda9d7SSteve Wise { 9262f25e9a5SSteve Wise c4iw_rdev_close(&ctx->dev->rdev); 92737eb816cSSteve Wise WARN_ON_ONCE(!idr_is_empty(&ctx->dev->cqidr)); 9282f25e9a5SSteve Wise idr_destroy(&ctx->dev->cqidr); 92937eb816cSSteve Wise WARN_ON_ONCE(!idr_is_empty(&ctx->dev->qpidr)); 9302f25e9a5SSteve Wise idr_destroy(&ctx->dev->qpidr); 93137eb816cSSteve Wise WARN_ON_ONCE(!idr_is_empty(&ctx->dev->mmidr)); 9322f25e9a5SSteve Wise idr_destroy(&ctx->dev->mmidr); 93337eb816cSSteve Wise wait_event(ctx->dev->wait, idr_is_empty(&ctx->dev->hwtid_idr)); 934793dad94SVipul Pandya idr_destroy(&ctx->dev->hwtid_idr); 935793dad94SVipul Pandya idr_destroy(&ctx->dev->stid_idr); 936793dad94SVipul Pandya idr_destroy(&ctx->dev->atid_idr); 937fa658a98SSteve Wise if (ctx->dev->rdev.bar2_kva) 938fa658a98SSteve Wise iounmap(ctx->dev->rdev.bar2_kva); 939fa658a98SSteve Wise if (ctx->dev->rdev.oc_mw_kva) 9402f25e9a5SSteve Wise iounmap(ctx->dev->rdev.oc_mw_kva); 9412f25e9a5SSteve Wise ib_dealloc_device(&ctx->dev->ibdev); 9422f25e9a5SSteve Wise ctx->dev = NULL; 943cfdda9d7SSteve Wise } 944cfdda9d7SSteve Wise 9459efe10a1SSteve Wise static void c4iw_remove(struct uld_ctx *ctx) 9469efe10a1SSteve Wise { 947548ddb19SBharat Potnuri pr_debug("c4iw_dev %p\n", ctx->dev); 9489efe10a1SSteve Wise c4iw_unregister_device(ctx->dev); 9499efe10a1SSteve Wise c4iw_dealloc(ctx); 9509efe10a1SSteve Wise } 9519efe10a1SSteve Wise 9529efe10a1SSteve Wise static int rdma_supported(const struct cxgb4_lld_info *infop) 9539efe10a1SSteve Wise { 9549efe10a1SSteve Wise return infop->vr->stag.size > 0 && infop->vr->pbl.size > 0 && 9559efe10a1SSteve Wise infop->vr->rq.size > 0 && infop->vr->qp.size > 0 && 956f079af7aSVipul Pandya infop->vr->cq.size > 0; 9579efe10a1SSteve Wise } 9589efe10a1SSteve Wise 959cfdda9d7SSteve Wise static struct c4iw_dev *c4iw_alloc(const struct cxgb4_lld_info *infop) 960cfdda9d7SSteve Wise { 961cfdda9d7SSteve Wise struct c4iw_dev *devp; 962cfdda9d7SSteve Wise int ret; 963cfdda9d7SSteve Wise 9649efe10a1SSteve Wise if (!rdma_supported(infop)) { 965700456bdSJoe Perches pr_info("%s: RDMA not supported on this device\n", 9669efe10a1SSteve Wise pci_name(infop->pdev)); 9679efe10a1SSteve Wise return ERR_PTR(-ENOSYS); 9689efe10a1SSteve Wise } 969f079af7aSVipul Pandya if (!ocqp_supported(infop)) 970700456bdSJoe Perches pr_info("%s: On-Chip Queues not supported on this device\n", 971f079af7aSVipul Pandya pci_name(infop->pdev)); 97280ccdd60SVipul Pandya 973cfdda9d7SSteve Wise devp = (struct c4iw_dev *)ib_alloc_device(sizeof(*devp)); 974cfdda9d7SSteve Wise if (!devp) { 975700456bdSJoe Perches pr_err("Cannot allocate ib device\n"); 976bbe9a0a2SSteve Wise return ERR_PTR(-ENOMEM); 977cfdda9d7SSteve Wise } 978cfdda9d7SSteve Wise devp->rdev.lldi = *infop; 979cfdda9d7SSteve Wise 98004e10e21SHariprasad Shenai /* init various hw-queue params based on lld info */ 981548ddb19SBharat Potnuri pr_debug("Ing. padding boundary is %d, egrsstatuspagesize = %d\n", 982548ddb19SBharat Potnuri devp->rdev.lldi.sge_ingpadboundary, 98304e10e21SHariprasad Shenai devp->rdev.lldi.sge_egrstatuspagesize); 98404e10e21SHariprasad Shenai 98504e10e21SHariprasad Shenai devp->rdev.hw_queue.t4_eq_status_entries = 9864bbfabedSGanesh Goudar devp->rdev.lldi.sge_egrstatuspagesize / 64; 98766eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_max_eq_size = 65520; 98866eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_max_iq_size = 65520; 98966eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_max_rq_size = 8192 - 99066eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_eq_status_entries - 1; 99104e10e21SHariprasad Shenai devp->rdev.hw_queue.t4_max_sq_size = 99266eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_max_eq_size - 99366eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_eq_status_entries - 1; 99404e10e21SHariprasad Shenai devp->rdev.hw_queue.t4_max_qp_depth = 99566eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_max_rq_size; 99604e10e21SHariprasad Shenai devp->rdev.hw_queue.t4_max_cq_depth = 99766eb19afSHariprasad Shenai devp->rdev.hw_queue.t4_max_iq_size - 2; 99804e10e21SHariprasad Shenai devp->rdev.hw_queue.t4_stat_len = 99904e10e21SHariprasad Shenai devp->rdev.lldi.sge_egrstatuspagesize; 100004e10e21SHariprasad Shenai 1001fa658a98SSteve Wise /* 1002963cab50SHariprasad S * For T5/T6 devices, we map all of BAR2 with WC. 1003fa658a98SSteve Wise * For T4 devices with onchip qp mem, we map only that part 1004fa658a98SSteve Wise * of BAR2 with WC. 1005fa658a98SSteve Wise */ 1006fa658a98SSteve Wise devp->rdev.bar2_pa = pci_resource_start(devp->rdev.lldi.pdev, 2); 1007963cab50SHariprasad S if (!is_t4(devp->rdev.lldi.adapter_type)) { 1008fa658a98SSteve Wise devp->rdev.bar2_kva = ioremap_wc(devp->rdev.bar2_pa, 1009fa658a98SSteve Wise pci_resource_len(devp->rdev.lldi.pdev, 2)); 1010fa658a98SSteve Wise if (!devp->rdev.bar2_kva) { 1011700456bdSJoe Perches pr_err("Unable to ioremap BAR2\n"); 101265b302adSChristoph Jaeger ib_dealloc_device(&devp->ibdev); 1013fa658a98SSteve Wise return ERR_PTR(-EINVAL); 1014fa658a98SSteve Wise } 1015fa658a98SSteve Wise } else if (ocqp_supported(infop)) { 1016fa658a98SSteve Wise devp->rdev.oc_mw_pa = 1017fa658a98SSteve Wise pci_resource_start(devp->rdev.lldi.pdev, 2) + 1018fa658a98SSteve Wise pci_resource_len(devp->rdev.lldi.pdev, 2) - 1019fa658a98SSteve Wise roundup_pow_of_two(devp->rdev.lldi.vr->ocq.size); 1020c6d7b267SSteve Wise devp->rdev.oc_mw_kva = ioremap_wc(devp->rdev.oc_mw_pa, 1021c6d7b267SSteve Wise devp->rdev.lldi.vr->ocq.size); 1022fa658a98SSteve Wise if (!devp->rdev.oc_mw_kva) { 1023700456bdSJoe Perches pr_err("Unable to ioremap onchip mem\n"); 102465b302adSChristoph Jaeger ib_dealloc_device(&devp->ibdev); 1025fa658a98SSteve Wise return ERR_PTR(-EINVAL); 1026fa658a98SSteve Wise } 1027fa658a98SSteve Wise } 1028c6d7b267SSteve Wise 1029a9a42886SJoe Perches pr_debug("ocq memory: hw_start 0x%x size %u mw_pa 0x%lx mw_kva %p\n", 1030c6d7b267SSteve Wise devp->rdev.lldi.vr->ocq.start, devp->rdev.lldi.vr->ocq.size, 1031c6d7b267SSteve Wise devp->rdev.oc_mw_pa, devp->rdev.oc_mw_kva); 1032c6d7b267SSteve Wise 1033cfdda9d7SSteve Wise ret = c4iw_rdev_open(&devp->rdev); 1034cfdda9d7SSteve Wise if (ret) { 1035700456bdSJoe Perches pr_err("Unable to open CXIO rdev err %d\n", ret); 1036cfdda9d7SSteve Wise ib_dealloc_device(&devp->ibdev); 1037bbe9a0a2SSteve Wise return ERR_PTR(ret); 1038cfdda9d7SSteve Wise } 1039cfdda9d7SSteve Wise 1040cfdda9d7SSteve Wise idr_init(&devp->cqidr); 1041cfdda9d7SSteve Wise idr_init(&devp->qpidr); 1042cfdda9d7SSteve Wise idr_init(&devp->mmidr); 1043793dad94SVipul Pandya idr_init(&devp->hwtid_idr); 1044793dad94SVipul Pandya idr_init(&devp->stid_idr); 1045793dad94SVipul Pandya idr_init(&devp->atid_idr); 1046cfdda9d7SSteve Wise spin_lock_init(&devp->lock); 10478d81ef34SVipul Pandya mutex_init(&devp->rdev.stats.lock); 10482c974781SVipul Pandya mutex_init(&devp->db_mutex); 104905eb2389SSteve Wise INIT_LIST_HEAD(&devp->db_fc_list); 105037eb816cSSteve Wise init_waitqueue_head(&devp->wait); 10514c2c5763SHariprasad Shenai devp->avail_ird = devp->rdev.lldi.max_ird_adapter; 1052cfdda9d7SSteve Wise 1053cfdda9d7SSteve Wise if (c4iw_debugfs_root) { 1054cfdda9d7SSteve Wise devp->debugfs_root = debugfs_create_dir( 1055cfdda9d7SSteve Wise pci_name(devp->rdev.lldi.pdev), 1056cfdda9d7SSteve Wise c4iw_debugfs_root); 1057cfdda9d7SSteve Wise setup_debugfs(devp); 1058cfdda9d7SSteve Wise } 10599eccfe10SSteve Wise 10609eccfe10SSteve Wise 1061cfdda9d7SSteve Wise return devp; 1062cfdda9d7SSteve Wise } 1063cfdda9d7SSteve Wise 1064cfdda9d7SSteve Wise static void *c4iw_uld_add(const struct cxgb4_lld_info *infop) 1065cfdda9d7SSteve Wise { 10662f25e9a5SSteve Wise struct uld_ctx *ctx; 1067cfdda9d7SSteve Wise static int vers_printed; 1068cfdda9d7SSteve Wise int i; 1069cfdda9d7SSteve Wise 1070cfdda9d7SSteve Wise if (!vers_printed++) 1071f079af7aSVipul Pandya pr_info("Chelsio T4/T5 RDMA Driver - version %s\n", 1072cfdda9d7SSteve Wise DRV_VERSION); 1073cfdda9d7SSteve Wise 10742f25e9a5SSteve Wise ctx = kzalloc(sizeof *ctx, GFP_KERNEL); 10752f25e9a5SSteve Wise if (!ctx) { 10762f25e9a5SSteve Wise ctx = ERR_PTR(-ENOMEM); 1077cfdda9d7SSteve Wise goto out; 10782f25e9a5SSteve Wise } 10792f25e9a5SSteve Wise ctx->lldi = *infop; 1080cfdda9d7SSteve Wise 1081548ddb19SBharat Potnuri pr_debug("found device %s nchan %u nrxq %u ntxq %u nports %u\n", 1082548ddb19SBharat Potnuri pci_name(ctx->lldi.pdev), 10832f25e9a5SSteve Wise ctx->lldi.nchan, ctx->lldi.nrxq, 10842f25e9a5SSteve Wise ctx->lldi.ntxq, ctx->lldi.nports); 1085cfdda9d7SSteve Wise 10862f25e9a5SSteve Wise mutex_lock(&dev_mutex); 10872f25e9a5SSteve Wise list_add_tail(&ctx->entry, &uld_ctx_list); 10882f25e9a5SSteve Wise mutex_unlock(&dev_mutex); 10892f25e9a5SSteve Wise 10902f25e9a5SSteve Wise for (i = 0; i < ctx->lldi.nrxq; i++) 1091a9a42886SJoe Perches pr_debug("rxqid[%u] %u\n", i, ctx->lldi.rxq_ids[i]); 1092cfdda9d7SSteve Wise out: 10932f25e9a5SSteve Wise return ctx; 1094cfdda9d7SSteve Wise } 1095cfdda9d7SSteve Wise 10961cab775cSVipul Pandya static inline struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl, 10971cab775cSVipul Pandya const __be64 *rsp, 10981cab775cSVipul Pandya u32 pktshift) 10991cab775cSVipul Pandya { 11001cab775cSVipul Pandya struct sk_buff *skb; 11011cab775cSVipul Pandya 11021cab775cSVipul Pandya /* 11031cab775cSVipul Pandya * Allocate space for cpl_pass_accept_req which will be synthesized by 11041cab775cSVipul Pandya * driver. Once the driver synthesizes the request the skb will go 11051cab775cSVipul Pandya * through the regular cpl_pass_accept_req processing. 11061cab775cSVipul Pandya * The math here assumes sizeof cpl_pass_accept_req >= sizeof 11071cab775cSVipul Pandya * cpl_rx_pkt. 11081cab775cSVipul Pandya */ 11091cab775cSVipul Pandya skb = alloc_skb(gl->tot_len + sizeof(struct cpl_pass_accept_req) + 11101cab775cSVipul Pandya sizeof(struct rss_header) - pktshift, GFP_ATOMIC); 11111cab775cSVipul Pandya if (unlikely(!skb)) 11121cab775cSVipul Pandya return NULL; 11131cab775cSVipul Pandya 11141cab775cSVipul Pandya __skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req) + 11151cab775cSVipul Pandya sizeof(struct rss_header) - pktshift); 11161cab775cSVipul Pandya 11171cab775cSVipul Pandya /* 11181cab775cSVipul Pandya * This skb will contain: 11191cab775cSVipul Pandya * rss_header from the rspq descriptor (1 flit) 11201cab775cSVipul Pandya * cpl_rx_pkt struct from the rspq descriptor (2 flits) 11211cab775cSVipul Pandya * space for the difference between the size of an 11221cab775cSVipul Pandya * rx_pkt and pass_accept_req cpl (1 flit) 11231cab775cSVipul Pandya * the packet data from the gl 11241cab775cSVipul Pandya */ 11251cab775cSVipul Pandya skb_copy_to_linear_data(skb, rsp, sizeof(struct cpl_pass_accept_req) + 11261cab775cSVipul Pandya sizeof(struct rss_header)); 11271cab775cSVipul Pandya skb_copy_to_linear_data_offset(skb, sizeof(struct rss_header) + 11281cab775cSVipul Pandya sizeof(struct cpl_pass_accept_req), 11291cab775cSVipul Pandya gl->va + pktshift, 11301cab775cSVipul Pandya gl->tot_len - pktshift); 11311cab775cSVipul Pandya return skb; 11321cab775cSVipul Pandya } 11331cab775cSVipul Pandya 11341cab775cSVipul Pandya static inline int recv_rx_pkt(struct c4iw_dev *dev, const struct pkt_gl *gl, 11351cab775cSVipul Pandya const __be64 *rsp) 11361cab775cSVipul Pandya { 11371cab775cSVipul Pandya unsigned int opcode = *(u8 *)rsp; 11381cab775cSVipul Pandya struct sk_buff *skb; 11391cab775cSVipul Pandya 11401cab775cSVipul Pandya if (opcode != CPL_RX_PKT) 11411cab775cSVipul Pandya goto out; 11421cab775cSVipul Pandya 11431cab775cSVipul Pandya skb = copy_gl_to_skb_pkt(gl , rsp, dev->rdev.lldi.sge_pktshift); 11441cab775cSVipul Pandya if (skb == NULL) 11451cab775cSVipul Pandya goto out; 11461cab775cSVipul Pandya 11471cab775cSVipul Pandya if (c4iw_handlers[opcode] == NULL) { 1148700456bdSJoe Perches pr_info("%s no handler opcode 0x%x...\n", __func__, opcode); 11491cab775cSVipul Pandya kfree_skb(skb); 11501cab775cSVipul Pandya goto out; 11511cab775cSVipul Pandya } 11521cab775cSVipul Pandya c4iw_handlers[opcode](dev, skb); 11531cab775cSVipul Pandya return 1; 11541cab775cSVipul Pandya out: 11551cab775cSVipul Pandya return 0; 11561cab775cSVipul Pandya } 11571cab775cSVipul Pandya 1158cfdda9d7SSteve Wise static int c4iw_uld_rx_handler(void *handle, const __be64 *rsp, 1159cfdda9d7SSteve Wise const struct pkt_gl *gl) 1160cfdda9d7SSteve Wise { 11612f25e9a5SSteve Wise struct uld_ctx *ctx = handle; 11622f25e9a5SSteve Wise struct c4iw_dev *dev = ctx->dev; 1163cfdda9d7SSteve Wise struct sk_buff *skb; 11641cab775cSVipul Pandya u8 opcode; 1165cfdda9d7SSteve Wise 1166cfdda9d7SSteve Wise if (gl == NULL) { 1167cfdda9d7SSteve Wise /* omit RSS and rsp_ctrl at end of descriptor */ 1168cfdda9d7SSteve Wise unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8; 1169cfdda9d7SSteve Wise 1170cfdda9d7SSteve Wise skb = alloc_skb(256, GFP_ATOMIC); 1171cfdda9d7SSteve Wise if (!skb) 1172cfdda9d7SSteve Wise goto nomem; 1173cfdda9d7SSteve Wise __skb_put(skb, len); 1174cfdda9d7SSteve Wise skb_copy_to_linear_data(skb, &rsp[1], len); 1175cfdda9d7SSteve Wise } else if (gl == CXGB4_MSG_AN) { 1176cfdda9d7SSteve Wise const struct rsp_ctrl *rc = (void *)rsp; 1177cfdda9d7SSteve Wise 1178cfdda9d7SSteve Wise u32 qid = be32_to_cpu(rc->pldbuflen_qid); 1179cfdda9d7SSteve Wise c4iw_ev_handler(dev, qid); 1180cfdda9d7SSteve Wise return 0; 11811cab775cSVipul Pandya } else if (unlikely(*(u8 *)rsp != *(u8 *)gl->va)) { 11821cab775cSVipul Pandya if (recv_rx_pkt(dev, gl, rsp)) 11831cab775cSVipul Pandya return 0; 11841cab775cSVipul Pandya 1185700456bdSJoe Perches pr_info("%s: unexpected FL contents at %p, RSS %#llx, FL %#llx, len %u\n", 11861cab775cSVipul Pandya pci_name(ctx->lldi.pdev), gl->va, 1187700456bdSJoe Perches be64_to_cpu(*rsp), 1188700456bdSJoe Perches be64_to_cpu(*(__force __be64 *)gl->va), 11891cab775cSVipul Pandya gl->tot_len); 11901cab775cSVipul Pandya 11911cab775cSVipul Pandya return 0; 1192cfdda9d7SSteve Wise } else { 1193da411ba1SSteve Wise skb = cxgb4_pktgl_to_skb(gl, 128, 128); 1194cfdda9d7SSteve Wise if (unlikely(!skb)) 1195cfdda9d7SSteve Wise goto nomem; 1196cfdda9d7SSteve Wise } 1197cfdda9d7SSteve Wise 11981cab775cSVipul Pandya opcode = *(u8 *)rsp; 1199dbb084ccSSteve Wise if (c4iw_handlers[opcode]) { 1200cfdda9d7SSteve Wise c4iw_handlers[opcode](dev, skb); 1201dbb084ccSSteve Wise } else { 1202700456bdSJoe Perches pr_info("%s no handler opcode 0x%x...\n", __func__, opcode); 1203dbb084ccSSteve Wise kfree_skb(skb); 1204dbb084ccSSteve Wise } 1205cfdda9d7SSteve Wise 1206cfdda9d7SSteve Wise return 0; 1207cfdda9d7SSteve Wise nomem: 1208cfdda9d7SSteve Wise return -1; 1209cfdda9d7SSteve Wise } 1210cfdda9d7SSteve Wise 1211cfdda9d7SSteve Wise static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state) 1212cfdda9d7SSteve Wise { 12132f25e9a5SSteve Wise struct uld_ctx *ctx = handle; 12141c01c538SSteve Wise 1215548ddb19SBharat Potnuri pr_debug("new_state %u\n", new_state); 12161c01c538SSteve Wise switch (new_state) { 12171c01c538SSteve Wise case CXGB4_STATE_UP: 1218700456bdSJoe Perches pr_info("%s: Up\n", pci_name(ctx->lldi.pdev)); 12192f25e9a5SSteve Wise if (!ctx->dev) { 12202f25e9a5SSteve Wise ctx->dev = c4iw_alloc(&ctx->lldi); 12219efe10a1SSteve Wise if (IS_ERR(ctx->dev)) { 1222700456bdSJoe Perches pr_err("%s: initialization failed: %ld\n", 12239efe10a1SSteve Wise pci_name(ctx->lldi.pdev), 12249efe10a1SSteve Wise PTR_ERR(ctx->dev)); 12259efe10a1SSteve Wise ctx->dev = NULL; 12269efe10a1SSteve Wise break; 12279efe10a1SSteve Wise } 12281c8f1da5SBharat Potnuri 12291c8f1da5SBharat Potnuri INIT_WORK(&ctx->reg_work, c4iw_register_device); 12301c8f1da5SBharat Potnuri queue_work(reg_workq, &ctx->reg_work); 12311c01c538SSteve Wise } 12321c01c538SSteve Wise break; 12331c01c538SSteve Wise case CXGB4_STATE_DOWN: 1234700456bdSJoe Perches pr_info("%s: Down\n", pci_name(ctx->lldi.pdev)); 12352f25e9a5SSteve Wise if (ctx->dev) 12362f25e9a5SSteve Wise c4iw_remove(ctx); 12371c01c538SSteve Wise break; 12388b7372c1SGanesh Goudar case CXGB4_STATE_FATAL_ERROR: 12391c01c538SSteve Wise case CXGB4_STATE_START_RECOVERY: 1240700456bdSJoe Perches pr_info("%s: Fatal Error\n", pci_name(ctx->lldi.pdev)); 12412f25e9a5SSteve Wise if (ctx->dev) { 1242767fbe81SSteve Wise struct ib_event event; 1243767fbe81SSteve Wise 12442f25e9a5SSteve Wise ctx->dev->rdev.flags |= T4_FATAL_ERROR; 1245767fbe81SSteve Wise memset(&event, 0, sizeof event); 1246767fbe81SSteve Wise event.event = IB_EVENT_DEVICE_FATAL; 12472f25e9a5SSteve Wise event.device = &ctx->dev->ibdev; 1248767fbe81SSteve Wise ib_dispatch_event(&event); 12492f25e9a5SSteve Wise c4iw_remove(ctx); 1250767fbe81SSteve Wise } 12511c01c538SSteve Wise break; 12521c01c538SSteve Wise case CXGB4_STATE_DETACH: 1253700456bdSJoe Perches pr_info("%s: Detach\n", pci_name(ctx->lldi.pdev)); 12542f25e9a5SSteve Wise if (ctx->dev) 12552f25e9a5SSteve Wise c4iw_remove(ctx); 12561c01c538SSteve Wise break; 12571c01c538SSteve Wise } 1258cfdda9d7SSteve Wise return 0; 1259cfdda9d7SSteve Wise } 1260cfdda9d7SSteve Wise 12612c974781SVipul Pandya static int disable_qp_db(int id, void *p, void *data) 12622c974781SVipul Pandya { 12632c974781SVipul Pandya struct c4iw_qp *qp = p; 12642c974781SVipul Pandya 12652c974781SVipul Pandya t4_disable_wq_db(&qp->wq); 12662c974781SVipul Pandya return 0; 12672c974781SVipul Pandya } 12682c974781SVipul Pandya 12692c974781SVipul Pandya static void stop_queues(struct uld_ctx *ctx) 12702c974781SVipul Pandya { 127105eb2389SSteve Wise unsigned long flags; 127205eb2389SSteve Wise 127305eb2389SSteve Wise spin_lock_irqsave(&ctx->dev->lock, flags); 1274422eea0aSVipul Pandya ctx->dev->rdev.stats.db_state_transitions++; 127505eb2389SSteve Wise ctx->dev->db_state = STOPPED; 127605eb2389SSteve Wise if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED) 12772c974781SVipul Pandya idr_for_each(&ctx->dev->qpidr, disable_qp_db, NULL); 127805eb2389SSteve Wise else 127905eb2389SSteve Wise ctx->dev->rdev.status_page->db_off = 1; 128005eb2389SSteve Wise spin_unlock_irqrestore(&ctx->dev->lock, flags); 12812c974781SVipul Pandya } 12822c974781SVipul Pandya 12832c974781SVipul Pandya static int enable_qp_db(int id, void *p, void *data) 12842c974781SVipul Pandya { 12852c974781SVipul Pandya struct c4iw_qp *qp = p; 12862c974781SVipul Pandya 12872c974781SVipul Pandya t4_enable_wq_db(&qp->wq); 12882c974781SVipul Pandya return 0; 12892c974781SVipul Pandya } 12902c974781SVipul Pandya 129105eb2389SSteve Wise static void resume_rc_qp(struct c4iw_qp *qp) 129205eb2389SSteve Wise { 129305eb2389SSteve Wise spin_lock(&qp->lock); 1294963cab50SHariprasad S t4_ring_sq_db(&qp->wq, qp->wq.sq.wq_pidx_inc, NULL); 129505eb2389SSteve Wise qp->wq.sq.wq_pidx_inc = 0; 1296963cab50SHariprasad S t4_ring_rq_db(&qp->wq, qp->wq.rq.wq_pidx_inc, NULL); 129705eb2389SSteve Wise qp->wq.rq.wq_pidx_inc = 0; 129805eb2389SSteve Wise spin_unlock(&qp->lock); 129905eb2389SSteve Wise } 130005eb2389SSteve Wise 130105eb2389SSteve Wise static void resume_a_chunk(struct uld_ctx *ctx) 130205eb2389SSteve Wise { 130305eb2389SSteve Wise int i; 130405eb2389SSteve Wise struct c4iw_qp *qp; 130505eb2389SSteve Wise 130605eb2389SSteve Wise for (i = 0; i < DB_FC_RESUME_SIZE; i++) { 130705eb2389SSteve Wise qp = list_first_entry(&ctx->dev->db_fc_list, struct c4iw_qp, 130805eb2389SSteve Wise db_fc_entry); 130905eb2389SSteve Wise list_del_init(&qp->db_fc_entry); 131005eb2389SSteve Wise resume_rc_qp(qp); 131105eb2389SSteve Wise if (list_empty(&ctx->dev->db_fc_list)) 131205eb2389SSteve Wise break; 131305eb2389SSteve Wise } 131405eb2389SSteve Wise } 131505eb2389SSteve Wise 13162c974781SVipul Pandya static void resume_queues(struct uld_ctx *ctx) 13172c974781SVipul Pandya { 13182c974781SVipul Pandya spin_lock_irq(&ctx->dev->lock); 131905eb2389SSteve Wise if (ctx->dev->db_state != STOPPED) 132005eb2389SSteve Wise goto out; 132105eb2389SSteve Wise ctx->dev->db_state = FLOW_CONTROL; 132205eb2389SSteve Wise while (1) { 132305eb2389SSteve Wise if (list_empty(&ctx->dev->db_fc_list)) { 132405eb2389SSteve Wise WARN_ON(ctx->dev->db_state != FLOW_CONTROL); 1325422eea0aSVipul Pandya ctx->dev->db_state = NORMAL; 1326422eea0aSVipul Pandya ctx->dev->rdev.stats.db_state_transitions++; 132705eb2389SSteve Wise if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED) { 132805eb2389SSteve Wise idr_for_each(&ctx->dev->qpidr, enable_qp_db, 132905eb2389SSteve Wise NULL); 133005eb2389SSteve Wise } else { 133105eb2389SSteve Wise ctx->dev->rdev.status_page->db_off = 0; 1332422eea0aSVipul Pandya } 133305eb2389SSteve Wise break; 133405eb2389SSteve Wise } else { 133505eb2389SSteve Wise if (cxgb4_dbfifo_count(ctx->dev->rdev.lldi.ports[0], 1) 133605eb2389SSteve Wise < (ctx->dev->rdev.lldi.dbfifo_int_thresh << 133705eb2389SSteve Wise DB_FC_DRAIN_THRESH)) { 133805eb2389SSteve Wise resume_a_chunk(ctx); 133905eb2389SSteve Wise } 134005eb2389SSteve Wise if (!list_empty(&ctx->dev->db_fc_list)) { 134105eb2389SSteve Wise spin_unlock_irq(&ctx->dev->lock); 134205eb2389SSteve Wise if (DB_FC_RESUME_DELAY) { 134305eb2389SSteve Wise set_current_state(TASK_UNINTERRUPTIBLE); 134405eb2389SSteve Wise schedule_timeout(DB_FC_RESUME_DELAY); 134505eb2389SSteve Wise } 134605eb2389SSteve Wise spin_lock_irq(&ctx->dev->lock); 134705eb2389SSteve Wise if (ctx->dev->db_state != FLOW_CONTROL) 134805eb2389SSteve Wise break; 134905eb2389SSteve Wise } 135005eb2389SSteve Wise } 135105eb2389SSteve Wise } 135205eb2389SSteve Wise out: 135305eb2389SSteve Wise if (ctx->dev->db_state != NORMAL) 135405eb2389SSteve Wise ctx->dev->rdev.stats.db_fc_interruptions++; 1355422eea0aSVipul Pandya spin_unlock_irq(&ctx->dev->lock); 1356422eea0aSVipul Pandya } 1357422eea0aSVipul Pandya 1358422eea0aSVipul Pandya struct qp_list { 1359422eea0aSVipul Pandya unsigned idx; 1360422eea0aSVipul Pandya struct c4iw_qp **qps; 1361422eea0aSVipul Pandya }; 1362422eea0aSVipul Pandya 1363422eea0aSVipul Pandya static int add_and_ref_qp(int id, void *p, void *data) 1364422eea0aSVipul Pandya { 1365422eea0aSVipul Pandya struct qp_list *qp_listp = data; 1366422eea0aSVipul Pandya struct c4iw_qp *qp = p; 1367422eea0aSVipul Pandya 1368422eea0aSVipul Pandya c4iw_qp_add_ref(&qp->ibqp); 1369422eea0aSVipul Pandya qp_listp->qps[qp_listp->idx++] = qp; 1370422eea0aSVipul Pandya return 0; 1371422eea0aSVipul Pandya } 1372422eea0aSVipul Pandya 1373422eea0aSVipul Pandya static int count_qps(int id, void *p, void *data) 1374422eea0aSVipul Pandya { 1375422eea0aSVipul Pandya unsigned *countp = data; 1376422eea0aSVipul Pandya (*countp)++; 1377422eea0aSVipul Pandya return 0; 1378422eea0aSVipul Pandya } 1379422eea0aSVipul Pandya 138005eb2389SSteve Wise static void deref_qps(struct qp_list *qp_list) 1381422eea0aSVipul Pandya { 1382422eea0aSVipul Pandya int idx; 1383422eea0aSVipul Pandya 138405eb2389SSteve Wise for (idx = 0; idx < qp_list->idx; idx++) 138505eb2389SSteve Wise c4iw_qp_rem_ref(&qp_list->qps[idx]->ibqp); 1386422eea0aSVipul Pandya } 1387422eea0aSVipul Pandya 1388422eea0aSVipul Pandya static void recover_lost_dbs(struct uld_ctx *ctx, struct qp_list *qp_list) 1389422eea0aSVipul Pandya { 1390422eea0aSVipul Pandya int idx; 1391422eea0aSVipul Pandya int ret; 1392422eea0aSVipul Pandya 1393422eea0aSVipul Pandya for (idx = 0; idx < qp_list->idx; idx++) { 1394422eea0aSVipul Pandya struct c4iw_qp *qp = qp_list->qps[idx]; 1395422eea0aSVipul Pandya 139605eb2389SSteve Wise spin_lock_irq(&qp->rhp->lock); 139705eb2389SSteve Wise spin_lock(&qp->lock); 1398422eea0aSVipul Pandya ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0], 1399422eea0aSVipul Pandya qp->wq.sq.qid, 1400422eea0aSVipul Pandya t4_sq_host_wq_pidx(&qp->wq), 1401422eea0aSVipul Pandya t4_sq_wq_size(&qp->wq)); 1402422eea0aSVipul Pandya if (ret) { 1403700456bdSJoe Perches pr_err("%s: Fatal error - DB overflow recovery failed - error syncing SQ qid %u\n", 1404422eea0aSVipul Pandya pci_name(ctx->lldi.pdev), qp->wq.sq.qid); 140505eb2389SSteve Wise spin_unlock(&qp->lock); 140605eb2389SSteve Wise spin_unlock_irq(&qp->rhp->lock); 1407422eea0aSVipul Pandya return; 1408422eea0aSVipul Pandya } 140905eb2389SSteve Wise qp->wq.sq.wq_pidx_inc = 0; 1410422eea0aSVipul Pandya 1411422eea0aSVipul Pandya ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0], 1412422eea0aSVipul Pandya qp->wq.rq.qid, 1413422eea0aSVipul Pandya t4_rq_host_wq_pidx(&qp->wq), 1414422eea0aSVipul Pandya t4_rq_wq_size(&qp->wq)); 1415422eea0aSVipul Pandya 1416422eea0aSVipul Pandya if (ret) { 1417700456bdSJoe Perches pr_err("%s: Fatal error - DB overflow recovery failed - error syncing RQ qid %u\n", 1418422eea0aSVipul Pandya pci_name(ctx->lldi.pdev), qp->wq.rq.qid); 141905eb2389SSteve Wise spin_unlock(&qp->lock); 142005eb2389SSteve Wise spin_unlock_irq(&qp->rhp->lock); 1421422eea0aSVipul Pandya return; 1422422eea0aSVipul Pandya } 142305eb2389SSteve Wise qp->wq.rq.wq_pidx_inc = 0; 142405eb2389SSteve Wise spin_unlock(&qp->lock); 142505eb2389SSteve Wise spin_unlock_irq(&qp->rhp->lock); 1426422eea0aSVipul Pandya 1427422eea0aSVipul Pandya /* Wait for the dbfifo to drain */ 1428422eea0aSVipul Pandya while (cxgb4_dbfifo_count(qp->rhp->rdev.lldi.ports[0], 1) > 0) { 1429422eea0aSVipul Pandya set_current_state(TASK_UNINTERRUPTIBLE); 1430422eea0aSVipul Pandya schedule_timeout(usecs_to_jiffies(10)); 1431422eea0aSVipul Pandya } 1432422eea0aSVipul Pandya } 1433422eea0aSVipul Pandya } 1434422eea0aSVipul Pandya 1435422eea0aSVipul Pandya static void recover_queues(struct uld_ctx *ctx) 1436422eea0aSVipul Pandya { 1437422eea0aSVipul Pandya int count = 0; 1438422eea0aSVipul Pandya struct qp_list qp_list; 1439422eea0aSVipul Pandya int ret; 1440422eea0aSVipul Pandya 1441422eea0aSVipul Pandya /* slow everybody down */ 1442422eea0aSVipul Pandya set_current_state(TASK_UNINTERRUPTIBLE); 1443422eea0aSVipul Pandya schedule_timeout(usecs_to_jiffies(1000)); 1444422eea0aSVipul Pandya 1445422eea0aSVipul Pandya /* flush the SGE contexts */ 1446422eea0aSVipul Pandya ret = cxgb4_flush_eq_cache(ctx->dev->rdev.lldi.ports[0]); 1447422eea0aSVipul Pandya if (ret) { 1448700456bdSJoe Perches pr_err("%s: Fatal error - DB overflow recovery failed\n", 1449422eea0aSVipul Pandya pci_name(ctx->lldi.pdev)); 145005eb2389SSteve Wise return; 1451422eea0aSVipul Pandya } 1452422eea0aSVipul Pandya 1453422eea0aSVipul Pandya /* Count active queues so we can build a list of queues to recover */ 1454422eea0aSVipul Pandya spin_lock_irq(&ctx->dev->lock); 145505eb2389SSteve Wise WARN_ON(ctx->dev->db_state != STOPPED); 145605eb2389SSteve Wise ctx->dev->db_state = RECOVERY; 1457422eea0aSVipul Pandya idr_for_each(&ctx->dev->qpidr, count_qps, &count); 1458422eea0aSVipul Pandya 14596396bb22SKees Cook qp_list.qps = kcalloc(count, sizeof(*qp_list.qps), GFP_ATOMIC); 1460422eea0aSVipul Pandya if (!qp_list.qps) { 1461422eea0aSVipul Pandya spin_unlock_irq(&ctx->dev->lock); 146205eb2389SSteve Wise return; 1463422eea0aSVipul Pandya } 1464422eea0aSVipul Pandya qp_list.idx = 0; 1465422eea0aSVipul Pandya 1466422eea0aSVipul Pandya /* add and ref each qp so it doesn't get freed */ 1467422eea0aSVipul Pandya idr_for_each(&ctx->dev->qpidr, add_and_ref_qp, &qp_list); 1468422eea0aSVipul Pandya 1469422eea0aSVipul Pandya spin_unlock_irq(&ctx->dev->lock); 1470422eea0aSVipul Pandya 1471422eea0aSVipul Pandya /* now traverse the list in a safe context to recover the db state*/ 1472422eea0aSVipul Pandya recover_lost_dbs(ctx, &qp_list); 1473422eea0aSVipul Pandya 1474422eea0aSVipul Pandya /* we're almost done! deref the qps and clean up */ 147505eb2389SSteve Wise deref_qps(&qp_list); 1476422eea0aSVipul Pandya kfree(qp_list.qps); 1477422eea0aSVipul Pandya 1478422eea0aSVipul Pandya spin_lock_irq(&ctx->dev->lock); 147905eb2389SSteve Wise WARN_ON(ctx->dev->db_state != RECOVERY); 148005eb2389SSteve Wise ctx->dev->db_state = STOPPED; 14812c974781SVipul Pandya spin_unlock_irq(&ctx->dev->lock); 14822c974781SVipul Pandya } 14832c974781SVipul Pandya 14842c974781SVipul Pandya static int c4iw_uld_control(void *handle, enum cxgb4_control control, ...) 14852c974781SVipul Pandya { 14862c974781SVipul Pandya struct uld_ctx *ctx = handle; 14872c974781SVipul Pandya 14882c974781SVipul Pandya switch (control) { 14892c974781SVipul Pandya case CXGB4_CONTROL_DB_FULL: 14902c974781SVipul Pandya stop_queues(ctx); 14912c974781SVipul Pandya ctx->dev->rdev.stats.db_full++; 14922c974781SVipul Pandya break; 14932c974781SVipul Pandya case CXGB4_CONTROL_DB_EMPTY: 14942c974781SVipul Pandya resume_queues(ctx); 14952c974781SVipul Pandya mutex_lock(&ctx->dev->rdev.stats.lock); 14962c974781SVipul Pandya ctx->dev->rdev.stats.db_empty++; 14972c974781SVipul Pandya mutex_unlock(&ctx->dev->rdev.stats.lock); 14982c974781SVipul Pandya break; 14992c974781SVipul Pandya case CXGB4_CONTROL_DB_DROP: 1500422eea0aSVipul Pandya recover_queues(ctx); 15012c974781SVipul Pandya mutex_lock(&ctx->dev->rdev.stats.lock); 15022c974781SVipul Pandya ctx->dev->rdev.stats.db_drop++; 15032c974781SVipul Pandya mutex_unlock(&ctx->dev->rdev.stats.lock); 15042c974781SVipul Pandya break; 15052c974781SVipul Pandya default: 1506700456bdSJoe Perches pr_warn("%s: unknown control cmd %u\n", 15072c974781SVipul Pandya pci_name(ctx->lldi.pdev), control); 15082c974781SVipul Pandya break; 15092c974781SVipul Pandya } 15102c974781SVipul Pandya return 0; 15112c974781SVipul Pandya } 15122c974781SVipul Pandya 1513cfdda9d7SSteve Wise static struct cxgb4_uld_info c4iw_uld_info = { 1514cfdda9d7SSteve Wise .name = DRV_NAME, 15150fbc81b3SHariprasad Shenai .nrxq = MAX_ULD_QSETS, 1516ab677ff4SHariprasad Shenai .ntxq = MAX_ULD_QSETS, 15170fbc81b3SHariprasad Shenai .rxq_size = 511, 15180fbc81b3SHariprasad Shenai .ciq = true, 15190fbc81b3SHariprasad Shenai .lro = false, 1520cfdda9d7SSteve Wise .add = c4iw_uld_add, 1521cfdda9d7SSteve Wise .rx_handler = c4iw_uld_rx_handler, 1522cfdda9d7SSteve Wise .state_change = c4iw_uld_state_change, 15232c974781SVipul Pandya .control = c4iw_uld_control, 1524cfdda9d7SSteve Wise }; 1525cfdda9d7SSteve Wise 15262015f26cSSteve Wise void _c4iw_free_wr_wait(struct kref *kref) 15272015f26cSSteve Wise { 15282015f26cSSteve Wise struct c4iw_wr_wait *wr_waitp; 15292015f26cSSteve Wise 15302015f26cSSteve Wise wr_waitp = container_of(kref, struct c4iw_wr_wait, kref); 15312015f26cSSteve Wise pr_debug("Free wr_wait %p\n", wr_waitp); 15322015f26cSSteve Wise kfree(wr_waitp); 15332015f26cSSteve Wise } 15342015f26cSSteve Wise 15352015f26cSSteve Wise struct c4iw_wr_wait *c4iw_alloc_wr_wait(gfp_t gfp) 15362015f26cSSteve Wise { 15372015f26cSSteve Wise struct c4iw_wr_wait *wr_waitp; 15382015f26cSSteve Wise 15392015f26cSSteve Wise wr_waitp = kzalloc(sizeof(*wr_waitp), gfp); 15402015f26cSSteve Wise if (wr_waitp) { 15412015f26cSSteve Wise kref_init(&wr_waitp->kref); 15422015f26cSSteve Wise pr_debug("wr_wait %p\n", wr_waitp); 15432015f26cSSteve Wise } 15442015f26cSSteve Wise return wr_waitp; 15452015f26cSSteve Wise } 15462015f26cSSteve Wise 1547cfdda9d7SSteve Wise static int __init c4iw_init_module(void) 1548cfdda9d7SSteve Wise { 1549cfdda9d7SSteve Wise int err; 1550cfdda9d7SSteve Wise 1551cfdda9d7SSteve Wise err = c4iw_cm_init(); 1552cfdda9d7SSteve Wise if (err) 1553cfdda9d7SSteve Wise return err; 1554cfdda9d7SSteve Wise 1555cfdda9d7SSteve Wise c4iw_debugfs_root = debugfs_create_dir(DRV_NAME, NULL); 1556cfdda9d7SSteve Wise if (!c4iw_debugfs_root) 1557700456bdSJoe Perches pr_warn("could not create debugfs entry, continuing\n"); 1558cfdda9d7SSteve Wise 15591c8f1da5SBharat Potnuri reg_workq = create_singlethread_workqueue("Register_iWARP_device"); 15601c8f1da5SBharat Potnuri if (!reg_workq) { 15611c8f1da5SBharat Potnuri pr_err("Failed creating workqueue to register iwarp device\n"); 15621c8f1da5SBharat Potnuri return -ENOMEM; 15631c8f1da5SBharat Potnuri } 15641c8f1da5SBharat Potnuri 1565cfdda9d7SSteve Wise cxgb4_register_uld(CXGB4_ULD_RDMA, &c4iw_uld_info); 1566cfdda9d7SSteve Wise 1567cfdda9d7SSteve Wise return 0; 1568cfdda9d7SSteve Wise } 1569cfdda9d7SSteve Wise 1570cfdda9d7SSteve Wise static void __exit c4iw_exit_module(void) 1571cfdda9d7SSteve Wise { 15722f25e9a5SSteve Wise struct uld_ctx *ctx, *tmp; 1573cfdda9d7SSteve Wise 1574cfdda9d7SSteve Wise mutex_lock(&dev_mutex); 15752f25e9a5SSteve Wise list_for_each_entry_safe(ctx, tmp, &uld_ctx_list, entry) { 15762f25e9a5SSteve Wise if (ctx->dev) 15772f25e9a5SSteve Wise c4iw_remove(ctx); 15782f25e9a5SSteve Wise kfree(ctx); 1579cfdda9d7SSteve Wise } 1580cfdda9d7SSteve Wise mutex_unlock(&dev_mutex); 15811c8f1da5SBharat Potnuri flush_workqueue(reg_workq); 15821c8f1da5SBharat Potnuri destroy_workqueue(reg_workq); 1583fd388ce6SSteve Wise cxgb4_unregister_uld(CXGB4_ULD_RDMA); 1584cfdda9d7SSteve Wise c4iw_cm_term(); 1585cfdda9d7SSteve Wise debugfs_remove_recursive(c4iw_debugfs_root); 1586cfdda9d7SSteve Wise } 1587cfdda9d7SSteve Wise 1588cfdda9d7SSteve Wise module_init(c4iw_init_module); 1589cfdda9d7SSteve Wise module_exit(c4iw_exit_module); 1590