1 /* 2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/module.h> 33 #include <linux/moduleparam.h> 34 #include <linux/debugfs.h> 35 #include <linux/vmalloc.h> 36 #include <linux/math64.h> 37 38 #include <rdma/ib_verbs.h> 39 40 #include "iw_cxgb4.h" 41 42 #define DRV_VERSION "0.1" 43 44 MODULE_AUTHOR("Steve Wise"); 45 MODULE_DESCRIPTION("Chelsio T4/T5 RDMA Driver"); 46 MODULE_LICENSE("Dual BSD/GPL"); 47 MODULE_VERSION(DRV_VERSION); 48 49 static int allow_db_fc_on_t5; 50 module_param(allow_db_fc_on_t5, int, 0644); 51 MODULE_PARM_DESC(allow_db_fc_on_t5, 52 "Allow DB Flow Control on T5 (default = 0)"); 53 54 static int allow_db_coalescing_on_t5; 55 module_param(allow_db_coalescing_on_t5, int, 0644); 56 MODULE_PARM_DESC(allow_db_coalescing_on_t5, 57 "Allow DB Coalescing on T5 (default = 0)"); 58 59 int c4iw_wr_log = 0; 60 module_param(c4iw_wr_log, int, 0444); 61 MODULE_PARM_DESC(c4iw_wr_log, "Enables logging of work request timing data."); 62 63 static int c4iw_wr_log_size_order = 12; 64 module_param(c4iw_wr_log_size_order, int, 0444); 65 MODULE_PARM_DESC(c4iw_wr_log_size_order, 66 "Number of entries (log2) in the work request timing log."); 67 68 struct uld_ctx { 69 struct list_head entry; 70 struct cxgb4_lld_info lldi; 71 struct c4iw_dev *dev; 72 }; 73 74 static LIST_HEAD(uld_ctx_list); 75 static DEFINE_MUTEX(dev_mutex); 76 77 #define DB_FC_RESUME_SIZE 64 78 #define DB_FC_RESUME_DELAY 1 79 #define DB_FC_DRAIN_THRESH 0 80 81 static struct dentry *c4iw_debugfs_root; 82 83 struct c4iw_debugfs_data { 84 struct c4iw_dev *devp; 85 char *buf; 86 int bufsize; 87 int pos; 88 }; 89 90 /* registered cxgb4 netlink callbacks */ 91 static struct ibnl_client_cbs c4iw_nl_cb_table[] = { 92 [RDMA_NL_IWPM_REG_PID] = {.dump = iwpm_register_pid_cb}, 93 [RDMA_NL_IWPM_ADD_MAPPING] = {.dump = iwpm_add_mapping_cb}, 94 [RDMA_NL_IWPM_QUERY_MAPPING] = {.dump = iwpm_add_and_query_mapping_cb}, 95 [RDMA_NL_IWPM_HANDLE_ERR] = {.dump = iwpm_mapping_error_cb}, 96 [RDMA_NL_IWPM_REMOTE_INFO] = {.dump = iwpm_remote_info_cb}, 97 [RDMA_NL_IWPM_MAPINFO] = {.dump = iwpm_mapping_info_cb}, 98 [RDMA_NL_IWPM_MAPINFO_NUM] = {.dump = iwpm_ack_mapping_info_cb} 99 }; 100 101 static int count_idrs(int id, void *p, void *data) 102 { 103 int *countp = data; 104 105 *countp = *countp + 1; 106 return 0; 107 } 108 109 static ssize_t debugfs_read(struct file *file, char __user *buf, size_t count, 110 loff_t *ppos) 111 { 112 struct c4iw_debugfs_data *d = file->private_data; 113 114 return simple_read_from_buffer(buf, count, ppos, d->buf, d->pos); 115 } 116 117 void c4iw_log_wr_stats(struct t4_wq *wq, struct t4_cqe *cqe) 118 { 119 struct wr_log_entry le; 120 int idx; 121 122 if (!wq->rdev->wr_log) 123 return; 124 125 idx = (atomic_inc_return(&wq->rdev->wr_log_idx) - 1) & 126 (wq->rdev->wr_log_size - 1); 127 le.poll_sge_ts = cxgb4_read_sge_timestamp(wq->rdev->lldi.ports[0]); 128 getnstimeofday(&le.poll_host_ts); 129 le.valid = 1; 130 le.cqe_sge_ts = CQE_TS(cqe); 131 if (SQ_TYPE(cqe)) { 132 le.qid = wq->sq.qid; 133 le.opcode = CQE_OPCODE(cqe); 134 le.post_host_ts = wq->sq.sw_sq[wq->sq.cidx].host_ts; 135 le.post_sge_ts = wq->sq.sw_sq[wq->sq.cidx].sge_ts; 136 le.wr_id = CQE_WRID_SQ_IDX(cqe); 137 } else { 138 le.qid = wq->rq.qid; 139 le.opcode = FW_RI_RECEIVE; 140 le.post_host_ts = wq->rq.sw_rq[wq->rq.cidx].host_ts; 141 le.post_sge_ts = wq->rq.sw_rq[wq->rq.cidx].sge_ts; 142 le.wr_id = CQE_WRID_MSN(cqe); 143 } 144 wq->rdev->wr_log[idx] = le; 145 } 146 147 static int wr_log_show(struct seq_file *seq, void *v) 148 { 149 struct c4iw_dev *dev = seq->private; 150 struct timespec prev_ts = {0, 0}; 151 struct wr_log_entry *lep; 152 int prev_ts_set = 0; 153 int idx, end; 154 155 #define ts2ns(ts) div64_u64((ts) * dev->rdev.lldi.cclk_ps, 1000) 156 157 idx = atomic_read(&dev->rdev.wr_log_idx) & 158 (dev->rdev.wr_log_size - 1); 159 end = idx - 1; 160 if (end < 0) 161 end = dev->rdev.wr_log_size - 1; 162 lep = &dev->rdev.wr_log[idx]; 163 while (idx != end) { 164 if (lep->valid) { 165 if (!prev_ts_set) { 166 prev_ts_set = 1; 167 prev_ts = lep->poll_host_ts; 168 } 169 seq_printf(seq, "%04u: sec %lu nsec %lu qid %u opcode " 170 "%u %s 0x%x host_wr_delta sec %lu nsec %lu " 171 "post_sge_ts 0x%llx cqe_sge_ts 0x%llx " 172 "poll_sge_ts 0x%llx post_poll_delta_ns %llu " 173 "cqe_poll_delta_ns %llu\n", 174 idx, 175 timespec_sub(lep->poll_host_ts, 176 prev_ts).tv_sec, 177 timespec_sub(lep->poll_host_ts, 178 prev_ts).tv_nsec, 179 lep->qid, lep->opcode, 180 lep->opcode == FW_RI_RECEIVE ? 181 "msn" : "wrid", 182 lep->wr_id, 183 timespec_sub(lep->poll_host_ts, 184 lep->post_host_ts).tv_sec, 185 timespec_sub(lep->poll_host_ts, 186 lep->post_host_ts).tv_nsec, 187 lep->post_sge_ts, lep->cqe_sge_ts, 188 lep->poll_sge_ts, 189 ts2ns(lep->poll_sge_ts - lep->post_sge_ts), 190 ts2ns(lep->poll_sge_ts - lep->cqe_sge_ts)); 191 prev_ts = lep->poll_host_ts; 192 } 193 idx++; 194 if (idx > (dev->rdev.wr_log_size - 1)) 195 idx = 0; 196 lep = &dev->rdev.wr_log[idx]; 197 } 198 #undef ts2ns 199 return 0; 200 } 201 202 static int wr_log_open(struct inode *inode, struct file *file) 203 { 204 return single_open(file, wr_log_show, inode->i_private); 205 } 206 207 static ssize_t wr_log_clear(struct file *file, const char __user *buf, 208 size_t count, loff_t *pos) 209 { 210 struct c4iw_dev *dev = ((struct seq_file *)file->private_data)->private; 211 int i; 212 213 if (dev->rdev.wr_log) 214 for (i = 0; i < dev->rdev.wr_log_size; i++) 215 dev->rdev.wr_log[i].valid = 0; 216 return count; 217 } 218 219 static const struct file_operations wr_log_debugfs_fops = { 220 .owner = THIS_MODULE, 221 .open = wr_log_open, 222 .release = single_release, 223 .read = seq_read, 224 .llseek = seq_lseek, 225 .write = wr_log_clear, 226 }; 227 228 static int dump_qp(int id, void *p, void *data) 229 { 230 struct c4iw_qp *qp = p; 231 struct c4iw_debugfs_data *qpd = data; 232 int space; 233 int cc; 234 235 if (id != qp->wq.sq.qid) 236 return 0; 237 238 space = qpd->bufsize - qpd->pos - 1; 239 if (space == 0) 240 return 1; 241 242 if (qp->ep) { 243 if (qp->ep->com.local_addr.ss_family == AF_INET) { 244 struct sockaddr_in *lsin = (struct sockaddr_in *) 245 &qp->ep->com.local_addr; 246 struct sockaddr_in *rsin = (struct sockaddr_in *) 247 &qp->ep->com.remote_addr; 248 struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) 249 &qp->ep->com.mapped_local_addr; 250 struct sockaddr_in *mapped_rsin = (struct sockaddr_in *) 251 &qp->ep->com.mapped_remote_addr; 252 253 cc = snprintf(qpd->buf + qpd->pos, space, 254 "rc qp sq id %u rq id %u state %u " 255 "onchip %u ep tid %u state %u " 256 "%pI4:%u/%u->%pI4:%u/%u\n", 257 qp->wq.sq.qid, qp->wq.rq.qid, 258 (int)qp->attr.state, 259 qp->wq.sq.flags & T4_SQ_ONCHIP, 260 qp->ep->hwtid, (int)qp->ep->com.state, 261 &lsin->sin_addr, ntohs(lsin->sin_port), 262 ntohs(mapped_lsin->sin_port), 263 &rsin->sin_addr, ntohs(rsin->sin_port), 264 ntohs(mapped_rsin->sin_port)); 265 } else { 266 struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) 267 &qp->ep->com.local_addr; 268 struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *) 269 &qp->ep->com.remote_addr; 270 struct sockaddr_in6 *mapped_lsin6 = 271 (struct sockaddr_in6 *) 272 &qp->ep->com.mapped_local_addr; 273 struct sockaddr_in6 *mapped_rsin6 = 274 (struct sockaddr_in6 *) 275 &qp->ep->com.mapped_remote_addr; 276 277 cc = snprintf(qpd->buf + qpd->pos, space, 278 "rc qp sq id %u rq id %u state %u " 279 "onchip %u ep tid %u state %u " 280 "%pI6:%u/%u->%pI6:%u/%u\n", 281 qp->wq.sq.qid, qp->wq.rq.qid, 282 (int)qp->attr.state, 283 qp->wq.sq.flags & T4_SQ_ONCHIP, 284 qp->ep->hwtid, (int)qp->ep->com.state, 285 &lsin6->sin6_addr, 286 ntohs(lsin6->sin6_port), 287 ntohs(mapped_lsin6->sin6_port), 288 &rsin6->sin6_addr, 289 ntohs(rsin6->sin6_port), 290 ntohs(mapped_rsin6->sin6_port)); 291 } 292 } else 293 cc = snprintf(qpd->buf + qpd->pos, space, 294 "qp sq id %u rq id %u state %u onchip %u\n", 295 qp->wq.sq.qid, qp->wq.rq.qid, 296 (int)qp->attr.state, 297 qp->wq.sq.flags & T4_SQ_ONCHIP); 298 if (cc < space) 299 qpd->pos += cc; 300 return 0; 301 } 302 303 static int qp_release(struct inode *inode, struct file *file) 304 { 305 struct c4iw_debugfs_data *qpd = file->private_data; 306 if (!qpd) { 307 printk(KERN_INFO "%s null qpd?\n", __func__); 308 return 0; 309 } 310 vfree(qpd->buf); 311 kfree(qpd); 312 return 0; 313 } 314 315 static int qp_open(struct inode *inode, struct file *file) 316 { 317 struct c4iw_debugfs_data *qpd; 318 int ret = 0; 319 int count = 1; 320 321 qpd = kmalloc(sizeof *qpd, GFP_KERNEL); 322 if (!qpd) { 323 ret = -ENOMEM; 324 goto out; 325 } 326 qpd->devp = inode->i_private; 327 qpd->pos = 0; 328 329 spin_lock_irq(&qpd->devp->lock); 330 idr_for_each(&qpd->devp->qpidr, count_idrs, &count); 331 spin_unlock_irq(&qpd->devp->lock); 332 333 qpd->bufsize = count * 128; 334 qpd->buf = vmalloc(qpd->bufsize); 335 if (!qpd->buf) { 336 ret = -ENOMEM; 337 goto err1; 338 } 339 340 spin_lock_irq(&qpd->devp->lock); 341 idr_for_each(&qpd->devp->qpidr, dump_qp, qpd); 342 spin_unlock_irq(&qpd->devp->lock); 343 344 qpd->buf[qpd->pos++] = 0; 345 file->private_data = qpd; 346 goto out; 347 err1: 348 kfree(qpd); 349 out: 350 return ret; 351 } 352 353 static const struct file_operations qp_debugfs_fops = { 354 .owner = THIS_MODULE, 355 .open = qp_open, 356 .release = qp_release, 357 .read = debugfs_read, 358 .llseek = default_llseek, 359 }; 360 361 static int dump_stag(int id, void *p, void *data) 362 { 363 struct c4iw_debugfs_data *stagd = data; 364 int space; 365 int cc; 366 struct fw_ri_tpte tpte; 367 int ret; 368 369 space = stagd->bufsize - stagd->pos - 1; 370 if (space == 0) 371 return 1; 372 373 ret = cxgb4_read_tpte(stagd->devp->rdev.lldi.ports[0], (u32)id<<8, 374 (__be32 *)&tpte); 375 if (ret) { 376 dev_err(&stagd->devp->rdev.lldi.pdev->dev, 377 "%s cxgb4_read_tpte err %d\n", __func__, ret); 378 return ret; 379 } 380 cc = snprintf(stagd->buf + stagd->pos, space, 381 "stag: idx 0x%x valid %d key 0x%x state %d pdid %d " 382 "perm 0x%x ps %d len 0x%llx va 0x%llx\n", 383 (u32)id<<8, 384 FW_RI_TPTE_VALID_G(ntohl(tpte.valid_to_pdid)), 385 FW_RI_TPTE_STAGKEY_G(ntohl(tpte.valid_to_pdid)), 386 FW_RI_TPTE_STAGSTATE_G(ntohl(tpte.valid_to_pdid)), 387 FW_RI_TPTE_PDID_G(ntohl(tpte.valid_to_pdid)), 388 FW_RI_TPTE_PERM_G(ntohl(tpte.locread_to_qpid)), 389 FW_RI_TPTE_PS_G(ntohl(tpte.locread_to_qpid)), 390 ((u64)ntohl(tpte.len_hi) << 32) | ntohl(tpte.len_lo), 391 ((u64)ntohl(tpte.va_hi) << 32) | ntohl(tpte.va_lo_fbo)); 392 if (cc < space) 393 stagd->pos += cc; 394 return 0; 395 } 396 397 static int stag_release(struct inode *inode, struct file *file) 398 { 399 struct c4iw_debugfs_data *stagd = file->private_data; 400 if (!stagd) { 401 printk(KERN_INFO "%s null stagd?\n", __func__); 402 return 0; 403 } 404 vfree(stagd->buf); 405 kfree(stagd); 406 return 0; 407 } 408 409 static int stag_open(struct inode *inode, struct file *file) 410 { 411 struct c4iw_debugfs_data *stagd; 412 int ret = 0; 413 int count = 1; 414 415 stagd = kmalloc(sizeof *stagd, GFP_KERNEL); 416 if (!stagd) { 417 ret = -ENOMEM; 418 goto out; 419 } 420 stagd->devp = inode->i_private; 421 stagd->pos = 0; 422 423 spin_lock_irq(&stagd->devp->lock); 424 idr_for_each(&stagd->devp->mmidr, count_idrs, &count); 425 spin_unlock_irq(&stagd->devp->lock); 426 427 stagd->bufsize = count * 256; 428 stagd->buf = vmalloc(stagd->bufsize); 429 if (!stagd->buf) { 430 ret = -ENOMEM; 431 goto err1; 432 } 433 434 spin_lock_irq(&stagd->devp->lock); 435 idr_for_each(&stagd->devp->mmidr, dump_stag, stagd); 436 spin_unlock_irq(&stagd->devp->lock); 437 438 stagd->buf[stagd->pos++] = 0; 439 file->private_data = stagd; 440 goto out; 441 err1: 442 kfree(stagd); 443 out: 444 return ret; 445 } 446 447 static const struct file_operations stag_debugfs_fops = { 448 .owner = THIS_MODULE, 449 .open = stag_open, 450 .release = stag_release, 451 .read = debugfs_read, 452 .llseek = default_llseek, 453 }; 454 455 static char *db_state_str[] = {"NORMAL", "FLOW_CONTROL", "RECOVERY", "STOPPED"}; 456 457 static int stats_show(struct seq_file *seq, void *v) 458 { 459 struct c4iw_dev *dev = seq->private; 460 461 seq_printf(seq, " Object: %10s %10s %10s %10s\n", "Total", "Current", 462 "Max", "Fail"); 463 seq_printf(seq, " PDID: %10llu %10llu %10llu %10llu\n", 464 dev->rdev.stats.pd.total, dev->rdev.stats.pd.cur, 465 dev->rdev.stats.pd.max, dev->rdev.stats.pd.fail); 466 seq_printf(seq, " QID: %10llu %10llu %10llu %10llu\n", 467 dev->rdev.stats.qid.total, dev->rdev.stats.qid.cur, 468 dev->rdev.stats.qid.max, dev->rdev.stats.qid.fail); 469 seq_printf(seq, " TPTMEM: %10llu %10llu %10llu %10llu\n", 470 dev->rdev.stats.stag.total, dev->rdev.stats.stag.cur, 471 dev->rdev.stats.stag.max, dev->rdev.stats.stag.fail); 472 seq_printf(seq, " PBLMEM: %10llu %10llu %10llu %10llu\n", 473 dev->rdev.stats.pbl.total, dev->rdev.stats.pbl.cur, 474 dev->rdev.stats.pbl.max, dev->rdev.stats.pbl.fail); 475 seq_printf(seq, " RQTMEM: %10llu %10llu %10llu %10llu\n", 476 dev->rdev.stats.rqt.total, dev->rdev.stats.rqt.cur, 477 dev->rdev.stats.rqt.max, dev->rdev.stats.rqt.fail); 478 seq_printf(seq, " OCQPMEM: %10llu %10llu %10llu %10llu\n", 479 dev->rdev.stats.ocqp.total, dev->rdev.stats.ocqp.cur, 480 dev->rdev.stats.ocqp.max, dev->rdev.stats.ocqp.fail); 481 seq_printf(seq, " DB FULL: %10llu\n", dev->rdev.stats.db_full); 482 seq_printf(seq, " DB EMPTY: %10llu\n", dev->rdev.stats.db_empty); 483 seq_printf(seq, " DB DROP: %10llu\n", dev->rdev.stats.db_drop); 484 seq_printf(seq, " DB State: %s Transitions %llu FC Interruptions %llu\n", 485 db_state_str[dev->db_state], 486 dev->rdev.stats.db_state_transitions, 487 dev->rdev.stats.db_fc_interruptions); 488 seq_printf(seq, "TCAM_FULL: %10llu\n", dev->rdev.stats.tcam_full); 489 seq_printf(seq, "ACT_OFLD_CONN_FAILS: %10llu\n", 490 dev->rdev.stats.act_ofld_conn_fails); 491 seq_printf(seq, "PAS_OFLD_CONN_FAILS: %10llu\n", 492 dev->rdev.stats.pas_ofld_conn_fails); 493 seq_printf(seq, "NEG_ADV_RCVD: %10llu\n", dev->rdev.stats.neg_adv); 494 seq_printf(seq, "AVAILABLE IRD: %10u\n", dev->avail_ird); 495 return 0; 496 } 497 498 static int stats_open(struct inode *inode, struct file *file) 499 { 500 return single_open(file, stats_show, inode->i_private); 501 } 502 503 static ssize_t stats_clear(struct file *file, const char __user *buf, 504 size_t count, loff_t *pos) 505 { 506 struct c4iw_dev *dev = ((struct seq_file *)file->private_data)->private; 507 508 mutex_lock(&dev->rdev.stats.lock); 509 dev->rdev.stats.pd.max = 0; 510 dev->rdev.stats.pd.fail = 0; 511 dev->rdev.stats.qid.max = 0; 512 dev->rdev.stats.qid.fail = 0; 513 dev->rdev.stats.stag.max = 0; 514 dev->rdev.stats.stag.fail = 0; 515 dev->rdev.stats.pbl.max = 0; 516 dev->rdev.stats.pbl.fail = 0; 517 dev->rdev.stats.rqt.max = 0; 518 dev->rdev.stats.rqt.fail = 0; 519 dev->rdev.stats.ocqp.max = 0; 520 dev->rdev.stats.ocqp.fail = 0; 521 dev->rdev.stats.db_full = 0; 522 dev->rdev.stats.db_empty = 0; 523 dev->rdev.stats.db_drop = 0; 524 dev->rdev.stats.db_state_transitions = 0; 525 dev->rdev.stats.tcam_full = 0; 526 dev->rdev.stats.act_ofld_conn_fails = 0; 527 dev->rdev.stats.pas_ofld_conn_fails = 0; 528 mutex_unlock(&dev->rdev.stats.lock); 529 return count; 530 } 531 532 static const struct file_operations stats_debugfs_fops = { 533 .owner = THIS_MODULE, 534 .open = stats_open, 535 .release = single_release, 536 .read = seq_read, 537 .llseek = seq_lseek, 538 .write = stats_clear, 539 }; 540 541 static int dump_ep(int id, void *p, void *data) 542 { 543 struct c4iw_ep *ep = p; 544 struct c4iw_debugfs_data *epd = data; 545 int space; 546 int cc; 547 548 space = epd->bufsize - epd->pos - 1; 549 if (space == 0) 550 return 1; 551 552 if (ep->com.local_addr.ss_family == AF_INET) { 553 struct sockaddr_in *lsin = (struct sockaddr_in *) 554 &ep->com.local_addr; 555 struct sockaddr_in *rsin = (struct sockaddr_in *) 556 &ep->com.remote_addr; 557 struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) 558 &ep->com.mapped_local_addr; 559 struct sockaddr_in *mapped_rsin = (struct sockaddr_in *) 560 &ep->com.mapped_remote_addr; 561 562 cc = snprintf(epd->buf + epd->pos, space, 563 "ep %p cm_id %p qp %p state %d flags 0x%lx " 564 "history 0x%lx hwtid %d atid %d " 565 "conn_na %u abort_na %u " 566 "%pI4:%d/%d <-> %pI4:%d/%d\n", 567 ep, ep->com.cm_id, ep->com.qp, 568 (int)ep->com.state, ep->com.flags, 569 ep->com.history, ep->hwtid, ep->atid, 570 ep->stats.connect_neg_adv, 571 ep->stats.abort_neg_adv, 572 &lsin->sin_addr, ntohs(lsin->sin_port), 573 ntohs(mapped_lsin->sin_port), 574 &rsin->sin_addr, ntohs(rsin->sin_port), 575 ntohs(mapped_rsin->sin_port)); 576 } else { 577 struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) 578 &ep->com.local_addr; 579 struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *) 580 &ep->com.remote_addr; 581 struct sockaddr_in6 *mapped_lsin6 = (struct sockaddr_in6 *) 582 &ep->com.mapped_local_addr; 583 struct sockaddr_in6 *mapped_rsin6 = (struct sockaddr_in6 *) 584 &ep->com.mapped_remote_addr; 585 586 cc = snprintf(epd->buf + epd->pos, space, 587 "ep %p cm_id %p qp %p state %d flags 0x%lx " 588 "history 0x%lx hwtid %d atid %d " 589 "conn_na %u abort_na %u " 590 "%pI6:%d/%d <-> %pI6:%d/%d\n", 591 ep, ep->com.cm_id, ep->com.qp, 592 (int)ep->com.state, ep->com.flags, 593 ep->com.history, ep->hwtid, ep->atid, 594 ep->stats.connect_neg_adv, 595 ep->stats.abort_neg_adv, 596 &lsin6->sin6_addr, ntohs(lsin6->sin6_port), 597 ntohs(mapped_lsin6->sin6_port), 598 &rsin6->sin6_addr, ntohs(rsin6->sin6_port), 599 ntohs(mapped_rsin6->sin6_port)); 600 } 601 if (cc < space) 602 epd->pos += cc; 603 return 0; 604 } 605 606 static int dump_listen_ep(int id, void *p, void *data) 607 { 608 struct c4iw_listen_ep *ep = p; 609 struct c4iw_debugfs_data *epd = data; 610 int space; 611 int cc; 612 613 space = epd->bufsize - epd->pos - 1; 614 if (space == 0) 615 return 1; 616 617 if (ep->com.local_addr.ss_family == AF_INET) { 618 struct sockaddr_in *lsin = (struct sockaddr_in *) 619 &ep->com.local_addr; 620 struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) 621 &ep->com.mapped_local_addr; 622 623 cc = snprintf(epd->buf + epd->pos, space, 624 "ep %p cm_id %p state %d flags 0x%lx stid %d " 625 "backlog %d %pI4:%d/%d\n", 626 ep, ep->com.cm_id, (int)ep->com.state, 627 ep->com.flags, ep->stid, ep->backlog, 628 &lsin->sin_addr, ntohs(lsin->sin_port), 629 ntohs(mapped_lsin->sin_port)); 630 } else { 631 struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) 632 &ep->com.local_addr; 633 struct sockaddr_in6 *mapped_lsin6 = (struct sockaddr_in6 *) 634 &ep->com.mapped_local_addr; 635 636 cc = snprintf(epd->buf + epd->pos, space, 637 "ep %p cm_id %p state %d flags 0x%lx stid %d " 638 "backlog %d %pI6:%d/%d\n", 639 ep, ep->com.cm_id, (int)ep->com.state, 640 ep->com.flags, ep->stid, ep->backlog, 641 &lsin6->sin6_addr, ntohs(lsin6->sin6_port), 642 ntohs(mapped_lsin6->sin6_port)); 643 } 644 if (cc < space) 645 epd->pos += cc; 646 return 0; 647 } 648 649 static int ep_release(struct inode *inode, struct file *file) 650 { 651 struct c4iw_debugfs_data *epd = file->private_data; 652 if (!epd) { 653 pr_info("%s null qpd?\n", __func__); 654 return 0; 655 } 656 vfree(epd->buf); 657 kfree(epd); 658 return 0; 659 } 660 661 static int ep_open(struct inode *inode, struct file *file) 662 { 663 struct c4iw_debugfs_data *epd; 664 int ret = 0; 665 int count = 1; 666 667 epd = kmalloc(sizeof(*epd), GFP_KERNEL); 668 if (!epd) { 669 ret = -ENOMEM; 670 goto out; 671 } 672 epd->devp = inode->i_private; 673 epd->pos = 0; 674 675 spin_lock_irq(&epd->devp->lock); 676 idr_for_each(&epd->devp->hwtid_idr, count_idrs, &count); 677 idr_for_each(&epd->devp->atid_idr, count_idrs, &count); 678 idr_for_each(&epd->devp->stid_idr, count_idrs, &count); 679 spin_unlock_irq(&epd->devp->lock); 680 681 epd->bufsize = count * 240; 682 epd->buf = vmalloc(epd->bufsize); 683 if (!epd->buf) { 684 ret = -ENOMEM; 685 goto err1; 686 } 687 688 spin_lock_irq(&epd->devp->lock); 689 idr_for_each(&epd->devp->hwtid_idr, dump_ep, epd); 690 idr_for_each(&epd->devp->atid_idr, dump_ep, epd); 691 idr_for_each(&epd->devp->stid_idr, dump_listen_ep, epd); 692 spin_unlock_irq(&epd->devp->lock); 693 694 file->private_data = epd; 695 goto out; 696 err1: 697 kfree(epd); 698 out: 699 return ret; 700 } 701 702 static const struct file_operations ep_debugfs_fops = { 703 .owner = THIS_MODULE, 704 .open = ep_open, 705 .release = ep_release, 706 .read = debugfs_read, 707 }; 708 709 static int setup_debugfs(struct c4iw_dev *devp) 710 { 711 if (!devp->debugfs_root) 712 return -1; 713 714 debugfs_create_file_size("qps", S_IWUSR, devp->debugfs_root, 715 (void *)devp, &qp_debugfs_fops, 4096); 716 717 debugfs_create_file_size("stags", S_IWUSR, devp->debugfs_root, 718 (void *)devp, &stag_debugfs_fops, 4096); 719 720 debugfs_create_file_size("stats", S_IWUSR, devp->debugfs_root, 721 (void *)devp, &stats_debugfs_fops, 4096); 722 723 debugfs_create_file_size("eps", S_IWUSR, devp->debugfs_root, 724 (void *)devp, &ep_debugfs_fops, 4096); 725 726 if (c4iw_wr_log) 727 debugfs_create_file_size("wr_log", S_IWUSR, devp->debugfs_root, 728 (void *)devp, &wr_log_debugfs_fops, 4096); 729 return 0; 730 } 731 732 void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev, 733 struct c4iw_dev_ucontext *uctx) 734 { 735 struct list_head *pos, *nxt; 736 struct c4iw_qid_list *entry; 737 738 mutex_lock(&uctx->lock); 739 list_for_each_safe(pos, nxt, &uctx->qpids) { 740 entry = list_entry(pos, struct c4iw_qid_list, entry); 741 list_del_init(&entry->entry); 742 if (!(entry->qid & rdev->qpmask)) { 743 c4iw_put_resource(&rdev->resource.qid_table, 744 entry->qid); 745 mutex_lock(&rdev->stats.lock); 746 rdev->stats.qid.cur -= rdev->qpmask + 1; 747 mutex_unlock(&rdev->stats.lock); 748 } 749 kfree(entry); 750 } 751 752 list_for_each_safe(pos, nxt, &uctx->qpids) { 753 entry = list_entry(pos, struct c4iw_qid_list, entry); 754 list_del_init(&entry->entry); 755 kfree(entry); 756 } 757 mutex_unlock(&uctx->lock); 758 } 759 760 void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev, 761 struct c4iw_dev_ucontext *uctx) 762 { 763 INIT_LIST_HEAD(&uctx->qpids); 764 INIT_LIST_HEAD(&uctx->cqids); 765 mutex_init(&uctx->lock); 766 } 767 768 /* Caller takes care of locking if needed */ 769 static int c4iw_rdev_open(struct c4iw_rdev *rdev) 770 { 771 int err; 772 773 c4iw_init_dev_ucontext(rdev, &rdev->uctx); 774 775 /* 776 * This implementation assumes udb_density == ucq_density! Eventually 777 * we might need to support this but for now fail the open. Also the 778 * cqid and qpid range must match for now. 779 */ 780 if (rdev->lldi.udb_density != rdev->lldi.ucq_density) { 781 pr_err(MOD "%s: unsupported udb/ucq densities %u/%u\n", 782 pci_name(rdev->lldi.pdev), rdev->lldi.udb_density, 783 rdev->lldi.ucq_density); 784 err = -EINVAL; 785 goto err1; 786 } 787 if (rdev->lldi.vr->qp.start != rdev->lldi.vr->cq.start || 788 rdev->lldi.vr->qp.size != rdev->lldi.vr->cq.size) { 789 pr_err(MOD "%s: unsupported qp and cq id ranges " 790 "qp start %u size %u cq start %u size %u\n", 791 pci_name(rdev->lldi.pdev), rdev->lldi.vr->qp.start, 792 rdev->lldi.vr->qp.size, rdev->lldi.vr->cq.size, 793 rdev->lldi.vr->cq.size); 794 err = -EINVAL; 795 goto err1; 796 } 797 798 rdev->qpmask = rdev->lldi.udb_density - 1; 799 rdev->cqmask = rdev->lldi.ucq_density - 1; 800 PDBG("%s dev %s stag start 0x%0x size 0x%0x num stags %d " 801 "pbl start 0x%0x size 0x%0x rq start 0x%0x size 0x%0x " 802 "qp qid start %u size %u cq qid start %u size %u\n", 803 __func__, pci_name(rdev->lldi.pdev), rdev->lldi.vr->stag.start, 804 rdev->lldi.vr->stag.size, c4iw_num_stags(rdev), 805 rdev->lldi.vr->pbl.start, 806 rdev->lldi.vr->pbl.size, rdev->lldi.vr->rq.start, 807 rdev->lldi.vr->rq.size, 808 rdev->lldi.vr->qp.start, 809 rdev->lldi.vr->qp.size, 810 rdev->lldi.vr->cq.start, 811 rdev->lldi.vr->cq.size); 812 PDBG("udb len 0x%x udb base %p db_reg %p gts_reg %p " 813 "qpmask 0x%x cqmask 0x%x\n", 814 (unsigned)pci_resource_len(rdev->lldi.pdev, 2), 815 (void *)pci_resource_start(rdev->lldi.pdev, 2), 816 rdev->lldi.db_reg, rdev->lldi.gts_reg, 817 rdev->qpmask, rdev->cqmask); 818 819 if (c4iw_num_stags(rdev) == 0) { 820 err = -EINVAL; 821 goto err1; 822 } 823 824 rdev->stats.pd.total = T4_MAX_NUM_PD; 825 rdev->stats.stag.total = rdev->lldi.vr->stag.size; 826 rdev->stats.pbl.total = rdev->lldi.vr->pbl.size; 827 rdev->stats.rqt.total = rdev->lldi.vr->rq.size; 828 rdev->stats.ocqp.total = rdev->lldi.vr->ocq.size; 829 rdev->stats.qid.total = rdev->lldi.vr->qp.size; 830 831 err = c4iw_init_resource(rdev, c4iw_num_stags(rdev), T4_MAX_NUM_PD); 832 if (err) { 833 printk(KERN_ERR MOD "error %d initializing resources\n", err); 834 goto err1; 835 } 836 err = c4iw_pblpool_create(rdev); 837 if (err) { 838 printk(KERN_ERR MOD "error %d initializing pbl pool\n", err); 839 goto err2; 840 } 841 err = c4iw_rqtpool_create(rdev); 842 if (err) { 843 printk(KERN_ERR MOD "error %d initializing rqt pool\n", err); 844 goto err3; 845 } 846 err = c4iw_ocqp_pool_create(rdev); 847 if (err) { 848 printk(KERN_ERR MOD "error %d initializing ocqp pool\n", err); 849 goto err4; 850 } 851 rdev->status_page = (struct t4_dev_status_page *) 852 __get_free_page(GFP_KERNEL); 853 if (!rdev->status_page) { 854 pr_err(MOD "error allocating status page\n"); 855 goto err4; 856 } 857 858 if (c4iw_wr_log) { 859 rdev->wr_log = kzalloc((1 << c4iw_wr_log_size_order) * 860 sizeof(*rdev->wr_log), GFP_KERNEL); 861 if (rdev->wr_log) { 862 rdev->wr_log_size = 1 << c4iw_wr_log_size_order; 863 atomic_set(&rdev->wr_log_idx, 0); 864 } else { 865 pr_err(MOD "error allocating wr_log. Logging disabled\n"); 866 } 867 } 868 869 rdev->status_page->db_off = 0; 870 871 return 0; 872 err4: 873 c4iw_rqtpool_destroy(rdev); 874 err3: 875 c4iw_pblpool_destroy(rdev); 876 err2: 877 c4iw_destroy_resource(&rdev->resource); 878 err1: 879 return err; 880 } 881 882 static void c4iw_rdev_close(struct c4iw_rdev *rdev) 883 { 884 kfree(rdev->wr_log); 885 free_page((unsigned long)rdev->status_page); 886 c4iw_pblpool_destroy(rdev); 887 c4iw_rqtpool_destroy(rdev); 888 c4iw_destroy_resource(&rdev->resource); 889 } 890 891 static void c4iw_dealloc(struct uld_ctx *ctx) 892 { 893 c4iw_rdev_close(&ctx->dev->rdev); 894 idr_destroy(&ctx->dev->cqidr); 895 idr_destroy(&ctx->dev->qpidr); 896 idr_destroy(&ctx->dev->mmidr); 897 idr_destroy(&ctx->dev->hwtid_idr); 898 idr_destroy(&ctx->dev->stid_idr); 899 idr_destroy(&ctx->dev->atid_idr); 900 if (ctx->dev->rdev.bar2_kva) 901 iounmap(ctx->dev->rdev.bar2_kva); 902 if (ctx->dev->rdev.oc_mw_kva) 903 iounmap(ctx->dev->rdev.oc_mw_kva); 904 ib_dealloc_device(&ctx->dev->ibdev); 905 ctx->dev = NULL; 906 } 907 908 static void c4iw_remove(struct uld_ctx *ctx) 909 { 910 PDBG("%s c4iw_dev %p\n", __func__, ctx->dev); 911 c4iw_unregister_device(ctx->dev); 912 c4iw_dealloc(ctx); 913 } 914 915 static int rdma_supported(const struct cxgb4_lld_info *infop) 916 { 917 return infop->vr->stag.size > 0 && infop->vr->pbl.size > 0 && 918 infop->vr->rq.size > 0 && infop->vr->qp.size > 0 && 919 infop->vr->cq.size > 0; 920 } 921 922 static struct c4iw_dev *c4iw_alloc(const struct cxgb4_lld_info *infop) 923 { 924 struct c4iw_dev *devp; 925 int ret; 926 927 if (!rdma_supported(infop)) { 928 printk(KERN_INFO MOD "%s: RDMA not supported on this device.\n", 929 pci_name(infop->pdev)); 930 return ERR_PTR(-ENOSYS); 931 } 932 if (!ocqp_supported(infop)) 933 pr_info("%s: On-Chip Queues not supported on this device.\n", 934 pci_name(infop->pdev)); 935 936 devp = (struct c4iw_dev *)ib_alloc_device(sizeof(*devp)); 937 if (!devp) { 938 printk(KERN_ERR MOD "Cannot allocate ib device\n"); 939 return ERR_PTR(-ENOMEM); 940 } 941 devp->rdev.lldi = *infop; 942 943 /* init various hw-queue params based on lld info */ 944 PDBG("%s: Ing. padding boundary is %d, egrsstatuspagesize = %d\n", 945 __func__, devp->rdev.lldi.sge_ingpadboundary, 946 devp->rdev.lldi.sge_egrstatuspagesize); 947 948 devp->rdev.hw_queue.t4_eq_status_entries = 949 devp->rdev.lldi.sge_ingpadboundary > 64 ? 2 : 1; 950 devp->rdev.hw_queue.t4_max_eq_size = 65520; 951 devp->rdev.hw_queue.t4_max_iq_size = 65520; 952 devp->rdev.hw_queue.t4_max_rq_size = 8192 - 953 devp->rdev.hw_queue.t4_eq_status_entries - 1; 954 devp->rdev.hw_queue.t4_max_sq_size = 955 devp->rdev.hw_queue.t4_max_eq_size - 956 devp->rdev.hw_queue.t4_eq_status_entries - 1; 957 devp->rdev.hw_queue.t4_max_qp_depth = 958 devp->rdev.hw_queue.t4_max_rq_size; 959 devp->rdev.hw_queue.t4_max_cq_depth = 960 devp->rdev.hw_queue.t4_max_iq_size - 2; 961 devp->rdev.hw_queue.t4_stat_len = 962 devp->rdev.lldi.sge_egrstatuspagesize; 963 964 /* 965 * For T5 devices, we map all of BAR2 with WC. 966 * For T4 devices with onchip qp mem, we map only that part 967 * of BAR2 with WC. 968 */ 969 devp->rdev.bar2_pa = pci_resource_start(devp->rdev.lldi.pdev, 2); 970 if (is_t5(devp->rdev.lldi.adapter_type)) { 971 devp->rdev.bar2_kva = ioremap_wc(devp->rdev.bar2_pa, 972 pci_resource_len(devp->rdev.lldi.pdev, 2)); 973 if (!devp->rdev.bar2_kva) { 974 pr_err(MOD "Unable to ioremap BAR2\n"); 975 ib_dealloc_device(&devp->ibdev); 976 return ERR_PTR(-EINVAL); 977 } 978 } else if (ocqp_supported(infop)) { 979 devp->rdev.oc_mw_pa = 980 pci_resource_start(devp->rdev.lldi.pdev, 2) + 981 pci_resource_len(devp->rdev.lldi.pdev, 2) - 982 roundup_pow_of_two(devp->rdev.lldi.vr->ocq.size); 983 devp->rdev.oc_mw_kva = ioremap_wc(devp->rdev.oc_mw_pa, 984 devp->rdev.lldi.vr->ocq.size); 985 if (!devp->rdev.oc_mw_kva) { 986 pr_err(MOD "Unable to ioremap onchip mem\n"); 987 ib_dealloc_device(&devp->ibdev); 988 return ERR_PTR(-EINVAL); 989 } 990 } 991 992 PDBG(KERN_INFO MOD "ocq memory: " 993 "hw_start 0x%x size %u mw_pa 0x%lx mw_kva %p\n", 994 devp->rdev.lldi.vr->ocq.start, devp->rdev.lldi.vr->ocq.size, 995 devp->rdev.oc_mw_pa, devp->rdev.oc_mw_kva); 996 997 ret = c4iw_rdev_open(&devp->rdev); 998 if (ret) { 999 printk(KERN_ERR MOD "Unable to open CXIO rdev err %d\n", ret); 1000 ib_dealloc_device(&devp->ibdev); 1001 return ERR_PTR(ret); 1002 } 1003 1004 idr_init(&devp->cqidr); 1005 idr_init(&devp->qpidr); 1006 idr_init(&devp->mmidr); 1007 idr_init(&devp->hwtid_idr); 1008 idr_init(&devp->stid_idr); 1009 idr_init(&devp->atid_idr); 1010 spin_lock_init(&devp->lock); 1011 mutex_init(&devp->rdev.stats.lock); 1012 mutex_init(&devp->db_mutex); 1013 INIT_LIST_HEAD(&devp->db_fc_list); 1014 devp->avail_ird = devp->rdev.lldi.max_ird_adapter; 1015 1016 if (c4iw_debugfs_root) { 1017 devp->debugfs_root = debugfs_create_dir( 1018 pci_name(devp->rdev.lldi.pdev), 1019 c4iw_debugfs_root); 1020 setup_debugfs(devp); 1021 } 1022 1023 1024 return devp; 1025 } 1026 1027 static void *c4iw_uld_add(const struct cxgb4_lld_info *infop) 1028 { 1029 struct uld_ctx *ctx; 1030 static int vers_printed; 1031 int i; 1032 1033 if (!vers_printed++) 1034 pr_info("Chelsio T4/T5 RDMA Driver - version %s\n", 1035 DRV_VERSION); 1036 1037 ctx = kzalloc(sizeof *ctx, GFP_KERNEL); 1038 if (!ctx) { 1039 ctx = ERR_PTR(-ENOMEM); 1040 goto out; 1041 } 1042 ctx->lldi = *infop; 1043 1044 PDBG("%s found device %s nchan %u nrxq %u ntxq %u nports %u\n", 1045 __func__, pci_name(ctx->lldi.pdev), 1046 ctx->lldi.nchan, ctx->lldi.nrxq, 1047 ctx->lldi.ntxq, ctx->lldi.nports); 1048 1049 mutex_lock(&dev_mutex); 1050 list_add_tail(&ctx->entry, &uld_ctx_list); 1051 mutex_unlock(&dev_mutex); 1052 1053 for (i = 0; i < ctx->lldi.nrxq; i++) 1054 PDBG("rxqid[%u] %u\n", i, ctx->lldi.rxq_ids[i]); 1055 out: 1056 return ctx; 1057 } 1058 1059 static inline struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl, 1060 const __be64 *rsp, 1061 u32 pktshift) 1062 { 1063 struct sk_buff *skb; 1064 1065 /* 1066 * Allocate space for cpl_pass_accept_req which will be synthesized by 1067 * driver. Once the driver synthesizes the request the skb will go 1068 * through the regular cpl_pass_accept_req processing. 1069 * The math here assumes sizeof cpl_pass_accept_req >= sizeof 1070 * cpl_rx_pkt. 1071 */ 1072 skb = alloc_skb(gl->tot_len + sizeof(struct cpl_pass_accept_req) + 1073 sizeof(struct rss_header) - pktshift, GFP_ATOMIC); 1074 if (unlikely(!skb)) 1075 return NULL; 1076 1077 __skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req) + 1078 sizeof(struct rss_header) - pktshift); 1079 1080 /* 1081 * This skb will contain: 1082 * rss_header from the rspq descriptor (1 flit) 1083 * cpl_rx_pkt struct from the rspq descriptor (2 flits) 1084 * space for the difference between the size of an 1085 * rx_pkt and pass_accept_req cpl (1 flit) 1086 * the packet data from the gl 1087 */ 1088 skb_copy_to_linear_data(skb, rsp, sizeof(struct cpl_pass_accept_req) + 1089 sizeof(struct rss_header)); 1090 skb_copy_to_linear_data_offset(skb, sizeof(struct rss_header) + 1091 sizeof(struct cpl_pass_accept_req), 1092 gl->va + pktshift, 1093 gl->tot_len - pktshift); 1094 return skb; 1095 } 1096 1097 static inline int recv_rx_pkt(struct c4iw_dev *dev, const struct pkt_gl *gl, 1098 const __be64 *rsp) 1099 { 1100 unsigned int opcode = *(u8 *)rsp; 1101 struct sk_buff *skb; 1102 1103 if (opcode != CPL_RX_PKT) 1104 goto out; 1105 1106 skb = copy_gl_to_skb_pkt(gl , rsp, dev->rdev.lldi.sge_pktshift); 1107 if (skb == NULL) 1108 goto out; 1109 1110 if (c4iw_handlers[opcode] == NULL) { 1111 pr_info("%s no handler opcode 0x%x...\n", __func__, 1112 opcode); 1113 kfree_skb(skb); 1114 goto out; 1115 } 1116 c4iw_handlers[opcode](dev, skb); 1117 return 1; 1118 out: 1119 return 0; 1120 } 1121 1122 static int c4iw_uld_rx_handler(void *handle, const __be64 *rsp, 1123 const struct pkt_gl *gl) 1124 { 1125 struct uld_ctx *ctx = handle; 1126 struct c4iw_dev *dev = ctx->dev; 1127 struct sk_buff *skb; 1128 u8 opcode; 1129 1130 if (gl == NULL) { 1131 /* omit RSS and rsp_ctrl at end of descriptor */ 1132 unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8; 1133 1134 skb = alloc_skb(256, GFP_ATOMIC); 1135 if (!skb) 1136 goto nomem; 1137 __skb_put(skb, len); 1138 skb_copy_to_linear_data(skb, &rsp[1], len); 1139 } else if (gl == CXGB4_MSG_AN) { 1140 const struct rsp_ctrl *rc = (void *)rsp; 1141 1142 u32 qid = be32_to_cpu(rc->pldbuflen_qid); 1143 c4iw_ev_handler(dev, qid); 1144 return 0; 1145 } else if (unlikely(*(u8 *)rsp != *(u8 *)gl->va)) { 1146 if (recv_rx_pkt(dev, gl, rsp)) 1147 return 0; 1148 1149 pr_info("%s: unexpected FL contents at %p, " \ 1150 "RSS %#llx, FL %#llx, len %u\n", 1151 pci_name(ctx->lldi.pdev), gl->va, 1152 (unsigned long long)be64_to_cpu(*rsp), 1153 (unsigned long long)be64_to_cpu( 1154 *(__force __be64 *)gl->va), 1155 gl->tot_len); 1156 1157 return 0; 1158 } else { 1159 skb = cxgb4_pktgl_to_skb(gl, 128, 128); 1160 if (unlikely(!skb)) 1161 goto nomem; 1162 } 1163 1164 opcode = *(u8 *)rsp; 1165 if (c4iw_handlers[opcode]) { 1166 c4iw_handlers[opcode](dev, skb); 1167 } else { 1168 pr_info("%s no handler opcode 0x%x...\n", __func__, 1169 opcode); 1170 kfree_skb(skb); 1171 } 1172 1173 return 0; 1174 nomem: 1175 return -1; 1176 } 1177 1178 static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state) 1179 { 1180 struct uld_ctx *ctx = handle; 1181 1182 PDBG("%s new_state %u\n", __func__, new_state); 1183 switch (new_state) { 1184 case CXGB4_STATE_UP: 1185 printk(KERN_INFO MOD "%s: Up\n", pci_name(ctx->lldi.pdev)); 1186 if (!ctx->dev) { 1187 int ret; 1188 1189 ctx->dev = c4iw_alloc(&ctx->lldi); 1190 if (IS_ERR(ctx->dev)) { 1191 printk(KERN_ERR MOD 1192 "%s: initialization failed: %ld\n", 1193 pci_name(ctx->lldi.pdev), 1194 PTR_ERR(ctx->dev)); 1195 ctx->dev = NULL; 1196 break; 1197 } 1198 ret = c4iw_register_device(ctx->dev); 1199 if (ret) { 1200 printk(KERN_ERR MOD 1201 "%s: RDMA registration failed: %d\n", 1202 pci_name(ctx->lldi.pdev), ret); 1203 c4iw_dealloc(ctx); 1204 } 1205 } 1206 break; 1207 case CXGB4_STATE_DOWN: 1208 printk(KERN_INFO MOD "%s: Down\n", 1209 pci_name(ctx->lldi.pdev)); 1210 if (ctx->dev) 1211 c4iw_remove(ctx); 1212 break; 1213 case CXGB4_STATE_START_RECOVERY: 1214 printk(KERN_INFO MOD "%s: Fatal Error\n", 1215 pci_name(ctx->lldi.pdev)); 1216 if (ctx->dev) { 1217 struct ib_event event; 1218 1219 ctx->dev->rdev.flags |= T4_FATAL_ERROR; 1220 memset(&event, 0, sizeof event); 1221 event.event = IB_EVENT_DEVICE_FATAL; 1222 event.device = &ctx->dev->ibdev; 1223 ib_dispatch_event(&event); 1224 c4iw_remove(ctx); 1225 } 1226 break; 1227 case CXGB4_STATE_DETACH: 1228 printk(KERN_INFO MOD "%s: Detach\n", 1229 pci_name(ctx->lldi.pdev)); 1230 if (ctx->dev) 1231 c4iw_remove(ctx); 1232 break; 1233 } 1234 return 0; 1235 } 1236 1237 static int disable_qp_db(int id, void *p, void *data) 1238 { 1239 struct c4iw_qp *qp = p; 1240 1241 t4_disable_wq_db(&qp->wq); 1242 return 0; 1243 } 1244 1245 static void stop_queues(struct uld_ctx *ctx) 1246 { 1247 unsigned long flags; 1248 1249 spin_lock_irqsave(&ctx->dev->lock, flags); 1250 ctx->dev->rdev.stats.db_state_transitions++; 1251 ctx->dev->db_state = STOPPED; 1252 if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED) 1253 idr_for_each(&ctx->dev->qpidr, disable_qp_db, NULL); 1254 else 1255 ctx->dev->rdev.status_page->db_off = 1; 1256 spin_unlock_irqrestore(&ctx->dev->lock, flags); 1257 } 1258 1259 static int enable_qp_db(int id, void *p, void *data) 1260 { 1261 struct c4iw_qp *qp = p; 1262 1263 t4_enable_wq_db(&qp->wq); 1264 return 0; 1265 } 1266 1267 static void resume_rc_qp(struct c4iw_qp *qp) 1268 { 1269 spin_lock(&qp->lock); 1270 t4_ring_sq_db(&qp->wq, qp->wq.sq.wq_pidx_inc, 1271 is_t5(qp->rhp->rdev.lldi.adapter_type), NULL); 1272 qp->wq.sq.wq_pidx_inc = 0; 1273 t4_ring_rq_db(&qp->wq, qp->wq.rq.wq_pidx_inc, 1274 is_t5(qp->rhp->rdev.lldi.adapter_type), NULL); 1275 qp->wq.rq.wq_pidx_inc = 0; 1276 spin_unlock(&qp->lock); 1277 } 1278 1279 static void resume_a_chunk(struct uld_ctx *ctx) 1280 { 1281 int i; 1282 struct c4iw_qp *qp; 1283 1284 for (i = 0; i < DB_FC_RESUME_SIZE; i++) { 1285 qp = list_first_entry(&ctx->dev->db_fc_list, struct c4iw_qp, 1286 db_fc_entry); 1287 list_del_init(&qp->db_fc_entry); 1288 resume_rc_qp(qp); 1289 if (list_empty(&ctx->dev->db_fc_list)) 1290 break; 1291 } 1292 } 1293 1294 static void resume_queues(struct uld_ctx *ctx) 1295 { 1296 spin_lock_irq(&ctx->dev->lock); 1297 if (ctx->dev->db_state != STOPPED) 1298 goto out; 1299 ctx->dev->db_state = FLOW_CONTROL; 1300 while (1) { 1301 if (list_empty(&ctx->dev->db_fc_list)) { 1302 WARN_ON(ctx->dev->db_state != FLOW_CONTROL); 1303 ctx->dev->db_state = NORMAL; 1304 ctx->dev->rdev.stats.db_state_transitions++; 1305 if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED) { 1306 idr_for_each(&ctx->dev->qpidr, enable_qp_db, 1307 NULL); 1308 } else { 1309 ctx->dev->rdev.status_page->db_off = 0; 1310 } 1311 break; 1312 } else { 1313 if (cxgb4_dbfifo_count(ctx->dev->rdev.lldi.ports[0], 1) 1314 < (ctx->dev->rdev.lldi.dbfifo_int_thresh << 1315 DB_FC_DRAIN_THRESH)) { 1316 resume_a_chunk(ctx); 1317 } 1318 if (!list_empty(&ctx->dev->db_fc_list)) { 1319 spin_unlock_irq(&ctx->dev->lock); 1320 if (DB_FC_RESUME_DELAY) { 1321 set_current_state(TASK_UNINTERRUPTIBLE); 1322 schedule_timeout(DB_FC_RESUME_DELAY); 1323 } 1324 spin_lock_irq(&ctx->dev->lock); 1325 if (ctx->dev->db_state != FLOW_CONTROL) 1326 break; 1327 } 1328 } 1329 } 1330 out: 1331 if (ctx->dev->db_state != NORMAL) 1332 ctx->dev->rdev.stats.db_fc_interruptions++; 1333 spin_unlock_irq(&ctx->dev->lock); 1334 } 1335 1336 struct qp_list { 1337 unsigned idx; 1338 struct c4iw_qp **qps; 1339 }; 1340 1341 static int add_and_ref_qp(int id, void *p, void *data) 1342 { 1343 struct qp_list *qp_listp = data; 1344 struct c4iw_qp *qp = p; 1345 1346 c4iw_qp_add_ref(&qp->ibqp); 1347 qp_listp->qps[qp_listp->idx++] = qp; 1348 return 0; 1349 } 1350 1351 static int count_qps(int id, void *p, void *data) 1352 { 1353 unsigned *countp = data; 1354 (*countp)++; 1355 return 0; 1356 } 1357 1358 static void deref_qps(struct qp_list *qp_list) 1359 { 1360 int idx; 1361 1362 for (idx = 0; idx < qp_list->idx; idx++) 1363 c4iw_qp_rem_ref(&qp_list->qps[idx]->ibqp); 1364 } 1365 1366 static void recover_lost_dbs(struct uld_ctx *ctx, struct qp_list *qp_list) 1367 { 1368 int idx; 1369 int ret; 1370 1371 for (idx = 0; idx < qp_list->idx; idx++) { 1372 struct c4iw_qp *qp = qp_list->qps[idx]; 1373 1374 spin_lock_irq(&qp->rhp->lock); 1375 spin_lock(&qp->lock); 1376 ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0], 1377 qp->wq.sq.qid, 1378 t4_sq_host_wq_pidx(&qp->wq), 1379 t4_sq_wq_size(&qp->wq)); 1380 if (ret) { 1381 pr_err(MOD "%s: Fatal error - " 1382 "DB overflow recovery failed - " 1383 "error syncing SQ qid %u\n", 1384 pci_name(ctx->lldi.pdev), qp->wq.sq.qid); 1385 spin_unlock(&qp->lock); 1386 spin_unlock_irq(&qp->rhp->lock); 1387 return; 1388 } 1389 qp->wq.sq.wq_pidx_inc = 0; 1390 1391 ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0], 1392 qp->wq.rq.qid, 1393 t4_rq_host_wq_pidx(&qp->wq), 1394 t4_rq_wq_size(&qp->wq)); 1395 1396 if (ret) { 1397 pr_err(MOD "%s: Fatal error - " 1398 "DB overflow recovery failed - " 1399 "error syncing RQ qid %u\n", 1400 pci_name(ctx->lldi.pdev), qp->wq.rq.qid); 1401 spin_unlock(&qp->lock); 1402 spin_unlock_irq(&qp->rhp->lock); 1403 return; 1404 } 1405 qp->wq.rq.wq_pidx_inc = 0; 1406 spin_unlock(&qp->lock); 1407 spin_unlock_irq(&qp->rhp->lock); 1408 1409 /* Wait for the dbfifo to drain */ 1410 while (cxgb4_dbfifo_count(qp->rhp->rdev.lldi.ports[0], 1) > 0) { 1411 set_current_state(TASK_UNINTERRUPTIBLE); 1412 schedule_timeout(usecs_to_jiffies(10)); 1413 } 1414 } 1415 } 1416 1417 static void recover_queues(struct uld_ctx *ctx) 1418 { 1419 int count = 0; 1420 struct qp_list qp_list; 1421 int ret; 1422 1423 /* slow everybody down */ 1424 set_current_state(TASK_UNINTERRUPTIBLE); 1425 schedule_timeout(usecs_to_jiffies(1000)); 1426 1427 /* flush the SGE contexts */ 1428 ret = cxgb4_flush_eq_cache(ctx->dev->rdev.lldi.ports[0]); 1429 if (ret) { 1430 printk(KERN_ERR MOD "%s: Fatal error - DB overflow recovery failed\n", 1431 pci_name(ctx->lldi.pdev)); 1432 return; 1433 } 1434 1435 /* Count active queues so we can build a list of queues to recover */ 1436 spin_lock_irq(&ctx->dev->lock); 1437 WARN_ON(ctx->dev->db_state != STOPPED); 1438 ctx->dev->db_state = RECOVERY; 1439 idr_for_each(&ctx->dev->qpidr, count_qps, &count); 1440 1441 qp_list.qps = kzalloc(count * sizeof *qp_list.qps, GFP_ATOMIC); 1442 if (!qp_list.qps) { 1443 printk(KERN_ERR MOD "%s: Fatal error - DB overflow recovery failed\n", 1444 pci_name(ctx->lldi.pdev)); 1445 spin_unlock_irq(&ctx->dev->lock); 1446 return; 1447 } 1448 qp_list.idx = 0; 1449 1450 /* add and ref each qp so it doesn't get freed */ 1451 idr_for_each(&ctx->dev->qpidr, add_and_ref_qp, &qp_list); 1452 1453 spin_unlock_irq(&ctx->dev->lock); 1454 1455 /* now traverse the list in a safe context to recover the db state*/ 1456 recover_lost_dbs(ctx, &qp_list); 1457 1458 /* we're almost done! deref the qps and clean up */ 1459 deref_qps(&qp_list); 1460 kfree(qp_list.qps); 1461 1462 spin_lock_irq(&ctx->dev->lock); 1463 WARN_ON(ctx->dev->db_state != RECOVERY); 1464 ctx->dev->db_state = STOPPED; 1465 spin_unlock_irq(&ctx->dev->lock); 1466 } 1467 1468 static int c4iw_uld_control(void *handle, enum cxgb4_control control, ...) 1469 { 1470 struct uld_ctx *ctx = handle; 1471 1472 switch (control) { 1473 case CXGB4_CONTROL_DB_FULL: 1474 stop_queues(ctx); 1475 ctx->dev->rdev.stats.db_full++; 1476 break; 1477 case CXGB4_CONTROL_DB_EMPTY: 1478 resume_queues(ctx); 1479 mutex_lock(&ctx->dev->rdev.stats.lock); 1480 ctx->dev->rdev.stats.db_empty++; 1481 mutex_unlock(&ctx->dev->rdev.stats.lock); 1482 break; 1483 case CXGB4_CONTROL_DB_DROP: 1484 recover_queues(ctx); 1485 mutex_lock(&ctx->dev->rdev.stats.lock); 1486 ctx->dev->rdev.stats.db_drop++; 1487 mutex_unlock(&ctx->dev->rdev.stats.lock); 1488 break; 1489 default: 1490 printk(KERN_WARNING MOD "%s: unknown control cmd %u\n", 1491 pci_name(ctx->lldi.pdev), control); 1492 break; 1493 } 1494 return 0; 1495 } 1496 1497 static struct cxgb4_uld_info c4iw_uld_info = { 1498 .name = DRV_NAME, 1499 .add = c4iw_uld_add, 1500 .rx_handler = c4iw_uld_rx_handler, 1501 .state_change = c4iw_uld_state_change, 1502 .control = c4iw_uld_control, 1503 }; 1504 1505 static int __init c4iw_init_module(void) 1506 { 1507 int err; 1508 1509 err = c4iw_cm_init(); 1510 if (err) 1511 return err; 1512 1513 c4iw_debugfs_root = debugfs_create_dir(DRV_NAME, NULL); 1514 if (!c4iw_debugfs_root) 1515 printk(KERN_WARNING MOD 1516 "could not create debugfs entry, continuing\n"); 1517 1518 if (ibnl_add_client(RDMA_NL_C4IW, RDMA_NL_IWPM_NUM_OPS, 1519 c4iw_nl_cb_table)) 1520 pr_err("%s[%u]: Failed to add netlink callback\n" 1521 , __func__, __LINE__); 1522 1523 err = iwpm_init(RDMA_NL_C4IW); 1524 if (err) { 1525 pr_err("port mapper initialization failed with %d\n", err); 1526 ibnl_remove_client(RDMA_NL_C4IW); 1527 c4iw_cm_term(); 1528 debugfs_remove_recursive(c4iw_debugfs_root); 1529 return err; 1530 } 1531 1532 cxgb4_register_uld(CXGB4_ULD_RDMA, &c4iw_uld_info); 1533 1534 return 0; 1535 } 1536 1537 static void __exit c4iw_exit_module(void) 1538 { 1539 struct uld_ctx *ctx, *tmp; 1540 1541 mutex_lock(&dev_mutex); 1542 list_for_each_entry_safe(ctx, tmp, &uld_ctx_list, entry) { 1543 if (ctx->dev) 1544 c4iw_remove(ctx); 1545 kfree(ctx); 1546 } 1547 mutex_unlock(&dev_mutex); 1548 cxgb4_unregister_uld(CXGB4_ULD_RDMA); 1549 iwpm_exit(RDMA_NL_C4IW); 1550 ibnl_remove_client(RDMA_NL_C4IW); 1551 c4iw_cm_term(); 1552 debugfs_remove_recursive(c4iw_debugfs_root); 1553 } 1554 1555 module_init(c4iw_init_module); 1556 module_exit(c4iw_exit_module); 1557