1 /* 2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/module.h> 33 #include <linux/moduleparam.h> 34 #include <linux/debugfs.h> 35 #include <linux/vmalloc.h> 36 37 #include <rdma/ib_verbs.h> 38 39 #include "iw_cxgb4.h" 40 41 #define DRV_VERSION "0.1" 42 43 MODULE_AUTHOR("Steve Wise"); 44 MODULE_DESCRIPTION("Chelsio T4/T5 RDMA Driver"); 45 MODULE_LICENSE("Dual BSD/GPL"); 46 MODULE_VERSION(DRV_VERSION); 47 48 static int allow_db_fc_on_t5; 49 module_param(allow_db_fc_on_t5, int, 0644); 50 MODULE_PARM_DESC(allow_db_fc_on_t5, 51 "Allow DB Flow Control on T5 (default = 0)"); 52 53 static int allow_db_coalescing_on_t5; 54 module_param(allow_db_coalescing_on_t5, int, 0644); 55 MODULE_PARM_DESC(allow_db_coalescing_on_t5, 56 "Allow DB Coalescing on T5 (default = 0)"); 57 58 struct uld_ctx { 59 struct list_head entry; 60 struct cxgb4_lld_info lldi; 61 struct c4iw_dev *dev; 62 }; 63 64 static LIST_HEAD(uld_ctx_list); 65 static DEFINE_MUTEX(dev_mutex); 66 67 #define DB_FC_RESUME_SIZE 64 68 #define DB_FC_RESUME_DELAY 1 69 #define DB_FC_DRAIN_THRESH 0 70 71 static struct dentry *c4iw_debugfs_root; 72 73 struct c4iw_debugfs_data { 74 struct c4iw_dev *devp; 75 char *buf; 76 int bufsize; 77 int pos; 78 }; 79 80 /* registered cxgb4 netlink callbacks */ 81 static struct ibnl_client_cbs c4iw_nl_cb_table[] = { 82 [RDMA_NL_IWPM_REG_PID] = {.dump = iwpm_register_pid_cb}, 83 [RDMA_NL_IWPM_ADD_MAPPING] = {.dump = iwpm_add_mapping_cb}, 84 [RDMA_NL_IWPM_QUERY_MAPPING] = {.dump = iwpm_add_and_query_mapping_cb}, 85 [RDMA_NL_IWPM_HANDLE_ERR] = {.dump = iwpm_mapping_error_cb}, 86 [RDMA_NL_IWPM_MAPINFO] = {.dump = iwpm_mapping_info_cb}, 87 [RDMA_NL_IWPM_MAPINFO_NUM] = {.dump = iwpm_ack_mapping_info_cb} 88 }; 89 90 static int count_idrs(int id, void *p, void *data) 91 { 92 int *countp = data; 93 94 *countp = *countp + 1; 95 return 0; 96 } 97 98 static ssize_t debugfs_read(struct file *file, char __user *buf, size_t count, 99 loff_t *ppos) 100 { 101 struct c4iw_debugfs_data *d = file->private_data; 102 103 return simple_read_from_buffer(buf, count, ppos, d->buf, d->pos); 104 } 105 106 static int dump_qp(int id, void *p, void *data) 107 { 108 struct c4iw_qp *qp = p; 109 struct c4iw_debugfs_data *qpd = data; 110 int space; 111 int cc; 112 113 if (id != qp->wq.sq.qid) 114 return 0; 115 116 space = qpd->bufsize - qpd->pos - 1; 117 if (space == 0) 118 return 1; 119 120 if (qp->ep) { 121 if (qp->ep->com.local_addr.ss_family == AF_INET) { 122 struct sockaddr_in *lsin = (struct sockaddr_in *) 123 &qp->ep->com.local_addr; 124 struct sockaddr_in *rsin = (struct sockaddr_in *) 125 &qp->ep->com.remote_addr; 126 struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) 127 &qp->ep->com.mapped_local_addr; 128 struct sockaddr_in *mapped_rsin = (struct sockaddr_in *) 129 &qp->ep->com.mapped_remote_addr; 130 131 cc = snprintf(qpd->buf + qpd->pos, space, 132 "rc qp sq id %u rq id %u state %u " 133 "onchip %u ep tid %u state %u " 134 "%pI4:%u/%u->%pI4:%u/%u\n", 135 qp->wq.sq.qid, qp->wq.rq.qid, 136 (int)qp->attr.state, 137 qp->wq.sq.flags & T4_SQ_ONCHIP, 138 qp->ep->hwtid, (int)qp->ep->com.state, 139 &lsin->sin_addr, ntohs(lsin->sin_port), 140 ntohs(mapped_lsin->sin_port), 141 &rsin->sin_addr, ntohs(rsin->sin_port), 142 ntohs(mapped_rsin->sin_port)); 143 } else { 144 struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) 145 &qp->ep->com.local_addr; 146 struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *) 147 &qp->ep->com.remote_addr; 148 struct sockaddr_in6 *mapped_lsin6 = 149 (struct sockaddr_in6 *) 150 &qp->ep->com.mapped_local_addr; 151 struct sockaddr_in6 *mapped_rsin6 = 152 (struct sockaddr_in6 *) 153 &qp->ep->com.mapped_remote_addr; 154 155 cc = snprintf(qpd->buf + qpd->pos, space, 156 "rc qp sq id %u rq id %u state %u " 157 "onchip %u ep tid %u state %u " 158 "%pI6:%u/%u->%pI6:%u/%u\n", 159 qp->wq.sq.qid, qp->wq.rq.qid, 160 (int)qp->attr.state, 161 qp->wq.sq.flags & T4_SQ_ONCHIP, 162 qp->ep->hwtid, (int)qp->ep->com.state, 163 &lsin6->sin6_addr, 164 ntohs(lsin6->sin6_port), 165 ntohs(mapped_lsin6->sin6_port), 166 &rsin6->sin6_addr, 167 ntohs(rsin6->sin6_port), 168 ntohs(mapped_rsin6->sin6_port)); 169 } 170 } else 171 cc = snprintf(qpd->buf + qpd->pos, space, 172 "qp sq id %u rq id %u state %u onchip %u\n", 173 qp->wq.sq.qid, qp->wq.rq.qid, 174 (int)qp->attr.state, 175 qp->wq.sq.flags & T4_SQ_ONCHIP); 176 if (cc < space) 177 qpd->pos += cc; 178 return 0; 179 } 180 181 static int qp_release(struct inode *inode, struct file *file) 182 { 183 struct c4iw_debugfs_data *qpd = file->private_data; 184 if (!qpd) { 185 printk(KERN_INFO "%s null qpd?\n", __func__); 186 return 0; 187 } 188 vfree(qpd->buf); 189 kfree(qpd); 190 return 0; 191 } 192 193 static int qp_open(struct inode *inode, struct file *file) 194 { 195 struct c4iw_debugfs_data *qpd; 196 int ret = 0; 197 int count = 1; 198 199 qpd = kmalloc(sizeof *qpd, GFP_KERNEL); 200 if (!qpd) { 201 ret = -ENOMEM; 202 goto out; 203 } 204 qpd->devp = inode->i_private; 205 qpd->pos = 0; 206 207 spin_lock_irq(&qpd->devp->lock); 208 idr_for_each(&qpd->devp->qpidr, count_idrs, &count); 209 spin_unlock_irq(&qpd->devp->lock); 210 211 qpd->bufsize = count * 128; 212 qpd->buf = vmalloc(qpd->bufsize); 213 if (!qpd->buf) { 214 ret = -ENOMEM; 215 goto err1; 216 } 217 218 spin_lock_irq(&qpd->devp->lock); 219 idr_for_each(&qpd->devp->qpidr, dump_qp, qpd); 220 spin_unlock_irq(&qpd->devp->lock); 221 222 qpd->buf[qpd->pos++] = 0; 223 file->private_data = qpd; 224 goto out; 225 err1: 226 kfree(qpd); 227 out: 228 return ret; 229 } 230 231 static const struct file_operations qp_debugfs_fops = { 232 .owner = THIS_MODULE, 233 .open = qp_open, 234 .release = qp_release, 235 .read = debugfs_read, 236 .llseek = default_llseek, 237 }; 238 239 static int dump_stag(int id, void *p, void *data) 240 { 241 struct c4iw_debugfs_data *stagd = data; 242 int space; 243 int cc; 244 245 space = stagd->bufsize - stagd->pos - 1; 246 if (space == 0) 247 return 1; 248 249 cc = snprintf(stagd->buf + stagd->pos, space, "0x%x\n", id<<8); 250 if (cc < space) 251 stagd->pos += cc; 252 return 0; 253 } 254 255 static int stag_release(struct inode *inode, struct file *file) 256 { 257 struct c4iw_debugfs_data *stagd = file->private_data; 258 if (!stagd) { 259 printk(KERN_INFO "%s null stagd?\n", __func__); 260 return 0; 261 } 262 kfree(stagd->buf); 263 kfree(stagd); 264 return 0; 265 } 266 267 static int stag_open(struct inode *inode, struct file *file) 268 { 269 struct c4iw_debugfs_data *stagd; 270 int ret = 0; 271 int count = 1; 272 273 stagd = kmalloc(sizeof *stagd, GFP_KERNEL); 274 if (!stagd) { 275 ret = -ENOMEM; 276 goto out; 277 } 278 stagd->devp = inode->i_private; 279 stagd->pos = 0; 280 281 spin_lock_irq(&stagd->devp->lock); 282 idr_for_each(&stagd->devp->mmidr, count_idrs, &count); 283 spin_unlock_irq(&stagd->devp->lock); 284 285 stagd->bufsize = count * sizeof("0x12345678\n"); 286 stagd->buf = kmalloc(stagd->bufsize, GFP_KERNEL); 287 if (!stagd->buf) { 288 ret = -ENOMEM; 289 goto err1; 290 } 291 292 spin_lock_irq(&stagd->devp->lock); 293 idr_for_each(&stagd->devp->mmidr, dump_stag, stagd); 294 spin_unlock_irq(&stagd->devp->lock); 295 296 stagd->buf[stagd->pos++] = 0; 297 file->private_data = stagd; 298 goto out; 299 err1: 300 kfree(stagd); 301 out: 302 return ret; 303 } 304 305 static const struct file_operations stag_debugfs_fops = { 306 .owner = THIS_MODULE, 307 .open = stag_open, 308 .release = stag_release, 309 .read = debugfs_read, 310 .llseek = default_llseek, 311 }; 312 313 static char *db_state_str[] = {"NORMAL", "FLOW_CONTROL", "RECOVERY", "STOPPED"}; 314 315 static int stats_show(struct seq_file *seq, void *v) 316 { 317 struct c4iw_dev *dev = seq->private; 318 319 seq_printf(seq, " Object: %10s %10s %10s %10s\n", "Total", "Current", 320 "Max", "Fail"); 321 seq_printf(seq, " PDID: %10llu %10llu %10llu %10llu\n", 322 dev->rdev.stats.pd.total, dev->rdev.stats.pd.cur, 323 dev->rdev.stats.pd.max, dev->rdev.stats.pd.fail); 324 seq_printf(seq, " QID: %10llu %10llu %10llu %10llu\n", 325 dev->rdev.stats.qid.total, dev->rdev.stats.qid.cur, 326 dev->rdev.stats.qid.max, dev->rdev.stats.qid.fail); 327 seq_printf(seq, " TPTMEM: %10llu %10llu %10llu %10llu\n", 328 dev->rdev.stats.stag.total, dev->rdev.stats.stag.cur, 329 dev->rdev.stats.stag.max, dev->rdev.stats.stag.fail); 330 seq_printf(seq, " PBLMEM: %10llu %10llu %10llu %10llu\n", 331 dev->rdev.stats.pbl.total, dev->rdev.stats.pbl.cur, 332 dev->rdev.stats.pbl.max, dev->rdev.stats.pbl.fail); 333 seq_printf(seq, " RQTMEM: %10llu %10llu %10llu %10llu\n", 334 dev->rdev.stats.rqt.total, dev->rdev.stats.rqt.cur, 335 dev->rdev.stats.rqt.max, dev->rdev.stats.rqt.fail); 336 seq_printf(seq, " OCQPMEM: %10llu %10llu %10llu %10llu\n", 337 dev->rdev.stats.ocqp.total, dev->rdev.stats.ocqp.cur, 338 dev->rdev.stats.ocqp.max, dev->rdev.stats.ocqp.fail); 339 seq_printf(seq, " DB FULL: %10llu\n", dev->rdev.stats.db_full); 340 seq_printf(seq, " DB EMPTY: %10llu\n", dev->rdev.stats.db_empty); 341 seq_printf(seq, " DB DROP: %10llu\n", dev->rdev.stats.db_drop); 342 seq_printf(seq, " DB State: %s Transitions %llu FC Interruptions %llu\n", 343 db_state_str[dev->db_state], 344 dev->rdev.stats.db_state_transitions, 345 dev->rdev.stats.db_fc_interruptions); 346 seq_printf(seq, "TCAM_FULL: %10llu\n", dev->rdev.stats.tcam_full); 347 seq_printf(seq, "ACT_OFLD_CONN_FAILS: %10llu\n", 348 dev->rdev.stats.act_ofld_conn_fails); 349 seq_printf(seq, "PAS_OFLD_CONN_FAILS: %10llu\n", 350 dev->rdev.stats.pas_ofld_conn_fails); 351 return 0; 352 } 353 354 static int stats_open(struct inode *inode, struct file *file) 355 { 356 return single_open(file, stats_show, inode->i_private); 357 } 358 359 static ssize_t stats_clear(struct file *file, const char __user *buf, 360 size_t count, loff_t *pos) 361 { 362 struct c4iw_dev *dev = ((struct seq_file *)file->private_data)->private; 363 364 mutex_lock(&dev->rdev.stats.lock); 365 dev->rdev.stats.pd.max = 0; 366 dev->rdev.stats.pd.fail = 0; 367 dev->rdev.stats.qid.max = 0; 368 dev->rdev.stats.qid.fail = 0; 369 dev->rdev.stats.stag.max = 0; 370 dev->rdev.stats.stag.fail = 0; 371 dev->rdev.stats.pbl.max = 0; 372 dev->rdev.stats.pbl.fail = 0; 373 dev->rdev.stats.rqt.max = 0; 374 dev->rdev.stats.rqt.fail = 0; 375 dev->rdev.stats.ocqp.max = 0; 376 dev->rdev.stats.ocqp.fail = 0; 377 dev->rdev.stats.db_full = 0; 378 dev->rdev.stats.db_empty = 0; 379 dev->rdev.stats.db_drop = 0; 380 dev->rdev.stats.db_state_transitions = 0; 381 dev->rdev.stats.tcam_full = 0; 382 dev->rdev.stats.act_ofld_conn_fails = 0; 383 dev->rdev.stats.pas_ofld_conn_fails = 0; 384 mutex_unlock(&dev->rdev.stats.lock); 385 return count; 386 } 387 388 static const struct file_operations stats_debugfs_fops = { 389 .owner = THIS_MODULE, 390 .open = stats_open, 391 .release = single_release, 392 .read = seq_read, 393 .llseek = seq_lseek, 394 .write = stats_clear, 395 }; 396 397 static int dump_ep(int id, void *p, void *data) 398 { 399 struct c4iw_ep *ep = p; 400 struct c4iw_debugfs_data *epd = data; 401 int space; 402 int cc; 403 404 space = epd->bufsize - epd->pos - 1; 405 if (space == 0) 406 return 1; 407 408 if (ep->com.local_addr.ss_family == AF_INET) { 409 struct sockaddr_in *lsin = (struct sockaddr_in *) 410 &ep->com.local_addr; 411 struct sockaddr_in *rsin = (struct sockaddr_in *) 412 &ep->com.remote_addr; 413 struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) 414 &ep->com.mapped_local_addr; 415 struct sockaddr_in *mapped_rsin = (struct sockaddr_in *) 416 &ep->com.mapped_remote_addr; 417 418 cc = snprintf(epd->buf + epd->pos, space, 419 "ep %p cm_id %p qp %p state %d flags 0x%lx " 420 "history 0x%lx hwtid %d atid %d " 421 "%pI4:%d/%d <-> %pI4:%d/%d\n", 422 ep, ep->com.cm_id, ep->com.qp, 423 (int)ep->com.state, ep->com.flags, 424 ep->com.history, ep->hwtid, ep->atid, 425 &lsin->sin_addr, ntohs(lsin->sin_port), 426 ntohs(mapped_lsin->sin_port), 427 &rsin->sin_addr, ntohs(rsin->sin_port), 428 ntohs(mapped_rsin->sin_port)); 429 } else { 430 struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) 431 &ep->com.local_addr; 432 struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *) 433 &ep->com.remote_addr; 434 struct sockaddr_in6 *mapped_lsin6 = (struct sockaddr_in6 *) 435 &ep->com.mapped_local_addr; 436 struct sockaddr_in6 *mapped_rsin6 = (struct sockaddr_in6 *) 437 &ep->com.mapped_remote_addr; 438 439 cc = snprintf(epd->buf + epd->pos, space, 440 "ep %p cm_id %p qp %p state %d flags 0x%lx " 441 "history 0x%lx hwtid %d atid %d " 442 "%pI6:%d/%d <-> %pI6:%d/%d\n", 443 ep, ep->com.cm_id, ep->com.qp, 444 (int)ep->com.state, ep->com.flags, 445 ep->com.history, ep->hwtid, ep->atid, 446 &lsin6->sin6_addr, ntohs(lsin6->sin6_port), 447 ntohs(mapped_lsin6->sin6_port), 448 &rsin6->sin6_addr, ntohs(rsin6->sin6_port), 449 ntohs(mapped_rsin6->sin6_port)); 450 } 451 if (cc < space) 452 epd->pos += cc; 453 return 0; 454 } 455 456 static int dump_listen_ep(int id, void *p, void *data) 457 { 458 struct c4iw_listen_ep *ep = p; 459 struct c4iw_debugfs_data *epd = data; 460 int space; 461 int cc; 462 463 space = epd->bufsize - epd->pos - 1; 464 if (space == 0) 465 return 1; 466 467 if (ep->com.local_addr.ss_family == AF_INET) { 468 struct sockaddr_in *lsin = (struct sockaddr_in *) 469 &ep->com.local_addr; 470 struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) 471 &ep->com.mapped_local_addr; 472 473 cc = snprintf(epd->buf + epd->pos, space, 474 "ep %p cm_id %p state %d flags 0x%lx stid %d " 475 "backlog %d %pI4:%d/%d\n", 476 ep, ep->com.cm_id, (int)ep->com.state, 477 ep->com.flags, ep->stid, ep->backlog, 478 &lsin->sin_addr, ntohs(lsin->sin_port), 479 ntohs(mapped_lsin->sin_port)); 480 } else { 481 struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) 482 &ep->com.local_addr; 483 struct sockaddr_in6 *mapped_lsin6 = (struct sockaddr_in6 *) 484 &ep->com.mapped_local_addr; 485 486 cc = snprintf(epd->buf + epd->pos, space, 487 "ep %p cm_id %p state %d flags 0x%lx stid %d " 488 "backlog %d %pI6:%d/%d\n", 489 ep, ep->com.cm_id, (int)ep->com.state, 490 ep->com.flags, ep->stid, ep->backlog, 491 &lsin6->sin6_addr, ntohs(lsin6->sin6_port), 492 ntohs(mapped_lsin6->sin6_port)); 493 } 494 if (cc < space) 495 epd->pos += cc; 496 return 0; 497 } 498 499 static int ep_release(struct inode *inode, struct file *file) 500 { 501 struct c4iw_debugfs_data *epd = file->private_data; 502 if (!epd) { 503 pr_info("%s null qpd?\n", __func__); 504 return 0; 505 } 506 vfree(epd->buf); 507 kfree(epd); 508 return 0; 509 } 510 511 static int ep_open(struct inode *inode, struct file *file) 512 { 513 struct c4iw_debugfs_data *epd; 514 int ret = 0; 515 int count = 1; 516 517 epd = kmalloc(sizeof(*epd), GFP_KERNEL); 518 if (!epd) { 519 ret = -ENOMEM; 520 goto out; 521 } 522 epd->devp = inode->i_private; 523 epd->pos = 0; 524 525 spin_lock_irq(&epd->devp->lock); 526 idr_for_each(&epd->devp->hwtid_idr, count_idrs, &count); 527 idr_for_each(&epd->devp->atid_idr, count_idrs, &count); 528 idr_for_each(&epd->devp->stid_idr, count_idrs, &count); 529 spin_unlock_irq(&epd->devp->lock); 530 531 epd->bufsize = count * 160; 532 epd->buf = vmalloc(epd->bufsize); 533 if (!epd->buf) { 534 ret = -ENOMEM; 535 goto err1; 536 } 537 538 spin_lock_irq(&epd->devp->lock); 539 idr_for_each(&epd->devp->hwtid_idr, dump_ep, epd); 540 idr_for_each(&epd->devp->atid_idr, dump_ep, epd); 541 idr_for_each(&epd->devp->stid_idr, dump_listen_ep, epd); 542 spin_unlock_irq(&epd->devp->lock); 543 544 file->private_data = epd; 545 goto out; 546 err1: 547 kfree(epd); 548 out: 549 return ret; 550 } 551 552 static const struct file_operations ep_debugfs_fops = { 553 .owner = THIS_MODULE, 554 .open = ep_open, 555 .release = ep_release, 556 .read = debugfs_read, 557 }; 558 559 static int setup_debugfs(struct c4iw_dev *devp) 560 { 561 struct dentry *de; 562 563 if (!devp->debugfs_root) 564 return -1; 565 566 de = debugfs_create_file("qps", S_IWUSR, devp->debugfs_root, 567 (void *)devp, &qp_debugfs_fops); 568 if (de && de->d_inode) 569 de->d_inode->i_size = 4096; 570 571 de = debugfs_create_file("stags", S_IWUSR, devp->debugfs_root, 572 (void *)devp, &stag_debugfs_fops); 573 if (de && de->d_inode) 574 de->d_inode->i_size = 4096; 575 576 de = debugfs_create_file("stats", S_IWUSR, devp->debugfs_root, 577 (void *)devp, &stats_debugfs_fops); 578 if (de && de->d_inode) 579 de->d_inode->i_size = 4096; 580 581 de = debugfs_create_file("eps", S_IWUSR, devp->debugfs_root, 582 (void *)devp, &ep_debugfs_fops); 583 if (de && de->d_inode) 584 de->d_inode->i_size = 4096; 585 586 return 0; 587 } 588 589 void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev, 590 struct c4iw_dev_ucontext *uctx) 591 { 592 struct list_head *pos, *nxt; 593 struct c4iw_qid_list *entry; 594 595 mutex_lock(&uctx->lock); 596 list_for_each_safe(pos, nxt, &uctx->qpids) { 597 entry = list_entry(pos, struct c4iw_qid_list, entry); 598 list_del_init(&entry->entry); 599 if (!(entry->qid & rdev->qpmask)) { 600 c4iw_put_resource(&rdev->resource.qid_table, 601 entry->qid); 602 mutex_lock(&rdev->stats.lock); 603 rdev->stats.qid.cur -= rdev->qpmask + 1; 604 mutex_unlock(&rdev->stats.lock); 605 } 606 kfree(entry); 607 } 608 609 list_for_each_safe(pos, nxt, &uctx->qpids) { 610 entry = list_entry(pos, struct c4iw_qid_list, entry); 611 list_del_init(&entry->entry); 612 kfree(entry); 613 } 614 mutex_unlock(&uctx->lock); 615 } 616 617 void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev, 618 struct c4iw_dev_ucontext *uctx) 619 { 620 INIT_LIST_HEAD(&uctx->qpids); 621 INIT_LIST_HEAD(&uctx->cqids); 622 mutex_init(&uctx->lock); 623 } 624 625 /* Caller takes care of locking if needed */ 626 static int c4iw_rdev_open(struct c4iw_rdev *rdev) 627 { 628 int err; 629 630 c4iw_init_dev_ucontext(rdev, &rdev->uctx); 631 632 /* 633 * qpshift is the number of bits to shift the qpid left in order 634 * to get the correct address of the doorbell for that qp. 635 */ 636 rdev->qpshift = PAGE_SHIFT - ilog2(rdev->lldi.udb_density); 637 rdev->qpmask = rdev->lldi.udb_density - 1; 638 rdev->cqshift = PAGE_SHIFT - ilog2(rdev->lldi.ucq_density); 639 rdev->cqmask = rdev->lldi.ucq_density - 1; 640 PDBG("%s dev %s stag start 0x%0x size 0x%0x num stags %d " 641 "pbl start 0x%0x size 0x%0x rq start 0x%0x size 0x%0x " 642 "qp qid start %u size %u cq qid start %u size %u\n", 643 __func__, pci_name(rdev->lldi.pdev), rdev->lldi.vr->stag.start, 644 rdev->lldi.vr->stag.size, c4iw_num_stags(rdev), 645 rdev->lldi.vr->pbl.start, 646 rdev->lldi.vr->pbl.size, rdev->lldi.vr->rq.start, 647 rdev->lldi.vr->rq.size, 648 rdev->lldi.vr->qp.start, 649 rdev->lldi.vr->qp.size, 650 rdev->lldi.vr->cq.start, 651 rdev->lldi.vr->cq.size); 652 PDBG("udb len 0x%x udb base %llx db_reg %p gts_reg %p qpshift %lu " 653 "qpmask 0x%x cqshift %lu cqmask 0x%x\n", 654 (unsigned)pci_resource_len(rdev->lldi.pdev, 2), 655 (u64)pci_resource_start(rdev->lldi.pdev, 2), 656 rdev->lldi.db_reg, 657 rdev->lldi.gts_reg, 658 rdev->qpshift, rdev->qpmask, 659 rdev->cqshift, rdev->cqmask); 660 661 if (c4iw_num_stags(rdev) == 0) { 662 err = -EINVAL; 663 goto err1; 664 } 665 666 rdev->stats.pd.total = T4_MAX_NUM_PD; 667 rdev->stats.stag.total = rdev->lldi.vr->stag.size; 668 rdev->stats.pbl.total = rdev->lldi.vr->pbl.size; 669 rdev->stats.rqt.total = rdev->lldi.vr->rq.size; 670 rdev->stats.ocqp.total = rdev->lldi.vr->ocq.size; 671 rdev->stats.qid.total = rdev->lldi.vr->qp.size; 672 673 err = c4iw_init_resource(rdev, c4iw_num_stags(rdev), T4_MAX_NUM_PD); 674 if (err) { 675 printk(KERN_ERR MOD "error %d initializing resources\n", err); 676 goto err1; 677 } 678 err = c4iw_pblpool_create(rdev); 679 if (err) { 680 printk(KERN_ERR MOD "error %d initializing pbl pool\n", err); 681 goto err2; 682 } 683 err = c4iw_rqtpool_create(rdev); 684 if (err) { 685 printk(KERN_ERR MOD "error %d initializing rqt pool\n", err); 686 goto err3; 687 } 688 err = c4iw_ocqp_pool_create(rdev); 689 if (err) { 690 printk(KERN_ERR MOD "error %d initializing ocqp pool\n", err); 691 goto err4; 692 } 693 rdev->status_page = (struct t4_dev_status_page *) 694 __get_free_page(GFP_KERNEL); 695 if (!rdev->status_page) { 696 pr_err(MOD "error allocating status page\n"); 697 goto err4; 698 } 699 rdev->status_page->db_off = 0; 700 return 0; 701 err4: 702 c4iw_rqtpool_destroy(rdev); 703 err3: 704 c4iw_pblpool_destroy(rdev); 705 err2: 706 c4iw_destroy_resource(&rdev->resource); 707 err1: 708 return err; 709 } 710 711 static void c4iw_rdev_close(struct c4iw_rdev *rdev) 712 { 713 free_page((unsigned long)rdev->status_page); 714 c4iw_pblpool_destroy(rdev); 715 c4iw_rqtpool_destroy(rdev); 716 c4iw_destroy_resource(&rdev->resource); 717 } 718 719 static void c4iw_dealloc(struct uld_ctx *ctx) 720 { 721 c4iw_rdev_close(&ctx->dev->rdev); 722 idr_destroy(&ctx->dev->cqidr); 723 idr_destroy(&ctx->dev->qpidr); 724 idr_destroy(&ctx->dev->mmidr); 725 idr_destroy(&ctx->dev->hwtid_idr); 726 idr_destroy(&ctx->dev->stid_idr); 727 idr_destroy(&ctx->dev->atid_idr); 728 if (ctx->dev->rdev.bar2_kva) 729 iounmap(ctx->dev->rdev.bar2_kva); 730 if (ctx->dev->rdev.oc_mw_kva) 731 iounmap(ctx->dev->rdev.oc_mw_kva); 732 ib_dealloc_device(&ctx->dev->ibdev); 733 ctx->dev = NULL; 734 } 735 736 static void c4iw_remove(struct uld_ctx *ctx) 737 { 738 PDBG("%s c4iw_dev %p\n", __func__, ctx->dev); 739 c4iw_unregister_device(ctx->dev); 740 c4iw_dealloc(ctx); 741 } 742 743 static int rdma_supported(const struct cxgb4_lld_info *infop) 744 { 745 return infop->vr->stag.size > 0 && infop->vr->pbl.size > 0 && 746 infop->vr->rq.size > 0 && infop->vr->qp.size > 0 && 747 infop->vr->cq.size > 0; 748 } 749 750 static struct c4iw_dev *c4iw_alloc(const struct cxgb4_lld_info *infop) 751 { 752 struct c4iw_dev *devp; 753 int ret; 754 755 if (!rdma_supported(infop)) { 756 printk(KERN_INFO MOD "%s: RDMA not supported on this device.\n", 757 pci_name(infop->pdev)); 758 return ERR_PTR(-ENOSYS); 759 } 760 if (!ocqp_supported(infop)) 761 pr_info("%s: On-Chip Queues not supported on this device.\n", 762 pci_name(infop->pdev)); 763 764 devp = (struct c4iw_dev *)ib_alloc_device(sizeof(*devp)); 765 if (!devp) { 766 printk(KERN_ERR MOD "Cannot allocate ib device\n"); 767 return ERR_PTR(-ENOMEM); 768 } 769 devp->rdev.lldi = *infop; 770 771 /* 772 * For T5 devices, we map all of BAR2 with WC. 773 * For T4 devices with onchip qp mem, we map only that part 774 * of BAR2 with WC. 775 */ 776 devp->rdev.bar2_pa = pci_resource_start(devp->rdev.lldi.pdev, 2); 777 if (is_t5(devp->rdev.lldi.adapter_type)) { 778 devp->rdev.bar2_kva = ioremap_wc(devp->rdev.bar2_pa, 779 pci_resource_len(devp->rdev.lldi.pdev, 2)); 780 if (!devp->rdev.bar2_kva) { 781 pr_err(MOD "Unable to ioremap BAR2\n"); 782 ib_dealloc_device(&devp->ibdev); 783 return ERR_PTR(-EINVAL); 784 } 785 } else if (ocqp_supported(infop)) { 786 devp->rdev.oc_mw_pa = 787 pci_resource_start(devp->rdev.lldi.pdev, 2) + 788 pci_resource_len(devp->rdev.lldi.pdev, 2) - 789 roundup_pow_of_two(devp->rdev.lldi.vr->ocq.size); 790 devp->rdev.oc_mw_kva = ioremap_wc(devp->rdev.oc_mw_pa, 791 devp->rdev.lldi.vr->ocq.size); 792 if (!devp->rdev.oc_mw_kva) { 793 pr_err(MOD "Unable to ioremap onchip mem\n"); 794 ib_dealloc_device(&devp->ibdev); 795 return ERR_PTR(-EINVAL); 796 } 797 } 798 799 PDBG(KERN_INFO MOD "ocq memory: " 800 "hw_start 0x%x size %u mw_pa 0x%lx mw_kva %p\n", 801 devp->rdev.lldi.vr->ocq.start, devp->rdev.lldi.vr->ocq.size, 802 devp->rdev.oc_mw_pa, devp->rdev.oc_mw_kva); 803 804 ret = c4iw_rdev_open(&devp->rdev); 805 if (ret) { 806 printk(KERN_ERR MOD "Unable to open CXIO rdev err %d\n", ret); 807 ib_dealloc_device(&devp->ibdev); 808 return ERR_PTR(ret); 809 } 810 811 idr_init(&devp->cqidr); 812 idr_init(&devp->qpidr); 813 idr_init(&devp->mmidr); 814 idr_init(&devp->hwtid_idr); 815 idr_init(&devp->stid_idr); 816 idr_init(&devp->atid_idr); 817 spin_lock_init(&devp->lock); 818 mutex_init(&devp->rdev.stats.lock); 819 mutex_init(&devp->db_mutex); 820 INIT_LIST_HEAD(&devp->db_fc_list); 821 822 if (c4iw_debugfs_root) { 823 devp->debugfs_root = debugfs_create_dir( 824 pci_name(devp->rdev.lldi.pdev), 825 c4iw_debugfs_root); 826 setup_debugfs(devp); 827 } 828 829 830 return devp; 831 } 832 833 static void *c4iw_uld_add(const struct cxgb4_lld_info *infop) 834 { 835 struct uld_ctx *ctx; 836 static int vers_printed; 837 int i; 838 839 if (!vers_printed++) 840 pr_info("Chelsio T4/T5 RDMA Driver - version %s\n", 841 DRV_VERSION); 842 843 ctx = kzalloc(sizeof *ctx, GFP_KERNEL); 844 if (!ctx) { 845 ctx = ERR_PTR(-ENOMEM); 846 goto out; 847 } 848 ctx->lldi = *infop; 849 850 PDBG("%s found device %s nchan %u nrxq %u ntxq %u nports %u\n", 851 __func__, pci_name(ctx->lldi.pdev), 852 ctx->lldi.nchan, ctx->lldi.nrxq, 853 ctx->lldi.ntxq, ctx->lldi.nports); 854 855 mutex_lock(&dev_mutex); 856 list_add_tail(&ctx->entry, &uld_ctx_list); 857 mutex_unlock(&dev_mutex); 858 859 for (i = 0; i < ctx->lldi.nrxq; i++) 860 PDBG("rxqid[%u] %u\n", i, ctx->lldi.rxq_ids[i]); 861 out: 862 return ctx; 863 } 864 865 static inline struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl, 866 const __be64 *rsp, 867 u32 pktshift) 868 { 869 struct sk_buff *skb; 870 871 /* 872 * Allocate space for cpl_pass_accept_req which will be synthesized by 873 * driver. Once the driver synthesizes the request the skb will go 874 * through the regular cpl_pass_accept_req processing. 875 * The math here assumes sizeof cpl_pass_accept_req >= sizeof 876 * cpl_rx_pkt. 877 */ 878 skb = alloc_skb(gl->tot_len + sizeof(struct cpl_pass_accept_req) + 879 sizeof(struct rss_header) - pktshift, GFP_ATOMIC); 880 if (unlikely(!skb)) 881 return NULL; 882 883 __skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req) + 884 sizeof(struct rss_header) - pktshift); 885 886 /* 887 * This skb will contain: 888 * rss_header from the rspq descriptor (1 flit) 889 * cpl_rx_pkt struct from the rspq descriptor (2 flits) 890 * space for the difference between the size of an 891 * rx_pkt and pass_accept_req cpl (1 flit) 892 * the packet data from the gl 893 */ 894 skb_copy_to_linear_data(skb, rsp, sizeof(struct cpl_pass_accept_req) + 895 sizeof(struct rss_header)); 896 skb_copy_to_linear_data_offset(skb, sizeof(struct rss_header) + 897 sizeof(struct cpl_pass_accept_req), 898 gl->va + pktshift, 899 gl->tot_len - pktshift); 900 return skb; 901 } 902 903 static inline int recv_rx_pkt(struct c4iw_dev *dev, const struct pkt_gl *gl, 904 const __be64 *rsp) 905 { 906 unsigned int opcode = *(u8 *)rsp; 907 struct sk_buff *skb; 908 909 if (opcode != CPL_RX_PKT) 910 goto out; 911 912 skb = copy_gl_to_skb_pkt(gl , rsp, dev->rdev.lldi.sge_pktshift); 913 if (skb == NULL) 914 goto out; 915 916 if (c4iw_handlers[opcode] == NULL) { 917 pr_info("%s no handler opcode 0x%x...\n", __func__, 918 opcode); 919 kfree_skb(skb); 920 goto out; 921 } 922 c4iw_handlers[opcode](dev, skb); 923 return 1; 924 out: 925 return 0; 926 } 927 928 static int c4iw_uld_rx_handler(void *handle, const __be64 *rsp, 929 const struct pkt_gl *gl) 930 { 931 struct uld_ctx *ctx = handle; 932 struct c4iw_dev *dev = ctx->dev; 933 struct sk_buff *skb; 934 u8 opcode; 935 936 if (gl == NULL) { 937 /* omit RSS and rsp_ctrl at end of descriptor */ 938 unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8; 939 940 skb = alloc_skb(256, GFP_ATOMIC); 941 if (!skb) 942 goto nomem; 943 __skb_put(skb, len); 944 skb_copy_to_linear_data(skb, &rsp[1], len); 945 } else if (gl == CXGB4_MSG_AN) { 946 const struct rsp_ctrl *rc = (void *)rsp; 947 948 u32 qid = be32_to_cpu(rc->pldbuflen_qid); 949 c4iw_ev_handler(dev, qid); 950 return 0; 951 } else if (unlikely(*(u8 *)rsp != *(u8 *)gl->va)) { 952 if (recv_rx_pkt(dev, gl, rsp)) 953 return 0; 954 955 pr_info("%s: unexpected FL contents at %p, " \ 956 "RSS %#llx, FL %#llx, len %u\n", 957 pci_name(ctx->lldi.pdev), gl->va, 958 (unsigned long long)be64_to_cpu(*rsp), 959 (unsigned long long)be64_to_cpu( 960 *(__force __be64 *)gl->va), 961 gl->tot_len); 962 963 return 0; 964 } else { 965 skb = cxgb4_pktgl_to_skb(gl, 128, 128); 966 if (unlikely(!skb)) 967 goto nomem; 968 } 969 970 opcode = *(u8 *)rsp; 971 if (c4iw_handlers[opcode]) { 972 c4iw_handlers[opcode](dev, skb); 973 } else { 974 pr_info("%s no handler opcode 0x%x...\n", __func__, 975 opcode); 976 kfree_skb(skb); 977 } 978 979 return 0; 980 nomem: 981 return -1; 982 } 983 984 static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state) 985 { 986 struct uld_ctx *ctx = handle; 987 988 PDBG("%s new_state %u\n", __func__, new_state); 989 switch (new_state) { 990 case CXGB4_STATE_UP: 991 printk(KERN_INFO MOD "%s: Up\n", pci_name(ctx->lldi.pdev)); 992 if (!ctx->dev) { 993 int ret; 994 995 ctx->dev = c4iw_alloc(&ctx->lldi); 996 if (IS_ERR(ctx->dev)) { 997 printk(KERN_ERR MOD 998 "%s: initialization failed: %ld\n", 999 pci_name(ctx->lldi.pdev), 1000 PTR_ERR(ctx->dev)); 1001 ctx->dev = NULL; 1002 break; 1003 } 1004 ret = c4iw_register_device(ctx->dev); 1005 if (ret) { 1006 printk(KERN_ERR MOD 1007 "%s: RDMA registration failed: %d\n", 1008 pci_name(ctx->lldi.pdev), ret); 1009 c4iw_dealloc(ctx); 1010 } 1011 } 1012 break; 1013 case CXGB4_STATE_DOWN: 1014 printk(KERN_INFO MOD "%s: Down\n", 1015 pci_name(ctx->lldi.pdev)); 1016 if (ctx->dev) 1017 c4iw_remove(ctx); 1018 break; 1019 case CXGB4_STATE_START_RECOVERY: 1020 printk(KERN_INFO MOD "%s: Fatal Error\n", 1021 pci_name(ctx->lldi.pdev)); 1022 if (ctx->dev) { 1023 struct ib_event event; 1024 1025 ctx->dev->rdev.flags |= T4_FATAL_ERROR; 1026 memset(&event, 0, sizeof event); 1027 event.event = IB_EVENT_DEVICE_FATAL; 1028 event.device = &ctx->dev->ibdev; 1029 ib_dispatch_event(&event); 1030 c4iw_remove(ctx); 1031 } 1032 break; 1033 case CXGB4_STATE_DETACH: 1034 printk(KERN_INFO MOD "%s: Detach\n", 1035 pci_name(ctx->lldi.pdev)); 1036 if (ctx->dev) 1037 c4iw_remove(ctx); 1038 break; 1039 } 1040 return 0; 1041 } 1042 1043 static int disable_qp_db(int id, void *p, void *data) 1044 { 1045 struct c4iw_qp *qp = p; 1046 1047 t4_disable_wq_db(&qp->wq); 1048 return 0; 1049 } 1050 1051 static void stop_queues(struct uld_ctx *ctx) 1052 { 1053 unsigned long flags; 1054 1055 spin_lock_irqsave(&ctx->dev->lock, flags); 1056 ctx->dev->rdev.stats.db_state_transitions++; 1057 ctx->dev->db_state = STOPPED; 1058 if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED) 1059 idr_for_each(&ctx->dev->qpidr, disable_qp_db, NULL); 1060 else 1061 ctx->dev->rdev.status_page->db_off = 1; 1062 spin_unlock_irqrestore(&ctx->dev->lock, flags); 1063 } 1064 1065 static int enable_qp_db(int id, void *p, void *data) 1066 { 1067 struct c4iw_qp *qp = p; 1068 1069 t4_enable_wq_db(&qp->wq); 1070 return 0; 1071 } 1072 1073 static void resume_rc_qp(struct c4iw_qp *qp) 1074 { 1075 spin_lock(&qp->lock); 1076 t4_ring_sq_db(&qp->wq, qp->wq.sq.wq_pidx_inc, 1077 is_t5(qp->rhp->rdev.lldi.adapter_type), NULL); 1078 qp->wq.sq.wq_pidx_inc = 0; 1079 t4_ring_rq_db(&qp->wq, qp->wq.rq.wq_pidx_inc, 1080 is_t5(qp->rhp->rdev.lldi.adapter_type), NULL); 1081 qp->wq.rq.wq_pidx_inc = 0; 1082 spin_unlock(&qp->lock); 1083 } 1084 1085 static void resume_a_chunk(struct uld_ctx *ctx) 1086 { 1087 int i; 1088 struct c4iw_qp *qp; 1089 1090 for (i = 0; i < DB_FC_RESUME_SIZE; i++) { 1091 qp = list_first_entry(&ctx->dev->db_fc_list, struct c4iw_qp, 1092 db_fc_entry); 1093 list_del_init(&qp->db_fc_entry); 1094 resume_rc_qp(qp); 1095 if (list_empty(&ctx->dev->db_fc_list)) 1096 break; 1097 } 1098 } 1099 1100 static void resume_queues(struct uld_ctx *ctx) 1101 { 1102 spin_lock_irq(&ctx->dev->lock); 1103 if (ctx->dev->db_state != STOPPED) 1104 goto out; 1105 ctx->dev->db_state = FLOW_CONTROL; 1106 while (1) { 1107 if (list_empty(&ctx->dev->db_fc_list)) { 1108 WARN_ON(ctx->dev->db_state != FLOW_CONTROL); 1109 ctx->dev->db_state = NORMAL; 1110 ctx->dev->rdev.stats.db_state_transitions++; 1111 if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED) { 1112 idr_for_each(&ctx->dev->qpidr, enable_qp_db, 1113 NULL); 1114 } else { 1115 ctx->dev->rdev.status_page->db_off = 0; 1116 } 1117 break; 1118 } else { 1119 if (cxgb4_dbfifo_count(ctx->dev->rdev.lldi.ports[0], 1) 1120 < (ctx->dev->rdev.lldi.dbfifo_int_thresh << 1121 DB_FC_DRAIN_THRESH)) { 1122 resume_a_chunk(ctx); 1123 } 1124 if (!list_empty(&ctx->dev->db_fc_list)) { 1125 spin_unlock_irq(&ctx->dev->lock); 1126 if (DB_FC_RESUME_DELAY) { 1127 set_current_state(TASK_UNINTERRUPTIBLE); 1128 schedule_timeout(DB_FC_RESUME_DELAY); 1129 } 1130 spin_lock_irq(&ctx->dev->lock); 1131 if (ctx->dev->db_state != FLOW_CONTROL) 1132 break; 1133 } 1134 } 1135 } 1136 out: 1137 if (ctx->dev->db_state != NORMAL) 1138 ctx->dev->rdev.stats.db_fc_interruptions++; 1139 spin_unlock_irq(&ctx->dev->lock); 1140 } 1141 1142 struct qp_list { 1143 unsigned idx; 1144 struct c4iw_qp **qps; 1145 }; 1146 1147 static int add_and_ref_qp(int id, void *p, void *data) 1148 { 1149 struct qp_list *qp_listp = data; 1150 struct c4iw_qp *qp = p; 1151 1152 c4iw_qp_add_ref(&qp->ibqp); 1153 qp_listp->qps[qp_listp->idx++] = qp; 1154 return 0; 1155 } 1156 1157 static int count_qps(int id, void *p, void *data) 1158 { 1159 unsigned *countp = data; 1160 (*countp)++; 1161 return 0; 1162 } 1163 1164 static void deref_qps(struct qp_list *qp_list) 1165 { 1166 int idx; 1167 1168 for (idx = 0; idx < qp_list->idx; idx++) 1169 c4iw_qp_rem_ref(&qp_list->qps[idx]->ibqp); 1170 } 1171 1172 static void recover_lost_dbs(struct uld_ctx *ctx, struct qp_list *qp_list) 1173 { 1174 int idx; 1175 int ret; 1176 1177 for (idx = 0; idx < qp_list->idx; idx++) { 1178 struct c4iw_qp *qp = qp_list->qps[idx]; 1179 1180 spin_lock_irq(&qp->rhp->lock); 1181 spin_lock(&qp->lock); 1182 ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0], 1183 qp->wq.sq.qid, 1184 t4_sq_host_wq_pidx(&qp->wq), 1185 t4_sq_wq_size(&qp->wq)); 1186 if (ret) { 1187 pr_err(KERN_ERR MOD "%s: Fatal error - " 1188 "DB overflow recovery failed - " 1189 "error syncing SQ qid %u\n", 1190 pci_name(ctx->lldi.pdev), qp->wq.sq.qid); 1191 spin_unlock(&qp->lock); 1192 spin_unlock_irq(&qp->rhp->lock); 1193 return; 1194 } 1195 qp->wq.sq.wq_pidx_inc = 0; 1196 1197 ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0], 1198 qp->wq.rq.qid, 1199 t4_rq_host_wq_pidx(&qp->wq), 1200 t4_rq_wq_size(&qp->wq)); 1201 1202 if (ret) { 1203 pr_err(KERN_ERR MOD "%s: Fatal error - " 1204 "DB overflow recovery failed - " 1205 "error syncing RQ qid %u\n", 1206 pci_name(ctx->lldi.pdev), qp->wq.rq.qid); 1207 spin_unlock(&qp->lock); 1208 spin_unlock_irq(&qp->rhp->lock); 1209 return; 1210 } 1211 qp->wq.rq.wq_pidx_inc = 0; 1212 spin_unlock(&qp->lock); 1213 spin_unlock_irq(&qp->rhp->lock); 1214 1215 /* Wait for the dbfifo to drain */ 1216 while (cxgb4_dbfifo_count(qp->rhp->rdev.lldi.ports[0], 1) > 0) { 1217 set_current_state(TASK_UNINTERRUPTIBLE); 1218 schedule_timeout(usecs_to_jiffies(10)); 1219 } 1220 } 1221 } 1222 1223 static void recover_queues(struct uld_ctx *ctx) 1224 { 1225 int count = 0; 1226 struct qp_list qp_list; 1227 int ret; 1228 1229 /* slow everybody down */ 1230 set_current_state(TASK_UNINTERRUPTIBLE); 1231 schedule_timeout(usecs_to_jiffies(1000)); 1232 1233 /* flush the SGE contexts */ 1234 ret = cxgb4_flush_eq_cache(ctx->dev->rdev.lldi.ports[0]); 1235 if (ret) { 1236 printk(KERN_ERR MOD "%s: Fatal error - DB overflow recovery failed\n", 1237 pci_name(ctx->lldi.pdev)); 1238 return; 1239 } 1240 1241 /* Count active queues so we can build a list of queues to recover */ 1242 spin_lock_irq(&ctx->dev->lock); 1243 WARN_ON(ctx->dev->db_state != STOPPED); 1244 ctx->dev->db_state = RECOVERY; 1245 idr_for_each(&ctx->dev->qpidr, count_qps, &count); 1246 1247 qp_list.qps = kzalloc(count * sizeof *qp_list.qps, GFP_ATOMIC); 1248 if (!qp_list.qps) { 1249 printk(KERN_ERR MOD "%s: Fatal error - DB overflow recovery failed\n", 1250 pci_name(ctx->lldi.pdev)); 1251 spin_unlock_irq(&ctx->dev->lock); 1252 return; 1253 } 1254 qp_list.idx = 0; 1255 1256 /* add and ref each qp so it doesn't get freed */ 1257 idr_for_each(&ctx->dev->qpidr, add_and_ref_qp, &qp_list); 1258 1259 spin_unlock_irq(&ctx->dev->lock); 1260 1261 /* now traverse the list in a safe context to recover the db state*/ 1262 recover_lost_dbs(ctx, &qp_list); 1263 1264 /* we're almost done! deref the qps and clean up */ 1265 deref_qps(&qp_list); 1266 kfree(qp_list.qps); 1267 1268 spin_lock_irq(&ctx->dev->lock); 1269 WARN_ON(ctx->dev->db_state != RECOVERY); 1270 ctx->dev->db_state = STOPPED; 1271 spin_unlock_irq(&ctx->dev->lock); 1272 } 1273 1274 static int c4iw_uld_control(void *handle, enum cxgb4_control control, ...) 1275 { 1276 struct uld_ctx *ctx = handle; 1277 1278 switch (control) { 1279 case CXGB4_CONTROL_DB_FULL: 1280 stop_queues(ctx); 1281 ctx->dev->rdev.stats.db_full++; 1282 break; 1283 case CXGB4_CONTROL_DB_EMPTY: 1284 resume_queues(ctx); 1285 mutex_lock(&ctx->dev->rdev.stats.lock); 1286 ctx->dev->rdev.stats.db_empty++; 1287 mutex_unlock(&ctx->dev->rdev.stats.lock); 1288 break; 1289 case CXGB4_CONTROL_DB_DROP: 1290 recover_queues(ctx); 1291 mutex_lock(&ctx->dev->rdev.stats.lock); 1292 ctx->dev->rdev.stats.db_drop++; 1293 mutex_unlock(&ctx->dev->rdev.stats.lock); 1294 break; 1295 default: 1296 printk(KERN_WARNING MOD "%s: unknown control cmd %u\n", 1297 pci_name(ctx->lldi.pdev), control); 1298 break; 1299 } 1300 return 0; 1301 } 1302 1303 static struct cxgb4_uld_info c4iw_uld_info = { 1304 .name = DRV_NAME, 1305 .add = c4iw_uld_add, 1306 .rx_handler = c4iw_uld_rx_handler, 1307 .state_change = c4iw_uld_state_change, 1308 .control = c4iw_uld_control, 1309 }; 1310 1311 static int __init c4iw_init_module(void) 1312 { 1313 int err; 1314 1315 err = c4iw_cm_init(); 1316 if (err) 1317 return err; 1318 1319 c4iw_debugfs_root = debugfs_create_dir(DRV_NAME, NULL); 1320 if (!c4iw_debugfs_root) 1321 printk(KERN_WARNING MOD 1322 "could not create debugfs entry, continuing\n"); 1323 1324 if (ibnl_add_client(RDMA_NL_C4IW, RDMA_NL_IWPM_NUM_OPS, 1325 c4iw_nl_cb_table)) 1326 pr_err("%s[%u]: Failed to add netlink callback\n" 1327 , __func__, __LINE__); 1328 1329 err = iwpm_init(RDMA_NL_C4IW); 1330 if (err) { 1331 pr_err("port mapper initialization failed with %d\n", err); 1332 ibnl_remove_client(RDMA_NL_C4IW); 1333 c4iw_cm_term(); 1334 debugfs_remove_recursive(c4iw_debugfs_root); 1335 return err; 1336 } 1337 1338 cxgb4_register_uld(CXGB4_ULD_RDMA, &c4iw_uld_info); 1339 1340 return 0; 1341 } 1342 1343 static void __exit c4iw_exit_module(void) 1344 { 1345 struct uld_ctx *ctx, *tmp; 1346 1347 mutex_lock(&dev_mutex); 1348 list_for_each_entry_safe(ctx, tmp, &uld_ctx_list, entry) { 1349 if (ctx->dev) 1350 c4iw_remove(ctx); 1351 kfree(ctx); 1352 } 1353 mutex_unlock(&dev_mutex); 1354 cxgb4_unregister_uld(CXGB4_ULD_RDMA); 1355 iwpm_exit(RDMA_NL_C4IW); 1356 ibnl_remove_client(RDMA_NL_C4IW); 1357 c4iw_cm_term(); 1358 debugfs_remove_recursive(c4iw_debugfs_root); 1359 } 1360 1361 module_init(c4iw_init_module); 1362 module_exit(c4iw_exit_module); 1363