1 /* 2 * Copyright (c) 2012, 2013 Intel Corporation. All rights reserved. 3 * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved. 4 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include <rdma/ib_mad.h> 36 #include <rdma/ib_user_verbs.h> 37 #include <linux/io.h> 38 #include <linux/module.h> 39 #include <linux/utsname.h> 40 #include <linux/rculist.h> 41 #include <linux/mm.h> 42 #include <linux/random.h> 43 #include <linux/vmalloc.h> 44 45 #include "qib.h" 46 #include "qib_common.h" 47 48 static unsigned int ib_qib_qp_table_size = 256; 49 module_param_named(qp_table_size, ib_qib_qp_table_size, uint, S_IRUGO); 50 MODULE_PARM_DESC(qp_table_size, "QP table size"); 51 52 unsigned int ib_qib_lkey_table_size = 16; 53 module_param_named(lkey_table_size, ib_qib_lkey_table_size, uint, 54 S_IRUGO); 55 MODULE_PARM_DESC(lkey_table_size, 56 "LKEY table size in bits (2^n, 1 <= n <= 23)"); 57 58 static unsigned int ib_qib_max_pds = 0xFFFF; 59 module_param_named(max_pds, ib_qib_max_pds, uint, S_IRUGO); 60 MODULE_PARM_DESC(max_pds, 61 "Maximum number of protection domains to support"); 62 63 static unsigned int ib_qib_max_ahs = 0xFFFF; 64 module_param_named(max_ahs, ib_qib_max_ahs, uint, S_IRUGO); 65 MODULE_PARM_DESC(max_ahs, "Maximum number of address handles to support"); 66 67 unsigned int ib_qib_max_cqes = 0x2FFFF; 68 module_param_named(max_cqes, ib_qib_max_cqes, uint, S_IRUGO); 69 MODULE_PARM_DESC(max_cqes, 70 "Maximum number of completion queue entries to support"); 71 72 unsigned int ib_qib_max_cqs = 0x1FFFF; 73 module_param_named(max_cqs, ib_qib_max_cqs, uint, S_IRUGO); 74 MODULE_PARM_DESC(max_cqs, "Maximum number of completion queues to support"); 75 76 unsigned int ib_qib_max_qp_wrs = 0x3FFF; 77 module_param_named(max_qp_wrs, ib_qib_max_qp_wrs, uint, S_IRUGO); 78 MODULE_PARM_DESC(max_qp_wrs, "Maximum number of QP WRs to support"); 79 80 unsigned int ib_qib_max_qps = 16384; 81 module_param_named(max_qps, ib_qib_max_qps, uint, S_IRUGO); 82 MODULE_PARM_DESC(max_qps, "Maximum number of QPs to support"); 83 84 unsigned int ib_qib_max_sges = 0x60; 85 module_param_named(max_sges, ib_qib_max_sges, uint, S_IRUGO); 86 MODULE_PARM_DESC(max_sges, "Maximum number of SGEs to support"); 87 88 unsigned int ib_qib_max_mcast_grps = 16384; 89 module_param_named(max_mcast_grps, ib_qib_max_mcast_grps, uint, S_IRUGO); 90 MODULE_PARM_DESC(max_mcast_grps, 91 "Maximum number of multicast groups to support"); 92 93 unsigned int ib_qib_max_mcast_qp_attached = 16; 94 module_param_named(max_mcast_qp_attached, ib_qib_max_mcast_qp_attached, 95 uint, S_IRUGO); 96 MODULE_PARM_DESC(max_mcast_qp_attached, 97 "Maximum number of attached QPs to support"); 98 99 unsigned int ib_qib_max_srqs = 1024; 100 module_param_named(max_srqs, ib_qib_max_srqs, uint, S_IRUGO); 101 MODULE_PARM_DESC(max_srqs, "Maximum number of SRQs to support"); 102 103 unsigned int ib_qib_max_srq_sges = 128; 104 module_param_named(max_srq_sges, ib_qib_max_srq_sges, uint, S_IRUGO); 105 MODULE_PARM_DESC(max_srq_sges, "Maximum number of SRQ SGEs to support"); 106 107 unsigned int ib_qib_max_srq_wrs = 0x1FFFF; 108 module_param_named(max_srq_wrs, ib_qib_max_srq_wrs, uint, S_IRUGO); 109 MODULE_PARM_DESC(max_srq_wrs, "Maximum number of SRQ WRs support"); 110 111 static unsigned int ib_qib_disable_sma; 112 module_param_named(disable_sma, ib_qib_disable_sma, uint, S_IWUSR | S_IRUGO); 113 MODULE_PARM_DESC(disable_sma, "Disable the SMA"); 114 115 /* 116 * Note that it is OK to post send work requests in the SQE and ERR 117 * states; qib_do_send() will process them and generate error 118 * completions as per IB 1.2 C10-96. 119 */ 120 const int ib_qib_state_ops[IB_QPS_ERR + 1] = { 121 [IB_QPS_RESET] = 0, 122 [IB_QPS_INIT] = QIB_POST_RECV_OK, 123 [IB_QPS_RTR] = QIB_POST_RECV_OK | QIB_PROCESS_RECV_OK, 124 [IB_QPS_RTS] = QIB_POST_RECV_OK | QIB_PROCESS_RECV_OK | 125 QIB_POST_SEND_OK | QIB_PROCESS_SEND_OK | 126 QIB_PROCESS_NEXT_SEND_OK, 127 [IB_QPS_SQD] = QIB_POST_RECV_OK | QIB_PROCESS_RECV_OK | 128 QIB_POST_SEND_OK | QIB_PROCESS_SEND_OK, 129 [IB_QPS_SQE] = QIB_POST_RECV_OK | QIB_PROCESS_RECV_OK | 130 QIB_POST_SEND_OK | QIB_FLUSH_SEND, 131 [IB_QPS_ERR] = QIB_POST_RECV_OK | QIB_FLUSH_RECV | 132 QIB_POST_SEND_OK | QIB_FLUSH_SEND, 133 }; 134 135 struct qib_ucontext { 136 struct ib_ucontext ibucontext; 137 }; 138 139 static inline struct qib_ucontext *to_iucontext(struct ib_ucontext 140 *ibucontext) 141 { 142 return container_of(ibucontext, struct qib_ucontext, ibucontext); 143 } 144 145 /* 146 * Translate ib_wr_opcode into ib_wc_opcode. 147 */ 148 const enum ib_wc_opcode ib_qib_wc_opcode[] = { 149 [IB_WR_RDMA_WRITE] = IB_WC_RDMA_WRITE, 150 [IB_WR_RDMA_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE, 151 [IB_WR_SEND] = IB_WC_SEND, 152 [IB_WR_SEND_WITH_IMM] = IB_WC_SEND, 153 [IB_WR_RDMA_READ] = IB_WC_RDMA_READ, 154 [IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP, 155 [IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD 156 }; 157 158 /* 159 * System image GUID. 160 */ 161 __be64 ib_qib_sys_image_guid; 162 163 /** 164 * qib_copy_sge - copy data to SGE memory 165 * @ss: the SGE state 166 * @data: the data to copy 167 * @length: the length of the data 168 */ 169 void qib_copy_sge(struct qib_sge_state *ss, void *data, u32 length, int release) 170 { 171 struct qib_sge *sge = &ss->sge; 172 173 while (length) { 174 u32 len = sge->length; 175 176 if (len > length) 177 len = length; 178 if (len > sge->sge_length) 179 len = sge->sge_length; 180 BUG_ON(len == 0); 181 memcpy(sge->vaddr, data, len); 182 sge->vaddr += len; 183 sge->length -= len; 184 sge->sge_length -= len; 185 if (sge->sge_length == 0) { 186 if (release) 187 qib_put_mr(sge->mr); 188 if (--ss->num_sge) 189 *sge = *ss->sg_list++; 190 } else if (sge->length == 0 && sge->mr->lkey) { 191 if (++sge->n >= QIB_SEGSZ) { 192 if (++sge->m >= sge->mr->mapsz) 193 break; 194 sge->n = 0; 195 } 196 sge->vaddr = 197 sge->mr->map[sge->m]->segs[sge->n].vaddr; 198 sge->length = 199 sge->mr->map[sge->m]->segs[sge->n].length; 200 } 201 data += len; 202 length -= len; 203 } 204 } 205 206 /** 207 * qib_skip_sge - skip over SGE memory - XXX almost dup of prev func 208 * @ss: the SGE state 209 * @length: the number of bytes to skip 210 */ 211 void qib_skip_sge(struct qib_sge_state *ss, u32 length, int release) 212 { 213 struct qib_sge *sge = &ss->sge; 214 215 while (length) { 216 u32 len = sge->length; 217 218 if (len > length) 219 len = length; 220 if (len > sge->sge_length) 221 len = sge->sge_length; 222 BUG_ON(len == 0); 223 sge->vaddr += len; 224 sge->length -= len; 225 sge->sge_length -= len; 226 if (sge->sge_length == 0) { 227 if (release) 228 qib_put_mr(sge->mr); 229 if (--ss->num_sge) 230 *sge = *ss->sg_list++; 231 } else if (sge->length == 0 && sge->mr->lkey) { 232 if (++sge->n >= QIB_SEGSZ) { 233 if (++sge->m >= sge->mr->mapsz) 234 break; 235 sge->n = 0; 236 } 237 sge->vaddr = 238 sge->mr->map[sge->m]->segs[sge->n].vaddr; 239 sge->length = 240 sge->mr->map[sge->m]->segs[sge->n].length; 241 } 242 length -= len; 243 } 244 } 245 246 /* 247 * Count the number of DMA descriptors needed to send length bytes of data. 248 * Don't modify the qib_sge_state to get the count. 249 * Return zero if any of the segments is not aligned. 250 */ 251 static u32 qib_count_sge(struct qib_sge_state *ss, u32 length) 252 { 253 struct qib_sge *sg_list = ss->sg_list; 254 struct qib_sge sge = ss->sge; 255 u8 num_sge = ss->num_sge; 256 u32 ndesc = 1; /* count the header */ 257 258 while (length) { 259 u32 len = sge.length; 260 261 if (len > length) 262 len = length; 263 if (len > sge.sge_length) 264 len = sge.sge_length; 265 BUG_ON(len == 0); 266 if (((long) sge.vaddr & (sizeof(u32) - 1)) || 267 (len != length && (len & (sizeof(u32) - 1)))) { 268 ndesc = 0; 269 break; 270 } 271 ndesc++; 272 sge.vaddr += len; 273 sge.length -= len; 274 sge.sge_length -= len; 275 if (sge.sge_length == 0) { 276 if (--num_sge) 277 sge = *sg_list++; 278 } else if (sge.length == 0 && sge.mr->lkey) { 279 if (++sge.n >= QIB_SEGSZ) { 280 if (++sge.m >= sge.mr->mapsz) 281 break; 282 sge.n = 0; 283 } 284 sge.vaddr = 285 sge.mr->map[sge.m]->segs[sge.n].vaddr; 286 sge.length = 287 sge.mr->map[sge.m]->segs[sge.n].length; 288 } 289 length -= len; 290 } 291 return ndesc; 292 } 293 294 /* 295 * Copy from the SGEs to the data buffer. 296 */ 297 static void qib_copy_from_sge(void *data, struct qib_sge_state *ss, u32 length) 298 { 299 struct qib_sge *sge = &ss->sge; 300 301 while (length) { 302 u32 len = sge->length; 303 304 if (len > length) 305 len = length; 306 if (len > sge->sge_length) 307 len = sge->sge_length; 308 BUG_ON(len == 0); 309 memcpy(data, sge->vaddr, len); 310 sge->vaddr += len; 311 sge->length -= len; 312 sge->sge_length -= len; 313 if (sge->sge_length == 0) { 314 if (--ss->num_sge) 315 *sge = *ss->sg_list++; 316 } else if (sge->length == 0 && sge->mr->lkey) { 317 if (++sge->n >= QIB_SEGSZ) { 318 if (++sge->m >= sge->mr->mapsz) 319 break; 320 sge->n = 0; 321 } 322 sge->vaddr = 323 sge->mr->map[sge->m]->segs[sge->n].vaddr; 324 sge->length = 325 sge->mr->map[sge->m]->segs[sge->n].length; 326 } 327 data += len; 328 length -= len; 329 } 330 } 331 332 /** 333 * qib_post_one_send - post one RC, UC, or UD send work request 334 * @qp: the QP to post on 335 * @wr: the work request to send 336 */ 337 static int qib_post_one_send(struct qib_qp *qp, struct ib_send_wr *wr, 338 int *scheduled) 339 { 340 struct qib_swqe *wqe; 341 u32 next; 342 int i; 343 int j; 344 int acc; 345 int ret; 346 unsigned long flags; 347 struct qib_lkey_table *rkt; 348 struct qib_pd *pd; 349 int avoid_schedule = 0; 350 351 spin_lock_irqsave(&qp->s_lock, flags); 352 353 /* Check that state is OK to post send. */ 354 if (unlikely(!(ib_qib_state_ops[qp->state] & QIB_POST_SEND_OK))) 355 goto bail_inval; 356 357 /* IB spec says that num_sge == 0 is OK. */ 358 if (wr->num_sge > qp->s_max_sge) 359 goto bail_inval; 360 361 /* 362 * Don't allow RDMA reads or atomic operations on UC or 363 * undefined operations. 364 * Make sure buffer is large enough to hold the result for atomics. 365 */ 366 if (wr->opcode == IB_WR_REG_MR) { 367 if (qib_reg_mr(qp, reg_wr(wr))) 368 goto bail_inval; 369 } else if (qp->ibqp.qp_type == IB_QPT_UC) { 370 if ((unsigned) wr->opcode >= IB_WR_RDMA_READ) 371 goto bail_inval; 372 } else if (qp->ibqp.qp_type != IB_QPT_RC) { 373 /* Check IB_QPT_SMI, IB_QPT_GSI, IB_QPT_UD opcode */ 374 if (wr->opcode != IB_WR_SEND && 375 wr->opcode != IB_WR_SEND_WITH_IMM) 376 goto bail_inval; 377 /* Check UD destination address PD */ 378 if (qp->ibqp.pd != ud_wr(wr)->ah->pd) 379 goto bail_inval; 380 } else if ((unsigned) wr->opcode > IB_WR_ATOMIC_FETCH_AND_ADD) 381 goto bail_inval; 382 else if (wr->opcode >= IB_WR_ATOMIC_CMP_AND_SWP && 383 (wr->num_sge == 0 || 384 wr->sg_list[0].length < sizeof(u64) || 385 wr->sg_list[0].addr & (sizeof(u64) - 1))) 386 goto bail_inval; 387 else if (wr->opcode >= IB_WR_RDMA_READ && !qp->s_max_rd_atomic) 388 goto bail_inval; 389 390 next = qp->s_head + 1; 391 if (next >= qp->s_size) 392 next = 0; 393 if (next == qp->s_last) { 394 ret = -ENOMEM; 395 goto bail; 396 } 397 398 rkt = &to_idev(qp->ibqp.device)->lk_table; 399 pd = to_ipd(qp->ibqp.pd); 400 wqe = get_swqe_ptr(qp, qp->s_head); 401 402 if (qp->ibqp.qp_type != IB_QPT_UC && 403 qp->ibqp.qp_type != IB_QPT_RC) 404 memcpy(&wqe->ud_wr, ud_wr(wr), sizeof(wqe->ud_wr)); 405 else if (wr->opcode == IB_WR_REG_MR) 406 memcpy(&wqe->reg_wr, reg_wr(wr), 407 sizeof(wqe->reg_wr)); 408 else if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 409 wr->opcode == IB_WR_RDMA_WRITE || 410 wr->opcode == IB_WR_RDMA_READ) 411 memcpy(&wqe->rdma_wr, rdma_wr(wr), sizeof(wqe->rdma_wr)); 412 else if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || 413 wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) 414 memcpy(&wqe->atomic_wr, atomic_wr(wr), sizeof(wqe->atomic_wr)); 415 else 416 memcpy(&wqe->wr, wr, sizeof(wqe->wr)); 417 418 wqe->length = 0; 419 j = 0; 420 if (wr->num_sge) { 421 acc = wr->opcode >= IB_WR_RDMA_READ ? 422 IB_ACCESS_LOCAL_WRITE : 0; 423 for (i = 0; i < wr->num_sge; i++) { 424 u32 length = wr->sg_list[i].length; 425 int ok; 426 427 if (length == 0) 428 continue; 429 ok = qib_lkey_ok(rkt, pd, &wqe->sg_list[j], 430 &wr->sg_list[i], acc); 431 if (!ok) 432 goto bail_inval_free; 433 wqe->length += length; 434 j++; 435 } 436 wqe->wr.num_sge = j; 437 } 438 if (qp->ibqp.qp_type == IB_QPT_UC || 439 qp->ibqp.qp_type == IB_QPT_RC) { 440 if (wqe->length > 0x80000000U) 441 goto bail_inval_free; 442 if (wqe->length <= qp->pmtu) 443 avoid_schedule = 1; 444 } else if (wqe->length > (dd_from_ibdev(qp->ibqp.device)->pport + 445 qp->port_num - 1)->ibmtu) { 446 goto bail_inval_free; 447 } else { 448 atomic_inc(&to_iah(ud_wr(wr)->ah)->refcount); 449 avoid_schedule = 1; 450 } 451 wqe->ssn = qp->s_ssn++; 452 qp->s_head = next; 453 454 ret = 0; 455 goto bail; 456 457 bail_inval_free: 458 while (j) { 459 struct qib_sge *sge = &wqe->sg_list[--j]; 460 461 qib_put_mr(sge->mr); 462 } 463 bail_inval: 464 ret = -EINVAL; 465 bail: 466 if (!ret && !wr->next && !avoid_schedule && 467 !qib_sdma_empty( 468 dd_from_ibdev(qp->ibqp.device)->pport + qp->port_num - 1)) { 469 qib_schedule_send(qp); 470 *scheduled = 1; 471 } 472 spin_unlock_irqrestore(&qp->s_lock, flags); 473 return ret; 474 } 475 476 /** 477 * qib_post_send - post a send on a QP 478 * @ibqp: the QP to post the send on 479 * @wr: the list of work requests to post 480 * @bad_wr: the first bad WR is put here 481 * 482 * This may be called from interrupt context. 483 */ 484 static int qib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 485 struct ib_send_wr **bad_wr) 486 { 487 struct qib_qp *qp = to_iqp(ibqp); 488 int err = 0; 489 int scheduled = 0; 490 491 for (; wr; wr = wr->next) { 492 err = qib_post_one_send(qp, wr, &scheduled); 493 if (err) { 494 *bad_wr = wr; 495 goto bail; 496 } 497 } 498 499 /* Try to do the send work in the caller's context. */ 500 if (!scheduled) 501 qib_do_send(&qp->s_work); 502 503 bail: 504 return err; 505 } 506 507 /** 508 * qib_post_receive - post a receive on a QP 509 * @ibqp: the QP to post the receive on 510 * @wr: the WR to post 511 * @bad_wr: the first bad WR is put here 512 * 513 * This may be called from interrupt context. 514 */ 515 static int qib_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, 516 struct ib_recv_wr **bad_wr) 517 { 518 struct qib_qp *qp = to_iqp(ibqp); 519 struct qib_rwq *wq = qp->r_rq.wq; 520 unsigned long flags; 521 int ret; 522 523 /* Check that state is OK to post receive. */ 524 if (!(ib_qib_state_ops[qp->state] & QIB_POST_RECV_OK) || !wq) { 525 *bad_wr = wr; 526 ret = -EINVAL; 527 goto bail; 528 } 529 530 for (; wr; wr = wr->next) { 531 struct qib_rwqe *wqe; 532 u32 next; 533 int i; 534 535 if ((unsigned) wr->num_sge > qp->r_rq.max_sge) { 536 *bad_wr = wr; 537 ret = -EINVAL; 538 goto bail; 539 } 540 541 spin_lock_irqsave(&qp->r_rq.lock, flags); 542 next = wq->head + 1; 543 if (next >= qp->r_rq.size) 544 next = 0; 545 if (next == wq->tail) { 546 spin_unlock_irqrestore(&qp->r_rq.lock, flags); 547 *bad_wr = wr; 548 ret = -ENOMEM; 549 goto bail; 550 } 551 552 wqe = get_rwqe_ptr(&qp->r_rq, wq->head); 553 wqe->wr_id = wr->wr_id; 554 wqe->num_sge = wr->num_sge; 555 for (i = 0; i < wr->num_sge; i++) 556 wqe->sg_list[i] = wr->sg_list[i]; 557 /* Make sure queue entry is written before the head index. */ 558 smp_wmb(); 559 wq->head = next; 560 spin_unlock_irqrestore(&qp->r_rq.lock, flags); 561 } 562 ret = 0; 563 564 bail: 565 return ret; 566 } 567 568 /** 569 * qib_qp_rcv - processing an incoming packet on a QP 570 * @rcd: the context pointer 571 * @hdr: the packet header 572 * @has_grh: true if the packet has a GRH 573 * @data: the packet data 574 * @tlen: the packet length 575 * @qp: the QP the packet came on 576 * 577 * This is called from qib_ib_rcv() to process an incoming packet 578 * for the given QP. 579 * Called at interrupt level. 580 */ 581 static void qib_qp_rcv(struct qib_ctxtdata *rcd, struct qib_ib_header *hdr, 582 int has_grh, void *data, u32 tlen, struct qib_qp *qp) 583 { 584 struct qib_ibport *ibp = &rcd->ppd->ibport_data; 585 586 spin_lock(&qp->r_lock); 587 588 /* Check for valid receive state. */ 589 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) { 590 ibp->n_pkt_drops++; 591 goto unlock; 592 } 593 594 switch (qp->ibqp.qp_type) { 595 case IB_QPT_SMI: 596 case IB_QPT_GSI: 597 if (ib_qib_disable_sma) 598 break; 599 /* FALLTHROUGH */ 600 case IB_QPT_UD: 601 qib_ud_rcv(ibp, hdr, has_grh, data, tlen, qp); 602 break; 603 604 case IB_QPT_RC: 605 qib_rc_rcv(rcd, hdr, has_grh, data, tlen, qp); 606 break; 607 608 case IB_QPT_UC: 609 qib_uc_rcv(ibp, hdr, has_grh, data, tlen, qp); 610 break; 611 612 default: 613 break; 614 } 615 616 unlock: 617 spin_unlock(&qp->r_lock); 618 } 619 620 /** 621 * qib_ib_rcv - process an incoming packet 622 * @rcd: the context pointer 623 * @rhdr: the header of the packet 624 * @data: the packet payload 625 * @tlen: the packet length 626 * 627 * This is called from qib_kreceive() to process an incoming packet at 628 * interrupt level. Tlen is the length of the header + data + CRC in bytes. 629 */ 630 void qib_ib_rcv(struct qib_ctxtdata *rcd, void *rhdr, void *data, u32 tlen) 631 { 632 struct qib_pportdata *ppd = rcd->ppd; 633 struct qib_ibport *ibp = &ppd->ibport_data; 634 struct qib_ib_header *hdr = rhdr; 635 struct qib_other_headers *ohdr; 636 struct qib_qp *qp; 637 u32 qp_num; 638 int lnh; 639 u8 opcode; 640 u16 lid; 641 642 /* 24 == LRH+BTH+CRC */ 643 if (unlikely(tlen < 24)) 644 goto drop; 645 646 /* Check for a valid destination LID (see ch. 7.11.1). */ 647 lid = be16_to_cpu(hdr->lrh[1]); 648 if (lid < QIB_MULTICAST_LID_BASE) { 649 lid &= ~((1 << ppd->lmc) - 1); 650 if (unlikely(lid != ppd->lid)) 651 goto drop; 652 } 653 654 /* Check for GRH */ 655 lnh = be16_to_cpu(hdr->lrh[0]) & 3; 656 if (lnh == QIB_LRH_BTH) 657 ohdr = &hdr->u.oth; 658 else if (lnh == QIB_LRH_GRH) { 659 u32 vtf; 660 661 ohdr = &hdr->u.l.oth; 662 if (hdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR) 663 goto drop; 664 vtf = be32_to_cpu(hdr->u.l.grh.version_tclass_flow); 665 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) 666 goto drop; 667 } else 668 goto drop; 669 670 opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0x7f; 671 #ifdef CONFIG_DEBUG_FS 672 rcd->opstats->stats[opcode].n_bytes += tlen; 673 rcd->opstats->stats[opcode].n_packets++; 674 #endif 675 676 /* Get the destination QP number. */ 677 qp_num = be32_to_cpu(ohdr->bth[1]) & QIB_QPN_MASK; 678 if (qp_num == QIB_MULTICAST_QPN) { 679 struct qib_mcast *mcast; 680 struct qib_mcast_qp *p; 681 682 if (lnh != QIB_LRH_GRH) 683 goto drop; 684 mcast = qib_mcast_find(ibp, &hdr->u.l.grh.dgid); 685 if (mcast == NULL) 686 goto drop; 687 this_cpu_inc(ibp->pmastats->n_multicast_rcv); 688 list_for_each_entry_rcu(p, &mcast->qp_list, list) 689 qib_qp_rcv(rcd, hdr, 1, data, tlen, p->qp); 690 /* 691 * Notify qib_multicast_detach() if it is waiting for us 692 * to finish. 693 */ 694 if (atomic_dec_return(&mcast->refcount) <= 1) 695 wake_up(&mcast->wait); 696 } else { 697 if (rcd->lookaside_qp) { 698 if (rcd->lookaside_qpn != qp_num) { 699 if (atomic_dec_and_test( 700 &rcd->lookaside_qp->refcount)) 701 wake_up( 702 &rcd->lookaside_qp->wait); 703 rcd->lookaside_qp = NULL; 704 } 705 } 706 if (!rcd->lookaside_qp) { 707 qp = qib_lookup_qpn(ibp, qp_num); 708 if (!qp) 709 goto drop; 710 rcd->lookaside_qp = qp; 711 rcd->lookaside_qpn = qp_num; 712 } else 713 qp = rcd->lookaside_qp; 714 this_cpu_inc(ibp->pmastats->n_unicast_rcv); 715 qib_qp_rcv(rcd, hdr, lnh == QIB_LRH_GRH, data, tlen, qp); 716 } 717 return; 718 719 drop: 720 ibp->n_pkt_drops++; 721 } 722 723 /* 724 * This is called from a timer to check for QPs 725 * which need kernel memory in order to send a packet. 726 */ 727 static void mem_timer(unsigned long data) 728 { 729 struct qib_ibdev *dev = (struct qib_ibdev *) data; 730 struct list_head *list = &dev->memwait; 731 struct qib_qp *qp = NULL; 732 unsigned long flags; 733 734 spin_lock_irqsave(&dev->pending_lock, flags); 735 if (!list_empty(list)) { 736 qp = list_entry(list->next, struct qib_qp, iowait); 737 list_del_init(&qp->iowait); 738 atomic_inc(&qp->refcount); 739 if (!list_empty(list)) 740 mod_timer(&dev->mem_timer, jiffies + 1); 741 } 742 spin_unlock_irqrestore(&dev->pending_lock, flags); 743 744 if (qp) { 745 spin_lock_irqsave(&qp->s_lock, flags); 746 if (qp->s_flags & QIB_S_WAIT_KMEM) { 747 qp->s_flags &= ~QIB_S_WAIT_KMEM; 748 qib_schedule_send(qp); 749 } 750 spin_unlock_irqrestore(&qp->s_lock, flags); 751 if (atomic_dec_and_test(&qp->refcount)) 752 wake_up(&qp->wait); 753 } 754 } 755 756 static void update_sge(struct qib_sge_state *ss, u32 length) 757 { 758 struct qib_sge *sge = &ss->sge; 759 760 sge->vaddr += length; 761 sge->length -= length; 762 sge->sge_length -= length; 763 if (sge->sge_length == 0) { 764 if (--ss->num_sge) 765 *sge = *ss->sg_list++; 766 } else if (sge->length == 0 && sge->mr->lkey) { 767 if (++sge->n >= QIB_SEGSZ) { 768 if (++sge->m >= sge->mr->mapsz) 769 return; 770 sge->n = 0; 771 } 772 sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr; 773 sge->length = sge->mr->map[sge->m]->segs[sge->n].length; 774 } 775 } 776 777 #ifdef __LITTLE_ENDIAN 778 static inline u32 get_upper_bits(u32 data, u32 shift) 779 { 780 return data >> shift; 781 } 782 783 static inline u32 set_upper_bits(u32 data, u32 shift) 784 { 785 return data << shift; 786 } 787 788 static inline u32 clear_upper_bytes(u32 data, u32 n, u32 off) 789 { 790 data <<= ((sizeof(u32) - n) * BITS_PER_BYTE); 791 data >>= ((sizeof(u32) - n - off) * BITS_PER_BYTE); 792 return data; 793 } 794 #else 795 static inline u32 get_upper_bits(u32 data, u32 shift) 796 { 797 return data << shift; 798 } 799 800 static inline u32 set_upper_bits(u32 data, u32 shift) 801 { 802 return data >> shift; 803 } 804 805 static inline u32 clear_upper_bytes(u32 data, u32 n, u32 off) 806 { 807 data >>= ((sizeof(u32) - n) * BITS_PER_BYTE); 808 data <<= ((sizeof(u32) - n - off) * BITS_PER_BYTE); 809 return data; 810 } 811 #endif 812 813 static void copy_io(u32 __iomem *piobuf, struct qib_sge_state *ss, 814 u32 length, unsigned flush_wc) 815 { 816 u32 extra = 0; 817 u32 data = 0; 818 u32 last; 819 820 while (1) { 821 u32 len = ss->sge.length; 822 u32 off; 823 824 if (len > length) 825 len = length; 826 if (len > ss->sge.sge_length) 827 len = ss->sge.sge_length; 828 BUG_ON(len == 0); 829 /* If the source address is not aligned, try to align it. */ 830 off = (unsigned long)ss->sge.vaddr & (sizeof(u32) - 1); 831 if (off) { 832 u32 *addr = (u32 *)((unsigned long)ss->sge.vaddr & 833 ~(sizeof(u32) - 1)); 834 u32 v = get_upper_bits(*addr, off * BITS_PER_BYTE); 835 u32 y; 836 837 y = sizeof(u32) - off; 838 if (len > y) 839 len = y; 840 if (len + extra >= sizeof(u32)) { 841 data |= set_upper_bits(v, extra * 842 BITS_PER_BYTE); 843 len = sizeof(u32) - extra; 844 if (len == length) { 845 last = data; 846 break; 847 } 848 __raw_writel(data, piobuf); 849 piobuf++; 850 extra = 0; 851 data = 0; 852 } else { 853 /* Clear unused upper bytes */ 854 data |= clear_upper_bytes(v, len, extra); 855 if (len == length) { 856 last = data; 857 break; 858 } 859 extra += len; 860 } 861 } else if (extra) { 862 /* Source address is aligned. */ 863 u32 *addr = (u32 *) ss->sge.vaddr; 864 int shift = extra * BITS_PER_BYTE; 865 int ushift = 32 - shift; 866 u32 l = len; 867 868 while (l >= sizeof(u32)) { 869 u32 v = *addr; 870 871 data |= set_upper_bits(v, shift); 872 __raw_writel(data, piobuf); 873 data = get_upper_bits(v, ushift); 874 piobuf++; 875 addr++; 876 l -= sizeof(u32); 877 } 878 /* 879 * We still have 'extra' number of bytes leftover. 880 */ 881 if (l) { 882 u32 v = *addr; 883 884 if (l + extra >= sizeof(u32)) { 885 data |= set_upper_bits(v, shift); 886 len -= l + extra - sizeof(u32); 887 if (len == length) { 888 last = data; 889 break; 890 } 891 __raw_writel(data, piobuf); 892 piobuf++; 893 extra = 0; 894 data = 0; 895 } else { 896 /* Clear unused upper bytes */ 897 data |= clear_upper_bytes(v, l, extra); 898 if (len == length) { 899 last = data; 900 break; 901 } 902 extra += l; 903 } 904 } else if (len == length) { 905 last = data; 906 break; 907 } 908 } else if (len == length) { 909 u32 w; 910 911 /* 912 * Need to round up for the last dword in the 913 * packet. 914 */ 915 w = (len + 3) >> 2; 916 qib_pio_copy(piobuf, ss->sge.vaddr, w - 1); 917 piobuf += w - 1; 918 last = ((u32 *) ss->sge.vaddr)[w - 1]; 919 break; 920 } else { 921 u32 w = len >> 2; 922 923 qib_pio_copy(piobuf, ss->sge.vaddr, w); 924 piobuf += w; 925 926 extra = len & (sizeof(u32) - 1); 927 if (extra) { 928 u32 v = ((u32 *) ss->sge.vaddr)[w]; 929 930 /* Clear unused upper bytes */ 931 data = clear_upper_bytes(v, extra, 0); 932 } 933 } 934 update_sge(ss, len); 935 length -= len; 936 } 937 /* Update address before sending packet. */ 938 update_sge(ss, length); 939 if (flush_wc) { 940 /* must flush early everything before trigger word */ 941 qib_flush_wc(); 942 __raw_writel(last, piobuf); 943 /* be sure trigger word is written */ 944 qib_flush_wc(); 945 } else 946 __raw_writel(last, piobuf); 947 } 948 949 static noinline struct qib_verbs_txreq *__get_txreq(struct qib_ibdev *dev, 950 struct qib_qp *qp) 951 { 952 struct qib_verbs_txreq *tx; 953 unsigned long flags; 954 955 spin_lock_irqsave(&qp->s_lock, flags); 956 spin_lock(&dev->pending_lock); 957 958 if (!list_empty(&dev->txreq_free)) { 959 struct list_head *l = dev->txreq_free.next; 960 961 list_del(l); 962 spin_unlock(&dev->pending_lock); 963 spin_unlock_irqrestore(&qp->s_lock, flags); 964 tx = list_entry(l, struct qib_verbs_txreq, txreq.list); 965 } else { 966 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK && 967 list_empty(&qp->iowait)) { 968 dev->n_txwait++; 969 qp->s_flags |= QIB_S_WAIT_TX; 970 list_add_tail(&qp->iowait, &dev->txwait); 971 } 972 qp->s_flags &= ~QIB_S_BUSY; 973 spin_unlock(&dev->pending_lock); 974 spin_unlock_irqrestore(&qp->s_lock, flags); 975 tx = ERR_PTR(-EBUSY); 976 } 977 return tx; 978 } 979 980 static inline struct qib_verbs_txreq *get_txreq(struct qib_ibdev *dev, 981 struct qib_qp *qp) 982 { 983 struct qib_verbs_txreq *tx; 984 unsigned long flags; 985 986 spin_lock_irqsave(&dev->pending_lock, flags); 987 /* assume the list non empty */ 988 if (likely(!list_empty(&dev->txreq_free))) { 989 struct list_head *l = dev->txreq_free.next; 990 991 list_del(l); 992 spin_unlock_irqrestore(&dev->pending_lock, flags); 993 tx = list_entry(l, struct qib_verbs_txreq, txreq.list); 994 } else { 995 /* call slow path to get the extra lock */ 996 spin_unlock_irqrestore(&dev->pending_lock, flags); 997 tx = __get_txreq(dev, qp); 998 } 999 return tx; 1000 } 1001 1002 void qib_put_txreq(struct qib_verbs_txreq *tx) 1003 { 1004 struct qib_ibdev *dev; 1005 struct qib_qp *qp; 1006 unsigned long flags; 1007 1008 qp = tx->qp; 1009 dev = to_idev(qp->ibqp.device); 1010 1011 if (atomic_dec_and_test(&qp->refcount)) 1012 wake_up(&qp->wait); 1013 if (tx->mr) { 1014 qib_put_mr(tx->mr); 1015 tx->mr = NULL; 1016 } 1017 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_FREEBUF) { 1018 tx->txreq.flags &= ~QIB_SDMA_TXREQ_F_FREEBUF; 1019 dma_unmap_single(&dd_from_dev(dev)->pcidev->dev, 1020 tx->txreq.addr, tx->hdr_dwords << 2, 1021 DMA_TO_DEVICE); 1022 kfree(tx->align_buf); 1023 } 1024 1025 spin_lock_irqsave(&dev->pending_lock, flags); 1026 1027 /* Put struct back on free list */ 1028 list_add(&tx->txreq.list, &dev->txreq_free); 1029 1030 if (!list_empty(&dev->txwait)) { 1031 /* Wake up first QP wanting a free struct */ 1032 qp = list_entry(dev->txwait.next, struct qib_qp, iowait); 1033 list_del_init(&qp->iowait); 1034 atomic_inc(&qp->refcount); 1035 spin_unlock_irqrestore(&dev->pending_lock, flags); 1036 1037 spin_lock_irqsave(&qp->s_lock, flags); 1038 if (qp->s_flags & QIB_S_WAIT_TX) { 1039 qp->s_flags &= ~QIB_S_WAIT_TX; 1040 qib_schedule_send(qp); 1041 } 1042 spin_unlock_irqrestore(&qp->s_lock, flags); 1043 1044 if (atomic_dec_and_test(&qp->refcount)) 1045 wake_up(&qp->wait); 1046 } else 1047 spin_unlock_irqrestore(&dev->pending_lock, flags); 1048 } 1049 1050 /* 1051 * This is called when there are send DMA descriptors that might be 1052 * available. 1053 * 1054 * This is called with ppd->sdma_lock held. 1055 */ 1056 void qib_verbs_sdma_desc_avail(struct qib_pportdata *ppd, unsigned avail) 1057 { 1058 struct qib_qp *qp, *nqp; 1059 struct qib_qp *qps[20]; 1060 struct qib_ibdev *dev; 1061 unsigned i, n; 1062 1063 n = 0; 1064 dev = &ppd->dd->verbs_dev; 1065 spin_lock(&dev->pending_lock); 1066 1067 /* Search wait list for first QP wanting DMA descriptors. */ 1068 list_for_each_entry_safe(qp, nqp, &dev->dmawait, iowait) { 1069 if (qp->port_num != ppd->port) 1070 continue; 1071 if (n == ARRAY_SIZE(qps)) 1072 break; 1073 if (qp->s_tx->txreq.sg_count > avail) 1074 break; 1075 avail -= qp->s_tx->txreq.sg_count; 1076 list_del_init(&qp->iowait); 1077 atomic_inc(&qp->refcount); 1078 qps[n++] = qp; 1079 } 1080 1081 spin_unlock(&dev->pending_lock); 1082 1083 for (i = 0; i < n; i++) { 1084 qp = qps[i]; 1085 spin_lock(&qp->s_lock); 1086 if (qp->s_flags & QIB_S_WAIT_DMA_DESC) { 1087 qp->s_flags &= ~QIB_S_WAIT_DMA_DESC; 1088 qib_schedule_send(qp); 1089 } 1090 spin_unlock(&qp->s_lock); 1091 if (atomic_dec_and_test(&qp->refcount)) 1092 wake_up(&qp->wait); 1093 } 1094 } 1095 1096 /* 1097 * This is called with ppd->sdma_lock held. 1098 */ 1099 static void sdma_complete(struct qib_sdma_txreq *cookie, int status) 1100 { 1101 struct qib_verbs_txreq *tx = 1102 container_of(cookie, struct qib_verbs_txreq, txreq); 1103 struct qib_qp *qp = tx->qp; 1104 1105 spin_lock(&qp->s_lock); 1106 if (tx->wqe) 1107 qib_send_complete(qp, tx->wqe, IB_WC_SUCCESS); 1108 else if (qp->ibqp.qp_type == IB_QPT_RC) { 1109 struct qib_ib_header *hdr; 1110 1111 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_FREEBUF) 1112 hdr = &tx->align_buf->hdr; 1113 else { 1114 struct qib_ibdev *dev = to_idev(qp->ibqp.device); 1115 1116 hdr = &dev->pio_hdrs[tx->hdr_inx].hdr; 1117 } 1118 qib_rc_send_complete(qp, hdr); 1119 } 1120 if (atomic_dec_and_test(&qp->s_dma_busy)) { 1121 if (qp->state == IB_QPS_RESET) 1122 wake_up(&qp->wait_dma); 1123 else if (qp->s_flags & QIB_S_WAIT_DMA) { 1124 qp->s_flags &= ~QIB_S_WAIT_DMA; 1125 qib_schedule_send(qp); 1126 } 1127 } 1128 spin_unlock(&qp->s_lock); 1129 1130 qib_put_txreq(tx); 1131 } 1132 1133 static int wait_kmem(struct qib_ibdev *dev, struct qib_qp *qp) 1134 { 1135 unsigned long flags; 1136 int ret = 0; 1137 1138 spin_lock_irqsave(&qp->s_lock, flags); 1139 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) { 1140 spin_lock(&dev->pending_lock); 1141 if (list_empty(&qp->iowait)) { 1142 if (list_empty(&dev->memwait)) 1143 mod_timer(&dev->mem_timer, jiffies + 1); 1144 qp->s_flags |= QIB_S_WAIT_KMEM; 1145 list_add_tail(&qp->iowait, &dev->memwait); 1146 } 1147 spin_unlock(&dev->pending_lock); 1148 qp->s_flags &= ~QIB_S_BUSY; 1149 ret = -EBUSY; 1150 } 1151 spin_unlock_irqrestore(&qp->s_lock, flags); 1152 1153 return ret; 1154 } 1155 1156 static int qib_verbs_send_dma(struct qib_qp *qp, struct qib_ib_header *hdr, 1157 u32 hdrwords, struct qib_sge_state *ss, u32 len, 1158 u32 plen, u32 dwords) 1159 { 1160 struct qib_ibdev *dev = to_idev(qp->ibqp.device); 1161 struct qib_devdata *dd = dd_from_dev(dev); 1162 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 1163 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 1164 struct qib_verbs_txreq *tx; 1165 struct qib_pio_header *phdr; 1166 u32 control; 1167 u32 ndesc; 1168 int ret; 1169 1170 tx = qp->s_tx; 1171 if (tx) { 1172 qp->s_tx = NULL; 1173 /* resend previously constructed packet */ 1174 ret = qib_sdma_verbs_send(ppd, tx->ss, tx->dwords, tx); 1175 goto bail; 1176 } 1177 1178 tx = get_txreq(dev, qp); 1179 if (IS_ERR(tx)) 1180 goto bail_tx; 1181 1182 control = dd->f_setpbc_control(ppd, plen, qp->s_srate, 1183 be16_to_cpu(hdr->lrh[0]) >> 12); 1184 tx->qp = qp; 1185 atomic_inc(&qp->refcount); 1186 tx->wqe = qp->s_wqe; 1187 tx->mr = qp->s_rdma_mr; 1188 if (qp->s_rdma_mr) 1189 qp->s_rdma_mr = NULL; 1190 tx->txreq.callback = sdma_complete; 1191 if (dd->flags & QIB_HAS_SDMA_TIMEOUT) 1192 tx->txreq.flags = QIB_SDMA_TXREQ_F_HEADTOHOST; 1193 else 1194 tx->txreq.flags = QIB_SDMA_TXREQ_F_INTREQ; 1195 if (plen + 1 > dd->piosize2kmax_dwords) 1196 tx->txreq.flags |= QIB_SDMA_TXREQ_F_USELARGEBUF; 1197 1198 if (len) { 1199 /* 1200 * Don't try to DMA if it takes more descriptors than 1201 * the queue holds. 1202 */ 1203 ndesc = qib_count_sge(ss, len); 1204 if (ndesc >= ppd->sdma_descq_cnt) 1205 ndesc = 0; 1206 } else 1207 ndesc = 1; 1208 if (ndesc) { 1209 phdr = &dev->pio_hdrs[tx->hdr_inx]; 1210 phdr->pbc[0] = cpu_to_le32(plen); 1211 phdr->pbc[1] = cpu_to_le32(control); 1212 memcpy(&phdr->hdr, hdr, hdrwords << 2); 1213 tx->txreq.flags |= QIB_SDMA_TXREQ_F_FREEDESC; 1214 tx->txreq.sg_count = ndesc; 1215 tx->txreq.addr = dev->pio_hdrs_phys + 1216 tx->hdr_inx * sizeof(struct qib_pio_header); 1217 tx->hdr_dwords = hdrwords + 2; /* add PBC length */ 1218 ret = qib_sdma_verbs_send(ppd, ss, dwords, tx); 1219 goto bail; 1220 } 1221 1222 /* Allocate a buffer and copy the header and payload to it. */ 1223 tx->hdr_dwords = plen + 1; 1224 phdr = kmalloc(tx->hdr_dwords << 2, GFP_ATOMIC); 1225 if (!phdr) 1226 goto err_tx; 1227 phdr->pbc[0] = cpu_to_le32(plen); 1228 phdr->pbc[1] = cpu_to_le32(control); 1229 memcpy(&phdr->hdr, hdr, hdrwords << 2); 1230 qib_copy_from_sge((u32 *) &phdr->hdr + hdrwords, ss, len); 1231 1232 tx->txreq.addr = dma_map_single(&dd->pcidev->dev, phdr, 1233 tx->hdr_dwords << 2, DMA_TO_DEVICE); 1234 if (dma_mapping_error(&dd->pcidev->dev, tx->txreq.addr)) 1235 goto map_err; 1236 tx->align_buf = phdr; 1237 tx->txreq.flags |= QIB_SDMA_TXREQ_F_FREEBUF; 1238 tx->txreq.sg_count = 1; 1239 ret = qib_sdma_verbs_send(ppd, NULL, 0, tx); 1240 goto unaligned; 1241 1242 map_err: 1243 kfree(phdr); 1244 err_tx: 1245 qib_put_txreq(tx); 1246 ret = wait_kmem(dev, qp); 1247 unaligned: 1248 ibp->n_unaligned++; 1249 bail: 1250 return ret; 1251 bail_tx: 1252 ret = PTR_ERR(tx); 1253 goto bail; 1254 } 1255 1256 /* 1257 * If we are now in the error state, return zero to flush the 1258 * send work request. 1259 */ 1260 static int no_bufs_available(struct qib_qp *qp) 1261 { 1262 struct qib_ibdev *dev = to_idev(qp->ibqp.device); 1263 struct qib_devdata *dd; 1264 unsigned long flags; 1265 int ret = 0; 1266 1267 /* 1268 * Note that as soon as want_buffer() is called and 1269 * possibly before it returns, qib_ib_piobufavail() 1270 * could be called. Therefore, put QP on the I/O wait list before 1271 * enabling the PIO avail interrupt. 1272 */ 1273 spin_lock_irqsave(&qp->s_lock, flags); 1274 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) { 1275 spin_lock(&dev->pending_lock); 1276 if (list_empty(&qp->iowait)) { 1277 dev->n_piowait++; 1278 qp->s_flags |= QIB_S_WAIT_PIO; 1279 list_add_tail(&qp->iowait, &dev->piowait); 1280 dd = dd_from_dev(dev); 1281 dd->f_wantpiobuf_intr(dd, 1); 1282 } 1283 spin_unlock(&dev->pending_lock); 1284 qp->s_flags &= ~QIB_S_BUSY; 1285 ret = -EBUSY; 1286 } 1287 spin_unlock_irqrestore(&qp->s_lock, flags); 1288 return ret; 1289 } 1290 1291 static int qib_verbs_send_pio(struct qib_qp *qp, struct qib_ib_header *ibhdr, 1292 u32 hdrwords, struct qib_sge_state *ss, u32 len, 1293 u32 plen, u32 dwords) 1294 { 1295 struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); 1296 struct qib_pportdata *ppd = dd->pport + qp->port_num - 1; 1297 u32 *hdr = (u32 *) ibhdr; 1298 u32 __iomem *piobuf_orig; 1299 u32 __iomem *piobuf; 1300 u64 pbc; 1301 unsigned long flags; 1302 unsigned flush_wc; 1303 u32 control; 1304 u32 pbufn; 1305 1306 control = dd->f_setpbc_control(ppd, plen, qp->s_srate, 1307 be16_to_cpu(ibhdr->lrh[0]) >> 12); 1308 pbc = ((u64) control << 32) | plen; 1309 piobuf = dd->f_getsendbuf(ppd, pbc, &pbufn); 1310 if (unlikely(piobuf == NULL)) 1311 return no_bufs_available(qp); 1312 1313 /* 1314 * Write the pbc. 1315 * We have to flush after the PBC for correctness on some cpus 1316 * or WC buffer can be written out of order. 1317 */ 1318 writeq(pbc, piobuf); 1319 piobuf_orig = piobuf; 1320 piobuf += 2; 1321 1322 flush_wc = dd->flags & QIB_PIO_FLUSH_WC; 1323 if (len == 0) { 1324 /* 1325 * If there is just the header portion, must flush before 1326 * writing last word of header for correctness, and after 1327 * the last header word (trigger word). 1328 */ 1329 if (flush_wc) { 1330 qib_flush_wc(); 1331 qib_pio_copy(piobuf, hdr, hdrwords - 1); 1332 qib_flush_wc(); 1333 __raw_writel(hdr[hdrwords - 1], piobuf + hdrwords - 1); 1334 qib_flush_wc(); 1335 } else 1336 qib_pio_copy(piobuf, hdr, hdrwords); 1337 goto done; 1338 } 1339 1340 if (flush_wc) 1341 qib_flush_wc(); 1342 qib_pio_copy(piobuf, hdr, hdrwords); 1343 piobuf += hdrwords; 1344 1345 /* The common case is aligned and contained in one segment. */ 1346 if (likely(ss->num_sge == 1 && len <= ss->sge.length && 1347 !((unsigned long)ss->sge.vaddr & (sizeof(u32) - 1)))) { 1348 u32 *addr = (u32 *) ss->sge.vaddr; 1349 1350 /* Update address before sending packet. */ 1351 update_sge(ss, len); 1352 if (flush_wc) { 1353 qib_pio_copy(piobuf, addr, dwords - 1); 1354 /* must flush early everything before trigger word */ 1355 qib_flush_wc(); 1356 __raw_writel(addr[dwords - 1], piobuf + dwords - 1); 1357 /* be sure trigger word is written */ 1358 qib_flush_wc(); 1359 } else 1360 qib_pio_copy(piobuf, addr, dwords); 1361 goto done; 1362 } 1363 copy_io(piobuf, ss, len, flush_wc); 1364 done: 1365 if (dd->flags & QIB_USE_SPCL_TRIG) { 1366 u32 spcl_off = (pbufn >= dd->piobcnt2k) ? 2047 : 1023; 1367 1368 qib_flush_wc(); 1369 __raw_writel(0xaebecede, piobuf_orig + spcl_off); 1370 } 1371 qib_sendbuf_done(dd, pbufn); 1372 if (qp->s_rdma_mr) { 1373 qib_put_mr(qp->s_rdma_mr); 1374 qp->s_rdma_mr = NULL; 1375 } 1376 if (qp->s_wqe) { 1377 spin_lock_irqsave(&qp->s_lock, flags); 1378 qib_send_complete(qp, qp->s_wqe, IB_WC_SUCCESS); 1379 spin_unlock_irqrestore(&qp->s_lock, flags); 1380 } else if (qp->ibqp.qp_type == IB_QPT_RC) { 1381 spin_lock_irqsave(&qp->s_lock, flags); 1382 qib_rc_send_complete(qp, ibhdr); 1383 spin_unlock_irqrestore(&qp->s_lock, flags); 1384 } 1385 return 0; 1386 } 1387 1388 /** 1389 * qib_verbs_send - send a packet 1390 * @qp: the QP to send on 1391 * @hdr: the packet header 1392 * @hdrwords: the number of 32-bit words in the header 1393 * @ss: the SGE to send 1394 * @len: the length of the packet in bytes 1395 * 1396 * Return zero if packet is sent or queued OK. 1397 * Return non-zero and clear qp->s_flags QIB_S_BUSY otherwise. 1398 */ 1399 int qib_verbs_send(struct qib_qp *qp, struct qib_ib_header *hdr, 1400 u32 hdrwords, struct qib_sge_state *ss, u32 len) 1401 { 1402 struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); 1403 u32 plen; 1404 int ret; 1405 u32 dwords = (len + 3) >> 2; 1406 1407 /* 1408 * Calculate the send buffer trigger address. 1409 * The +1 counts for the pbc control dword following the pbc length. 1410 */ 1411 plen = hdrwords + dwords + 1; 1412 1413 /* 1414 * VL15 packets (IB_QPT_SMI) will always use PIO, so we 1415 * can defer SDMA restart until link goes ACTIVE without 1416 * worrying about just how we got there. 1417 */ 1418 if (qp->ibqp.qp_type == IB_QPT_SMI || 1419 !(dd->flags & QIB_HAS_SEND_DMA)) 1420 ret = qib_verbs_send_pio(qp, hdr, hdrwords, ss, len, 1421 plen, dwords); 1422 else 1423 ret = qib_verbs_send_dma(qp, hdr, hdrwords, ss, len, 1424 plen, dwords); 1425 1426 return ret; 1427 } 1428 1429 int qib_snapshot_counters(struct qib_pportdata *ppd, u64 *swords, 1430 u64 *rwords, u64 *spkts, u64 *rpkts, 1431 u64 *xmit_wait) 1432 { 1433 int ret; 1434 struct qib_devdata *dd = ppd->dd; 1435 1436 if (!(dd->flags & QIB_PRESENT)) { 1437 /* no hardware, freeze, etc. */ 1438 ret = -EINVAL; 1439 goto bail; 1440 } 1441 *swords = dd->f_portcntr(ppd, QIBPORTCNTR_WORDSEND); 1442 *rwords = dd->f_portcntr(ppd, QIBPORTCNTR_WORDRCV); 1443 *spkts = dd->f_portcntr(ppd, QIBPORTCNTR_PKTSEND); 1444 *rpkts = dd->f_portcntr(ppd, QIBPORTCNTR_PKTRCV); 1445 *xmit_wait = dd->f_portcntr(ppd, QIBPORTCNTR_SENDSTALL); 1446 1447 ret = 0; 1448 1449 bail: 1450 return ret; 1451 } 1452 1453 /** 1454 * qib_get_counters - get various chip counters 1455 * @dd: the qlogic_ib device 1456 * @cntrs: counters are placed here 1457 * 1458 * Return the counters needed by recv_pma_get_portcounters(). 1459 */ 1460 int qib_get_counters(struct qib_pportdata *ppd, 1461 struct qib_verbs_counters *cntrs) 1462 { 1463 int ret; 1464 1465 if (!(ppd->dd->flags & QIB_PRESENT)) { 1466 /* no hardware, freeze, etc. */ 1467 ret = -EINVAL; 1468 goto bail; 1469 } 1470 cntrs->symbol_error_counter = 1471 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_IBSYMBOLERR); 1472 cntrs->link_error_recovery_counter = 1473 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_IBLINKERRRECOV); 1474 /* 1475 * The link downed counter counts when the other side downs the 1476 * connection. We add in the number of times we downed the link 1477 * due to local link integrity errors to compensate. 1478 */ 1479 cntrs->link_downed_counter = 1480 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_IBLINKDOWN); 1481 cntrs->port_rcv_errors = 1482 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RXDROPPKT) + 1483 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RCVOVFL) + 1484 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERR_RLEN) + 1485 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_INVALIDRLEN) + 1486 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRLINK) + 1487 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRICRC) + 1488 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRVCRC) + 1489 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_ERRLPCRC) + 1490 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_BADFORMAT); 1491 cntrs->port_rcv_errors += 1492 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RXLOCALPHYERR); 1493 cntrs->port_rcv_errors += 1494 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RXVLERR); 1495 cntrs->port_rcv_remphys_errors = 1496 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_RCVEBP); 1497 cntrs->port_xmit_discards = 1498 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_UNSUPVL); 1499 cntrs->port_xmit_data = ppd->dd->f_portcntr(ppd, 1500 QIBPORTCNTR_WORDSEND); 1501 cntrs->port_rcv_data = ppd->dd->f_portcntr(ppd, 1502 QIBPORTCNTR_WORDRCV); 1503 cntrs->port_xmit_packets = ppd->dd->f_portcntr(ppd, 1504 QIBPORTCNTR_PKTSEND); 1505 cntrs->port_rcv_packets = ppd->dd->f_portcntr(ppd, 1506 QIBPORTCNTR_PKTRCV); 1507 cntrs->local_link_integrity_errors = 1508 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_LLI); 1509 cntrs->excessive_buffer_overrun_errors = 1510 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_EXCESSBUFOVFL); 1511 cntrs->vl15_dropped = 1512 ppd->dd->f_portcntr(ppd, QIBPORTCNTR_VL15PKTDROP); 1513 1514 ret = 0; 1515 1516 bail: 1517 return ret; 1518 } 1519 1520 /** 1521 * qib_ib_piobufavail - callback when a PIO buffer is available 1522 * @dd: the device pointer 1523 * 1524 * This is called from qib_intr() at interrupt level when a PIO buffer is 1525 * available after qib_verbs_send() returned an error that no buffers were 1526 * available. Disable the interrupt if there are no more QPs waiting. 1527 */ 1528 void qib_ib_piobufavail(struct qib_devdata *dd) 1529 { 1530 struct qib_ibdev *dev = &dd->verbs_dev; 1531 struct list_head *list; 1532 struct qib_qp *qps[5]; 1533 struct qib_qp *qp; 1534 unsigned long flags; 1535 unsigned i, n; 1536 1537 list = &dev->piowait; 1538 n = 0; 1539 1540 /* 1541 * Note: checking that the piowait list is empty and clearing 1542 * the buffer available interrupt needs to be atomic or we 1543 * could end up with QPs on the wait list with the interrupt 1544 * disabled. 1545 */ 1546 spin_lock_irqsave(&dev->pending_lock, flags); 1547 while (!list_empty(list)) { 1548 if (n == ARRAY_SIZE(qps)) 1549 goto full; 1550 qp = list_entry(list->next, struct qib_qp, iowait); 1551 list_del_init(&qp->iowait); 1552 atomic_inc(&qp->refcount); 1553 qps[n++] = qp; 1554 } 1555 dd->f_wantpiobuf_intr(dd, 0); 1556 full: 1557 spin_unlock_irqrestore(&dev->pending_lock, flags); 1558 1559 for (i = 0; i < n; i++) { 1560 qp = qps[i]; 1561 1562 spin_lock_irqsave(&qp->s_lock, flags); 1563 if (qp->s_flags & QIB_S_WAIT_PIO) { 1564 qp->s_flags &= ~QIB_S_WAIT_PIO; 1565 qib_schedule_send(qp); 1566 } 1567 spin_unlock_irqrestore(&qp->s_lock, flags); 1568 1569 /* Notify qib_destroy_qp() if it is waiting. */ 1570 if (atomic_dec_and_test(&qp->refcount)) 1571 wake_up(&qp->wait); 1572 } 1573 } 1574 1575 static int qib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, 1576 struct ib_udata *uhw) 1577 { 1578 struct qib_devdata *dd = dd_from_ibdev(ibdev); 1579 struct qib_ibdev *dev = to_idev(ibdev); 1580 1581 if (uhw->inlen || uhw->outlen) 1582 return -EINVAL; 1583 memset(props, 0, sizeof(*props)); 1584 1585 props->device_cap_flags = IB_DEVICE_BAD_PKEY_CNTR | 1586 IB_DEVICE_BAD_QKEY_CNTR | IB_DEVICE_SHUTDOWN_PORT | 1587 IB_DEVICE_SYS_IMAGE_GUID | IB_DEVICE_RC_RNR_NAK_GEN | 1588 IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_SRQ_RESIZE; 1589 props->page_size_cap = PAGE_SIZE; 1590 props->vendor_id = 1591 QIB_SRC_OUI_1 << 16 | QIB_SRC_OUI_2 << 8 | QIB_SRC_OUI_3; 1592 props->vendor_part_id = dd->deviceid; 1593 props->hw_ver = dd->minrev; 1594 props->sys_image_guid = ib_qib_sys_image_guid; 1595 props->max_mr_size = ~0ULL; 1596 props->max_qp = ib_qib_max_qps; 1597 props->max_qp_wr = ib_qib_max_qp_wrs; 1598 props->max_sge = ib_qib_max_sges; 1599 props->max_sge_rd = ib_qib_max_sges; 1600 props->max_cq = ib_qib_max_cqs; 1601 props->max_ah = ib_qib_max_ahs; 1602 props->max_cqe = ib_qib_max_cqes; 1603 props->max_mr = dev->lk_table.max; 1604 props->max_fmr = dev->lk_table.max; 1605 props->max_map_per_fmr = 32767; 1606 props->max_pd = ib_qib_max_pds; 1607 props->max_qp_rd_atom = QIB_MAX_RDMA_ATOMIC; 1608 props->max_qp_init_rd_atom = 255; 1609 /* props->max_res_rd_atom */ 1610 props->max_srq = ib_qib_max_srqs; 1611 props->max_srq_wr = ib_qib_max_srq_wrs; 1612 props->max_srq_sge = ib_qib_max_srq_sges; 1613 /* props->local_ca_ack_delay */ 1614 props->atomic_cap = IB_ATOMIC_GLOB; 1615 props->max_pkeys = qib_get_npkeys(dd); 1616 props->max_mcast_grp = ib_qib_max_mcast_grps; 1617 props->max_mcast_qp_attach = ib_qib_max_mcast_qp_attached; 1618 props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * 1619 props->max_mcast_grp; 1620 1621 return 0; 1622 } 1623 1624 static int qib_query_port(struct ib_device *ibdev, u8 port, 1625 struct ib_port_attr *props) 1626 { 1627 struct qib_devdata *dd = dd_from_ibdev(ibdev); 1628 struct qib_ibport *ibp = to_iport(ibdev, port); 1629 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 1630 enum ib_mtu mtu; 1631 u16 lid = ppd->lid; 1632 1633 memset(props, 0, sizeof(*props)); 1634 props->lid = lid ? lid : be16_to_cpu(IB_LID_PERMISSIVE); 1635 props->lmc = ppd->lmc; 1636 props->sm_lid = ibp->sm_lid; 1637 props->sm_sl = ibp->sm_sl; 1638 props->state = dd->f_iblink_state(ppd->lastibcstat); 1639 props->phys_state = dd->f_ibphys_portstate(ppd->lastibcstat); 1640 props->port_cap_flags = ibp->port_cap_flags; 1641 props->gid_tbl_len = QIB_GUIDS_PER_PORT; 1642 props->max_msg_sz = 0x80000000; 1643 props->pkey_tbl_len = qib_get_npkeys(dd); 1644 props->bad_pkey_cntr = ibp->pkey_violations; 1645 props->qkey_viol_cntr = ibp->qkey_violations; 1646 props->active_width = ppd->link_width_active; 1647 /* See rate_show() */ 1648 props->active_speed = ppd->link_speed_active; 1649 props->max_vl_num = qib_num_vls(ppd->vls_supported); 1650 props->init_type_reply = 0; 1651 1652 props->max_mtu = qib_ibmtu ? qib_ibmtu : IB_MTU_4096; 1653 switch (ppd->ibmtu) { 1654 case 4096: 1655 mtu = IB_MTU_4096; 1656 break; 1657 case 2048: 1658 mtu = IB_MTU_2048; 1659 break; 1660 case 1024: 1661 mtu = IB_MTU_1024; 1662 break; 1663 case 512: 1664 mtu = IB_MTU_512; 1665 break; 1666 case 256: 1667 mtu = IB_MTU_256; 1668 break; 1669 default: 1670 mtu = IB_MTU_2048; 1671 } 1672 props->active_mtu = mtu; 1673 props->subnet_timeout = ibp->subnet_timeout; 1674 1675 return 0; 1676 } 1677 1678 static int qib_modify_device(struct ib_device *device, 1679 int device_modify_mask, 1680 struct ib_device_modify *device_modify) 1681 { 1682 struct qib_devdata *dd = dd_from_ibdev(device); 1683 unsigned i; 1684 int ret; 1685 1686 if (device_modify_mask & ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID | 1687 IB_DEVICE_MODIFY_NODE_DESC)) { 1688 ret = -EOPNOTSUPP; 1689 goto bail; 1690 } 1691 1692 if (device_modify_mask & IB_DEVICE_MODIFY_NODE_DESC) { 1693 memcpy(device->node_desc, device_modify->node_desc, 64); 1694 for (i = 0; i < dd->num_pports; i++) { 1695 struct qib_ibport *ibp = &dd->pport[i].ibport_data; 1696 1697 qib_node_desc_chg(ibp); 1698 } 1699 } 1700 1701 if (device_modify_mask & IB_DEVICE_MODIFY_SYS_IMAGE_GUID) { 1702 ib_qib_sys_image_guid = 1703 cpu_to_be64(device_modify->sys_image_guid); 1704 for (i = 0; i < dd->num_pports; i++) { 1705 struct qib_ibport *ibp = &dd->pport[i].ibport_data; 1706 1707 qib_sys_guid_chg(ibp); 1708 } 1709 } 1710 1711 ret = 0; 1712 1713 bail: 1714 return ret; 1715 } 1716 1717 static int qib_modify_port(struct ib_device *ibdev, u8 port, 1718 int port_modify_mask, struct ib_port_modify *props) 1719 { 1720 struct qib_ibport *ibp = to_iport(ibdev, port); 1721 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 1722 1723 ibp->port_cap_flags |= props->set_port_cap_mask; 1724 ibp->port_cap_flags &= ~props->clr_port_cap_mask; 1725 if (props->set_port_cap_mask || props->clr_port_cap_mask) 1726 qib_cap_mask_chg(ibp); 1727 if (port_modify_mask & IB_PORT_SHUTDOWN) 1728 qib_set_linkstate(ppd, QIB_IB_LINKDOWN); 1729 if (port_modify_mask & IB_PORT_RESET_QKEY_CNTR) 1730 ibp->qkey_violations = 0; 1731 return 0; 1732 } 1733 1734 static int qib_query_gid(struct ib_device *ibdev, u8 port, 1735 int index, union ib_gid *gid) 1736 { 1737 struct qib_devdata *dd = dd_from_ibdev(ibdev); 1738 int ret = 0; 1739 1740 if (!port || port > dd->num_pports) 1741 ret = -EINVAL; 1742 else { 1743 struct qib_ibport *ibp = to_iport(ibdev, port); 1744 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 1745 1746 gid->global.subnet_prefix = ibp->gid_prefix; 1747 if (index == 0) 1748 gid->global.interface_id = ppd->guid; 1749 else if (index < QIB_GUIDS_PER_PORT) 1750 gid->global.interface_id = ibp->guids[index - 1]; 1751 else 1752 ret = -EINVAL; 1753 } 1754 1755 return ret; 1756 } 1757 1758 static struct ib_pd *qib_alloc_pd(struct ib_device *ibdev, 1759 struct ib_ucontext *context, 1760 struct ib_udata *udata) 1761 { 1762 struct qib_ibdev *dev = to_idev(ibdev); 1763 struct qib_pd *pd; 1764 struct ib_pd *ret; 1765 1766 /* 1767 * This is actually totally arbitrary. Some correctness tests 1768 * assume there's a maximum number of PDs that can be allocated. 1769 * We don't actually have this limit, but we fail the test if 1770 * we allow allocations of more than we report for this value. 1771 */ 1772 1773 pd = kmalloc(sizeof(*pd), GFP_KERNEL); 1774 if (!pd) { 1775 ret = ERR_PTR(-ENOMEM); 1776 goto bail; 1777 } 1778 1779 spin_lock(&dev->n_pds_lock); 1780 if (dev->n_pds_allocated == ib_qib_max_pds) { 1781 spin_unlock(&dev->n_pds_lock); 1782 kfree(pd); 1783 ret = ERR_PTR(-ENOMEM); 1784 goto bail; 1785 } 1786 1787 dev->n_pds_allocated++; 1788 spin_unlock(&dev->n_pds_lock); 1789 1790 /* ib_alloc_pd() will initialize pd->ibpd. */ 1791 pd->user = udata != NULL; 1792 1793 ret = &pd->ibpd; 1794 1795 bail: 1796 return ret; 1797 } 1798 1799 static int qib_dealloc_pd(struct ib_pd *ibpd) 1800 { 1801 struct qib_pd *pd = to_ipd(ibpd); 1802 struct qib_ibdev *dev = to_idev(ibpd->device); 1803 1804 spin_lock(&dev->n_pds_lock); 1805 dev->n_pds_allocated--; 1806 spin_unlock(&dev->n_pds_lock); 1807 1808 kfree(pd); 1809 1810 return 0; 1811 } 1812 1813 int qib_check_ah(struct ib_device *ibdev, struct ib_ah_attr *ah_attr) 1814 { 1815 /* A multicast address requires a GRH (see ch. 8.4.1). */ 1816 if (ah_attr->dlid >= QIB_MULTICAST_LID_BASE && 1817 ah_attr->dlid != QIB_PERMISSIVE_LID && 1818 !(ah_attr->ah_flags & IB_AH_GRH)) 1819 goto bail; 1820 if ((ah_attr->ah_flags & IB_AH_GRH) && 1821 ah_attr->grh.sgid_index >= QIB_GUIDS_PER_PORT) 1822 goto bail; 1823 if (ah_attr->dlid == 0) 1824 goto bail; 1825 if (ah_attr->port_num < 1 || 1826 ah_attr->port_num > ibdev->phys_port_cnt) 1827 goto bail; 1828 if (ah_attr->static_rate != IB_RATE_PORT_CURRENT && 1829 ib_rate_to_mult(ah_attr->static_rate) < 0) 1830 goto bail; 1831 if (ah_attr->sl > 15) 1832 goto bail; 1833 return 0; 1834 bail: 1835 return -EINVAL; 1836 } 1837 1838 /** 1839 * qib_create_ah - create an address handle 1840 * @pd: the protection domain 1841 * @ah_attr: the attributes of the AH 1842 * 1843 * This may be called from interrupt context. 1844 */ 1845 static struct ib_ah *qib_create_ah(struct ib_pd *pd, 1846 struct ib_ah_attr *ah_attr) 1847 { 1848 struct qib_ah *ah; 1849 struct ib_ah *ret; 1850 struct qib_ibdev *dev = to_idev(pd->device); 1851 unsigned long flags; 1852 1853 if (qib_check_ah(pd->device, ah_attr)) { 1854 ret = ERR_PTR(-EINVAL); 1855 goto bail; 1856 } 1857 1858 ah = kmalloc(sizeof(*ah), GFP_ATOMIC); 1859 if (!ah) { 1860 ret = ERR_PTR(-ENOMEM); 1861 goto bail; 1862 } 1863 1864 spin_lock_irqsave(&dev->n_ahs_lock, flags); 1865 if (dev->n_ahs_allocated == ib_qib_max_ahs) { 1866 spin_unlock_irqrestore(&dev->n_ahs_lock, flags); 1867 kfree(ah); 1868 ret = ERR_PTR(-ENOMEM); 1869 goto bail; 1870 } 1871 1872 dev->n_ahs_allocated++; 1873 spin_unlock_irqrestore(&dev->n_ahs_lock, flags); 1874 1875 /* ib_create_ah() will initialize ah->ibah. */ 1876 ah->attr = *ah_attr; 1877 atomic_set(&ah->refcount, 0); 1878 1879 ret = &ah->ibah; 1880 1881 bail: 1882 return ret; 1883 } 1884 1885 struct ib_ah *qib_create_qp0_ah(struct qib_ibport *ibp, u16 dlid) 1886 { 1887 struct ib_ah_attr attr; 1888 struct ib_ah *ah = ERR_PTR(-EINVAL); 1889 struct qib_qp *qp0; 1890 1891 memset(&attr, 0, sizeof(attr)); 1892 attr.dlid = dlid; 1893 attr.port_num = ppd_from_ibp(ibp)->port; 1894 rcu_read_lock(); 1895 qp0 = rcu_dereference(ibp->qp0); 1896 if (qp0) 1897 ah = ib_create_ah(qp0->ibqp.pd, &attr); 1898 rcu_read_unlock(); 1899 return ah; 1900 } 1901 1902 /** 1903 * qib_destroy_ah - destroy an address handle 1904 * @ibah: the AH to destroy 1905 * 1906 * This may be called from interrupt context. 1907 */ 1908 static int qib_destroy_ah(struct ib_ah *ibah) 1909 { 1910 struct qib_ibdev *dev = to_idev(ibah->device); 1911 struct qib_ah *ah = to_iah(ibah); 1912 unsigned long flags; 1913 1914 if (atomic_read(&ah->refcount) != 0) 1915 return -EBUSY; 1916 1917 spin_lock_irqsave(&dev->n_ahs_lock, flags); 1918 dev->n_ahs_allocated--; 1919 spin_unlock_irqrestore(&dev->n_ahs_lock, flags); 1920 1921 kfree(ah); 1922 1923 return 0; 1924 } 1925 1926 static int qib_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr) 1927 { 1928 struct qib_ah *ah = to_iah(ibah); 1929 1930 if (qib_check_ah(ibah->device, ah_attr)) 1931 return -EINVAL; 1932 1933 ah->attr = *ah_attr; 1934 1935 return 0; 1936 } 1937 1938 static int qib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr) 1939 { 1940 struct qib_ah *ah = to_iah(ibah); 1941 1942 *ah_attr = ah->attr; 1943 1944 return 0; 1945 } 1946 1947 /** 1948 * qib_get_npkeys - return the size of the PKEY table for context 0 1949 * @dd: the qlogic_ib device 1950 */ 1951 unsigned qib_get_npkeys(struct qib_devdata *dd) 1952 { 1953 return ARRAY_SIZE(dd->rcd[0]->pkeys); 1954 } 1955 1956 /* 1957 * Return the indexed PKEY from the port PKEY table. 1958 * No need to validate rcd[ctxt]; the port is setup if we are here. 1959 */ 1960 unsigned qib_get_pkey(struct qib_ibport *ibp, unsigned index) 1961 { 1962 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 1963 struct qib_devdata *dd = ppd->dd; 1964 unsigned ctxt = ppd->hw_pidx; 1965 unsigned ret; 1966 1967 /* dd->rcd null if mini_init or some init failures */ 1968 if (!dd->rcd || index >= ARRAY_SIZE(dd->rcd[ctxt]->pkeys)) 1969 ret = 0; 1970 else 1971 ret = dd->rcd[ctxt]->pkeys[index]; 1972 1973 return ret; 1974 } 1975 1976 static int qib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, 1977 u16 *pkey) 1978 { 1979 struct qib_devdata *dd = dd_from_ibdev(ibdev); 1980 int ret; 1981 1982 if (index >= qib_get_npkeys(dd)) { 1983 ret = -EINVAL; 1984 goto bail; 1985 } 1986 1987 *pkey = qib_get_pkey(to_iport(ibdev, port), index); 1988 ret = 0; 1989 1990 bail: 1991 return ret; 1992 } 1993 1994 /** 1995 * qib_alloc_ucontext - allocate a ucontest 1996 * @ibdev: the infiniband device 1997 * @udata: not used by the QLogic_IB driver 1998 */ 1999 2000 static struct ib_ucontext *qib_alloc_ucontext(struct ib_device *ibdev, 2001 struct ib_udata *udata) 2002 { 2003 struct qib_ucontext *context; 2004 struct ib_ucontext *ret; 2005 2006 context = kmalloc(sizeof(*context), GFP_KERNEL); 2007 if (!context) { 2008 ret = ERR_PTR(-ENOMEM); 2009 goto bail; 2010 } 2011 2012 ret = &context->ibucontext; 2013 2014 bail: 2015 return ret; 2016 } 2017 2018 static int qib_dealloc_ucontext(struct ib_ucontext *context) 2019 { 2020 kfree(to_iucontext(context)); 2021 return 0; 2022 } 2023 2024 static void init_ibport(struct qib_pportdata *ppd) 2025 { 2026 struct qib_verbs_counters cntrs; 2027 struct qib_ibport *ibp = &ppd->ibport_data; 2028 2029 spin_lock_init(&ibp->lock); 2030 /* Set the prefix to the default value (see ch. 4.1.1) */ 2031 ibp->gid_prefix = IB_DEFAULT_GID_PREFIX; 2032 ibp->sm_lid = be16_to_cpu(IB_LID_PERMISSIVE); 2033 ibp->port_cap_flags = IB_PORT_SYS_IMAGE_GUID_SUP | 2034 IB_PORT_CLIENT_REG_SUP | IB_PORT_SL_MAP_SUP | 2035 IB_PORT_TRAP_SUP | IB_PORT_AUTO_MIGR_SUP | 2036 IB_PORT_DR_NOTICE_SUP | IB_PORT_CAP_MASK_NOTICE_SUP | 2037 IB_PORT_OTHER_LOCAL_CHANGES_SUP; 2038 if (ppd->dd->flags & QIB_HAS_LINK_LATENCY) 2039 ibp->port_cap_flags |= IB_PORT_LINK_LATENCY_SUP; 2040 ibp->pma_counter_select[0] = IB_PMA_PORT_XMIT_DATA; 2041 ibp->pma_counter_select[1] = IB_PMA_PORT_RCV_DATA; 2042 ibp->pma_counter_select[2] = IB_PMA_PORT_XMIT_PKTS; 2043 ibp->pma_counter_select[3] = IB_PMA_PORT_RCV_PKTS; 2044 ibp->pma_counter_select[4] = IB_PMA_PORT_XMIT_WAIT; 2045 2046 /* Snapshot current HW counters to "clear" them. */ 2047 qib_get_counters(ppd, &cntrs); 2048 ibp->z_symbol_error_counter = cntrs.symbol_error_counter; 2049 ibp->z_link_error_recovery_counter = 2050 cntrs.link_error_recovery_counter; 2051 ibp->z_link_downed_counter = cntrs.link_downed_counter; 2052 ibp->z_port_rcv_errors = cntrs.port_rcv_errors; 2053 ibp->z_port_rcv_remphys_errors = cntrs.port_rcv_remphys_errors; 2054 ibp->z_port_xmit_discards = cntrs.port_xmit_discards; 2055 ibp->z_port_xmit_data = cntrs.port_xmit_data; 2056 ibp->z_port_rcv_data = cntrs.port_rcv_data; 2057 ibp->z_port_xmit_packets = cntrs.port_xmit_packets; 2058 ibp->z_port_rcv_packets = cntrs.port_rcv_packets; 2059 ibp->z_local_link_integrity_errors = 2060 cntrs.local_link_integrity_errors; 2061 ibp->z_excessive_buffer_overrun_errors = 2062 cntrs.excessive_buffer_overrun_errors; 2063 ibp->z_vl15_dropped = cntrs.vl15_dropped; 2064 RCU_INIT_POINTER(ibp->qp0, NULL); 2065 RCU_INIT_POINTER(ibp->qp1, NULL); 2066 } 2067 2068 static int qib_port_immutable(struct ib_device *ibdev, u8 port_num, 2069 struct ib_port_immutable *immutable) 2070 { 2071 struct ib_port_attr attr; 2072 int err; 2073 2074 err = qib_query_port(ibdev, port_num, &attr); 2075 if (err) 2076 return err; 2077 2078 immutable->pkey_tbl_len = attr.pkey_tbl_len; 2079 immutable->gid_tbl_len = attr.gid_tbl_len; 2080 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB; 2081 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 2082 2083 return 0; 2084 } 2085 2086 /** 2087 * qib_register_ib_device - register our device with the infiniband core 2088 * @dd: the device data structure 2089 * Return the allocated qib_ibdev pointer or NULL on error. 2090 */ 2091 int qib_register_ib_device(struct qib_devdata *dd) 2092 { 2093 struct qib_ibdev *dev = &dd->verbs_dev; 2094 struct ib_device *ibdev = &dev->ibdev; 2095 struct qib_pportdata *ppd = dd->pport; 2096 unsigned i, lk_tab_size; 2097 int ret; 2098 2099 dev->qp_table_size = ib_qib_qp_table_size; 2100 get_random_bytes(&dev->qp_rnd, sizeof(dev->qp_rnd)); 2101 dev->qp_table = kmalloc_array( 2102 dev->qp_table_size, 2103 sizeof(*dev->qp_table), 2104 GFP_KERNEL); 2105 if (!dev->qp_table) { 2106 ret = -ENOMEM; 2107 goto err_qpt; 2108 } 2109 for (i = 0; i < dev->qp_table_size; i++) 2110 RCU_INIT_POINTER(dev->qp_table[i], NULL); 2111 2112 for (i = 0; i < dd->num_pports; i++) 2113 init_ibport(ppd + i); 2114 2115 /* Only need to initialize non-zero fields. */ 2116 spin_lock_init(&dev->qpt_lock); 2117 spin_lock_init(&dev->n_pds_lock); 2118 spin_lock_init(&dev->n_ahs_lock); 2119 spin_lock_init(&dev->n_cqs_lock); 2120 spin_lock_init(&dev->n_qps_lock); 2121 spin_lock_init(&dev->n_srqs_lock); 2122 spin_lock_init(&dev->n_mcast_grps_lock); 2123 init_timer(&dev->mem_timer); 2124 dev->mem_timer.function = mem_timer; 2125 dev->mem_timer.data = (unsigned long) dev; 2126 2127 qib_init_qpn_table(dd, &dev->qpn_table); 2128 2129 /* 2130 * The top ib_qib_lkey_table_size bits are used to index the 2131 * table. The lower 8 bits can be owned by the user (copied from 2132 * the LKEY). The remaining bits act as a generation number or tag. 2133 */ 2134 spin_lock_init(&dev->lk_table.lock); 2135 /* insure generation is at least 4 bits see keys.c */ 2136 if (ib_qib_lkey_table_size > MAX_LKEY_TABLE_BITS) { 2137 qib_dev_warn(dd, "lkey bits %u too large, reduced to %u\n", 2138 ib_qib_lkey_table_size, MAX_LKEY_TABLE_BITS); 2139 ib_qib_lkey_table_size = MAX_LKEY_TABLE_BITS; 2140 } 2141 dev->lk_table.max = 1 << ib_qib_lkey_table_size; 2142 lk_tab_size = dev->lk_table.max * sizeof(*dev->lk_table.table); 2143 dev->lk_table.table = (struct qib_mregion __rcu **) 2144 vmalloc(lk_tab_size); 2145 if (dev->lk_table.table == NULL) { 2146 ret = -ENOMEM; 2147 goto err_lk; 2148 } 2149 RCU_INIT_POINTER(dev->dma_mr, NULL); 2150 for (i = 0; i < dev->lk_table.max; i++) 2151 RCU_INIT_POINTER(dev->lk_table.table[i], NULL); 2152 INIT_LIST_HEAD(&dev->pending_mmaps); 2153 spin_lock_init(&dev->pending_lock); 2154 dev->mmap_offset = PAGE_SIZE; 2155 spin_lock_init(&dev->mmap_offset_lock); 2156 INIT_LIST_HEAD(&dev->piowait); 2157 INIT_LIST_HEAD(&dev->dmawait); 2158 INIT_LIST_HEAD(&dev->txwait); 2159 INIT_LIST_HEAD(&dev->memwait); 2160 INIT_LIST_HEAD(&dev->txreq_free); 2161 2162 if (ppd->sdma_descq_cnt) { 2163 dev->pio_hdrs = dma_alloc_coherent(&dd->pcidev->dev, 2164 ppd->sdma_descq_cnt * 2165 sizeof(struct qib_pio_header), 2166 &dev->pio_hdrs_phys, 2167 GFP_KERNEL); 2168 if (!dev->pio_hdrs) { 2169 ret = -ENOMEM; 2170 goto err_hdrs; 2171 } 2172 } 2173 2174 for (i = 0; i < ppd->sdma_descq_cnt; i++) { 2175 struct qib_verbs_txreq *tx; 2176 2177 tx = kzalloc(sizeof(*tx), GFP_KERNEL); 2178 if (!tx) { 2179 ret = -ENOMEM; 2180 goto err_tx; 2181 } 2182 tx->hdr_inx = i; 2183 list_add(&tx->txreq.list, &dev->txreq_free); 2184 } 2185 2186 /* 2187 * The system image GUID is supposed to be the same for all 2188 * IB HCAs in a single system but since there can be other 2189 * device types in the system, we can't be sure this is unique. 2190 */ 2191 if (!ib_qib_sys_image_guid) 2192 ib_qib_sys_image_guid = ppd->guid; 2193 2194 strlcpy(ibdev->name, "qib%d", IB_DEVICE_NAME_MAX); 2195 ibdev->owner = THIS_MODULE; 2196 ibdev->node_guid = ppd->guid; 2197 ibdev->uverbs_abi_ver = QIB_UVERBS_ABI_VERSION; 2198 ibdev->uverbs_cmd_mask = 2199 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | 2200 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | 2201 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | 2202 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | 2203 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | 2204 (1ull << IB_USER_VERBS_CMD_CREATE_AH) | 2205 (1ull << IB_USER_VERBS_CMD_MODIFY_AH) | 2206 (1ull << IB_USER_VERBS_CMD_QUERY_AH) | 2207 (1ull << IB_USER_VERBS_CMD_DESTROY_AH) | 2208 (1ull << IB_USER_VERBS_CMD_REG_MR) | 2209 (1ull << IB_USER_VERBS_CMD_DEREG_MR) | 2210 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | 2211 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | 2212 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | 2213 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | 2214 (1ull << IB_USER_VERBS_CMD_POLL_CQ) | 2215 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | 2216 (1ull << IB_USER_VERBS_CMD_CREATE_QP) | 2217 (1ull << IB_USER_VERBS_CMD_QUERY_QP) | 2218 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | 2219 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | 2220 (1ull << IB_USER_VERBS_CMD_POST_SEND) | 2221 (1ull << IB_USER_VERBS_CMD_POST_RECV) | 2222 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) | 2223 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) | 2224 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | 2225 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | 2226 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | 2227 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | 2228 (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV); 2229 ibdev->node_type = RDMA_NODE_IB_CA; 2230 ibdev->phys_port_cnt = dd->num_pports; 2231 ibdev->num_comp_vectors = 1; 2232 ibdev->dma_device = &dd->pcidev->dev; 2233 ibdev->query_device = qib_query_device; 2234 ibdev->modify_device = qib_modify_device; 2235 ibdev->query_port = qib_query_port; 2236 ibdev->modify_port = qib_modify_port; 2237 ibdev->query_pkey = qib_query_pkey; 2238 ibdev->query_gid = qib_query_gid; 2239 ibdev->alloc_ucontext = qib_alloc_ucontext; 2240 ibdev->dealloc_ucontext = qib_dealloc_ucontext; 2241 ibdev->alloc_pd = qib_alloc_pd; 2242 ibdev->dealloc_pd = qib_dealloc_pd; 2243 ibdev->create_ah = qib_create_ah; 2244 ibdev->destroy_ah = qib_destroy_ah; 2245 ibdev->modify_ah = qib_modify_ah; 2246 ibdev->query_ah = qib_query_ah; 2247 ibdev->create_srq = qib_create_srq; 2248 ibdev->modify_srq = qib_modify_srq; 2249 ibdev->query_srq = qib_query_srq; 2250 ibdev->destroy_srq = qib_destroy_srq; 2251 ibdev->create_qp = qib_create_qp; 2252 ibdev->modify_qp = qib_modify_qp; 2253 ibdev->query_qp = qib_query_qp; 2254 ibdev->destroy_qp = qib_destroy_qp; 2255 ibdev->post_send = qib_post_send; 2256 ibdev->post_recv = qib_post_receive; 2257 ibdev->post_srq_recv = qib_post_srq_receive; 2258 ibdev->create_cq = qib_create_cq; 2259 ibdev->destroy_cq = qib_destroy_cq; 2260 ibdev->resize_cq = qib_resize_cq; 2261 ibdev->poll_cq = qib_poll_cq; 2262 ibdev->req_notify_cq = qib_req_notify_cq; 2263 ibdev->get_dma_mr = qib_get_dma_mr; 2264 ibdev->reg_user_mr = qib_reg_user_mr; 2265 ibdev->dereg_mr = qib_dereg_mr; 2266 ibdev->alloc_mr = qib_alloc_mr; 2267 ibdev->map_mr_sg = qib_map_mr_sg; 2268 ibdev->alloc_fmr = qib_alloc_fmr; 2269 ibdev->map_phys_fmr = qib_map_phys_fmr; 2270 ibdev->unmap_fmr = qib_unmap_fmr; 2271 ibdev->dealloc_fmr = qib_dealloc_fmr; 2272 ibdev->attach_mcast = qib_multicast_attach; 2273 ibdev->detach_mcast = qib_multicast_detach; 2274 ibdev->process_mad = qib_process_mad; 2275 ibdev->mmap = qib_mmap; 2276 ibdev->dma_ops = &qib_dma_mapping_ops; 2277 ibdev->get_port_immutable = qib_port_immutable; 2278 2279 snprintf(ibdev->node_desc, sizeof(ibdev->node_desc), 2280 "Intel Infiniband HCA %s", init_utsname()->nodename); 2281 2282 ret = ib_register_device(ibdev, qib_create_port_files); 2283 if (ret) 2284 goto err_reg; 2285 2286 ret = qib_create_agents(dev); 2287 if (ret) 2288 goto err_agents; 2289 2290 ret = qib_verbs_register_sysfs(dd); 2291 if (ret) 2292 goto err_class; 2293 2294 goto bail; 2295 2296 err_class: 2297 qib_free_agents(dev); 2298 err_agents: 2299 ib_unregister_device(ibdev); 2300 err_reg: 2301 err_tx: 2302 while (!list_empty(&dev->txreq_free)) { 2303 struct list_head *l = dev->txreq_free.next; 2304 struct qib_verbs_txreq *tx; 2305 2306 list_del(l); 2307 tx = list_entry(l, struct qib_verbs_txreq, txreq.list); 2308 kfree(tx); 2309 } 2310 if (ppd->sdma_descq_cnt) 2311 dma_free_coherent(&dd->pcidev->dev, 2312 ppd->sdma_descq_cnt * 2313 sizeof(struct qib_pio_header), 2314 dev->pio_hdrs, dev->pio_hdrs_phys); 2315 err_hdrs: 2316 vfree(dev->lk_table.table); 2317 err_lk: 2318 kfree(dev->qp_table); 2319 err_qpt: 2320 qib_dev_err(dd, "cannot register verbs: %d!\n", -ret); 2321 bail: 2322 return ret; 2323 } 2324 2325 void qib_unregister_ib_device(struct qib_devdata *dd) 2326 { 2327 struct qib_ibdev *dev = &dd->verbs_dev; 2328 struct ib_device *ibdev = &dev->ibdev; 2329 u32 qps_inuse; 2330 unsigned lk_tab_size; 2331 2332 qib_verbs_unregister_sysfs(dd); 2333 2334 qib_free_agents(dev); 2335 2336 ib_unregister_device(ibdev); 2337 2338 if (!list_empty(&dev->piowait)) 2339 qib_dev_err(dd, "piowait list not empty!\n"); 2340 if (!list_empty(&dev->dmawait)) 2341 qib_dev_err(dd, "dmawait list not empty!\n"); 2342 if (!list_empty(&dev->txwait)) 2343 qib_dev_err(dd, "txwait list not empty!\n"); 2344 if (!list_empty(&dev->memwait)) 2345 qib_dev_err(dd, "memwait list not empty!\n"); 2346 if (dev->dma_mr) 2347 qib_dev_err(dd, "DMA MR not NULL!\n"); 2348 2349 qps_inuse = qib_free_all_qps(dd); 2350 if (qps_inuse) 2351 qib_dev_err(dd, "QP memory leak! %u still in use\n", 2352 qps_inuse); 2353 2354 del_timer_sync(&dev->mem_timer); 2355 qib_free_qpn_table(&dev->qpn_table); 2356 while (!list_empty(&dev->txreq_free)) { 2357 struct list_head *l = dev->txreq_free.next; 2358 struct qib_verbs_txreq *tx; 2359 2360 list_del(l); 2361 tx = list_entry(l, struct qib_verbs_txreq, txreq.list); 2362 kfree(tx); 2363 } 2364 if (dd->pport->sdma_descq_cnt) 2365 dma_free_coherent(&dd->pcidev->dev, 2366 dd->pport->sdma_descq_cnt * 2367 sizeof(struct qib_pio_header), 2368 dev->pio_hdrs, dev->pio_hdrs_phys); 2369 lk_tab_size = dev->lk_table.max * sizeof(*dev->lk_table.table); 2370 vfree(dev->lk_table.table); 2371 kfree(dev->qp_table); 2372 } 2373 2374 /* 2375 * This must be called with s_lock held. 2376 */ 2377 void qib_schedule_send(struct qib_qp *qp) 2378 { 2379 if (qib_send_ok(qp)) { 2380 struct qib_ibport *ibp = 2381 to_iport(qp->ibqp.device, qp->port_num); 2382 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 2383 2384 queue_work(ppd->qib_wq, &qp->s_work); 2385 } 2386 } 2387