1 /* 2 * Copyright(c) 2016, 2017 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 48 #include <linux/hash.h> 49 #include <linux/bitops.h> 50 #include <linux/lockdep.h> 51 #include <linux/vmalloc.h> 52 #include <linux/slab.h> 53 #include <rdma/ib_verbs.h> 54 #include <rdma/ib_hdrs.h> 55 #include <rdma/opa_addr.h> 56 #include "qp.h" 57 #include "vt.h" 58 #include "trace.h" 59 60 static void rvt_rc_timeout(struct timer_list *t); 61 62 /* 63 * Convert the AETH RNR timeout code into the number of microseconds. 64 */ 65 static const u32 ib_rvt_rnr_table[32] = { 66 655360, /* 00: 655.36 */ 67 10, /* 01: .01 */ 68 20, /* 02 .02 */ 69 30, /* 03: .03 */ 70 40, /* 04: .04 */ 71 60, /* 05: .06 */ 72 80, /* 06: .08 */ 73 120, /* 07: .12 */ 74 160, /* 08: .16 */ 75 240, /* 09: .24 */ 76 320, /* 0A: .32 */ 77 480, /* 0B: .48 */ 78 640, /* 0C: .64 */ 79 960, /* 0D: .96 */ 80 1280, /* 0E: 1.28 */ 81 1920, /* 0F: 1.92 */ 82 2560, /* 10: 2.56 */ 83 3840, /* 11: 3.84 */ 84 5120, /* 12: 5.12 */ 85 7680, /* 13: 7.68 */ 86 10240, /* 14: 10.24 */ 87 15360, /* 15: 15.36 */ 88 20480, /* 16: 20.48 */ 89 30720, /* 17: 30.72 */ 90 40960, /* 18: 40.96 */ 91 61440, /* 19: 61.44 */ 92 81920, /* 1A: 81.92 */ 93 122880, /* 1B: 122.88 */ 94 163840, /* 1C: 163.84 */ 95 245760, /* 1D: 245.76 */ 96 327680, /* 1E: 327.68 */ 97 491520 /* 1F: 491.52 */ 98 }; 99 100 /* 101 * Note that it is OK to post send work requests in the SQE and ERR 102 * states; rvt_do_send() will process them and generate error 103 * completions as per IB 1.2 C10-96. 104 */ 105 const int ib_rvt_state_ops[IB_QPS_ERR + 1] = { 106 [IB_QPS_RESET] = 0, 107 [IB_QPS_INIT] = RVT_POST_RECV_OK, 108 [IB_QPS_RTR] = RVT_POST_RECV_OK | RVT_PROCESS_RECV_OK, 109 [IB_QPS_RTS] = RVT_POST_RECV_OK | RVT_PROCESS_RECV_OK | 110 RVT_POST_SEND_OK | RVT_PROCESS_SEND_OK | 111 RVT_PROCESS_NEXT_SEND_OK, 112 [IB_QPS_SQD] = RVT_POST_RECV_OK | RVT_PROCESS_RECV_OK | 113 RVT_POST_SEND_OK | RVT_PROCESS_SEND_OK, 114 [IB_QPS_SQE] = RVT_POST_RECV_OK | RVT_PROCESS_RECV_OK | 115 RVT_POST_SEND_OK | RVT_FLUSH_SEND, 116 [IB_QPS_ERR] = RVT_POST_RECV_OK | RVT_FLUSH_RECV | 117 RVT_POST_SEND_OK | RVT_FLUSH_SEND, 118 }; 119 EXPORT_SYMBOL(ib_rvt_state_ops); 120 121 static void get_map_page(struct rvt_qpn_table *qpt, 122 struct rvt_qpn_map *map) 123 { 124 unsigned long page = get_zeroed_page(GFP_KERNEL); 125 126 /* 127 * Free the page if someone raced with us installing it. 128 */ 129 130 spin_lock(&qpt->lock); 131 if (map->page) 132 free_page(page); 133 else 134 map->page = (void *)page; 135 spin_unlock(&qpt->lock); 136 } 137 138 /** 139 * init_qpn_table - initialize the QP number table for a device 140 * @qpt: the QPN table 141 */ 142 static int init_qpn_table(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt) 143 { 144 u32 offset, i; 145 struct rvt_qpn_map *map; 146 int ret = 0; 147 148 if (!(rdi->dparms.qpn_res_end >= rdi->dparms.qpn_res_start)) 149 return -EINVAL; 150 151 spin_lock_init(&qpt->lock); 152 153 qpt->last = rdi->dparms.qpn_start; 154 qpt->incr = rdi->dparms.qpn_inc << rdi->dparms.qos_shift; 155 156 /* 157 * Drivers may want some QPs beyond what we need for verbs let them use 158 * our qpn table. No need for two. Lets go ahead and mark the bitmaps 159 * for those. The reserved range must be *after* the range which verbs 160 * will pick from. 161 */ 162 163 /* Figure out number of bit maps needed before reserved range */ 164 qpt->nmaps = rdi->dparms.qpn_res_start / RVT_BITS_PER_PAGE; 165 166 /* This should always be zero */ 167 offset = rdi->dparms.qpn_res_start & RVT_BITS_PER_PAGE_MASK; 168 169 /* Starting with the first reserved bit map */ 170 map = &qpt->map[qpt->nmaps]; 171 172 rvt_pr_info(rdi, "Reserving QPNs from 0x%x to 0x%x for non-verbs use\n", 173 rdi->dparms.qpn_res_start, rdi->dparms.qpn_res_end); 174 for (i = rdi->dparms.qpn_res_start; i <= rdi->dparms.qpn_res_end; i++) { 175 if (!map->page) { 176 get_map_page(qpt, map); 177 if (!map->page) { 178 ret = -ENOMEM; 179 break; 180 } 181 } 182 set_bit(offset, map->page); 183 offset++; 184 if (offset == RVT_BITS_PER_PAGE) { 185 /* next page */ 186 qpt->nmaps++; 187 map++; 188 offset = 0; 189 } 190 } 191 return ret; 192 } 193 194 /** 195 * free_qpn_table - free the QP number table for a device 196 * @qpt: the QPN table 197 */ 198 static void free_qpn_table(struct rvt_qpn_table *qpt) 199 { 200 int i; 201 202 for (i = 0; i < ARRAY_SIZE(qpt->map); i++) 203 free_page((unsigned long)qpt->map[i].page); 204 } 205 206 /** 207 * rvt_driver_qp_init - Init driver qp resources 208 * @rdi: rvt dev strucutre 209 * 210 * Return: 0 on success 211 */ 212 int rvt_driver_qp_init(struct rvt_dev_info *rdi) 213 { 214 int i; 215 int ret = -ENOMEM; 216 217 if (!rdi->dparms.qp_table_size) 218 return -EINVAL; 219 220 /* 221 * If driver is not doing any QP allocation then make sure it is 222 * providing the necessary QP functions. 223 */ 224 if (!rdi->driver_f.free_all_qps || 225 !rdi->driver_f.qp_priv_alloc || 226 !rdi->driver_f.qp_priv_free || 227 !rdi->driver_f.notify_qp_reset || 228 !rdi->driver_f.notify_restart_rc) 229 return -EINVAL; 230 231 /* allocate parent object */ 232 rdi->qp_dev = kzalloc_node(sizeof(*rdi->qp_dev), GFP_KERNEL, 233 rdi->dparms.node); 234 if (!rdi->qp_dev) 235 return -ENOMEM; 236 237 /* allocate hash table */ 238 rdi->qp_dev->qp_table_size = rdi->dparms.qp_table_size; 239 rdi->qp_dev->qp_table_bits = ilog2(rdi->dparms.qp_table_size); 240 rdi->qp_dev->qp_table = 241 kmalloc_array_node(rdi->qp_dev->qp_table_size, 242 sizeof(*rdi->qp_dev->qp_table), 243 GFP_KERNEL, rdi->dparms.node); 244 if (!rdi->qp_dev->qp_table) 245 goto no_qp_table; 246 247 for (i = 0; i < rdi->qp_dev->qp_table_size; i++) 248 RCU_INIT_POINTER(rdi->qp_dev->qp_table[i], NULL); 249 250 spin_lock_init(&rdi->qp_dev->qpt_lock); 251 252 /* initialize qpn map */ 253 if (init_qpn_table(rdi, &rdi->qp_dev->qpn_table)) 254 goto fail_table; 255 256 spin_lock_init(&rdi->n_qps_lock); 257 258 return 0; 259 260 fail_table: 261 kfree(rdi->qp_dev->qp_table); 262 free_qpn_table(&rdi->qp_dev->qpn_table); 263 264 no_qp_table: 265 kfree(rdi->qp_dev); 266 267 return ret; 268 } 269 270 /** 271 * free_all_qps - check for QPs still in use 272 * @rdi: rvt device info structure 273 * 274 * There should not be any QPs still in use. 275 * Free memory for table. 276 */ 277 static unsigned rvt_free_all_qps(struct rvt_dev_info *rdi) 278 { 279 unsigned long flags; 280 struct rvt_qp *qp; 281 unsigned n, qp_inuse = 0; 282 spinlock_t *ql; /* work around too long line below */ 283 284 if (rdi->driver_f.free_all_qps) 285 qp_inuse = rdi->driver_f.free_all_qps(rdi); 286 287 qp_inuse += rvt_mcast_tree_empty(rdi); 288 289 if (!rdi->qp_dev) 290 return qp_inuse; 291 292 ql = &rdi->qp_dev->qpt_lock; 293 spin_lock_irqsave(ql, flags); 294 for (n = 0; n < rdi->qp_dev->qp_table_size; n++) { 295 qp = rcu_dereference_protected(rdi->qp_dev->qp_table[n], 296 lockdep_is_held(ql)); 297 RCU_INIT_POINTER(rdi->qp_dev->qp_table[n], NULL); 298 299 for (; qp; qp = rcu_dereference_protected(qp->next, 300 lockdep_is_held(ql))) 301 qp_inuse++; 302 } 303 spin_unlock_irqrestore(ql, flags); 304 synchronize_rcu(); 305 return qp_inuse; 306 } 307 308 /** 309 * rvt_qp_exit - clean up qps on device exit 310 * @rdi: rvt dev structure 311 * 312 * Check for qp leaks and free resources. 313 */ 314 void rvt_qp_exit(struct rvt_dev_info *rdi) 315 { 316 u32 qps_inuse = rvt_free_all_qps(rdi); 317 318 if (qps_inuse) 319 rvt_pr_err(rdi, "QP memory leak! %u still in use\n", 320 qps_inuse); 321 if (!rdi->qp_dev) 322 return; 323 324 kfree(rdi->qp_dev->qp_table); 325 free_qpn_table(&rdi->qp_dev->qpn_table); 326 kfree(rdi->qp_dev); 327 } 328 329 static inline unsigned mk_qpn(struct rvt_qpn_table *qpt, 330 struct rvt_qpn_map *map, unsigned off) 331 { 332 return (map - qpt->map) * RVT_BITS_PER_PAGE + off; 333 } 334 335 /** 336 * alloc_qpn - Allocate the next available qpn or zero/one for QP type 337 * IB_QPT_SMI/IB_QPT_GSI 338 * @rdi: rvt device info structure 339 * @qpt: queue pair number table pointer 340 * @port_num: IB port number, 1 based, comes from core 341 * 342 * Return: The queue pair number 343 */ 344 static int alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt, 345 enum ib_qp_type type, u8 port_num) 346 { 347 u32 i, offset, max_scan, qpn; 348 struct rvt_qpn_map *map; 349 u32 ret; 350 351 if (rdi->driver_f.alloc_qpn) 352 return rdi->driver_f.alloc_qpn(rdi, qpt, type, port_num); 353 354 if (type == IB_QPT_SMI || type == IB_QPT_GSI) { 355 unsigned n; 356 357 ret = type == IB_QPT_GSI; 358 n = 1 << (ret + 2 * (port_num - 1)); 359 spin_lock(&qpt->lock); 360 if (qpt->flags & n) 361 ret = -EINVAL; 362 else 363 qpt->flags |= n; 364 spin_unlock(&qpt->lock); 365 goto bail; 366 } 367 368 qpn = qpt->last + qpt->incr; 369 if (qpn >= RVT_QPN_MAX) 370 qpn = qpt->incr | ((qpt->last & 1) ^ 1); 371 /* offset carries bit 0 */ 372 offset = qpn & RVT_BITS_PER_PAGE_MASK; 373 map = &qpt->map[qpn / RVT_BITS_PER_PAGE]; 374 max_scan = qpt->nmaps - !offset; 375 for (i = 0;;) { 376 if (unlikely(!map->page)) { 377 get_map_page(qpt, map); 378 if (unlikely(!map->page)) 379 break; 380 } 381 do { 382 if (!test_and_set_bit(offset, map->page)) { 383 qpt->last = qpn; 384 ret = qpn; 385 goto bail; 386 } 387 offset += qpt->incr; 388 /* 389 * This qpn might be bogus if offset >= BITS_PER_PAGE. 390 * That is OK. It gets re-assigned below 391 */ 392 qpn = mk_qpn(qpt, map, offset); 393 } while (offset < RVT_BITS_PER_PAGE && qpn < RVT_QPN_MAX); 394 /* 395 * In order to keep the number of pages allocated to a 396 * minimum, we scan the all existing pages before increasing 397 * the size of the bitmap table. 398 */ 399 if (++i > max_scan) { 400 if (qpt->nmaps == RVT_QPNMAP_ENTRIES) 401 break; 402 map = &qpt->map[qpt->nmaps++]; 403 /* start at incr with current bit 0 */ 404 offset = qpt->incr | (offset & 1); 405 } else if (map < &qpt->map[qpt->nmaps]) { 406 ++map; 407 /* start at incr with current bit 0 */ 408 offset = qpt->incr | (offset & 1); 409 } else { 410 map = &qpt->map[0]; 411 /* wrap to first map page, invert bit 0 */ 412 offset = qpt->incr | ((offset & 1) ^ 1); 413 } 414 /* there can be no set bits in low-order QoS bits */ 415 WARN_ON(offset & (BIT(rdi->dparms.qos_shift) - 1)); 416 qpn = mk_qpn(qpt, map, offset); 417 } 418 419 ret = -ENOMEM; 420 421 bail: 422 return ret; 423 } 424 425 /** 426 * rvt_clear_mr_refs - Drop help mr refs 427 * @qp: rvt qp data structure 428 * @clr_sends: If shoudl clear send side or not 429 */ 430 static void rvt_clear_mr_refs(struct rvt_qp *qp, int clr_sends) 431 { 432 unsigned n; 433 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 434 435 if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) 436 rvt_put_ss(&qp->s_rdma_read_sge); 437 438 rvt_put_ss(&qp->r_sge); 439 440 if (clr_sends) { 441 while (qp->s_last != qp->s_head) { 442 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_last); 443 444 rvt_put_swqe(wqe); 445 446 if (qp->ibqp.qp_type == IB_QPT_UD || 447 qp->ibqp.qp_type == IB_QPT_SMI || 448 qp->ibqp.qp_type == IB_QPT_GSI) 449 atomic_dec(&ibah_to_rvtah( 450 wqe->ud_wr.ah)->refcount); 451 if (++qp->s_last >= qp->s_size) 452 qp->s_last = 0; 453 smp_wmb(); /* see qp_set_savail */ 454 } 455 if (qp->s_rdma_mr) { 456 rvt_put_mr(qp->s_rdma_mr); 457 qp->s_rdma_mr = NULL; 458 } 459 } 460 461 for (n = 0; qp->s_ack_queue && n < rvt_max_atomic(rdi); n++) { 462 struct rvt_ack_entry *e = &qp->s_ack_queue[n]; 463 464 if (e->rdma_sge.mr) { 465 rvt_put_mr(e->rdma_sge.mr); 466 e->rdma_sge.mr = NULL; 467 } 468 } 469 } 470 471 /** 472 * rvt_swqe_has_lkey - return true if lkey is used by swqe 473 * @wqe - the send wqe 474 * @lkey - the lkey 475 * 476 * Test the swqe for using lkey 477 */ 478 static bool rvt_swqe_has_lkey(struct rvt_swqe *wqe, u32 lkey) 479 { 480 int i; 481 482 for (i = 0; i < wqe->wr.num_sge; i++) { 483 struct rvt_sge *sge = &wqe->sg_list[i]; 484 485 if (rvt_mr_has_lkey(sge->mr, lkey)) 486 return true; 487 } 488 return false; 489 } 490 491 /** 492 * rvt_qp_sends_has_lkey - return true is qp sends use lkey 493 * @qp - the rvt_qp 494 * @lkey - the lkey 495 */ 496 static bool rvt_qp_sends_has_lkey(struct rvt_qp *qp, u32 lkey) 497 { 498 u32 s_last = qp->s_last; 499 500 while (s_last != qp->s_head) { 501 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, s_last); 502 503 if (rvt_swqe_has_lkey(wqe, lkey)) 504 return true; 505 506 if (++s_last >= qp->s_size) 507 s_last = 0; 508 } 509 if (qp->s_rdma_mr) 510 if (rvt_mr_has_lkey(qp->s_rdma_mr, lkey)) 511 return true; 512 return false; 513 } 514 515 /** 516 * rvt_qp_acks_has_lkey - return true if acks have lkey 517 * @qp - the qp 518 * @lkey - the lkey 519 */ 520 static bool rvt_qp_acks_has_lkey(struct rvt_qp *qp, u32 lkey) 521 { 522 int i; 523 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 524 525 for (i = 0; qp->s_ack_queue && i < rvt_max_atomic(rdi); i++) { 526 struct rvt_ack_entry *e = &qp->s_ack_queue[i]; 527 528 if (rvt_mr_has_lkey(e->rdma_sge.mr, lkey)) 529 return true; 530 } 531 return false; 532 } 533 534 /* 535 * rvt_qp_mr_clean - clean up remote ops for lkey 536 * @qp - the qp 537 * @lkey - the lkey that is being de-registered 538 * 539 * This routine checks if the lkey is being used by 540 * the qp. 541 * 542 * If so, the qp is put into an error state to elminate 543 * any references from the qp. 544 */ 545 void rvt_qp_mr_clean(struct rvt_qp *qp, u32 lkey) 546 { 547 bool lastwqe = false; 548 549 if (qp->ibqp.qp_type == IB_QPT_SMI || 550 qp->ibqp.qp_type == IB_QPT_GSI) 551 /* avoid special QPs */ 552 return; 553 spin_lock_irq(&qp->r_lock); 554 spin_lock(&qp->s_hlock); 555 spin_lock(&qp->s_lock); 556 557 if (qp->state == IB_QPS_ERR || qp->state == IB_QPS_RESET) 558 goto check_lwqe; 559 560 if (rvt_ss_has_lkey(&qp->r_sge, lkey) || 561 rvt_qp_sends_has_lkey(qp, lkey) || 562 rvt_qp_acks_has_lkey(qp, lkey)) 563 lastwqe = rvt_error_qp(qp, IB_WC_LOC_PROT_ERR); 564 check_lwqe: 565 spin_unlock(&qp->s_lock); 566 spin_unlock(&qp->s_hlock); 567 spin_unlock_irq(&qp->r_lock); 568 if (lastwqe) { 569 struct ib_event ev; 570 571 ev.device = qp->ibqp.device; 572 ev.element.qp = &qp->ibqp; 573 ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 574 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 575 } 576 } 577 578 /** 579 * rvt_remove_qp - remove qp form table 580 * @rdi: rvt dev struct 581 * @qp: qp to remove 582 * 583 * Remove the QP from the table so it can't be found asynchronously by 584 * the receive routine. 585 */ 586 static void rvt_remove_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp) 587 { 588 struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1]; 589 u32 n = hash_32(qp->ibqp.qp_num, rdi->qp_dev->qp_table_bits); 590 unsigned long flags; 591 int removed = 1; 592 593 spin_lock_irqsave(&rdi->qp_dev->qpt_lock, flags); 594 595 if (rcu_dereference_protected(rvp->qp[0], 596 lockdep_is_held(&rdi->qp_dev->qpt_lock)) == qp) { 597 RCU_INIT_POINTER(rvp->qp[0], NULL); 598 } else if (rcu_dereference_protected(rvp->qp[1], 599 lockdep_is_held(&rdi->qp_dev->qpt_lock)) == qp) { 600 RCU_INIT_POINTER(rvp->qp[1], NULL); 601 } else { 602 struct rvt_qp *q; 603 struct rvt_qp __rcu **qpp; 604 605 removed = 0; 606 qpp = &rdi->qp_dev->qp_table[n]; 607 for (; (q = rcu_dereference_protected(*qpp, 608 lockdep_is_held(&rdi->qp_dev->qpt_lock))) != NULL; 609 qpp = &q->next) { 610 if (q == qp) { 611 RCU_INIT_POINTER(*qpp, 612 rcu_dereference_protected(qp->next, 613 lockdep_is_held(&rdi->qp_dev->qpt_lock))); 614 removed = 1; 615 trace_rvt_qpremove(qp, n); 616 break; 617 } 618 } 619 } 620 621 spin_unlock_irqrestore(&rdi->qp_dev->qpt_lock, flags); 622 if (removed) { 623 synchronize_rcu(); 624 rvt_put_qp(qp); 625 } 626 } 627 628 /** 629 * rvt_init_qp - initialize the QP state to the reset state 630 * @qp: the QP to init or reinit 631 * @type: the QP type 632 * 633 * This function is called from both rvt_create_qp() and 634 * rvt_reset_qp(). The difference is that the reset 635 * patch the necessary locks to protect against concurent 636 * access. 637 */ 638 static void rvt_init_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, 639 enum ib_qp_type type) 640 { 641 qp->remote_qpn = 0; 642 qp->qkey = 0; 643 qp->qp_access_flags = 0; 644 qp->s_flags &= RVT_S_SIGNAL_REQ_WR; 645 qp->s_hdrwords = 0; 646 qp->s_wqe = NULL; 647 qp->s_draining = 0; 648 qp->s_next_psn = 0; 649 qp->s_last_psn = 0; 650 qp->s_sending_psn = 0; 651 qp->s_sending_hpsn = 0; 652 qp->s_psn = 0; 653 qp->r_psn = 0; 654 qp->r_msn = 0; 655 if (type == IB_QPT_RC) { 656 qp->s_state = IB_OPCODE_RC_SEND_LAST; 657 qp->r_state = IB_OPCODE_RC_SEND_LAST; 658 } else { 659 qp->s_state = IB_OPCODE_UC_SEND_LAST; 660 qp->r_state = IB_OPCODE_UC_SEND_LAST; 661 } 662 qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE; 663 qp->r_nak_state = 0; 664 qp->r_aflags = 0; 665 qp->r_flags = 0; 666 qp->s_head = 0; 667 qp->s_tail = 0; 668 qp->s_cur = 0; 669 qp->s_acked = 0; 670 qp->s_last = 0; 671 qp->s_ssn = 1; 672 qp->s_lsn = 0; 673 qp->s_mig_state = IB_MIG_MIGRATED; 674 qp->r_head_ack_queue = 0; 675 qp->s_tail_ack_queue = 0; 676 qp->s_num_rd_atomic = 0; 677 if (qp->r_rq.wq) { 678 qp->r_rq.wq->head = 0; 679 qp->r_rq.wq->tail = 0; 680 } 681 qp->r_sge.num_sge = 0; 682 atomic_set(&qp->s_reserved_used, 0); 683 } 684 685 /** 686 * rvt_reset_qp - initialize the QP state to the reset state 687 * @qp: the QP to reset 688 * @type: the QP type 689 * 690 * r_lock, s_hlock, and s_lock are required to be held by the caller 691 */ 692 static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, 693 enum ib_qp_type type) 694 __must_hold(&qp->s_lock) 695 __must_hold(&qp->s_hlock) 696 __must_hold(&qp->r_lock) 697 { 698 lockdep_assert_held(&qp->r_lock); 699 lockdep_assert_held(&qp->s_hlock); 700 lockdep_assert_held(&qp->s_lock); 701 if (qp->state != IB_QPS_RESET) { 702 qp->state = IB_QPS_RESET; 703 704 /* Let drivers flush their waitlist */ 705 rdi->driver_f.flush_qp_waiters(qp); 706 rvt_stop_rc_timers(qp); 707 qp->s_flags &= ~(RVT_S_TIMER | RVT_S_ANY_WAIT); 708 spin_unlock(&qp->s_lock); 709 spin_unlock(&qp->s_hlock); 710 spin_unlock_irq(&qp->r_lock); 711 712 /* Stop the send queue and the retry timer */ 713 rdi->driver_f.stop_send_queue(qp); 714 rvt_del_timers_sync(qp); 715 /* Wait for things to stop */ 716 rdi->driver_f.quiesce_qp(qp); 717 718 /* take qp out the hash and wait for it to be unused */ 719 rvt_remove_qp(rdi, qp); 720 721 /* grab the lock b/c it was locked at call time */ 722 spin_lock_irq(&qp->r_lock); 723 spin_lock(&qp->s_hlock); 724 spin_lock(&qp->s_lock); 725 726 rvt_clear_mr_refs(qp, 1); 727 /* 728 * Let the driver do any tear down or re-init it needs to for 729 * a qp that has been reset 730 */ 731 rdi->driver_f.notify_qp_reset(qp); 732 } 733 rvt_init_qp(rdi, qp, type); 734 lockdep_assert_held(&qp->r_lock); 735 lockdep_assert_held(&qp->s_hlock); 736 lockdep_assert_held(&qp->s_lock); 737 } 738 739 /** rvt_free_qpn - Free a qpn from the bit map 740 * @qpt: QP table 741 * @qpn: queue pair number to free 742 */ 743 static void rvt_free_qpn(struct rvt_qpn_table *qpt, u32 qpn) 744 { 745 struct rvt_qpn_map *map; 746 747 map = qpt->map + (qpn & RVT_QPN_MASK) / RVT_BITS_PER_PAGE; 748 if (map->page) 749 clear_bit(qpn & RVT_BITS_PER_PAGE_MASK, map->page); 750 } 751 752 /** 753 * rvt_create_qp - create a queue pair for a device 754 * @ibpd: the protection domain who's device we create the queue pair for 755 * @init_attr: the attributes of the queue pair 756 * @udata: user data for libibverbs.so 757 * 758 * Queue pair creation is mostly an rvt issue. However, drivers have their own 759 * unique idea of what queue pair numbers mean. For instance there is a reserved 760 * range for PSM. 761 * 762 * Return: the queue pair on success, otherwise returns an errno. 763 * 764 * Called by the ib_create_qp() core verbs function. 765 */ 766 struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, 767 struct ib_qp_init_attr *init_attr, 768 struct ib_udata *udata) 769 { 770 struct rvt_qp *qp; 771 int err; 772 struct rvt_swqe *swq = NULL; 773 size_t sz; 774 size_t sg_list_sz; 775 struct ib_qp *ret = ERR_PTR(-ENOMEM); 776 struct rvt_dev_info *rdi = ib_to_rvt(ibpd->device); 777 void *priv = NULL; 778 size_t sqsize; 779 780 if (!rdi) 781 return ERR_PTR(-EINVAL); 782 783 if (init_attr->cap.max_send_sge > rdi->dparms.props.max_sge || 784 init_attr->cap.max_send_wr > rdi->dparms.props.max_qp_wr || 785 init_attr->create_flags) 786 return ERR_PTR(-EINVAL); 787 788 /* Check receive queue parameters if no SRQ is specified. */ 789 if (!init_attr->srq) { 790 if (init_attr->cap.max_recv_sge > rdi->dparms.props.max_sge || 791 init_attr->cap.max_recv_wr > rdi->dparms.props.max_qp_wr) 792 return ERR_PTR(-EINVAL); 793 794 if (init_attr->cap.max_send_sge + 795 init_attr->cap.max_send_wr + 796 init_attr->cap.max_recv_sge + 797 init_attr->cap.max_recv_wr == 0) 798 return ERR_PTR(-EINVAL); 799 } 800 sqsize = 801 init_attr->cap.max_send_wr + 1 + 802 rdi->dparms.reserved_operations; 803 switch (init_attr->qp_type) { 804 case IB_QPT_SMI: 805 case IB_QPT_GSI: 806 if (init_attr->port_num == 0 || 807 init_attr->port_num > ibpd->device->phys_port_cnt) 808 return ERR_PTR(-EINVAL); 809 /* fall through */ 810 case IB_QPT_UC: 811 case IB_QPT_RC: 812 case IB_QPT_UD: 813 sz = sizeof(struct rvt_sge) * 814 init_attr->cap.max_send_sge + 815 sizeof(struct rvt_swqe); 816 swq = vzalloc_node(array_size(sz, sqsize), rdi->dparms.node); 817 if (!swq) 818 return ERR_PTR(-ENOMEM); 819 820 sz = sizeof(*qp); 821 sg_list_sz = 0; 822 if (init_attr->srq) { 823 struct rvt_srq *srq = ibsrq_to_rvtsrq(init_attr->srq); 824 825 if (srq->rq.max_sge > 1) 826 sg_list_sz = sizeof(*qp->r_sg_list) * 827 (srq->rq.max_sge - 1); 828 } else if (init_attr->cap.max_recv_sge > 1) 829 sg_list_sz = sizeof(*qp->r_sg_list) * 830 (init_attr->cap.max_recv_sge - 1); 831 qp = kzalloc_node(sz + sg_list_sz, GFP_KERNEL, 832 rdi->dparms.node); 833 if (!qp) 834 goto bail_swq; 835 836 RCU_INIT_POINTER(qp->next, NULL); 837 if (init_attr->qp_type == IB_QPT_RC) { 838 qp->s_ack_queue = 839 kcalloc_node(rvt_max_atomic(rdi), 840 sizeof(*qp->s_ack_queue), 841 GFP_KERNEL, 842 rdi->dparms.node); 843 if (!qp->s_ack_queue) 844 goto bail_qp; 845 } 846 /* initialize timers needed for rc qp */ 847 timer_setup(&qp->s_timer, rvt_rc_timeout, 0); 848 hrtimer_init(&qp->s_rnr_timer, CLOCK_MONOTONIC, 849 HRTIMER_MODE_REL); 850 qp->s_rnr_timer.function = rvt_rc_rnr_retry; 851 852 /* 853 * Driver needs to set up it's private QP structure and do any 854 * initialization that is needed. 855 */ 856 priv = rdi->driver_f.qp_priv_alloc(rdi, qp); 857 if (IS_ERR(priv)) { 858 ret = priv; 859 goto bail_qp; 860 } 861 qp->priv = priv; 862 qp->timeout_jiffies = 863 usecs_to_jiffies((4096UL * (1UL << qp->timeout)) / 864 1000UL); 865 if (init_attr->srq) { 866 sz = 0; 867 } else { 868 qp->r_rq.size = init_attr->cap.max_recv_wr + 1; 869 qp->r_rq.max_sge = init_attr->cap.max_recv_sge; 870 sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) + 871 sizeof(struct rvt_rwqe); 872 if (udata) 873 qp->r_rq.wq = vmalloc_user( 874 sizeof(struct rvt_rwq) + 875 qp->r_rq.size * sz); 876 else 877 qp->r_rq.wq = vzalloc_node( 878 sizeof(struct rvt_rwq) + 879 qp->r_rq.size * sz, 880 rdi->dparms.node); 881 if (!qp->r_rq.wq) 882 goto bail_driver_priv; 883 } 884 885 /* 886 * ib_create_qp() will initialize qp->ibqp 887 * except for qp->ibqp.qp_num. 888 */ 889 spin_lock_init(&qp->r_lock); 890 spin_lock_init(&qp->s_hlock); 891 spin_lock_init(&qp->s_lock); 892 spin_lock_init(&qp->r_rq.lock); 893 atomic_set(&qp->refcount, 0); 894 atomic_set(&qp->local_ops_pending, 0); 895 init_waitqueue_head(&qp->wait); 896 INIT_LIST_HEAD(&qp->rspwait); 897 qp->state = IB_QPS_RESET; 898 qp->s_wq = swq; 899 qp->s_size = sqsize; 900 qp->s_avail = init_attr->cap.max_send_wr; 901 qp->s_max_sge = init_attr->cap.max_send_sge; 902 if (init_attr->sq_sig_type == IB_SIGNAL_REQ_WR) 903 qp->s_flags = RVT_S_SIGNAL_REQ_WR; 904 905 err = alloc_qpn(rdi, &rdi->qp_dev->qpn_table, 906 init_attr->qp_type, 907 init_attr->port_num); 908 if (err < 0) { 909 ret = ERR_PTR(err); 910 goto bail_rq_wq; 911 } 912 qp->ibqp.qp_num = err; 913 qp->port_num = init_attr->port_num; 914 rvt_init_qp(rdi, qp, init_attr->qp_type); 915 break; 916 917 default: 918 /* Don't support raw QPs */ 919 return ERR_PTR(-EINVAL); 920 } 921 922 init_attr->cap.max_inline_data = 0; 923 924 /* 925 * Return the address of the RWQ as the offset to mmap. 926 * See rvt_mmap() for details. 927 */ 928 if (udata && udata->outlen >= sizeof(__u64)) { 929 if (!qp->r_rq.wq) { 930 __u64 offset = 0; 931 932 err = ib_copy_to_udata(udata, &offset, 933 sizeof(offset)); 934 if (err) { 935 ret = ERR_PTR(err); 936 goto bail_qpn; 937 } 938 } else { 939 u32 s = sizeof(struct rvt_rwq) + qp->r_rq.size * sz; 940 941 qp->ip = rvt_create_mmap_info(rdi, s, 942 ibpd->uobject->context, 943 qp->r_rq.wq); 944 if (!qp->ip) { 945 ret = ERR_PTR(-ENOMEM); 946 goto bail_qpn; 947 } 948 949 err = ib_copy_to_udata(udata, &qp->ip->offset, 950 sizeof(qp->ip->offset)); 951 if (err) { 952 ret = ERR_PTR(err); 953 goto bail_ip; 954 } 955 } 956 qp->pid = current->pid; 957 } 958 959 spin_lock(&rdi->n_qps_lock); 960 if (rdi->n_qps_allocated == rdi->dparms.props.max_qp) { 961 spin_unlock(&rdi->n_qps_lock); 962 ret = ERR_PTR(-ENOMEM); 963 goto bail_ip; 964 } 965 966 rdi->n_qps_allocated++; 967 /* 968 * Maintain a busy_jiffies variable that will be added to the timeout 969 * period in mod_retry_timer and add_retry_timer. This busy jiffies 970 * is scaled by the number of rc qps created for the device to reduce 971 * the number of timeouts occurring when there is a large number of 972 * qps. busy_jiffies is incremented every rc qp scaling interval. 973 * The scaling interval is selected based on extensive performance 974 * evaluation of targeted workloads. 975 */ 976 if (init_attr->qp_type == IB_QPT_RC) { 977 rdi->n_rc_qps++; 978 rdi->busy_jiffies = rdi->n_rc_qps / RC_QP_SCALING_INTERVAL; 979 } 980 spin_unlock(&rdi->n_qps_lock); 981 982 if (qp->ip) { 983 spin_lock_irq(&rdi->pending_lock); 984 list_add(&qp->ip->pending_mmaps, &rdi->pending_mmaps); 985 spin_unlock_irq(&rdi->pending_lock); 986 } 987 988 ret = &qp->ibqp; 989 990 /* 991 * We have our QP and its good, now keep track of what types of opcodes 992 * can be processed on this QP. We do this by keeping track of what the 993 * 3 high order bits of the opcode are. 994 */ 995 switch (init_attr->qp_type) { 996 case IB_QPT_SMI: 997 case IB_QPT_GSI: 998 case IB_QPT_UD: 999 qp->allowed_ops = IB_OPCODE_UD; 1000 break; 1001 case IB_QPT_RC: 1002 qp->allowed_ops = IB_OPCODE_RC; 1003 break; 1004 case IB_QPT_UC: 1005 qp->allowed_ops = IB_OPCODE_UC; 1006 break; 1007 default: 1008 ret = ERR_PTR(-EINVAL); 1009 goto bail_ip; 1010 } 1011 1012 return ret; 1013 1014 bail_ip: 1015 if (qp->ip) 1016 kref_put(&qp->ip->ref, rvt_release_mmap_info); 1017 1018 bail_qpn: 1019 rvt_free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num); 1020 1021 bail_rq_wq: 1022 if (!qp->ip) 1023 vfree(qp->r_rq.wq); 1024 1025 bail_driver_priv: 1026 rdi->driver_f.qp_priv_free(rdi, qp); 1027 1028 bail_qp: 1029 kfree(qp->s_ack_queue); 1030 kfree(qp); 1031 1032 bail_swq: 1033 vfree(swq); 1034 1035 return ret; 1036 } 1037 1038 /** 1039 * rvt_error_qp - put a QP into the error state 1040 * @qp: the QP to put into the error state 1041 * @err: the receive completion error to signal if a RWQE is active 1042 * 1043 * Flushes both send and receive work queues. 1044 * 1045 * Return: true if last WQE event should be generated. 1046 * The QP r_lock and s_lock should be held and interrupts disabled. 1047 * If we are already in error state, just return. 1048 */ 1049 int rvt_error_qp(struct rvt_qp *qp, enum ib_wc_status err) 1050 { 1051 struct ib_wc wc; 1052 int ret = 0; 1053 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 1054 1055 lockdep_assert_held(&qp->r_lock); 1056 lockdep_assert_held(&qp->s_lock); 1057 if (qp->state == IB_QPS_ERR || qp->state == IB_QPS_RESET) 1058 goto bail; 1059 1060 qp->state = IB_QPS_ERR; 1061 1062 if (qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR)) { 1063 qp->s_flags &= ~(RVT_S_TIMER | RVT_S_WAIT_RNR); 1064 del_timer(&qp->s_timer); 1065 } 1066 1067 if (qp->s_flags & RVT_S_ANY_WAIT_SEND) 1068 qp->s_flags &= ~RVT_S_ANY_WAIT_SEND; 1069 1070 rdi->driver_f.notify_error_qp(qp); 1071 1072 /* Schedule the sending tasklet to drain the send work queue. */ 1073 if (READ_ONCE(qp->s_last) != qp->s_head) 1074 rdi->driver_f.schedule_send(qp); 1075 1076 rvt_clear_mr_refs(qp, 0); 1077 1078 memset(&wc, 0, sizeof(wc)); 1079 wc.qp = &qp->ibqp; 1080 wc.opcode = IB_WC_RECV; 1081 1082 if (test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) { 1083 wc.wr_id = qp->r_wr_id; 1084 wc.status = err; 1085 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1); 1086 } 1087 wc.status = IB_WC_WR_FLUSH_ERR; 1088 1089 if (qp->r_rq.wq) { 1090 struct rvt_rwq *wq; 1091 u32 head; 1092 u32 tail; 1093 1094 spin_lock(&qp->r_rq.lock); 1095 1096 /* sanity check pointers before trusting them */ 1097 wq = qp->r_rq.wq; 1098 head = wq->head; 1099 if (head >= qp->r_rq.size) 1100 head = 0; 1101 tail = wq->tail; 1102 if (tail >= qp->r_rq.size) 1103 tail = 0; 1104 while (tail != head) { 1105 wc.wr_id = rvt_get_rwqe_ptr(&qp->r_rq, tail)->wr_id; 1106 if (++tail >= qp->r_rq.size) 1107 tail = 0; 1108 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1); 1109 } 1110 wq->tail = tail; 1111 1112 spin_unlock(&qp->r_rq.lock); 1113 } else if (qp->ibqp.event_handler) { 1114 ret = 1; 1115 } 1116 1117 bail: 1118 return ret; 1119 } 1120 EXPORT_SYMBOL(rvt_error_qp); 1121 1122 /* 1123 * Put the QP into the hash table. 1124 * The hash table holds a reference to the QP. 1125 */ 1126 static void rvt_insert_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp) 1127 { 1128 struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1]; 1129 unsigned long flags; 1130 1131 rvt_get_qp(qp); 1132 spin_lock_irqsave(&rdi->qp_dev->qpt_lock, flags); 1133 1134 if (qp->ibqp.qp_num <= 1) { 1135 rcu_assign_pointer(rvp->qp[qp->ibqp.qp_num], qp); 1136 } else { 1137 u32 n = hash_32(qp->ibqp.qp_num, rdi->qp_dev->qp_table_bits); 1138 1139 qp->next = rdi->qp_dev->qp_table[n]; 1140 rcu_assign_pointer(rdi->qp_dev->qp_table[n], qp); 1141 trace_rvt_qpinsert(qp, n); 1142 } 1143 1144 spin_unlock_irqrestore(&rdi->qp_dev->qpt_lock, flags); 1145 } 1146 1147 /** 1148 * rvt_modify_qp - modify the attributes of a queue pair 1149 * @ibqp: the queue pair who's attributes we're modifying 1150 * @attr: the new attributes 1151 * @attr_mask: the mask of attributes to modify 1152 * @udata: user data for libibverbs.so 1153 * 1154 * Return: 0 on success, otherwise returns an errno. 1155 */ 1156 int rvt_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1157 int attr_mask, struct ib_udata *udata) 1158 { 1159 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); 1160 struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); 1161 enum ib_qp_state cur_state, new_state; 1162 struct ib_event ev; 1163 int lastwqe = 0; 1164 int mig = 0; 1165 int pmtu = 0; /* for gcc warning only */ 1166 enum rdma_link_layer link; 1167 int opa_ah; 1168 1169 link = rdma_port_get_link_layer(ibqp->device, qp->port_num); 1170 1171 spin_lock_irq(&qp->r_lock); 1172 spin_lock(&qp->s_hlock); 1173 spin_lock(&qp->s_lock); 1174 1175 cur_state = attr_mask & IB_QP_CUR_STATE ? 1176 attr->cur_qp_state : qp->state; 1177 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; 1178 opa_ah = rdma_cap_opa_ah(ibqp->device, qp->port_num); 1179 1180 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, 1181 attr_mask, link)) 1182 goto inval; 1183 1184 if (rdi->driver_f.check_modify_qp && 1185 rdi->driver_f.check_modify_qp(qp, attr, attr_mask, udata)) 1186 goto inval; 1187 1188 if (attr_mask & IB_QP_AV) { 1189 if (opa_ah) { 1190 if (rdma_ah_get_dlid(&attr->ah_attr) >= 1191 opa_get_mcast_base(OPA_MCAST_NR)) 1192 goto inval; 1193 } else { 1194 if (rdma_ah_get_dlid(&attr->ah_attr) >= 1195 be16_to_cpu(IB_MULTICAST_LID_BASE)) 1196 goto inval; 1197 } 1198 1199 if (rvt_check_ah(qp->ibqp.device, &attr->ah_attr)) 1200 goto inval; 1201 } 1202 1203 if (attr_mask & IB_QP_ALT_PATH) { 1204 if (opa_ah) { 1205 if (rdma_ah_get_dlid(&attr->alt_ah_attr) >= 1206 opa_get_mcast_base(OPA_MCAST_NR)) 1207 goto inval; 1208 } else { 1209 if (rdma_ah_get_dlid(&attr->alt_ah_attr) >= 1210 be16_to_cpu(IB_MULTICAST_LID_BASE)) 1211 goto inval; 1212 } 1213 1214 if (rvt_check_ah(qp->ibqp.device, &attr->alt_ah_attr)) 1215 goto inval; 1216 if (attr->alt_pkey_index >= rvt_get_npkeys(rdi)) 1217 goto inval; 1218 } 1219 1220 if (attr_mask & IB_QP_PKEY_INDEX) 1221 if (attr->pkey_index >= rvt_get_npkeys(rdi)) 1222 goto inval; 1223 1224 if (attr_mask & IB_QP_MIN_RNR_TIMER) 1225 if (attr->min_rnr_timer > 31) 1226 goto inval; 1227 1228 if (attr_mask & IB_QP_PORT) 1229 if (qp->ibqp.qp_type == IB_QPT_SMI || 1230 qp->ibqp.qp_type == IB_QPT_GSI || 1231 attr->port_num == 0 || 1232 attr->port_num > ibqp->device->phys_port_cnt) 1233 goto inval; 1234 1235 if (attr_mask & IB_QP_DEST_QPN) 1236 if (attr->dest_qp_num > RVT_QPN_MASK) 1237 goto inval; 1238 1239 if (attr_mask & IB_QP_RETRY_CNT) 1240 if (attr->retry_cnt > 7) 1241 goto inval; 1242 1243 if (attr_mask & IB_QP_RNR_RETRY) 1244 if (attr->rnr_retry > 7) 1245 goto inval; 1246 1247 /* 1248 * Don't allow invalid path_mtu values. OK to set greater 1249 * than the active mtu (or even the max_cap, if we have tuned 1250 * that to a small mtu. We'll set qp->path_mtu 1251 * to the lesser of requested attribute mtu and active, 1252 * for packetizing messages. 1253 * Note that the QP port has to be set in INIT and MTU in RTR. 1254 */ 1255 if (attr_mask & IB_QP_PATH_MTU) { 1256 pmtu = rdi->driver_f.get_pmtu_from_attr(rdi, qp, attr); 1257 if (pmtu < 0) 1258 goto inval; 1259 } 1260 1261 if (attr_mask & IB_QP_PATH_MIG_STATE) { 1262 if (attr->path_mig_state == IB_MIG_REARM) { 1263 if (qp->s_mig_state == IB_MIG_ARMED) 1264 goto inval; 1265 if (new_state != IB_QPS_RTS) 1266 goto inval; 1267 } else if (attr->path_mig_state == IB_MIG_MIGRATED) { 1268 if (qp->s_mig_state == IB_MIG_REARM) 1269 goto inval; 1270 if (new_state != IB_QPS_RTS && new_state != IB_QPS_SQD) 1271 goto inval; 1272 if (qp->s_mig_state == IB_MIG_ARMED) 1273 mig = 1; 1274 } else { 1275 goto inval; 1276 } 1277 } 1278 1279 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 1280 if (attr->max_dest_rd_atomic > rdi->dparms.max_rdma_atomic) 1281 goto inval; 1282 1283 switch (new_state) { 1284 case IB_QPS_RESET: 1285 if (qp->state != IB_QPS_RESET) 1286 rvt_reset_qp(rdi, qp, ibqp->qp_type); 1287 break; 1288 1289 case IB_QPS_RTR: 1290 /* Allow event to re-trigger if QP set to RTR more than once */ 1291 qp->r_flags &= ~RVT_R_COMM_EST; 1292 qp->state = new_state; 1293 break; 1294 1295 case IB_QPS_SQD: 1296 qp->s_draining = qp->s_last != qp->s_cur; 1297 qp->state = new_state; 1298 break; 1299 1300 case IB_QPS_SQE: 1301 if (qp->ibqp.qp_type == IB_QPT_RC) 1302 goto inval; 1303 qp->state = new_state; 1304 break; 1305 1306 case IB_QPS_ERR: 1307 lastwqe = rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); 1308 break; 1309 1310 default: 1311 qp->state = new_state; 1312 break; 1313 } 1314 1315 if (attr_mask & IB_QP_PKEY_INDEX) 1316 qp->s_pkey_index = attr->pkey_index; 1317 1318 if (attr_mask & IB_QP_PORT) 1319 qp->port_num = attr->port_num; 1320 1321 if (attr_mask & IB_QP_DEST_QPN) 1322 qp->remote_qpn = attr->dest_qp_num; 1323 1324 if (attr_mask & IB_QP_SQ_PSN) { 1325 qp->s_next_psn = attr->sq_psn & rdi->dparms.psn_modify_mask; 1326 qp->s_psn = qp->s_next_psn; 1327 qp->s_sending_psn = qp->s_next_psn; 1328 qp->s_last_psn = qp->s_next_psn - 1; 1329 qp->s_sending_hpsn = qp->s_last_psn; 1330 } 1331 1332 if (attr_mask & IB_QP_RQ_PSN) 1333 qp->r_psn = attr->rq_psn & rdi->dparms.psn_modify_mask; 1334 1335 if (attr_mask & IB_QP_ACCESS_FLAGS) 1336 qp->qp_access_flags = attr->qp_access_flags; 1337 1338 if (attr_mask & IB_QP_AV) { 1339 qp->remote_ah_attr = attr->ah_attr; 1340 qp->s_srate = rdma_ah_get_static_rate(&attr->ah_attr); 1341 qp->srate_mbps = ib_rate_to_mbps(qp->s_srate); 1342 } 1343 1344 if (attr_mask & IB_QP_ALT_PATH) { 1345 qp->alt_ah_attr = attr->alt_ah_attr; 1346 qp->s_alt_pkey_index = attr->alt_pkey_index; 1347 } 1348 1349 if (attr_mask & IB_QP_PATH_MIG_STATE) { 1350 qp->s_mig_state = attr->path_mig_state; 1351 if (mig) { 1352 qp->remote_ah_attr = qp->alt_ah_attr; 1353 qp->port_num = rdma_ah_get_port_num(&qp->alt_ah_attr); 1354 qp->s_pkey_index = qp->s_alt_pkey_index; 1355 } 1356 } 1357 1358 if (attr_mask & IB_QP_PATH_MTU) { 1359 qp->pmtu = rdi->driver_f.mtu_from_qp(rdi, qp, pmtu); 1360 qp->log_pmtu = ilog2(qp->pmtu); 1361 } 1362 1363 if (attr_mask & IB_QP_RETRY_CNT) { 1364 qp->s_retry_cnt = attr->retry_cnt; 1365 qp->s_retry = attr->retry_cnt; 1366 } 1367 1368 if (attr_mask & IB_QP_RNR_RETRY) { 1369 qp->s_rnr_retry_cnt = attr->rnr_retry; 1370 qp->s_rnr_retry = attr->rnr_retry; 1371 } 1372 1373 if (attr_mask & IB_QP_MIN_RNR_TIMER) 1374 qp->r_min_rnr_timer = attr->min_rnr_timer; 1375 1376 if (attr_mask & IB_QP_TIMEOUT) { 1377 qp->timeout = attr->timeout; 1378 qp->timeout_jiffies = rvt_timeout_to_jiffies(qp->timeout); 1379 } 1380 1381 if (attr_mask & IB_QP_QKEY) 1382 qp->qkey = attr->qkey; 1383 1384 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 1385 qp->r_max_rd_atomic = attr->max_dest_rd_atomic; 1386 1387 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) 1388 qp->s_max_rd_atomic = attr->max_rd_atomic; 1389 1390 if (rdi->driver_f.modify_qp) 1391 rdi->driver_f.modify_qp(qp, attr, attr_mask, udata); 1392 1393 spin_unlock(&qp->s_lock); 1394 spin_unlock(&qp->s_hlock); 1395 spin_unlock_irq(&qp->r_lock); 1396 1397 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) 1398 rvt_insert_qp(rdi, qp); 1399 1400 if (lastwqe) { 1401 ev.device = qp->ibqp.device; 1402 ev.element.qp = &qp->ibqp; 1403 ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 1404 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 1405 } 1406 if (mig) { 1407 ev.device = qp->ibqp.device; 1408 ev.element.qp = &qp->ibqp; 1409 ev.event = IB_EVENT_PATH_MIG; 1410 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 1411 } 1412 return 0; 1413 1414 inval: 1415 spin_unlock(&qp->s_lock); 1416 spin_unlock(&qp->s_hlock); 1417 spin_unlock_irq(&qp->r_lock); 1418 return -EINVAL; 1419 } 1420 1421 /** 1422 * rvt_destroy_qp - destroy a queue pair 1423 * @ibqp: the queue pair to destroy 1424 * 1425 * Note that this can be called while the QP is actively sending or 1426 * receiving! 1427 * 1428 * Return: 0 on success. 1429 */ 1430 int rvt_destroy_qp(struct ib_qp *ibqp) 1431 { 1432 struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); 1433 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); 1434 1435 spin_lock_irq(&qp->r_lock); 1436 spin_lock(&qp->s_hlock); 1437 spin_lock(&qp->s_lock); 1438 rvt_reset_qp(rdi, qp, ibqp->qp_type); 1439 spin_unlock(&qp->s_lock); 1440 spin_unlock(&qp->s_hlock); 1441 spin_unlock_irq(&qp->r_lock); 1442 1443 wait_event(qp->wait, !atomic_read(&qp->refcount)); 1444 /* qpn is now available for use again */ 1445 rvt_free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num); 1446 1447 spin_lock(&rdi->n_qps_lock); 1448 rdi->n_qps_allocated--; 1449 if (qp->ibqp.qp_type == IB_QPT_RC) { 1450 rdi->n_rc_qps--; 1451 rdi->busy_jiffies = rdi->n_rc_qps / RC_QP_SCALING_INTERVAL; 1452 } 1453 spin_unlock(&rdi->n_qps_lock); 1454 1455 if (qp->ip) 1456 kref_put(&qp->ip->ref, rvt_release_mmap_info); 1457 else 1458 vfree(qp->r_rq.wq); 1459 vfree(qp->s_wq); 1460 rdi->driver_f.qp_priv_free(rdi, qp); 1461 kfree(qp->s_ack_queue); 1462 kfree(qp); 1463 return 0; 1464 } 1465 1466 /** 1467 * rvt_query_qp - query an ipbq 1468 * @ibqp: IB qp to query 1469 * @attr: attr struct to fill in 1470 * @attr_mask: attr mask ignored 1471 * @init_attr: struct to fill in 1472 * 1473 * Return: always 0 1474 */ 1475 int rvt_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1476 int attr_mask, struct ib_qp_init_attr *init_attr) 1477 { 1478 struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); 1479 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); 1480 1481 attr->qp_state = qp->state; 1482 attr->cur_qp_state = attr->qp_state; 1483 attr->path_mtu = rdi->driver_f.mtu_to_path_mtu(qp->pmtu); 1484 attr->path_mig_state = qp->s_mig_state; 1485 attr->qkey = qp->qkey; 1486 attr->rq_psn = qp->r_psn & rdi->dparms.psn_mask; 1487 attr->sq_psn = qp->s_next_psn & rdi->dparms.psn_mask; 1488 attr->dest_qp_num = qp->remote_qpn; 1489 attr->qp_access_flags = qp->qp_access_flags; 1490 attr->cap.max_send_wr = qp->s_size - 1 - 1491 rdi->dparms.reserved_operations; 1492 attr->cap.max_recv_wr = qp->ibqp.srq ? 0 : qp->r_rq.size - 1; 1493 attr->cap.max_send_sge = qp->s_max_sge; 1494 attr->cap.max_recv_sge = qp->r_rq.max_sge; 1495 attr->cap.max_inline_data = 0; 1496 attr->ah_attr = qp->remote_ah_attr; 1497 attr->alt_ah_attr = qp->alt_ah_attr; 1498 attr->pkey_index = qp->s_pkey_index; 1499 attr->alt_pkey_index = qp->s_alt_pkey_index; 1500 attr->en_sqd_async_notify = 0; 1501 attr->sq_draining = qp->s_draining; 1502 attr->max_rd_atomic = qp->s_max_rd_atomic; 1503 attr->max_dest_rd_atomic = qp->r_max_rd_atomic; 1504 attr->min_rnr_timer = qp->r_min_rnr_timer; 1505 attr->port_num = qp->port_num; 1506 attr->timeout = qp->timeout; 1507 attr->retry_cnt = qp->s_retry_cnt; 1508 attr->rnr_retry = qp->s_rnr_retry_cnt; 1509 attr->alt_port_num = 1510 rdma_ah_get_port_num(&qp->alt_ah_attr); 1511 attr->alt_timeout = qp->alt_timeout; 1512 1513 init_attr->event_handler = qp->ibqp.event_handler; 1514 init_attr->qp_context = qp->ibqp.qp_context; 1515 init_attr->send_cq = qp->ibqp.send_cq; 1516 init_attr->recv_cq = qp->ibqp.recv_cq; 1517 init_attr->srq = qp->ibqp.srq; 1518 init_attr->cap = attr->cap; 1519 if (qp->s_flags & RVT_S_SIGNAL_REQ_WR) 1520 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR; 1521 else 1522 init_attr->sq_sig_type = IB_SIGNAL_ALL_WR; 1523 init_attr->qp_type = qp->ibqp.qp_type; 1524 init_attr->port_num = qp->port_num; 1525 return 0; 1526 } 1527 1528 /** 1529 * rvt_post_receive - post a receive on a QP 1530 * @ibqp: the QP to post the receive on 1531 * @wr: the WR to post 1532 * @bad_wr: the first bad WR is put here 1533 * 1534 * This may be called from interrupt context. 1535 * 1536 * Return: 0 on success otherwise errno 1537 */ 1538 int rvt_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, 1539 struct ib_recv_wr **bad_wr) 1540 { 1541 struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); 1542 struct rvt_rwq *wq = qp->r_rq.wq; 1543 unsigned long flags; 1544 int qp_err_flush = (ib_rvt_state_ops[qp->state] & RVT_FLUSH_RECV) && 1545 !qp->ibqp.srq; 1546 1547 /* Check that state is OK to post receive. */ 1548 if (!(ib_rvt_state_ops[qp->state] & RVT_POST_RECV_OK) || !wq) { 1549 *bad_wr = wr; 1550 return -EINVAL; 1551 } 1552 1553 for (; wr; wr = wr->next) { 1554 struct rvt_rwqe *wqe; 1555 u32 next; 1556 int i; 1557 1558 if ((unsigned)wr->num_sge > qp->r_rq.max_sge) { 1559 *bad_wr = wr; 1560 return -EINVAL; 1561 } 1562 1563 spin_lock_irqsave(&qp->r_rq.lock, flags); 1564 next = wq->head + 1; 1565 if (next >= qp->r_rq.size) 1566 next = 0; 1567 if (next == wq->tail) { 1568 spin_unlock_irqrestore(&qp->r_rq.lock, flags); 1569 *bad_wr = wr; 1570 return -ENOMEM; 1571 } 1572 if (unlikely(qp_err_flush)) { 1573 struct ib_wc wc; 1574 1575 memset(&wc, 0, sizeof(wc)); 1576 wc.qp = &qp->ibqp; 1577 wc.opcode = IB_WC_RECV; 1578 wc.wr_id = wr->wr_id; 1579 wc.status = IB_WC_WR_FLUSH_ERR; 1580 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1); 1581 } else { 1582 wqe = rvt_get_rwqe_ptr(&qp->r_rq, wq->head); 1583 wqe->wr_id = wr->wr_id; 1584 wqe->num_sge = wr->num_sge; 1585 for (i = 0; i < wr->num_sge; i++) 1586 wqe->sg_list[i] = wr->sg_list[i]; 1587 /* 1588 * Make sure queue entry is written 1589 * before the head index. 1590 */ 1591 smp_wmb(); 1592 wq->head = next; 1593 } 1594 spin_unlock_irqrestore(&qp->r_rq.lock, flags); 1595 } 1596 return 0; 1597 } 1598 1599 /** 1600 * rvt_qp_valid_operation - validate post send wr request 1601 * @qp - the qp 1602 * @post-parms - the post send table for the driver 1603 * @wr - the work request 1604 * 1605 * The routine validates the operation based on the 1606 * validation table an returns the length of the operation 1607 * which can extend beyond the ib_send_bw. Operation 1608 * dependent flags key atomic operation validation. 1609 * 1610 * There is an exception for UD qps that validates the pd and 1611 * overrides the length to include the additional UD specific 1612 * length. 1613 * 1614 * Returns a negative error or the length of the work request 1615 * for building the swqe. 1616 */ 1617 static inline int rvt_qp_valid_operation( 1618 struct rvt_qp *qp, 1619 const struct rvt_operation_params *post_parms, 1620 struct ib_send_wr *wr) 1621 { 1622 int len; 1623 1624 if (wr->opcode >= RVT_OPERATION_MAX || !post_parms[wr->opcode].length) 1625 return -EINVAL; 1626 if (!(post_parms[wr->opcode].qpt_support & BIT(qp->ibqp.qp_type))) 1627 return -EINVAL; 1628 if ((post_parms[wr->opcode].flags & RVT_OPERATION_PRIV) && 1629 ibpd_to_rvtpd(qp->ibqp.pd)->user) 1630 return -EINVAL; 1631 if (post_parms[wr->opcode].flags & RVT_OPERATION_ATOMIC_SGE && 1632 (wr->num_sge == 0 || 1633 wr->sg_list[0].length < sizeof(u64) || 1634 wr->sg_list[0].addr & (sizeof(u64) - 1))) 1635 return -EINVAL; 1636 if (post_parms[wr->opcode].flags & RVT_OPERATION_ATOMIC && 1637 !qp->s_max_rd_atomic) 1638 return -EINVAL; 1639 len = post_parms[wr->opcode].length; 1640 /* UD specific */ 1641 if (qp->ibqp.qp_type != IB_QPT_UC && 1642 qp->ibqp.qp_type != IB_QPT_RC) { 1643 if (qp->ibqp.pd != ud_wr(wr)->ah->pd) 1644 return -EINVAL; 1645 len = sizeof(struct ib_ud_wr); 1646 } 1647 return len; 1648 } 1649 1650 /** 1651 * rvt_qp_is_avail - determine queue capacity 1652 * @qp: the qp 1653 * @rdi: the rdmavt device 1654 * @reserved_op: is reserved operation 1655 * 1656 * This assumes the s_hlock is held but the s_last 1657 * qp variable is uncontrolled. 1658 * 1659 * For non reserved operations, the qp->s_avail 1660 * may be changed. 1661 * 1662 * The return value is zero or a -ENOMEM. 1663 */ 1664 static inline int rvt_qp_is_avail( 1665 struct rvt_qp *qp, 1666 struct rvt_dev_info *rdi, 1667 bool reserved_op) 1668 { 1669 u32 slast; 1670 u32 avail; 1671 u32 reserved_used; 1672 1673 /* see rvt_qp_wqe_unreserve() */ 1674 smp_mb__before_atomic(); 1675 reserved_used = atomic_read(&qp->s_reserved_used); 1676 if (unlikely(reserved_op)) { 1677 /* see rvt_qp_wqe_unreserve() */ 1678 smp_mb__before_atomic(); 1679 if (reserved_used >= rdi->dparms.reserved_operations) 1680 return -ENOMEM; 1681 return 0; 1682 } 1683 /* non-reserved operations */ 1684 if (likely(qp->s_avail)) 1685 return 0; 1686 slast = READ_ONCE(qp->s_last); 1687 if (qp->s_head >= slast) 1688 avail = qp->s_size - (qp->s_head - slast); 1689 else 1690 avail = slast - qp->s_head; 1691 1692 /* see rvt_qp_wqe_unreserve() */ 1693 smp_mb__before_atomic(); 1694 reserved_used = atomic_read(&qp->s_reserved_used); 1695 avail = avail - 1 - 1696 (rdi->dparms.reserved_operations - reserved_used); 1697 /* insure we don't assign a negative s_avail */ 1698 if ((s32)avail <= 0) 1699 return -ENOMEM; 1700 qp->s_avail = avail; 1701 if (WARN_ON(qp->s_avail > 1702 (qp->s_size - 1 - rdi->dparms.reserved_operations))) 1703 rvt_pr_err(rdi, 1704 "More avail entries than QP RB size.\nQP: %u, size: %u, avail: %u\nhead: %u, tail: %u, cur: %u, acked: %u, last: %u", 1705 qp->ibqp.qp_num, qp->s_size, qp->s_avail, 1706 qp->s_head, qp->s_tail, qp->s_cur, 1707 qp->s_acked, qp->s_last); 1708 return 0; 1709 } 1710 1711 /** 1712 * rvt_post_one_wr - post one RC, UC, or UD send work request 1713 * @qp: the QP to post on 1714 * @wr: the work request to send 1715 */ 1716 static int rvt_post_one_wr(struct rvt_qp *qp, 1717 struct ib_send_wr *wr, 1718 int *call_send) 1719 { 1720 struct rvt_swqe *wqe; 1721 u32 next; 1722 int i; 1723 int j; 1724 int acc; 1725 struct rvt_lkey_table *rkt; 1726 struct rvt_pd *pd; 1727 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 1728 u8 log_pmtu; 1729 int ret; 1730 size_t cplen; 1731 bool reserved_op; 1732 int local_ops_delayed = 0; 1733 1734 BUILD_BUG_ON(IB_QPT_MAX >= (sizeof(u32) * BITS_PER_BYTE)); 1735 1736 /* IB spec says that num_sge == 0 is OK. */ 1737 if (unlikely(wr->num_sge > qp->s_max_sge)) 1738 return -EINVAL; 1739 1740 ret = rvt_qp_valid_operation(qp, rdi->post_parms, wr); 1741 if (ret < 0) 1742 return ret; 1743 cplen = ret; 1744 1745 /* 1746 * Local operations include fast register and local invalidate. 1747 * Fast register needs to be processed immediately because the 1748 * registered lkey may be used by following work requests and the 1749 * lkey needs to be valid at the time those requests are posted. 1750 * Local invalidate can be processed immediately if fencing is 1751 * not required and no previous local invalidate ops are pending. 1752 * Signaled local operations that have been processed immediately 1753 * need to have requests with "completion only" flags set posted 1754 * to the send queue in order to generate completions. 1755 */ 1756 if ((rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL)) { 1757 switch (wr->opcode) { 1758 case IB_WR_REG_MR: 1759 ret = rvt_fast_reg_mr(qp, 1760 reg_wr(wr)->mr, 1761 reg_wr(wr)->key, 1762 reg_wr(wr)->access); 1763 if (ret || !(wr->send_flags & IB_SEND_SIGNALED)) 1764 return ret; 1765 break; 1766 case IB_WR_LOCAL_INV: 1767 if ((wr->send_flags & IB_SEND_FENCE) || 1768 atomic_read(&qp->local_ops_pending)) { 1769 local_ops_delayed = 1; 1770 } else { 1771 ret = rvt_invalidate_rkey( 1772 qp, wr->ex.invalidate_rkey); 1773 if (ret || !(wr->send_flags & IB_SEND_SIGNALED)) 1774 return ret; 1775 } 1776 break; 1777 default: 1778 return -EINVAL; 1779 } 1780 } 1781 1782 reserved_op = rdi->post_parms[wr->opcode].flags & 1783 RVT_OPERATION_USE_RESERVE; 1784 /* check for avail */ 1785 ret = rvt_qp_is_avail(qp, rdi, reserved_op); 1786 if (ret) 1787 return ret; 1788 next = qp->s_head + 1; 1789 if (next >= qp->s_size) 1790 next = 0; 1791 1792 rkt = &rdi->lkey_table; 1793 pd = ibpd_to_rvtpd(qp->ibqp.pd); 1794 wqe = rvt_get_swqe_ptr(qp, qp->s_head); 1795 1796 /* cplen has length from above */ 1797 memcpy(&wqe->wr, wr, cplen); 1798 1799 wqe->length = 0; 1800 j = 0; 1801 if (wr->num_sge) { 1802 struct rvt_sge *last_sge = NULL; 1803 1804 acc = wr->opcode >= IB_WR_RDMA_READ ? 1805 IB_ACCESS_LOCAL_WRITE : 0; 1806 for (i = 0; i < wr->num_sge; i++) { 1807 u32 length = wr->sg_list[i].length; 1808 1809 if (length == 0) 1810 continue; 1811 ret = rvt_lkey_ok(rkt, pd, &wqe->sg_list[j], last_sge, 1812 &wr->sg_list[i], acc); 1813 if (unlikely(ret < 0)) 1814 goto bail_inval_free; 1815 wqe->length += length; 1816 if (ret) 1817 last_sge = &wqe->sg_list[j]; 1818 j += ret; 1819 } 1820 wqe->wr.num_sge = j; 1821 } 1822 1823 /* general part of wqe valid - allow for driver checks */ 1824 if (rdi->driver_f.check_send_wqe) { 1825 ret = rdi->driver_f.check_send_wqe(qp, wqe); 1826 if (ret < 0) 1827 goto bail_inval_free; 1828 if (ret) 1829 *call_send = ret; 1830 } 1831 1832 log_pmtu = qp->log_pmtu; 1833 if (qp->ibqp.qp_type != IB_QPT_UC && 1834 qp->ibqp.qp_type != IB_QPT_RC) { 1835 struct rvt_ah *ah = ibah_to_rvtah(wqe->ud_wr.ah); 1836 1837 log_pmtu = ah->log_pmtu; 1838 atomic_inc(&ibah_to_rvtah(ud_wr(wr)->ah)->refcount); 1839 } 1840 1841 if (rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL) { 1842 if (local_ops_delayed) 1843 atomic_inc(&qp->local_ops_pending); 1844 else 1845 wqe->wr.send_flags |= RVT_SEND_COMPLETION_ONLY; 1846 wqe->ssn = 0; 1847 wqe->psn = 0; 1848 wqe->lpsn = 0; 1849 } else { 1850 wqe->ssn = qp->s_ssn++; 1851 wqe->psn = qp->s_next_psn; 1852 wqe->lpsn = wqe->psn + 1853 (wqe->length ? 1854 ((wqe->length - 1) >> log_pmtu) : 1855 0); 1856 qp->s_next_psn = wqe->lpsn + 1; 1857 } 1858 if (unlikely(reserved_op)) { 1859 wqe->wr.send_flags |= RVT_SEND_RESERVE_USED; 1860 rvt_qp_wqe_reserve(qp, wqe); 1861 } else { 1862 wqe->wr.send_flags &= ~RVT_SEND_RESERVE_USED; 1863 qp->s_avail--; 1864 } 1865 trace_rvt_post_one_wr(qp, wqe, wr->num_sge); 1866 smp_wmb(); /* see request builders */ 1867 qp->s_head = next; 1868 1869 return 0; 1870 1871 bail_inval_free: 1872 /* release mr holds */ 1873 while (j) { 1874 struct rvt_sge *sge = &wqe->sg_list[--j]; 1875 1876 rvt_put_mr(sge->mr); 1877 } 1878 return ret; 1879 } 1880 1881 /** 1882 * rvt_post_send - post a send on a QP 1883 * @ibqp: the QP to post the send on 1884 * @wr: the list of work requests to post 1885 * @bad_wr: the first bad WR is put here 1886 * 1887 * This may be called from interrupt context. 1888 * 1889 * Return: 0 on success else errno 1890 */ 1891 int rvt_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 1892 struct ib_send_wr **bad_wr) 1893 { 1894 struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); 1895 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); 1896 unsigned long flags = 0; 1897 int call_send; 1898 unsigned nreq = 0; 1899 int err = 0; 1900 1901 spin_lock_irqsave(&qp->s_hlock, flags); 1902 1903 /* 1904 * Ensure QP state is such that we can send. If not bail out early, 1905 * there is no need to do this every time we post a send. 1906 */ 1907 if (unlikely(!(ib_rvt_state_ops[qp->state] & RVT_POST_SEND_OK))) { 1908 spin_unlock_irqrestore(&qp->s_hlock, flags); 1909 return -EINVAL; 1910 } 1911 1912 /* 1913 * If the send queue is empty, and we only have a single WR then just go 1914 * ahead and kick the send engine into gear. Otherwise we will always 1915 * just schedule the send to happen later. 1916 */ 1917 call_send = qp->s_head == READ_ONCE(qp->s_last) && !wr->next; 1918 1919 for (; wr; wr = wr->next) { 1920 err = rvt_post_one_wr(qp, wr, &call_send); 1921 if (unlikely(err)) { 1922 *bad_wr = wr; 1923 goto bail; 1924 } 1925 nreq++; 1926 } 1927 bail: 1928 spin_unlock_irqrestore(&qp->s_hlock, flags); 1929 if (nreq) { 1930 if (call_send) 1931 rdi->driver_f.do_send(qp); 1932 else 1933 rdi->driver_f.schedule_send_no_lock(qp); 1934 } 1935 return err; 1936 } 1937 1938 /** 1939 * rvt_post_srq_receive - post a receive on a shared receive queue 1940 * @ibsrq: the SRQ to post the receive on 1941 * @wr: the list of work requests to post 1942 * @bad_wr: A pointer to the first WR to cause a problem is put here 1943 * 1944 * This may be called from interrupt context. 1945 * 1946 * Return: 0 on success else errno 1947 */ 1948 int rvt_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, 1949 struct ib_recv_wr **bad_wr) 1950 { 1951 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); 1952 struct rvt_rwq *wq; 1953 unsigned long flags; 1954 1955 for (; wr; wr = wr->next) { 1956 struct rvt_rwqe *wqe; 1957 u32 next; 1958 int i; 1959 1960 if ((unsigned)wr->num_sge > srq->rq.max_sge) { 1961 *bad_wr = wr; 1962 return -EINVAL; 1963 } 1964 1965 spin_lock_irqsave(&srq->rq.lock, flags); 1966 wq = srq->rq.wq; 1967 next = wq->head + 1; 1968 if (next >= srq->rq.size) 1969 next = 0; 1970 if (next == wq->tail) { 1971 spin_unlock_irqrestore(&srq->rq.lock, flags); 1972 *bad_wr = wr; 1973 return -ENOMEM; 1974 } 1975 1976 wqe = rvt_get_rwqe_ptr(&srq->rq, wq->head); 1977 wqe->wr_id = wr->wr_id; 1978 wqe->num_sge = wr->num_sge; 1979 for (i = 0; i < wr->num_sge; i++) 1980 wqe->sg_list[i] = wr->sg_list[i]; 1981 /* Make sure queue entry is written before the head index. */ 1982 smp_wmb(); 1983 wq->head = next; 1984 spin_unlock_irqrestore(&srq->rq.lock, flags); 1985 } 1986 return 0; 1987 } 1988 1989 /* 1990 * Validate a RWQE and fill in the SGE state. 1991 * Return 1 if OK. 1992 */ 1993 static int init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe) 1994 { 1995 int i, j, ret; 1996 struct ib_wc wc; 1997 struct rvt_lkey_table *rkt; 1998 struct rvt_pd *pd; 1999 struct rvt_sge_state *ss; 2000 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 2001 2002 rkt = &rdi->lkey_table; 2003 pd = ibpd_to_rvtpd(qp->ibqp.srq ? qp->ibqp.srq->pd : qp->ibqp.pd); 2004 ss = &qp->r_sge; 2005 ss->sg_list = qp->r_sg_list; 2006 qp->r_len = 0; 2007 for (i = j = 0; i < wqe->num_sge; i++) { 2008 if (wqe->sg_list[i].length == 0) 2009 continue; 2010 /* Check LKEY */ 2011 ret = rvt_lkey_ok(rkt, pd, j ? &ss->sg_list[j - 1] : &ss->sge, 2012 NULL, &wqe->sg_list[i], 2013 IB_ACCESS_LOCAL_WRITE); 2014 if (unlikely(ret <= 0)) 2015 goto bad_lkey; 2016 qp->r_len += wqe->sg_list[i].length; 2017 j++; 2018 } 2019 ss->num_sge = j; 2020 ss->total_len = qp->r_len; 2021 return 1; 2022 2023 bad_lkey: 2024 while (j) { 2025 struct rvt_sge *sge = --j ? &ss->sg_list[j - 1] : &ss->sge; 2026 2027 rvt_put_mr(sge->mr); 2028 } 2029 ss->num_sge = 0; 2030 memset(&wc, 0, sizeof(wc)); 2031 wc.wr_id = wqe->wr_id; 2032 wc.status = IB_WC_LOC_PROT_ERR; 2033 wc.opcode = IB_WC_RECV; 2034 wc.qp = &qp->ibqp; 2035 /* Signal solicited completion event. */ 2036 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1); 2037 return 0; 2038 } 2039 2040 /** 2041 * rvt_get_rwqe - copy the next RWQE into the QP's RWQE 2042 * @qp: the QP 2043 * @wr_id_only: update qp->r_wr_id only, not qp->r_sge 2044 * 2045 * Return -1 if there is a local error, 0 if no RWQE is available, 2046 * otherwise return 1. 2047 * 2048 * Can be called from interrupt level. 2049 */ 2050 int rvt_get_rwqe(struct rvt_qp *qp, bool wr_id_only) 2051 { 2052 unsigned long flags; 2053 struct rvt_rq *rq; 2054 struct rvt_rwq *wq; 2055 struct rvt_srq *srq; 2056 struct rvt_rwqe *wqe; 2057 void (*handler)(struct ib_event *, void *); 2058 u32 tail; 2059 int ret; 2060 2061 if (qp->ibqp.srq) { 2062 srq = ibsrq_to_rvtsrq(qp->ibqp.srq); 2063 handler = srq->ibsrq.event_handler; 2064 rq = &srq->rq; 2065 } else { 2066 srq = NULL; 2067 handler = NULL; 2068 rq = &qp->r_rq; 2069 } 2070 2071 spin_lock_irqsave(&rq->lock, flags); 2072 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { 2073 ret = 0; 2074 goto unlock; 2075 } 2076 2077 wq = rq->wq; 2078 tail = wq->tail; 2079 /* Validate tail before using it since it is user writable. */ 2080 if (tail >= rq->size) 2081 tail = 0; 2082 if (unlikely(tail == wq->head)) { 2083 ret = 0; 2084 goto unlock; 2085 } 2086 /* Make sure entry is read after head index is read. */ 2087 smp_rmb(); 2088 wqe = rvt_get_rwqe_ptr(rq, tail); 2089 /* 2090 * Even though we update the tail index in memory, the verbs 2091 * consumer is not supposed to post more entries until a 2092 * completion is generated. 2093 */ 2094 if (++tail >= rq->size) 2095 tail = 0; 2096 wq->tail = tail; 2097 if (!wr_id_only && !init_sge(qp, wqe)) { 2098 ret = -1; 2099 goto unlock; 2100 } 2101 qp->r_wr_id = wqe->wr_id; 2102 2103 ret = 1; 2104 set_bit(RVT_R_WRID_VALID, &qp->r_aflags); 2105 if (handler) { 2106 u32 n; 2107 2108 /* 2109 * Validate head pointer value and compute 2110 * the number of remaining WQEs. 2111 */ 2112 n = wq->head; 2113 if (n >= rq->size) 2114 n = 0; 2115 if (n < tail) 2116 n += rq->size - tail; 2117 else 2118 n -= tail; 2119 if (n < srq->limit) { 2120 struct ib_event ev; 2121 2122 srq->limit = 0; 2123 spin_unlock_irqrestore(&rq->lock, flags); 2124 ev.device = qp->ibqp.device; 2125 ev.element.srq = qp->ibqp.srq; 2126 ev.event = IB_EVENT_SRQ_LIMIT_REACHED; 2127 handler(&ev, srq->ibsrq.srq_context); 2128 goto bail; 2129 } 2130 } 2131 unlock: 2132 spin_unlock_irqrestore(&rq->lock, flags); 2133 bail: 2134 return ret; 2135 } 2136 EXPORT_SYMBOL(rvt_get_rwqe); 2137 2138 /** 2139 * qp_comm_est - handle trap with QP established 2140 * @qp: the QP 2141 */ 2142 void rvt_comm_est(struct rvt_qp *qp) 2143 { 2144 qp->r_flags |= RVT_R_COMM_EST; 2145 if (qp->ibqp.event_handler) { 2146 struct ib_event ev; 2147 2148 ev.device = qp->ibqp.device; 2149 ev.element.qp = &qp->ibqp; 2150 ev.event = IB_EVENT_COMM_EST; 2151 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 2152 } 2153 } 2154 EXPORT_SYMBOL(rvt_comm_est); 2155 2156 void rvt_rc_error(struct rvt_qp *qp, enum ib_wc_status err) 2157 { 2158 unsigned long flags; 2159 int lastwqe; 2160 2161 spin_lock_irqsave(&qp->s_lock, flags); 2162 lastwqe = rvt_error_qp(qp, err); 2163 spin_unlock_irqrestore(&qp->s_lock, flags); 2164 2165 if (lastwqe) { 2166 struct ib_event ev; 2167 2168 ev.device = qp->ibqp.device; 2169 ev.element.qp = &qp->ibqp; 2170 ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 2171 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 2172 } 2173 } 2174 EXPORT_SYMBOL(rvt_rc_error); 2175 2176 /* 2177 * rvt_rnr_tbl_to_usec - return index into ib_rvt_rnr_table 2178 * @index - the index 2179 * return usec from an index into ib_rvt_rnr_table 2180 */ 2181 unsigned long rvt_rnr_tbl_to_usec(u32 index) 2182 { 2183 return ib_rvt_rnr_table[(index & IB_AETH_CREDIT_MASK)]; 2184 } 2185 EXPORT_SYMBOL(rvt_rnr_tbl_to_usec); 2186 2187 static inline unsigned long rvt_aeth_to_usec(u32 aeth) 2188 { 2189 return ib_rvt_rnr_table[(aeth >> IB_AETH_CREDIT_SHIFT) & 2190 IB_AETH_CREDIT_MASK]; 2191 } 2192 2193 /* 2194 * rvt_add_retry_timer - add/start a retry timer 2195 * @qp - the QP 2196 * add a retry timer on the QP 2197 */ 2198 void rvt_add_retry_timer(struct rvt_qp *qp) 2199 { 2200 struct ib_qp *ibqp = &qp->ibqp; 2201 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); 2202 2203 lockdep_assert_held(&qp->s_lock); 2204 qp->s_flags |= RVT_S_TIMER; 2205 /* 4.096 usec. * (1 << qp->timeout) */ 2206 qp->s_timer.expires = jiffies + qp->timeout_jiffies + 2207 rdi->busy_jiffies; 2208 add_timer(&qp->s_timer); 2209 } 2210 EXPORT_SYMBOL(rvt_add_retry_timer); 2211 2212 /** 2213 * rvt_add_rnr_timer - add/start an rnr timer 2214 * @qp - the QP 2215 * @aeth - aeth of RNR timeout, simulated aeth for loopback 2216 * add an rnr timer on the QP 2217 */ 2218 void rvt_add_rnr_timer(struct rvt_qp *qp, u32 aeth) 2219 { 2220 u32 to; 2221 2222 lockdep_assert_held(&qp->s_lock); 2223 qp->s_flags |= RVT_S_WAIT_RNR; 2224 to = rvt_aeth_to_usec(aeth); 2225 trace_rvt_rnrnak_add(qp, to); 2226 hrtimer_start(&qp->s_rnr_timer, 2227 ns_to_ktime(1000 * to), HRTIMER_MODE_REL_PINNED); 2228 } 2229 EXPORT_SYMBOL(rvt_add_rnr_timer); 2230 2231 /** 2232 * rvt_stop_rc_timers - stop all timers 2233 * @qp - the QP 2234 * stop any pending timers 2235 */ 2236 void rvt_stop_rc_timers(struct rvt_qp *qp) 2237 { 2238 lockdep_assert_held(&qp->s_lock); 2239 /* Remove QP from all timers */ 2240 if (qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR)) { 2241 qp->s_flags &= ~(RVT_S_TIMER | RVT_S_WAIT_RNR); 2242 del_timer(&qp->s_timer); 2243 hrtimer_try_to_cancel(&qp->s_rnr_timer); 2244 } 2245 } 2246 EXPORT_SYMBOL(rvt_stop_rc_timers); 2247 2248 /** 2249 * rvt_stop_rnr_timer - stop an rnr timer 2250 * @qp - the QP 2251 * 2252 * stop an rnr timer and return if the timer 2253 * had been pending. 2254 */ 2255 static void rvt_stop_rnr_timer(struct rvt_qp *qp) 2256 { 2257 lockdep_assert_held(&qp->s_lock); 2258 /* Remove QP from rnr timer */ 2259 if (qp->s_flags & RVT_S_WAIT_RNR) { 2260 qp->s_flags &= ~RVT_S_WAIT_RNR; 2261 trace_rvt_rnrnak_stop(qp, 0); 2262 } 2263 } 2264 2265 /** 2266 * rvt_del_timers_sync - wait for any timeout routines to exit 2267 * @qp - the QP 2268 */ 2269 void rvt_del_timers_sync(struct rvt_qp *qp) 2270 { 2271 del_timer_sync(&qp->s_timer); 2272 hrtimer_cancel(&qp->s_rnr_timer); 2273 } 2274 EXPORT_SYMBOL(rvt_del_timers_sync); 2275 2276 /** 2277 * This is called from s_timer for missing responses. 2278 */ 2279 static void rvt_rc_timeout(struct timer_list *t) 2280 { 2281 struct rvt_qp *qp = from_timer(qp, t, s_timer); 2282 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 2283 unsigned long flags; 2284 2285 spin_lock_irqsave(&qp->r_lock, flags); 2286 spin_lock(&qp->s_lock); 2287 if (qp->s_flags & RVT_S_TIMER) { 2288 struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1]; 2289 2290 qp->s_flags &= ~RVT_S_TIMER; 2291 rvp->n_rc_timeouts++; 2292 del_timer(&qp->s_timer); 2293 trace_rvt_rc_timeout(qp, qp->s_last_psn + 1); 2294 if (rdi->driver_f.notify_restart_rc) 2295 rdi->driver_f.notify_restart_rc(qp, 2296 qp->s_last_psn + 1, 2297 1); 2298 rdi->driver_f.schedule_send(qp); 2299 } 2300 spin_unlock(&qp->s_lock); 2301 spin_unlock_irqrestore(&qp->r_lock, flags); 2302 } 2303 2304 /* 2305 * This is called from s_timer for RNR timeouts. 2306 */ 2307 enum hrtimer_restart rvt_rc_rnr_retry(struct hrtimer *t) 2308 { 2309 struct rvt_qp *qp = container_of(t, struct rvt_qp, s_rnr_timer); 2310 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 2311 unsigned long flags; 2312 2313 spin_lock_irqsave(&qp->s_lock, flags); 2314 rvt_stop_rnr_timer(qp); 2315 trace_rvt_rnrnak_timeout(qp, 0); 2316 rdi->driver_f.schedule_send(qp); 2317 spin_unlock_irqrestore(&qp->s_lock, flags); 2318 return HRTIMER_NORESTART; 2319 } 2320 EXPORT_SYMBOL(rvt_rc_rnr_retry); 2321 2322 /** 2323 * rvt_qp_iter_init - initial for QP iteration 2324 * @rdi: rvt devinfo 2325 * @v: u64 value 2326 * 2327 * This returns an iterator suitable for iterating QPs 2328 * in the system. 2329 * 2330 * The @cb is a user defined callback and @v is a 64 2331 * bit value passed to and relevant for processing in the 2332 * @cb. An example use case would be to alter QP processing 2333 * based on criteria not part of the rvt_qp. 2334 * 2335 * Use cases that require memory allocation to succeed 2336 * must preallocate appropriately. 2337 * 2338 * Return: a pointer to an rvt_qp_iter or NULL 2339 */ 2340 struct rvt_qp_iter *rvt_qp_iter_init(struct rvt_dev_info *rdi, 2341 u64 v, 2342 void (*cb)(struct rvt_qp *qp, u64 v)) 2343 { 2344 struct rvt_qp_iter *i; 2345 2346 i = kzalloc(sizeof(*i), GFP_KERNEL); 2347 if (!i) 2348 return NULL; 2349 2350 i->rdi = rdi; 2351 /* number of special QPs (SMI/GSI) for device */ 2352 i->specials = rdi->ibdev.phys_port_cnt * 2; 2353 i->v = v; 2354 i->cb = cb; 2355 2356 return i; 2357 } 2358 EXPORT_SYMBOL(rvt_qp_iter_init); 2359 2360 /** 2361 * rvt_qp_iter_next - return the next QP in iter 2362 * @iter - the iterator 2363 * 2364 * Fine grained QP iterator suitable for use 2365 * with debugfs seq_file mechanisms. 2366 * 2367 * Updates iter->qp with the current QP when the return 2368 * value is 0. 2369 * 2370 * Return: 0 - iter->qp is valid 1 - no more QPs 2371 */ 2372 int rvt_qp_iter_next(struct rvt_qp_iter *iter) 2373 __must_hold(RCU) 2374 { 2375 int n = iter->n; 2376 int ret = 1; 2377 struct rvt_qp *pqp = iter->qp; 2378 struct rvt_qp *qp; 2379 struct rvt_dev_info *rdi = iter->rdi; 2380 2381 /* 2382 * The approach is to consider the special qps 2383 * as additional table entries before the 2384 * real hash table. Since the qp code sets 2385 * the qp->next hash link to NULL, this works just fine. 2386 * 2387 * iter->specials is 2 * # ports 2388 * 2389 * n = 0..iter->specials is the special qp indices 2390 * 2391 * n = iter->specials..rdi->qp_dev->qp_table_size+iter->specials are 2392 * the potential hash bucket entries 2393 * 2394 */ 2395 for (; n < rdi->qp_dev->qp_table_size + iter->specials; n++) { 2396 if (pqp) { 2397 qp = rcu_dereference(pqp->next); 2398 } else { 2399 if (n < iter->specials) { 2400 struct rvt_ibport *rvp; 2401 int pidx; 2402 2403 pidx = n % rdi->ibdev.phys_port_cnt; 2404 rvp = rdi->ports[pidx]; 2405 qp = rcu_dereference(rvp->qp[n & 1]); 2406 } else { 2407 qp = rcu_dereference( 2408 rdi->qp_dev->qp_table[ 2409 (n - iter->specials)]); 2410 } 2411 } 2412 pqp = qp; 2413 if (qp) { 2414 iter->qp = qp; 2415 iter->n = n; 2416 return 0; 2417 } 2418 } 2419 return ret; 2420 } 2421 EXPORT_SYMBOL(rvt_qp_iter_next); 2422 2423 /** 2424 * rvt_qp_iter - iterate all QPs 2425 * @rdi - rvt devinfo 2426 * @v - a 64 bit value 2427 * @cb - a callback 2428 * 2429 * This provides a way for iterating all QPs. 2430 * 2431 * The @cb is a user defined callback and @v is a 64 2432 * bit value passed to and relevant for processing in the 2433 * cb. An example use case would be to alter QP processing 2434 * based on criteria not part of the rvt_qp. 2435 * 2436 * The code has an internal iterator to simplify 2437 * non seq_file use cases. 2438 */ 2439 void rvt_qp_iter(struct rvt_dev_info *rdi, 2440 u64 v, 2441 void (*cb)(struct rvt_qp *qp, u64 v)) 2442 { 2443 int ret; 2444 struct rvt_qp_iter i = { 2445 .rdi = rdi, 2446 .specials = rdi->ibdev.phys_port_cnt * 2, 2447 .v = v, 2448 .cb = cb 2449 }; 2450 2451 rcu_read_lock(); 2452 do { 2453 ret = rvt_qp_iter_next(&i); 2454 if (!ret) { 2455 rvt_get_qp(i.qp); 2456 rcu_read_unlock(); 2457 i.cb(i.qp, i.v); 2458 rcu_read_lock(); 2459 rvt_put_qp(i.qp); 2460 } 2461 } while (!ret); 2462 rcu_read_unlock(); 2463 } 2464 EXPORT_SYMBOL(rvt_qp_iter); 2465