1 /* 2 * Copyright(c) 2016 - 2020 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 48 #include <linux/hash.h> 49 #include <linux/bitops.h> 50 #include <linux/lockdep.h> 51 #include <linux/vmalloc.h> 52 #include <linux/slab.h> 53 #include <rdma/ib_verbs.h> 54 #include <rdma/ib_hdrs.h> 55 #include <rdma/opa_addr.h> 56 #include <rdma/uverbs_ioctl.h> 57 #include "qp.h" 58 #include "vt.h" 59 #include "trace.h" 60 61 #define RVT_RWQ_COUNT_THRESHOLD 16 62 63 static void rvt_rc_timeout(struct timer_list *t); 64 static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, 65 enum ib_qp_type type); 66 67 /* 68 * Convert the AETH RNR timeout code into the number of microseconds. 69 */ 70 static const u32 ib_rvt_rnr_table[32] = { 71 655360, /* 00: 655.36 */ 72 10, /* 01: .01 */ 73 20, /* 02 .02 */ 74 30, /* 03: .03 */ 75 40, /* 04: .04 */ 76 60, /* 05: .06 */ 77 80, /* 06: .08 */ 78 120, /* 07: .12 */ 79 160, /* 08: .16 */ 80 240, /* 09: .24 */ 81 320, /* 0A: .32 */ 82 480, /* 0B: .48 */ 83 640, /* 0C: .64 */ 84 960, /* 0D: .96 */ 85 1280, /* 0E: 1.28 */ 86 1920, /* 0F: 1.92 */ 87 2560, /* 10: 2.56 */ 88 3840, /* 11: 3.84 */ 89 5120, /* 12: 5.12 */ 90 7680, /* 13: 7.68 */ 91 10240, /* 14: 10.24 */ 92 15360, /* 15: 15.36 */ 93 20480, /* 16: 20.48 */ 94 30720, /* 17: 30.72 */ 95 40960, /* 18: 40.96 */ 96 61440, /* 19: 61.44 */ 97 81920, /* 1A: 81.92 */ 98 122880, /* 1B: 122.88 */ 99 163840, /* 1C: 163.84 */ 100 245760, /* 1D: 245.76 */ 101 327680, /* 1E: 327.68 */ 102 491520 /* 1F: 491.52 */ 103 }; 104 105 /* 106 * Note that it is OK to post send work requests in the SQE and ERR 107 * states; rvt_do_send() will process them and generate error 108 * completions as per IB 1.2 C10-96. 109 */ 110 const int ib_rvt_state_ops[IB_QPS_ERR + 1] = { 111 [IB_QPS_RESET] = 0, 112 [IB_QPS_INIT] = RVT_POST_RECV_OK, 113 [IB_QPS_RTR] = RVT_POST_RECV_OK | RVT_PROCESS_RECV_OK, 114 [IB_QPS_RTS] = RVT_POST_RECV_OK | RVT_PROCESS_RECV_OK | 115 RVT_POST_SEND_OK | RVT_PROCESS_SEND_OK | 116 RVT_PROCESS_NEXT_SEND_OK, 117 [IB_QPS_SQD] = RVT_POST_RECV_OK | RVT_PROCESS_RECV_OK | 118 RVT_POST_SEND_OK | RVT_PROCESS_SEND_OK, 119 [IB_QPS_SQE] = RVT_POST_RECV_OK | RVT_PROCESS_RECV_OK | 120 RVT_POST_SEND_OK | RVT_FLUSH_SEND, 121 [IB_QPS_ERR] = RVT_POST_RECV_OK | RVT_FLUSH_RECV | 122 RVT_POST_SEND_OK | RVT_FLUSH_SEND, 123 }; 124 EXPORT_SYMBOL(ib_rvt_state_ops); 125 126 /* platform specific: return the last level cache (llc) size, in KiB */ 127 static int rvt_wss_llc_size(void) 128 { 129 /* assume that the boot CPU value is universal for all CPUs */ 130 return boot_cpu_data.x86_cache_size; 131 } 132 133 /* platform specific: cacheless copy */ 134 static void cacheless_memcpy(void *dst, void *src, size_t n) 135 { 136 /* 137 * Use the only available X64 cacheless copy. Add a __user cast 138 * to quiet sparse. The src agument is already in the kernel so 139 * there are no security issues. The extra fault recovery machinery 140 * is not invoked. 141 */ 142 __copy_user_nocache(dst, (void __user *)src, n, 0); 143 } 144 145 void rvt_wss_exit(struct rvt_dev_info *rdi) 146 { 147 struct rvt_wss *wss = rdi->wss; 148 149 if (!wss) 150 return; 151 152 /* coded to handle partially initialized and repeat callers */ 153 kfree(wss->entries); 154 wss->entries = NULL; 155 kfree(rdi->wss); 156 rdi->wss = NULL; 157 } 158 159 /** 160 * rvt_wss_init - Init wss data structures 161 * 162 * Return: 0 on success 163 */ 164 int rvt_wss_init(struct rvt_dev_info *rdi) 165 { 166 unsigned int sge_copy_mode = rdi->dparms.sge_copy_mode; 167 unsigned int wss_threshold = rdi->dparms.wss_threshold; 168 unsigned int wss_clean_period = rdi->dparms.wss_clean_period; 169 long llc_size; 170 long llc_bits; 171 long table_size; 172 long table_bits; 173 struct rvt_wss *wss; 174 int node = rdi->dparms.node; 175 176 if (sge_copy_mode != RVT_SGE_COPY_ADAPTIVE) { 177 rdi->wss = NULL; 178 return 0; 179 } 180 181 rdi->wss = kzalloc_node(sizeof(*rdi->wss), GFP_KERNEL, node); 182 if (!rdi->wss) 183 return -ENOMEM; 184 wss = rdi->wss; 185 186 /* check for a valid percent range - default to 80 if none or invalid */ 187 if (wss_threshold < 1 || wss_threshold > 100) 188 wss_threshold = 80; 189 190 /* reject a wildly large period */ 191 if (wss_clean_period > 1000000) 192 wss_clean_period = 256; 193 194 /* reject a zero period */ 195 if (wss_clean_period == 0) 196 wss_clean_period = 1; 197 198 /* 199 * Calculate the table size - the next power of 2 larger than the 200 * LLC size. LLC size is in KiB. 201 */ 202 llc_size = rvt_wss_llc_size() * 1024; 203 table_size = roundup_pow_of_two(llc_size); 204 205 /* one bit per page in rounded up table */ 206 llc_bits = llc_size / PAGE_SIZE; 207 table_bits = table_size / PAGE_SIZE; 208 wss->pages_mask = table_bits - 1; 209 wss->num_entries = table_bits / BITS_PER_LONG; 210 211 wss->threshold = (llc_bits * wss_threshold) / 100; 212 if (wss->threshold == 0) 213 wss->threshold = 1; 214 215 wss->clean_period = wss_clean_period; 216 atomic_set(&wss->clean_counter, wss_clean_period); 217 218 wss->entries = kcalloc_node(wss->num_entries, sizeof(*wss->entries), 219 GFP_KERNEL, node); 220 if (!wss->entries) { 221 rvt_wss_exit(rdi); 222 return -ENOMEM; 223 } 224 225 return 0; 226 } 227 228 /* 229 * Advance the clean counter. When the clean period has expired, 230 * clean an entry. 231 * 232 * This is implemented in atomics to avoid locking. Because multiple 233 * variables are involved, it can be racy which can lead to slightly 234 * inaccurate information. Since this is only a heuristic, this is 235 * OK. Any innaccuracies will clean themselves out as the counter 236 * advances. That said, it is unlikely the entry clean operation will 237 * race - the next possible racer will not start until the next clean 238 * period. 239 * 240 * The clean counter is implemented as a decrement to zero. When zero 241 * is reached an entry is cleaned. 242 */ 243 static void wss_advance_clean_counter(struct rvt_wss *wss) 244 { 245 int entry; 246 int weight; 247 unsigned long bits; 248 249 /* become the cleaner if we decrement the counter to zero */ 250 if (atomic_dec_and_test(&wss->clean_counter)) { 251 /* 252 * Set, not add, the clean period. This avoids an issue 253 * where the counter could decrement below the clean period. 254 * Doing a set can result in lost decrements, slowing the 255 * clean advance. Since this a heuristic, this possible 256 * slowdown is OK. 257 * 258 * An alternative is to loop, advancing the counter by a 259 * clean period until the result is > 0. However, this could 260 * lead to several threads keeping another in the clean loop. 261 * This could be mitigated by limiting the number of times 262 * we stay in the loop. 263 */ 264 atomic_set(&wss->clean_counter, wss->clean_period); 265 266 /* 267 * Uniquely grab the entry to clean and move to next. 268 * The current entry is always the lower bits of 269 * wss.clean_entry. The table size, wss.num_entries, 270 * is always a power-of-2. 271 */ 272 entry = (atomic_inc_return(&wss->clean_entry) - 1) 273 & (wss->num_entries - 1); 274 275 /* clear the entry and count the bits */ 276 bits = xchg(&wss->entries[entry], 0); 277 weight = hweight64((u64)bits); 278 /* only adjust the contended total count if needed */ 279 if (weight) 280 atomic_sub(weight, &wss->total_count); 281 } 282 } 283 284 /* 285 * Insert the given address into the working set array. 286 */ 287 static void wss_insert(struct rvt_wss *wss, void *address) 288 { 289 u32 page = ((unsigned long)address >> PAGE_SHIFT) & wss->pages_mask; 290 u32 entry = page / BITS_PER_LONG; /* assumes this ends up a shift */ 291 u32 nr = page & (BITS_PER_LONG - 1); 292 293 if (!test_and_set_bit(nr, &wss->entries[entry])) 294 atomic_inc(&wss->total_count); 295 296 wss_advance_clean_counter(wss); 297 } 298 299 /* 300 * Is the working set larger than the threshold? 301 */ 302 static inline bool wss_exceeds_threshold(struct rvt_wss *wss) 303 { 304 return atomic_read(&wss->total_count) >= wss->threshold; 305 } 306 307 static void get_map_page(struct rvt_qpn_table *qpt, 308 struct rvt_qpn_map *map) 309 { 310 unsigned long page = get_zeroed_page(GFP_KERNEL); 311 312 /* 313 * Free the page if someone raced with us installing it. 314 */ 315 316 spin_lock(&qpt->lock); 317 if (map->page) 318 free_page(page); 319 else 320 map->page = (void *)page; 321 spin_unlock(&qpt->lock); 322 } 323 324 /** 325 * init_qpn_table - initialize the QP number table for a device 326 * @qpt: the QPN table 327 */ 328 static int init_qpn_table(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt) 329 { 330 u32 offset, i; 331 struct rvt_qpn_map *map; 332 int ret = 0; 333 334 if (!(rdi->dparms.qpn_res_end >= rdi->dparms.qpn_res_start)) 335 return -EINVAL; 336 337 spin_lock_init(&qpt->lock); 338 339 qpt->last = rdi->dparms.qpn_start; 340 qpt->incr = rdi->dparms.qpn_inc << rdi->dparms.qos_shift; 341 342 /* 343 * Drivers may want some QPs beyond what we need for verbs let them use 344 * our qpn table. No need for two. Lets go ahead and mark the bitmaps 345 * for those. The reserved range must be *after* the range which verbs 346 * will pick from. 347 */ 348 349 /* Figure out number of bit maps needed before reserved range */ 350 qpt->nmaps = rdi->dparms.qpn_res_start / RVT_BITS_PER_PAGE; 351 352 /* This should always be zero */ 353 offset = rdi->dparms.qpn_res_start & RVT_BITS_PER_PAGE_MASK; 354 355 /* Starting with the first reserved bit map */ 356 map = &qpt->map[qpt->nmaps]; 357 358 rvt_pr_info(rdi, "Reserving QPNs from 0x%x to 0x%x for non-verbs use\n", 359 rdi->dparms.qpn_res_start, rdi->dparms.qpn_res_end); 360 for (i = rdi->dparms.qpn_res_start; i <= rdi->dparms.qpn_res_end; i++) { 361 if (!map->page) { 362 get_map_page(qpt, map); 363 if (!map->page) { 364 ret = -ENOMEM; 365 break; 366 } 367 } 368 set_bit(offset, map->page); 369 offset++; 370 if (offset == RVT_BITS_PER_PAGE) { 371 /* next page */ 372 qpt->nmaps++; 373 map++; 374 offset = 0; 375 } 376 } 377 return ret; 378 } 379 380 /** 381 * free_qpn_table - free the QP number table for a device 382 * @qpt: the QPN table 383 */ 384 static void free_qpn_table(struct rvt_qpn_table *qpt) 385 { 386 int i; 387 388 for (i = 0; i < ARRAY_SIZE(qpt->map); i++) 389 free_page((unsigned long)qpt->map[i].page); 390 } 391 392 /** 393 * rvt_driver_qp_init - Init driver qp resources 394 * @rdi: rvt dev strucutre 395 * 396 * Return: 0 on success 397 */ 398 int rvt_driver_qp_init(struct rvt_dev_info *rdi) 399 { 400 int i; 401 int ret = -ENOMEM; 402 403 if (!rdi->dparms.qp_table_size) 404 return -EINVAL; 405 406 /* 407 * If driver is not doing any QP allocation then make sure it is 408 * providing the necessary QP functions. 409 */ 410 if (!rdi->driver_f.free_all_qps || 411 !rdi->driver_f.qp_priv_alloc || 412 !rdi->driver_f.qp_priv_free || 413 !rdi->driver_f.notify_qp_reset || 414 !rdi->driver_f.notify_restart_rc) 415 return -EINVAL; 416 417 /* allocate parent object */ 418 rdi->qp_dev = kzalloc_node(sizeof(*rdi->qp_dev), GFP_KERNEL, 419 rdi->dparms.node); 420 if (!rdi->qp_dev) 421 return -ENOMEM; 422 423 /* allocate hash table */ 424 rdi->qp_dev->qp_table_size = rdi->dparms.qp_table_size; 425 rdi->qp_dev->qp_table_bits = ilog2(rdi->dparms.qp_table_size); 426 rdi->qp_dev->qp_table = 427 kmalloc_array_node(rdi->qp_dev->qp_table_size, 428 sizeof(*rdi->qp_dev->qp_table), 429 GFP_KERNEL, rdi->dparms.node); 430 if (!rdi->qp_dev->qp_table) 431 goto no_qp_table; 432 433 for (i = 0; i < rdi->qp_dev->qp_table_size; i++) 434 RCU_INIT_POINTER(rdi->qp_dev->qp_table[i], NULL); 435 436 spin_lock_init(&rdi->qp_dev->qpt_lock); 437 438 /* initialize qpn map */ 439 if (init_qpn_table(rdi, &rdi->qp_dev->qpn_table)) 440 goto fail_table; 441 442 spin_lock_init(&rdi->n_qps_lock); 443 444 return 0; 445 446 fail_table: 447 kfree(rdi->qp_dev->qp_table); 448 free_qpn_table(&rdi->qp_dev->qpn_table); 449 450 no_qp_table: 451 kfree(rdi->qp_dev); 452 453 return ret; 454 } 455 456 /** 457 * rvt_free_qp_cb - callback function to reset a qp 458 * @qp: the qp to reset 459 * @v: a 64-bit value 460 * 461 * This function resets the qp and removes it from the 462 * qp hash table. 463 */ 464 static void rvt_free_qp_cb(struct rvt_qp *qp, u64 v) 465 { 466 unsigned int *qp_inuse = (unsigned int *)v; 467 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 468 469 /* Reset the qp and remove it from the qp hash list */ 470 rvt_reset_qp(rdi, qp, qp->ibqp.qp_type); 471 472 /* Increment the qp_inuse count */ 473 (*qp_inuse)++; 474 } 475 476 /** 477 * rvt_free_all_qps - check for QPs still in use 478 * @rdi: rvt device info structure 479 * 480 * There should not be any QPs still in use. 481 * Free memory for table. 482 * Return the number of QPs still in use. 483 */ 484 static unsigned rvt_free_all_qps(struct rvt_dev_info *rdi) 485 { 486 unsigned int qp_inuse = 0; 487 488 qp_inuse += rvt_mcast_tree_empty(rdi); 489 490 rvt_qp_iter(rdi, (u64)&qp_inuse, rvt_free_qp_cb); 491 492 return qp_inuse; 493 } 494 495 /** 496 * rvt_qp_exit - clean up qps on device exit 497 * @rdi: rvt dev structure 498 * 499 * Check for qp leaks and free resources. 500 */ 501 void rvt_qp_exit(struct rvt_dev_info *rdi) 502 { 503 u32 qps_inuse = rvt_free_all_qps(rdi); 504 505 if (qps_inuse) 506 rvt_pr_err(rdi, "QP memory leak! %u still in use\n", 507 qps_inuse); 508 if (!rdi->qp_dev) 509 return; 510 511 kfree(rdi->qp_dev->qp_table); 512 free_qpn_table(&rdi->qp_dev->qpn_table); 513 kfree(rdi->qp_dev); 514 } 515 516 static inline unsigned mk_qpn(struct rvt_qpn_table *qpt, 517 struct rvt_qpn_map *map, unsigned off) 518 { 519 return (map - qpt->map) * RVT_BITS_PER_PAGE + off; 520 } 521 522 /** 523 * alloc_qpn - Allocate the next available qpn or zero/one for QP type 524 * IB_QPT_SMI/IB_QPT_GSI 525 * @rdi: rvt device info structure 526 * @qpt: queue pair number table pointer 527 * @port_num: IB port number, 1 based, comes from core 528 * @exclude_prefix: prefix of special queue pair number being allocated 529 * 530 * Return: The queue pair number 531 */ 532 static int alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt, 533 enum ib_qp_type type, u8 port_num, u8 exclude_prefix) 534 { 535 u32 i, offset, max_scan, qpn; 536 struct rvt_qpn_map *map; 537 u32 ret; 538 u32 max_qpn = exclude_prefix == RVT_AIP_QP_PREFIX ? 539 RVT_AIP_QPN_MAX : RVT_QPN_MAX; 540 541 if (rdi->driver_f.alloc_qpn) 542 return rdi->driver_f.alloc_qpn(rdi, qpt, type, port_num); 543 544 if (type == IB_QPT_SMI || type == IB_QPT_GSI) { 545 unsigned n; 546 547 ret = type == IB_QPT_GSI; 548 n = 1 << (ret + 2 * (port_num - 1)); 549 spin_lock(&qpt->lock); 550 if (qpt->flags & n) 551 ret = -EINVAL; 552 else 553 qpt->flags |= n; 554 spin_unlock(&qpt->lock); 555 goto bail; 556 } 557 558 qpn = qpt->last + qpt->incr; 559 if (qpn >= max_qpn) 560 qpn = qpt->incr | ((qpt->last & 1) ^ 1); 561 /* offset carries bit 0 */ 562 offset = qpn & RVT_BITS_PER_PAGE_MASK; 563 map = &qpt->map[qpn / RVT_BITS_PER_PAGE]; 564 max_scan = qpt->nmaps - !offset; 565 for (i = 0;;) { 566 if (unlikely(!map->page)) { 567 get_map_page(qpt, map); 568 if (unlikely(!map->page)) 569 break; 570 } 571 do { 572 if (!test_and_set_bit(offset, map->page)) { 573 qpt->last = qpn; 574 ret = qpn; 575 goto bail; 576 } 577 offset += qpt->incr; 578 /* 579 * This qpn might be bogus if offset >= BITS_PER_PAGE. 580 * That is OK. It gets re-assigned below 581 */ 582 qpn = mk_qpn(qpt, map, offset); 583 } while (offset < RVT_BITS_PER_PAGE && qpn < RVT_QPN_MAX); 584 /* 585 * In order to keep the number of pages allocated to a 586 * minimum, we scan the all existing pages before increasing 587 * the size of the bitmap table. 588 */ 589 if (++i > max_scan) { 590 if (qpt->nmaps == RVT_QPNMAP_ENTRIES) 591 break; 592 map = &qpt->map[qpt->nmaps++]; 593 /* start at incr with current bit 0 */ 594 offset = qpt->incr | (offset & 1); 595 } else if (map < &qpt->map[qpt->nmaps]) { 596 ++map; 597 /* start at incr with current bit 0 */ 598 offset = qpt->incr | (offset & 1); 599 } else { 600 map = &qpt->map[0]; 601 /* wrap to first map page, invert bit 0 */ 602 offset = qpt->incr | ((offset & 1) ^ 1); 603 } 604 /* there can be no set bits in low-order QoS bits */ 605 WARN_ON(rdi->dparms.qos_shift > 1 && 606 offset & ((BIT(rdi->dparms.qos_shift - 1) - 1) << 1)); 607 qpn = mk_qpn(qpt, map, offset); 608 } 609 610 ret = -ENOMEM; 611 612 bail: 613 return ret; 614 } 615 616 /** 617 * rvt_clear_mr_refs - Drop help mr refs 618 * @qp: rvt qp data structure 619 * @clr_sends: If shoudl clear send side or not 620 */ 621 static void rvt_clear_mr_refs(struct rvt_qp *qp, int clr_sends) 622 { 623 unsigned n; 624 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 625 626 if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) 627 rvt_put_ss(&qp->s_rdma_read_sge); 628 629 rvt_put_ss(&qp->r_sge); 630 631 if (clr_sends) { 632 while (qp->s_last != qp->s_head) { 633 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_last); 634 635 rvt_put_qp_swqe(qp, wqe); 636 if (++qp->s_last >= qp->s_size) 637 qp->s_last = 0; 638 smp_wmb(); /* see qp_set_savail */ 639 } 640 if (qp->s_rdma_mr) { 641 rvt_put_mr(qp->s_rdma_mr); 642 qp->s_rdma_mr = NULL; 643 } 644 } 645 646 for (n = 0; qp->s_ack_queue && n < rvt_max_atomic(rdi); n++) { 647 struct rvt_ack_entry *e = &qp->s_ack_queue[n]; 648 649 if (e->rdma_sge.mr) { 650 rvt_put_mr(e->rdma_sge.mr); 651 e->rdma_sge.mr = NULL; 652 } 653 } 654 } 655 656 /** 657 * rvt_swqe_has_lkey - return true if lkey is used by swqe 658 * @wqe - the send wqe 659 * @lkey - the lkey 660 * 661 * Test the swqe for using lkey 662 */ 663 static bool rvt_swqe_has_lkey(struct rvt_swqe *wqe, u32 lkey) 664 { 665 int i; 666 667 for (i = 0; i < wqe->wr.num_sge; i++) { 668 struct rvt_sge *sge = &wqe->sg_list[i]; 669 670 if (rvt_mr_has_lkey(sge->mr, lkey)) 671 return true; 672 } 673 return false; 674 } 675 676 /** 677 * rvt_qp_sends_has_lkey - return true is qp sends use lkey 678 * @qp - the rvt_qp 679 * @lkey - the lkey 680 */ 681 static bool rvt_qp_sends_has_lkey(struct rvt_qp *qp, u32 lkey) 682 { 683 u32 s_last = qp->s_last; 684 685 while (s_last != qp->s_head) { 686 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, s_last); 687 688 if (rvt_swqe_has_lkey(wqe, lkey)) 689 return true; 690 691 if (++s_last >= qp->s_size) 692 s_last = 0; 693 } 694 if (qp->s_rdma_mr) 695 if (rvt_mr_has_lkey(qp->s_rdma_mr, lkey)) 696 return true; 697 return false; 698 } 699 700 /** 701 * rvt_qp_acks_has_lkey - return true if acks have lkey 702 * @qp - the qp 703 * @lkey - the lkey 704 */ 705 static bool rvt_qp_acks_has_lkey(struct rvt_qp *qp, u32 lkey) 706 { 707 int i; 708 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 709 710 for (i = 0; qp->s_ack_queue && i < rvt_max_atomic(rdi); i++) { 711 struct rvt_ack_entry *e = &qp->s_ack_queue[i]; 712 713 if (rvt_mr_has_lkey(e->rdma_sge.mr, lkey)) 714 return true; 715 } 716 return false; 717 } 718 719 /* 720 * rvt_qp_mr_clean - clean up remote ops for lkey 721 * @qp - the qp 722 * @lkey - the lkey that is being de-registered 723 * 724 * This routine checks if the lkey is being used by 725 * the qp. 726 * 727 * If so, the qp is put into an error state to elminate 728 * any references from the qp. 729 */ 730 void rvt_qp_mr_clean(struct rvt_qp *qp, u32 lkey) 731 { 732 bool lastwqe = false; 733 734 if (qp->ibqp.qp_type == IB_QPT_SMI || 735 qp->ibqp.qp_type == IB_QPT_GSI) 736 /* avoid special QPs */ 737 return; 738 spin_lock_irq(&qp->r_lock); 739 spin_lock(&qp->s_hlock); 740 spin_lock(&qp->s_lock); 741 742 if (qp->state == IB_QPS_ERR || qp->state == IB_QPS_RESET) 743 goto check_lwqe; 744 745 if (rvt_ss_has_lkey(&qp->r_sge, lkey) || 746 rvt_qp_sends_has_lkey(qp, lkey) || 747 rvt_qp_acks_has_lkey(qp, lkey)) 748 lastwqe = rvt_error_qp(qp, IB_WC_LOC_PROT_ERR); 749 check_lwqe: 750 spin_unlock(&qp->s_lock); 751 spin_unlock(&qp->s_hlock); 752 spin_unlock_irq(&qp->r_lock); 753 if (lastwqe) { 754 struct ib_event ev; 755 756 ev.device = qp->ibqp.device; 757 ev.element.qp = &qp->ibqp; 758 ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 759 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 760 } 761 } 762 763 /** 764 * rvt_remove_qp - remove qp form table 765 * @rdi: rvt dev struct 766 * @qp: qp to remove 767 * 768 * Remove the QP from the table so it can't be found asynchronously by 769 * the receive routine. 770 */ 771 static void rvt_remove_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp) 772 { 773 struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1]; 774 u32 n = hash_32(qp->ibqp.qp_num, rdi->qp_dev->qp_table_bits); 775 unsigned long flags; 776 int removed = 1; 777 778 spin_lock_irqsave(&rdi->qp_dev->qpt_lock, flags); 779 780 if (rcu_dereference_protected(rvp->qp[0], 781 lockdep_is_held(&rdi->qp_dev->qpt_lock)) == qp) { 782 RCU_INIT_POINTER(rvp->qp[0], NULL); 783 } else if (rcu_dereference_protected(rvp->qp[1], 784 lockdep_is_held(&rdi->qp_dev->qpt_lock)) == qp) { 785 RCU_INIT_POINTER(rvp->qp[1], NULL); 786 } else { 787 struct rvt_qp *q; 788 struct rvt_qp __rcu **qpp; 789 790 removed = 0; 791 qpp = &rdi->qp_dev->qp_table[n]; 792 for (; (q = rcu_dereference_protected(*qpp, 793 lockdep_is_held(&rdi->qp_dev->qpt_lock))) != NULL; 794 qpp = &q->next) { 795 if (q == qp) { 796 RCU_INIT_POINTER(*qpp, 797 rcu_dereference_protected(qp->next, 798 lockdep_is_held(&rdi->qp_dev->qpt_lock))); 799 removed = 1; 800 trace_rvt_qpremove(qp, n); 801 break; 802 } 803 } 804 } 805 806 spin_unlock_irqrestore(&rdi->qp_dev->qpt_lock, flags); 807 if (removed) { 808 synchronize_rcu(); 809 rvt_put_qp(qp); 810 } 811 } 812 813 /** 814 * rvt_alloc_rq - allocate memory for user or kernel buffer 815 * @rq: receive queue data structure 816 * @size: number of request queue entries 817 * @node: The NUMA node 818 * @udata: True if user data is available or not false 819 * 820 * Return: If memory allocation failed, return -ENONEM 821 * This function is used by both shared receive 822 * queues and non-shared receive queues to allocate 823 * memory. 824 */ 825 int rvt_alloc_rq(struct rvt_rq *rq, u32 size, int node, 826 struct ib_udata *udata) 827 { 828 if (udata) { 829 rq->wq = vmalloc_user(sizeof(struct rvt_rwq) + size); 830 if (!rq->wq) 831 goto bail; 832 /* need kwq with no buffers */ 833 rq->kwq = kzalloc_node(sizeof(*rq->kwq), GFP_KERNEL, node); 834 if (!rq->kwq) 835 goto bail; 836 rq->kwq->curr_wq = rq->wq->wq; 837 } else { 838 /* need kwq with buffers */ 839 rq->kwq = 840 vzalloc_node(sizeof(struct rvt_krwq) + size, node); 841 if (!rq->kwq) 842 goto bail; 843 rq->kwq->curr_wq = rq->kwq->wq; 844 } 845 846 spin_lock_init(&rq->kwq->p_lock); 847 spin_lock_init(&rq->kwq->c_lock); 848 return 0; 849 bail: 850 rvt_free_rq(rq); 851 return -ENOMEM; 852 } 853 854 /** 855 * rvt_init_qp - initialize the QP state to the reset state 856 * @qp: the QP to init or reinit 857 * @type: the QP type 858 * 859 * This function is called from both rvt_create_qp() and 860 * rvt_reset_qp(). The difference is that the reset 861 * patch the necessary locks to protect against concurent 862 * access. 863 */ 864 static void rvt_init_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, 865 enum ib_qp_type type) 866 { 867 qp->remote_qpn = 0; 868 qp->qkey = 0; 869 qp->qp_access_flags = 0; 870 qp->s_flags &= RVT_S_SIGNAL_REQ_WR; 871 qp->s_hdrwords = 0; 872 qp->s_wqe = NULL; 873 qp->s_draining = 0; 874 qp->s_next_psn = 0; 875 qp->s_last_psn = 0; 876 qp->s_sending_psn = 0; 877 qp->s_sending_hpsn = 0; 878 qp->s_psn = 0; 879 qp->r_psn = 0; 880 qp->r_msn = 0; 881 if (type == IB_QPT_RC) { 882 qp->s_state = IB_OPCODE_RC_SEND_LAST; 883 qp->r_state = IB_OPCODE_RC_SEND_LAST; 884 } else { 885 qp->s_state = IB_OPCODE_UC_SEND_LAST; 886 qp->r_state = IB_OPCODE_UC_SEND_LAST; 887 } 888 qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE; 889 qp->r_nak_state = 0; 890 qp->r_aflags = 0; 891 qp->r_flags = 0; 892 qp->s_head = 0; 893 qp->s_tail = 0; 894 qp->s_cur = 0; 895 qp->s_acked = 0; 896 qp->s_last = 0; 897 qp->s_ssn = 1; 898 qp->s_lsn = 0; 899 qp->s_mig_state = IB_MIG_MIGRATED; 900 qp->r_head_ack_queue = 0; 901 qp->s_tail_ack_queue = 0; 902 qp->s_acked_ack_queue = 0; 903 qp->s_num_rd_atomic = 0; 904 if (qp->r_rq.kwq) 905 qp->r_rq.kwq->count = qp->r_rq.size; 906 qp->r_sge.num_sge = 0; 907 atomic_set(&qp->s_reserved_used, 0); 908 } 909 910 /** 911 * _rvt_reset_qp - initialize the QP state to the reset state 912 * @qp: the QP to reset 913 * @type: the QP type 914 * 915 * r_lock, s_hlock, and s_lock are required to be held by the caller 916 */ 917 static void _rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, 918 enum ib_qp_type type) 919 __must_hold(&qp->s_lock) 920 __must_hold(&qp->s_hlock) 921 __must_hold(&qp->r_lock) 922 { 923 lockdep_assert_held(&qp->r_lock); 924 lockdep_assert_held(&qp->s_hlock); 925 lockdep_assert_held(&qp->s_lock); 926 if (qp->state != IB_QPS_RESET) { 927 qp->state = IB_QPS_RESET; 928 929 /* Let drivers flush their waitlist */ 930 rdi->driver_f.flush_qp_waiters(qp); 931 rvt_stop_rc_timers(qp); 932 qp->s_flags &= ~(RVT_S_TIMER | RVT_S_ANY_WAIT); 933 spin_unlock(&qp->s_lock); 934 spin_unlock(&qp->s_hlock); 935 spin_unlock_irq(&qp->r_lock); 936 937 /* Stop the send queue and the retry timer */ 938 rdi->driver_f.stop_send_queue(qp); 939 rvt_del_timers_sync(qp); 940 /* Wait for things to stop */ 941 rdi->driver_f.quiesce_qp(qp); 942 943 /* take qp out the hash and wait for it to be unused */ 944 rvt_remove_qp(rdi, qp); 945 946 /* grab the lock b/c it was locked at call time */ 947 spin_lock_irq(&qp->r_lock); 948 spin_lock(&qp->s_hlock); 949 spin_lock(&qp->s_lock); 950 951 rvt_clear_mr_refs(qp, 1); 952 /* 953 * Let the driver do any tear down or re-init it needs to for 954 * a qp that has been reset 955 */ 956 rdi->driver_f.notify_qp_reset(qp); 957 } 958 rvt_init_qp(rdi, qp, type); 959 lockdep_assert_held(&qp->r_lock); 960 lockdep_assert_held(&qp->s_hlock); 961 lockdep_assert_held(&qp->s_lock); 962 } 963 964 /** 965 * rvt_reset_qp - initialize the QP state to the reset state 966 * @rdi: the device info 967 * @qp: the QP to reset 968 * @type: the QP type 969 * 970 * This is the wrapper function to acquire the r_lock, s_hlock, and s_lock 971 * before calling _rvt_reset_qp(). 972 */ 973 static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, 974 enum ib_qp_type type) 975 { 976 spin_lock_irq(&qp->r_lock); 977 spin_lock(&qp->s_hlock); 978 spin_lock(&qp->s_lock); 979 _rvt_reset_qp(rdi, qp, type); 980 spin_unlock(&qp->s_lock); 981 spin_unlock(&qp->s_hlock); 982 spin_unlock_irq(&qp->r_lock); 983 } 984 985 /** rvt_free_qpn - Free a qpn from the bit map 986 * @qpt: QP table 987 * @qpn: queue pair number to free 988 */ 989 static void rvt_free_qpn(struct rvt_qpn_table *qpt, u32 qpn) 990 { 991 struct rvt_qpn_map *map; 992 993 if ((qpn & RVT_AIP_QP_PREFIX_MASK) == RVT_AIP_QP_BASE) 994 qpn &= RVT_AIP_QP_SUFFIX; 995 996 map = qpt->map + (qpn & RVT_QPN_MASK) / RVT_BITS_PER_PAGE; 997 if (map->page) 998 clear_bit(qpn & RVT_BITS_PER_PAGE_MASK, map->page); 999 } 1000 1001 /** 1002 * get_allowed_ops - Given a QP type return the appropriate allowed OP 1003 * @type: valid, supported, QP type 1004 */ 1005 static u8 get_allowed_ops(enum ib_qp_type type) 1006 { 1007 return type == IB_QPT_RC ? IB_OPCODE_RC : type == IB_QPT_UC ? 1008 IB_OPCODE_UC : IB_OPCODE_UD; 1009 } 1010 1011 /** 1012 * free_ud_wq_attr - Clean up AH attribute cache for UD QPs 1013 * @qp: Valid QP with allowed_ops set 1014 * 1015 * The rvt_swqe data structure being used is a union, so this is 1016 * only valid for UD QPs. 1017 */ 1018 static void free_ud_wq_attr(struct rvt_qp *qp) 1019 { 1020 struct rvt_swqe *wqe; 1021 int i; 1022 1023 for (i = 0; qp->allowed_ops == IB_OPCODE_UD && i < qp->s_size; i++) { 1024 wqe = rvt_get_swqe_ptr(qp, i); 1025 kfree(wqe->ud_wr.attr); 1026 wqe->ud_wr.attr = NULL; 1027 } 1028 } 1029 1030 /** 1031 * alloc_ud_wq_attr - AH attribute cache for UD QPs 1032 * @qp: Valid QP with allowed_ops set 1033 * @node: Numa node for allocation 1034 * 1035 * The rvt_swqe data structure being used is a union, so this is 1036 * only valid for UD QPs. 1037 */ 1038 static int alloc_ud_wq_attr(struct rvt_qp *qp, int node) 1039 { 1040 struct rvt_swqe *wqe; 1041 int i; 1042 1043 for (i = 0; qp->allowed_ops == IB_OPCODE_UD && i < qp->s_size; i++) { 1044 wqe = rvt_get_swqe_ptr(qp, i); 1045 wqe->ud_wr.attr = kzalloc_node(sizeof(*wqe->ud_wr.attr), 1046 GFP_KERNEL, node); 1047 if (!wqe->ud_wr.attr) { 1048 free_ud_wq_attr(qp); 1049 return -ENOMEM; 1050 } 1051 } 1052 1053 return 0; 1054 } 1055 1056 /** 1057 * rvt_create_qp - create a queue pair for a device 1058 * @ibpd: the protection domain who's device we create the queue pair for 1059 * @init_attr: the attributes of the queue pair 1060 * @udata: user data for libibverbs.so 1061 * 1062 * Queue pair creation is mostly an rvt issue. However, drivers have their own 1063 * unique idea of what queue pair numbers mean. For instance there is a reserved 1064 * range for PSM. 1065 * 1066 * Return: the queue pair on success, otherwise returns an errno. 1067 * 1068 * Called by the ib_create_qp() core verbs function. 1069 */ 1070 struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, 1071 struct ib_qp_init_attr *init_attr, 1072 struct ib_udata *udata) 1073 { 1074 struct rvt_qp *qp; 1075 int err; 1076 struct rvt_swqe *swq = NULL; 1077 size_t sz; 1078 size_t sg_list_sz; 1079 struct ib_qp *ret = ERR_PTR(-ENOMEM); 1080 struct rvt_dev_info *rdi = ib_to_rvt(ibpd->device); 1081 void *priv = NULL; 1082 size_t sqsize; 1083 u8 exclude_prefix = 0; 1084 1085 if (!rdi) 1086 return ERR_PTR(-EINVAL); 1087 1088 if (init_attr->cap.max_send_sge > rdi->dparms.props.max_send_sge || 1089 init_attr->cap.max_send_wr > rdi->dparms.props.max_qp_wr || 1090 (init_attr->create_flags && 1091 init_attr->create_flags != IB_QP_CREATE_NETDEV_USE)) 1092 return ERR_PTR(-EINVAL); 1093 1094 /* Check receive queue parameters if no SRQ is specified. */ 1095 if (!init_attr->srq) { 1096 if (init_attr->cap.max_recv_sge > 1097 rdi->dparms.props.max_recv_sge || 1098 init_attr->cap.max_recv_wr > rdi->dparms.props.max_qp_wr) 1099 return ERR_PTR(-EINVAL); 1100 1101 if (init_attr->cap.max_send_sge + 1102 init_attr->cap.max_send_wr + 1103 init_attr->cap.max_recv_sge + 1104 init_attr->cap.max_recv_wr == 0) 1105 return ERR_PTR(-EINVAL); 1106 } 1107 sqsize = 1108 init_attr->cap.max_send_wr + 1 + 1109 rdi->dparms.reserved_operations; 1110 switch (init_attr->qp_type) { 1111 case IB_QPT_SMI: 1112 case IB_QPT_GSI: 1113 if (init_attr->port_num == 0 || 1114 init_attr->port_num > ibpd->device->phys_port_cnt) 1115 return ERR_PTR(-EINVAL); 1116 /* fall through */ 1117 case IB_QPT_UC: 1118 case IB_QPT_RC: 1119 case IB_QPT_UD: 1120 sz = struct_size(swq, sg_list, init_attr->cap.max_send_sge); 1121 swq = vzalloc_node(array_size(sz, sqsize), rdi->dparms.node); 1122 if (!swq) 1123 return ERR_PTR(-ENOMEM); 1124 1125 sz = sizeof(*qp); 1126 sg_list_sz = 0; 1127 if (init_attr->srq) { 1128 struct rvt_srq *srq = ibsrq_to_rvtsrq(init_attr->srq); 1129 1130 if (srq->rq.max_sge > 1) 1131 sg_list_sz = sizeof(*qp->r_sg_list) * 1132 (srq->rq.max_sge - 1); 1133 } else if (init_attr->cap.max_recv_sge > 1) 1134 sg_list_sz = sizeof(*qp->r_sg_list) * 1135 (init_attr->cap.max_recv_sge - 1); 1136 qp = kzalloc_node(sz + sg_list_sz, GFP_KERNEL, 1137 rdi->dparms.node); 1138 if (!qp) 1139 goto bail_swq; 1140 qp->allowed_ops = get_allowed_ops(init_attr->qp_type); 1141 1142 RCU_INIT_POINTER(qp->next, NULL); 1143 if (init_attr->qp_type == IB_QPT_RC) { 1144 qp->s_ack_queue = 1145 kcalloc_node(rvt_max_atomic(rdi), 1146 sizeof(*qp->s_ack_queue), 1147 GFP_KERNEL, 1148 rdi->dparms.node); 1149 if (!qp->s_ack_queue) 1150 goto bail_qp; 1151 } 1152 /* initialize timers needed for rc qp */ 1153 timer_setup(&qp->s_timer, rvt_rc_timeout, 0); 1154 hrtimer_init(&qp->s_rnr_timer, CLOCK_MONOTONIC, 1155 HRTIMER_MODE_REL); 1156 qp->s_rnr_timer.function = rvt_rc_rnr_retry; 1157 1158 /* 1159 * Driver needs to set up it's private QP structure and do any 1160 * initialization that is needed. 1161 */ 1162 priv = rdi->driver_f.qp_priv_alloc(rdi, qp); 1163 if (IS_ERR(priv)) { 1164 ret = priv; 1165 goto bail_qp; 1166 } 1167 qp->priv = priv; 1168 qp->timeout_jiffies = 1169 usecs_to_jiffies((4096UL * (1UL << qp->timeout)) / 1170 1000UL); 1171 if (init_attr->srq) { 1172 sz = 0; 1173 } else { 1174 qp->r_rq.size = init_attr->cap.max_recv_wr + 1; 1175 qp->r_rq.max_sge = init_attr->cap.max_recv_sge; 1176 sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) + 1177 sizeof(struct rvt_rwqe); 1178 err = rvt_alloc_rq(&qp->r_rq, qp->r_rq.size * sz, 1179 rdi->dparms.node, udata); 1180 if (err) { 1181 ret = ERR_PTR(err); 1182 goto bail_driver_priv; 1183 } 1184 } 1185 1186 /* 1187 * ib_create_qp() will initialize qp->ibqp 1188 * except for qp->ibqp.qp_num. 1189 */ 1190 spin_lock_init(&qp->r_lock); 1191 spin_lock_init(&qp->s_hlock); 1192 spin_lock_init(&qp->s_lock); 1193 atomic_set(&qp->refcount, 0); 1194 atomic_set(&qp->local_ops_pending, 0); 1195 init_waitqueue_head(&qp->wait); 1196 INIT_LIST_HEAD(&qp->rspwait); 1197 qp->state = IB_QPS_RESET; 1198 qp->s_wq = swq; 1199 qp->s_size = sqsize; 1200 qp->s_avail = init_attr->cap.max_send_wr; 1201 qp->s_max_sge = init_attr->cap.max_send_sge; 1202 if (init_attr->sq_sig_type == IB_SIGNAL_REQ_WR) 1203 qp->s_flags = RVT_S_SIGNAL_REQ_WR; 1204 err = alloc_ud_wq_attr(qp, rdi->dparms.node); 1205 if (err) { 1206 ret = (ERR_PTR(err)); 1207 goto bail_rq_rvt; 1208 } 1209 1210 if (init_attr->create_flags & IB_QP_CREATE_NETDEV_USE) 1211 exclude_prefix = RVT_AIP_QP_PREFIX; 1212 1213 err = alloc_qpn(rdi, &rdi->qp_dev->qpn_table, 1214 init_attr->qp_type, 1215 init_attr->port_num, 1216 exclude_prefix); 1217 if (err < 0) { 1218 ret = ERR_PTR(err); 1219 goto bail_rq_wq; 1220 } 1221 qp->ibqp.qp_num = err; 1222 if (init_attr->create_flags & IB_QP_CREATE_NETDEV_USE) 1223 qp->ibqp.qp_num |= RVT_AIP_QP_BASE; 1224 qp->port_num = init_attr->port_num; 1225 rvt_init_qp(rdi, qp, init_attr->qp_type); 1226 if (rdi->driver_f.qp_priv_init) { 1227 err = rdi->driver_f.qp_priv_init(rdi, qp, init_attr); 1228 if (err) { 1229 ret = ERR_PTR(err); 1230 goto bail_rq_wq; 1231 } 1232 } 1233 break; 1234 1235 default: 1236 /* Don't support raw QPs */ 1237 return ERR_PTR(-EOPNOTSUPP); 1238 } 1239 1240 init_attr->cap.max_inline_data = 0; 1241 1242 /* 1243 * Return the address of the RWQ as the offset to mmap. 1244 * See rvt_mmap() for details. 1245 */ 1246 if (udata && udata->outlen >= sizeof(__u64)) { 1247 if (!qp->r_rq.wq) { 1248 __u64 offset = 0; 1249 1250 err = ib_copy_to_udata(udata, &offset, 1251 sizeof(offset)); 1252 if (err) { 1253 ret = ERR_PTR(err); 1254 goto bail_qpn; 1255 } 1256 } else { 1257 u32 s = sizeof(struct rvt_rwq) + qp->r_rq.size * sz; 1258 1259 qp->ip = rvt_create_mmap_info(rdi, s, udata, 1260 qp->r_rq.wq); 1261 if (IS_ERR(qp->ip)) { 1262 ret = ERR_CAST(qp->ip); 1263 goto bail_qpn; 1264 } 1265 1266 err = ib_copy_to_udata(udata, &qp->ip->offset, 1267 sizeof(qp->ip->offset)); 1268 if (err) { 1269 ret = ERR_PTR(err); 1270 goto bail_ip; 1271 } 1272 } 1273 qp->pid = current->pid; 1274 } 1275 1276 spin_lock(&rdi->n_qps_lock); 1277 if (rdi->n_qps_allocated == rdi->dparms.props.max_qp) { 1278 spin_unlock(&rdi->n_qps_lock); 1279 ret = ERR_PTR(-ENOMEM); 1280 goto bail_ip; 1281 } 1282 1283 rdi->n_qps_allocated++; 1284 /* 1285 * Maintain a busy_jiffies variable that will be added to the timeout 1286 * period in mod_retry_timer and add_retry_timer. This busy jiffies 1287 * is scaled by the number of rc qps created for the device to reduce 1288 * the number of timeouts occurring when there is a large number of 1289 * qps. busy_jiffies is incremented every rc qp scaling interval. 1290 * The scaling interval is selected based on extensive performance 1291 * evaluation of targeted workloads. 1292 */ 1293 if (init_attr->qp_type == IB_QPT_RC) { 1294 rdi->n_rc_qps++; 1295 rdi->busy_jiffies = rdi->n_rc_qps / RC_QP_SCALING_INTERVAL; 1296 } 1297 spin_unlock(&rdi->n_qps_lock); 1298 1299 if (qp->ip) { 1300 spin_lock_irq(&rdi->pending_lock); 1301 list_add(&qp->ip->pending_mmaps, &rdi->pending_mmaps); 1302 spin_unlock_irq(&rdi->pending_lock); 1303 } 1304 1305 ret = &qp->ibqp; 1306 1307 return ret; 1308 1309 bail_ip: 1310 if (qp->ip) 1311 kref_put(&qp->ip->ref, rvt_release_mmap_info); 1312 1313 bail_qpn: 1314 rvt_free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num); 1315 1316 bail_rq_wq: 1317 free_ud_wq_attr(qp); 1318 1319 bail_rq_rvt: 1320 rvt_free_rq(&qp->r_rq); 1321 1322 bail_driver_priv: 1323 rdi->driver_f.qp_priv_free(rdi, qp); 1324 1325 bail_qp: 1326 kfree(qp->s_ack_queue); 1327 kfree(qp); 1328 1329 bail_swq: 1330 vfree(swq); 1331 1332 return ret; 1333 } 1334 1335 /** 1336 * rvt_error_qp - put a QP into the error state 1337 * @qp: the QP to put into the error state 1338 * @err: the receive completion error to signal if a RWQE is active 1339 * 1340 * Flushes both send and receive work queues. 1341 * 1342 * Return: true if last WQE event should be generated. 1343 * The QP r_lock and s_lock should be held and interrupts disabled. 1344 * If we are already in error state, just return. 1345 */ 1346 int rvt_error_qp(struct rvt_qp *qp, enum ib_wc_status err) 1347 { 1348 struct ib_wc wc; 1349 int ret = 0; 1350 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 1351 1352 lockdep_assert_held(&qp->r_lock); 1353 lockdep_assert_held(&qp->s_lock); 1354 if (qp->state == IB_QPS_ERR || qp->state == IB_QPS_RESET) 1355 goto bail; 1356 1357 qp->state = IB_QPS_ERR; 1358 1359 if (qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR)) { 1360 qp->s_flags &= ~(RVT_S_TIMER | RVT_S_WAIT_RNR); 1361 del_timer(&qp->s_timer); 1362 } 1363 1364 if (qp->s_flags & RVT_S_ANY_WAIT_SEND) 1365 qp->s_flags &= ~RVT_S_ANY_WAIT_SEND; 1366 1367 rdi->driver_f.notify_error_qp(qp); 1368 1369 /* Schedule the sending tasklet to drain the send work queue. */ 1370 if (READ_ONCE(qp->s_last) != qp->s_head) 1371 rdi->driver_f.schedule_send(qp); 1372 1373 rvt_clear_mr_refs(qp, 0); 1374 1375 memset(&wc, 0, sizeof(wc)); 1376 wc.qp = &qp->ibqp; 1377 wc.opcode = IB_WC_RECV; 1378 1379 if (test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) { 1380 wc.wr_id = qp->r_wr_id; 1381 wc.status = err; 1382 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1); 1383 } 1384 wc.status = IB_WC_WR_FLUSH_ERR; 1385 1386 if (qp->r_rq.kwq) { 1387 u32 head; 1388 u32 tail; 1389 struct rvt_rwq *wq = NULL; 1390 struct rvt_krwq *kwq = NULL; 1391 1392 spin_lock(&qp->r_rq.kwq->c_lock); 1393 /* qp->ip used to validate if there is a user buffer mmaped */ 1394 if (qp->ip) { 1395 wq = qp->r_rq.wq; 1396 head = RDMA_READ_UAPI_ATOMIC(wq->head); 1397 tail = RDMA_READ_UAPI_ATOMIC(wq->tail); 1398 } else { 1399 kwq = qp->r_rq.kwq; 1400 head = kwq->head; 1401 tail = kwq->tail; 1402 } 1403 /* sanity check pointers before trusting them */ 1404 if (head >= qp->r_rq.size) 1405 head = 0; 1406 if (tail >= qp->r_rq.size) 1407 tail = 0; 1408 while (tail != head) { 1409 wc.wr_id = rvt_get_rwqe_ptr(&qp->r_rq, tail)->wr_id; 1410 if (++tail >= qp->r_rq.size) 1411 tail = 0; 1412 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1); 1413 } 1414 if (qp->ip) 1415 RDMA_WRITE_UAPI_ATOMIC(wq->tail, tail); 1416 else 1417 kwq->tail = tail; 1418 spin_unlock(&qp->r_rq.kwq->c_lock); 1419 } else if (qp->ibqp.event_handler) { 1420 ret = 1; 1421 } 1422 1423 bail: 1424 return ret; 1425 } 1426 EXPORT_SYMBOL(rvt_error_qp); 1427 1428 /* 1429 * Put the QP into the hash table. 1430 * The hash table holds a reference to the QP. 1431 */ 1432 static void rvt_insert_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp) 1433 { 1434 struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1]; 1435 unsigned long flags; 1436 1437 rvt_get_qp(qp); 1438 spin_lock_irqsave(&rdi->qp_dev->qpt_lock, flags); 1439 1440 if (qp->ibqp.qp_num <= 1) { 1441 rcu_assign_pointer(rvp->qp[qp->ibqp.qp_num], qp); 1442 } else { 1443 u32 n = hash_32(qp->ibqp.qp_num, rdi->qp_dev->qp_table_bits); 1444 1445 qp->next = rdi->qp_dev->qp_table[n]; 1446 rcu_assign_pointer(rdi->qp_dev->qp_table[n], qp); 1447 trace_rvt_qpinsert(qp, n); 1448 } 1449 1450 spin_unlock_irqrestore(&rdi->qp_dev->qpt_lock, flags); 1451 } 1452 1453 /** 1454 * rvt_modify_qp - modify the attributes of a queue pair 1455 * @ibqp: the queue pair who's attributes we're modifying 1456 * @attr: the new attributes 1457 * @attr_mask: the mask of attributes to modify 1458 * @udata: user data for libibverbs.so 1459 * 1460 * Return: 0 on success, otherwise returns an errno. 1461 */ 1462 int rvt_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1463 int attr_mask, struct ib_udata *udata) 1464 { 1465 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); 1466 struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); 1467 enum ib_qp_state cur_state, new_state; 1468 struct ib_event ev; 1469 int lastwqe = 0; 1470 int mig = 0; 1471 int pmtu = 0; /* for gcc warning only */ 1472 int opa_ah; 1473 1474 spin_lock_irq(&qp->r_lock); 1475 spin_lock(&qp->s_hlock); 1476 spin_lock(&qp->s_lock); 1477 1478 cur_state = attr_mask & IB_QP_CUR_STATE ? 1479 attr->cur_qp_state : qp->state; 1480 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; 1481 opa_ah = rdma_cap_opa_ah(ibqp->device, qp->port_num); 1482 1483 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, 1484 attr_mask)) 1485 goto inval; 1486 1487 if (rdi->driver_f.check_modify_qp && 1488 rdi->driver_f.check_modify_qp(qp, attr, attr_mask, udata)) 1489 goto inval; 1490 1491 if (attr_mask & IB_QP_AV) { 1492 if (opa_ah) { 1493 if (rdma_ah_get_dlid(&attr->ah_attr) >= 1494 opa_get_mcast_base(OPA_MCAST_NR)) 1495 goto inval; 1496 } else { 1497 if (rdma_ah_get_dlid(&attr->ah_attr) >= 1498 be16_to_cpu(IB_MULTICAST_LID_BASE)) 1499 goto inval; 1500 } 1501 1502 if (rvt_check_ah(qp->ibqp.device, &attr->ah_attr)) 1503 goto inval; 1504 } 1505 1506 if (attr_mask & IB_QP_ALT_PATH) { 1507 if (opa_ah) { 1508 if (rdma_ah_get_dlid(&attr->alt_ah_attr) >= 1509 opa_get_mcast_base(OPA_MCAST_NR)) 1510 goto inval; 1511 } else { 1512 if (rdma_ah_get_dlid(&attr->alt_ah_attr) >= 1513 be16_to_cpu(IB_MULTICAST_LID_BASE)) 1514 goto inval; 1515 } 1516 1517 if (rvt_check_ah(qp->ibqp.device, &attr->alt_ah_attr)) 1518 goto inval; 1519 if (attr->alt_pkey_index >= rvt_get_npkeys(rdi)) 1520 goto inval; 1521 } 1522 1523 if (attr_mask & IB_QP_PKEY_INDEX) 1524 if (attr->pkey_index >= rvt_get_npkeys(rdi)) 1525 goto inval; 1526 1527 if (attr_mask & IB_QP_MIN_RNR_TIMER) 1528 if (attr->min_rnr_timer > 31) 1529 goto inval; 1530 1531 if (attr_mask & IB_QP_PORT) 1532 if (qp->ibqp.qp_type == IB_QPT_SMI || 1533 qp->ibqp.qp_type == IB_QPT_GSI || 1534 attr->port_num == 0 || 1535 attr->port_num > ibqp->device->phys_port_cnt) 1536 goto inval; 1537 1538 if (attr_mask & IB_QP_DEST_QPN) 1539 if (attr->dest_qp_num > RVT_QPN_MASK) 1540 goto inval; 1541 1542 if (attr_mask & IB_QP_RETRY_CNT) 1543 if (attr->retry_cnt > 7) 1544 goto inval; 1545 1546 if (attr_mask & IB_QP_RNR_RETRY) 1547 if (attr->rnr_retry > 7) 1548 goto inval; 1549 1550 /* 1551 * Don't allow invalid path_mtu values. OK to set greater 1552 * than the active mtu (or even the max_cap, if we have tuned 1553 * that to a small mtu. We'll set qp->path_mtu 1554 * to the lesser of requested attribute mtu and active, 1555 * for packetizing messages. 1556 * Note that the QP port has to be set in INIT and MTU in RTR. 1557 */ 1558 if (attr_mask & IB_QP_PATH_MTU) { 1559 pmtu = rdi->driver_f.get_pmtu_from_attr(rdi, qp, attr); 1560 if (pmtu < 0) 1561 goto inval; 1562 } 1563 1564 if (attr_mask & IB_QP_PATH_MIG_STATE) { 1565 if (attr->path_mig_state == IB_MIG_REARM) { 1566 if (qp->s_mig_state == IB_MIG_ARMED) 1567 goto inval; 1568 if (new_state != IB_QPS_RTS) 1569 goto inval; 1570 } else if (attr->path_mig_state == IB_MIG_MIGRATED) { 1571 if (qp->s_mig_state == IB_MIG_REARM) 1572 goto inval; 1573 if (new_state != IB_QPS_RTS && new_state != IB_QPS_SQD) 1574 goto inval; 1575 if (qp->s_mig_state == IB_MIG_ARMED) 1576 mig = 1; 1577 } else { 1578 goto inval; 1579 } 1580 } 1581 1582 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 1583 if (attr->max_dest_rd_atomic > rdi->dparms.max_rdma_atomic) 1584 goto inval; 1585 1586 switch (new_state) { 1587 case IB_QPS_RESET: 1588 if (qp->state != IB_QPS_RESET) 1589 _rvt_reset_qp(rdi, qp, ibqp->qp_type); 1590 break; 1591 1592 case IB_QPS_RTR: 1593 /* Allow event to re-trigger if QP set to RTR more than once */ 1594 qp->r_flags &= ~RVT_R_COMM_EST; 1595 qp->state = new_state; 1596 break; 1597 1598 case IB_QPS_SQD: 1599 qp->s_draining = qp->s_last != qp->s_cur; 1600 qp->state = new_state; 1601 break; 1602 1603 case IB_QPS_SQE: 1604 if (qp->ibqp.qp_type == IB_QPT_RC) 1605 goto inval; 1606 qp->state = new_state; 1607 break; 1608 1609 case IB_QPS_ERR: 1610 lastwqe = rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); 1611 break; 1612 1613 default: 1614 qp->state = new_state; 1615 break; 1616 } 1617 1618 if (attr_mask & IB_QP_PKEY_INDEX) 1619 qp->s_pkey_index = attr->pkey_index; 1620 1621 if (attr_mask & IB_QP_PORT) 1622 qp->port_num = attr->port_num; 1623 1624 if (attr_mask & IB_QP_DEST_QPN) 1625 qp->remote_qpn = attr->dest_qp_num; 1626 1627 if (attr_mask & IB_QP_SQ_PSN) { 1628 qp->s_next_psn = attr->sq_psn & rdi->dparms.psn_modify_mask; 1629 qp->s_psn = qp->s_next_psn; 1630 qp->s_sending_psn = qp->s_next_psn; 1631 qp->s_last_psn = qp->s_next_psn - 1; 1632 qp->s_sending_hpsn = qp->s_last_psn; 1633 } 1634 1635 if (attr_mask & IB_QP_RQ_PSN) 1636 qp->r_psn = attr->rq_psn & rdi->dparms.psn_modify_mask; 1637 1638 if (attr_mask & IB_QP_ACCESS_FLAGS) 1639 qp->qp_access_flags = attr->qp_access_flags; 1640 1641 if (attr_mask & IB_QP_AV) { 1642 rdma_replace_ah_attr(&qp->remote_ah_attr, &attr->ah_attr); 1643 qp->s_srate = rdma_ah_get_static_rate(&attr->ah_attr); 1644 qp->srate_mbps = ib_rate_to_mbps(qp->s_srate); 1645 } 1646 1647 if (attr_mask & IB_QP_ALT_PATH) { 1648 rdma_replace_ah_attr(&qp->alt_ah_attr, &attr->alt_ah_attr); 1649 qp->s_alt_pkey_index = attr->alt_pkey_index; 1650 } 1651 1652 if (attr_mask & IB_QP_PATH_MIG_STATE) { 1653 qp->s_mig_state = attr->path_mig_state; 1654 if (mig) { 1655 qp->remote_ah_attr = qp->alt_ah_attr; 1656 qp->port_num = rdma_ah_get_port_num(&qp->alt_ah_attr); 1657 qp->s_pkey_index = qp->s_alt_pkey_index; 1658 } 1659 } 1660 1661 if (attr_mask & IB_QP_PATH_MTU) { 1662 qp->pmtu = rdi->driver_f.mtu_from_qp(rdi, qp, pmtu); 1663 qp->log_pmtu = ilog2(qp->pmtu); 1664 } 1665 1666 if (attr_mask & IB_QP_RETRY_CNT) { 1667 qp->s_retry_cnt = attr->retry_cnt; 1668 qp->s_retry = attr->retry_cnt; 1669 } 1670 1671 if (attr_mask & IB_QP_RNR_RETRY) { 1672 qp->s_rnr_retry_cnt = attr->rnr_retry; 1673 qp->s_rnr_retry = attr->rnr_retry; 1674 } 1675 1676 if (attr_mask & IB_QP_MIN_RNR_TIMER) 1677 qp->r_min_rnr_timer = attr->min_rnr_timer; 1678 1679 if (attr_mask & IB_QP_TIMEOUT) { 1680 qp->timeout = attr->timeout; 1681 qp->timeout_jiffies = rvt_timeout_to_jiffies(qp->timeout); 1682 } 1683 1684 if (attr_mask & IB_QP_QKEY) 1685 qp->qkey = attr->qkey; 1686 1687 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 1688 qp->r_max_rd_atomic = attr->max_dest_rd_atomic; 1689 1690 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) 1691 qp->s_max_rd_atomic = attr->max_rd_atomic; 1692 1693 if (rdi->driver_f.modify_qp) 1694 rdi->driver_f.modify_qp(qp, attr, attr_mask, udata); 1695 1696 spin_unlock(&qp->s_lock); 1697 spin_unlock(&qp->s_hlock); 1698 spin_unlock_irq(&qp->r_lock); 1699 1700 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) 1701 rvt_insert_qp(rdi, qp); 1702 1703 if (lastwqe) { 1704 ev.device = qp->ibqp.device; 1705 ev.element.qp = &qp->ibqp; 1706 ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 1707 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 1708 } 1709 if (mig) { 1710 ev.device = qp->ibqp.device; 1711 ev.element.qp = &qp->ibqp; 1712 ev.event = IB_EVENT_PATH_MIG; 1713 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 1714 } 1715 return 0; 1716 1717 inval: 1718 spin_unlock(&qp->s_lock); 1719 spin_unlock(&qp->s_hlock); 1720 spin_unlock_irq(&qp->r_lock); 1721 return -EINVAL; 1722 } 1723 1724 /** 1725 * rvt_destroy_qp - destroy a queue pair 1726 * @ibqp: the queue pair to destroy 1727 * 1728 * Note that this can be called while the QP is actively sending or 1729 * receiving! 1730 * 1731 * Return: 0 on success. 1732 */ 1733 int rvt_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 1734 { 1735 struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); 1736 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); 1737 1738 rvt_reset_qp(rdi, qp, ibqp->qp_type); 1739 1740 wait_event(qp->wait, !atomic_read(&qp->refcount)); 1741 /* qpn is now available for use again */ 1742 rvt_free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num); 1743 1744 spin_lock(&rdi->n_qps_lock); 1745 rdi->n_qps_allocated--; 1746 if (qp->ibqp.qp_type == IB_QPT_RC) { 1747 rdi->n_rc_qps--; 1748 rdi->busy_jiffies = rdi->n_rc_qps / RC_QP_SCALING_INTERVAL; 1749 } 1750 spin_unlock(&rdi->n_qps_lock); 1751 1752 if (qp->ip) 1753 kref_put(&qp->ip->ref, rvt_release_mmap_info); 1754 kvfree(qp->r_rq.kwq); 1755 rdi->driver_f.qp_priv_free(rdi, qp); 1756 kfree(qp->s_ack_queue); 1757 rdma_destroy_ah_attr(&qp->remote_ah_attr); 1758 rdma_destroy_ah_attr(&qp->alt_ah_attr); 1759 free_ud_wq_attr(qp); 1760 vfree(qp->s_wq); 1761 kfree(qp); 1762 return 0; 1763 } 1764 1765 /** 1766 * rvt_query_qp - query an ipbq 1767 * @ibqp: IB qp to query 1768 * @attr: attr struct to fill in 1769 * @attr_mask: attr mask ignored 1770 * @init_attr: struct to fill in 1771 * 1772 * Return: always 0 1773 */ 1774 int rvt_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1775 int attr_mask, struct ib_qp_init_attr *init_attr) 1776 { 1777 struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); 1778 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); 1779 1780 attr->qp_state = qp->state; 1781 attr->cur_qp_state = attr->qp_state; 1782 attr->path_mtu = rdi->driver_f.mtu_to_path_mtu(qp->pmtu); 1783 attr->path_mig_state = qp->s_mig_state; 1784 attr->qkey = qp->qkey; 1785 attr->rq_psn = qp->r_psn & rdi->dparms.psn_mask; 1786 attr->sq_psn = qp->s_next_psn & rdi->dparms.psn_mask; 1787 attr->dest_qp_num = qp->remote_qpn; 1788 attr->qp_access_flags = qp->qp_access_flags; 1789 attr->cap.max_send_wr = qp->s_size - 1 - 1790 rdi->dparms.reserved_operations; 1791 attr->cap.max_recv_wr = qp->ibqp.srq ? 0 : qp->r_rq.size - 1; 1792 attr->cap.max_send_sge = qp->s_max_sge; 1793 attr->cap.max_recv_sge = qp->r_rq.max_sge; 1794 attr->cap.max_inline_data = 0; 1795 attr->ah_attr = qp->remote_ah_attr; 1796 attr->alt_ah_attr = qp->alt_ah_attr; 1797 attr->pkey_index = qp->s_pkey_index; 1798 attr->alt_pkey_index = qp->s_alt_pkey_index; 1799 attr->en_sqd_async_notify = 0; 1800 attr->sq_draining = qp->s_draining; 1801 attr->max_rd_atomic = qp->s_max_rd_atomic; 1802 attr->max_dest_rd_atomic = qp->r_max_rd_atomic; 1803 attr->min_rnr_timer = qp->r_min_rnr_timer; 1804 attr->port_num = qp->port_num; 1805 attr->timeout = qp->timeout; 1806 attr->retry_cnt = qp->s_retry_cnt; 1807 attr->rnr_retry = qp->s_rnr_retry_cnt; 1808 attr->alt_port_num = 1809 rdma_ah_get_port_num(&qp->alt_ah_attr); 1810 attr->alt_timeout = qp->alt_timeout; 1811 1812 init_attr->event_handler = qp->ibqp.event_handler; 1813 init_attr->qp_context = qp->ibqp.qp_context; 1814 init_attr->send_cq = qp->ibqp.send_cq; 1815 init_attr->recv_cq = qp->ibqp.recv_cq; 1816 init_attr->srq = qp->ibqp.srq; 1817 init_attr->cap = attr->cap; 1818 if (qp->s_flags & RVT_S_SIGNAL_REQ_WR) 1819 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR; 1820 else 1821 init_attr->sq_sig_type = IB_SIGNAL_ALL_WR; 1822 init_attr->qp_type = qp->ibqp.qp_type; 1823 init_attr->port_num = qp->port_num; 1824 return 0; 1825 } 1826 1827 /** 1828 * rvt_post_receive - post a receive on a QP 1829 * @ibqp: the QP to post the receive on 1830 * @wr: the WR to post 1831 * @bad_wr: the first bad WR is put here 1832 * 1833 * This may be called from interrupt context. 1834 * 1835 * Return: 0 on success otherwise errno 1836 */ 1837 int rvt_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 1838 const struct ib_recv_wr **bad_wr) 1839 { 1840 struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); 1841 struct rvt_krwq *wq = qp->r_rq.kwq; 1842 unsigned long flags; 1843 int qp_err_flush = (ib_rvt_state_ops[qp->state] & RVT_FLUSH_RECV) && 1844 !qp->ibqp.srq; 1845 1846 /* Check that state is OK to post receive. */ 1847 if (!(ib_rvt_state_ops[qp->state] & RVT_POST_RECV_OK) || !wq) { 1848 *bad_wr = wr; 1849 return -EINVAL; 1850 } 1851 1852 for (; wr; wr = wr->next) { 1853 struct rvt_rwqe *wqe; 1854 u32 next; 1855 int i; 1856 1857 if ((unsigned)wr->num_sge > qp->r_rq.max_sge) { 1858 *bad_wr = wr; 1859 return -EINVAL; 1860 } 1861 1862 spin_lock_irqsave(&qp->r_rq.kwq->p_lock, flags); 1863 next = wq->head + 1; 1864 if (next >= qp->r_rq.size) 1865 next = 0; 1866 if (next == READ_ONCE(wq->tail)) { 1867 spin_unlock_irqrestore(&qp->r_rq.kwq->p_lock, flags); 1868 *bad_wr = wr; 1869 return -ENOMEM; 1870 } 1871 if (unlikely(qp_err_flush)) { 1872 struct ib_wc wc; 1873 1874 memset(&wc, 0, sizeof(wc)); 1875 wc.qp = &qp->ibqp; 1876 wc.opcode = IB_WC_RECV; 1877 wc.wr_id = wr->wr_id; 1878 wc.status = IB_WC_WR_FLUSH_ERR; 1879 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1); 1880 } else { 1881 wqe = rvt_get_rwqe_ptr(&qp->r_rq, wq->head); 1882 wqe->wr_id = wr->wr_id; 1883 wqe->num_sge = wr->num_sge; 1884 for (i = 0; i < wr->num_sge; i++) { 1885 wqe->sg_list[i].addr = wr->sg_list[i].addr; 1886 wqe->sg_list[i].length = wr->sg_list[i].length; 1887 wqe->sg_list[i].lkey = wr->sg_list[i].lkey; 1888 } 1889 /* 1890 * Make sure queue entry is written 1891 * before the head index. 1892 */ 1893 smp_store_release(&wq->head, next); 1894 } 1895 spin_unlock_irqrestore(&qp->r_rq.kwq->p_lock, flags); 1896 } 1897 return 0; 1898 } 1899 1900 /** 1901 * rvt_qp_valid_operation - validate post send wr request 1902 * @qp - the qp 1903 * @post-parms - the post send table for the driver 1904 * @wr - the work request 1905 * 1906 * The routine validates the operation based on the 1907 * validation table an returns the length of the operation 1908 * which can extend beyond the ib_send_bw. Operation 1909 * dependent flags key atomic operation validation. 1910 * 1911 * There is an exception for UD qps that validates the pd and 1912 * overrides the length to include the additional UD specific 1913 * length. 1914 * 1915 * Returns a negative error or the length of the work request 1916 * for building the swqe. 1917 */ 1918 static inline int rvt_qp_valid_operation( 1919 struct rvt_qp *qp, 1920 const struct rvt_operation_params *post_parms, 1921 const struct ib_send_wr *wr) 1922 { 1923 int len; 1924 1925 if (wr->opcode >= RVT_OPERATION_MAX || !post_parms[wr->opcode].length) 1926 return -EINVAL; 1927 if (!(post_parms[wr->opcode].qpt_support & BIT(qp->ibqp.qp_type))) 1928 return -EINVAL; 1929 if ((post_parms[wr->opcode].flags & RVT_OPERATION_PRIV) && 1930 ibpd_to_rvtpd(qp->ibqp.pd)->user) 1931 return -EINVAL; 1932 if (post_parms[wr->opcode].flags & RVT_OPERATION_ATOMIC_SGE && 1933 (wr->num_sge == 0 || 1934 wr->sg_list[0].length < sizeof(u64) || 1935 wr->sg_list[0].addr & (sizeof(u64) - 1))) 1936 return -EINVAL; 1937 if (post_parms[wr->opcode].flags & RVT_OPERATION_ATOMIC && 1938 !qp->s_max_rd_atomic) 1939 return -EINVAL; 1940 len = post_parms[wr->opcode].length; 1941 /* UD specific */ 1942 if (qp->ibqp.qp_type != IB_QPT_UC && 1943 qp->ibqp.qp_type != IB_QPT_RC) { 1944 if (qp->ibqp.pd != ud_wr(wr)->ah->pd) 1945 return -EINVAL; 1946 len = sizeof(struct ib_ud_wr); 1947 } 1948 return len; 1949 } 1950 1951 /** 1952 * rvt_qp_is_avail - determine queue capacity 1953 * @qp: the qp 1954 * @rdi: the rdmavt device 1955 * @reserved_op: is reserved operation 1956 * 1957 * This assumes the s_hlock is held but the s_last 1958 * qp variable is uncontrolled. 1959 * 1960 * For non reserved operations, the qp->s_avail 1961 * may be changed. 1962 * 1963 * The return value is zero or a -ENOMEM. 1964 */ 1965 static inline int rvt_qp_is_avail( 1966 struct rvt_qp *qp, 1967 struct rvt_dev_info *rdi, 1968 bool reserved_op) 1969 { 1970 u32 slast; 1971 u32 avail; 1972 u32 reserved_used; 1973 1974 /* see rvt_qp_wqe_unreserve() */ 1975 smp_mb__before_atomic(); 1976 if (unlikely(reserved_op)) { 1977 /* see rvt_qp_wqe_unreserve() */ 1978 reserved_used = atomic_read(&qp->s_reserved_used); 1979 if (reserved_used >= rdi->dparms.reserved_operations) 1980 return -ENOMEM; 1981 return 0; 1982 } 1983 /* non-reserved operations */ 1984 if (likely(qp->s_avail)) 1985 return 0; 1986 /* See rvt_qp_complete_swqe() */ 1987 slast = smp_load_acquire(&qp->s_last); 1988 if (qp->s_head >= slast) 1989 avail = qp->s_size - (qp->s_head - slast); 1990 else 1991 avail = slast - qp->s_head; 1992 1993 reserved_used = atomic_read(&qp->s_reserved_used); 1994 avail = avail - 1 - 1995 (rdi->dparms.reserved_operations - reserved_used); 1996 /* insure we don't assign a negative s_avail */ 1997 if ((s32)avail <= 0) 1998 return -ENOMEM; 1999 qp->s_avail = avail; 2000 if (WARN_ON(qp->s_avail > 2001 (qp->s_size - 1 - rdi->dparms.reserved_operations))) 2002 rvt_pr_err(rdi, 2003 "More avail entries than QP RB size.\nQP: %u, size: %u, avail: %u\nhead: %u, tail: %u, cur: %u, acked: %u, last: %u", 2004 qp->ibqp.qp_num, qp->s_size, qp->s_avail, 2005 qp->s_head, qp->s_tail, qp->s_cur, 2006 qp->s_acked, qp->s_last); 2007 return 0; 2008 } 2009 2010 /** 2011 * rvt_post_one_wr - post one RC, UC, or UD send work request 2012 * @qp: the QP to post on 2013 * @wr: the work request to send 2014 */ 2015 static int rvt_post_one_wr(struct rvt_qp *qp, 2016 const struct ib_send_wr *wr, 2017 bool *call_send) 2018 { 2019 struct rvt_swqe *wqe; 2020 u32 next; 2021 int i; 2022 int j; 2023 int acc; 2024 struct rvt_lkey_table *rkt; 2025 struct rvt_pd *pd; 2026 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 2027 u8 log_pmtu; 2028 int ret; 2029 size_t cplen; 2030 bool reserved_op; 2031 int local_ops_delayed = 0; 2032 2033 BUILD_BUG_ON(IB_QPT_MAX >= (sizeof(u32) * BITS_PER_BYTE)); 2034 2035 /* IB spec says that num_sge == 0 is OK. */ 2036 if (unlikely(wr->num_sge > qp->s_max_sge)) 2037 return -EINVAL; 2038 2039 ret = rvt_qp_valid_operation(qp, rdi->post_parms, wr); 2040 if (ret < 0) 2041 return ret; 2042 cplen = ret; 2043 2044 /* 2045 * Local operations include fast register and local invalidate. 2046 * Fast register needs to be processed immediately because the 2047 * registered lkey may be used by following work requests and the 2048 * lkey needs to be valid at the time those requests are posted. 2049 * Local invalidate can be processed immediately if fencing is 2050 * not required and no previous local invalidate ops are pending. 2051 * Signaled local operations that have been processed immediately 2052 * need to have requests with "completion only" flags set posted 2053 * to the send queue in order to generate completions. 2054 */ 2055 if ((rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL)) { 2056 switch (wr->opcode) { 2057 case IB_WR_REG_MR: 2058 ret = rvt_fast_reg_mr(qp, 2059 reg_wr(wr)->mr, 2060 reg_wr(wr)->key, 2061 reg_wr(wr)->access); 2062 if (ret || !(wr->send_flags & IB_SEND_SIGNALED)) 2063 return ret; 2064 break; 2065 case IB_WR_LOCAL_INV: 2066 if ((wr->send_flags & IB_SEND_FENCE) || 2067 atomic_read(&qp->local_ops_pending)) { 2068 local_ops_delayed = 1; 2069 } else { 2070 ret = rvt_invalidate_rkey( 2071 qp, wr->ex.invalidate_rkey); 2072 if (ret || !(wr->send_flags & IB_SEND_SIGNALED)) 2073 return ret; 2074 } 2075 break; 2076 default: 2077 return -EINVAL; 2078 } 2079 } 2080 2081 reserved_op = rdi->post_parms[wr->opcode].flags & 2082 RVT_OPERATION_USE_RESERVE; 2083 /* check for avail */ 2084 ret = rvt_qp_is_avail(qp, rdi, reserved_op); 2085 if (ret) 2086 return ret; 2087 next = qp->s_head + 1; 2088 if (next >= qp->s_size) 2089 next = 0; 2090 2091 rkt = &rdi->lkey_table; 2092 pd = ibpd_to_rvtpd(qp->ibqp.pd); 2093 wqe = rvt_get_swqe_ptr(qp, qp->s_head); 2094 2095 /* cplen has length from above */ 2096 memcpy(&wqe->wr, wr, cplen); 2097 2098 wqe->length = 0; 2099 j = 0; 2100 if (wr->num_sge) { 2101 struct rvt_sge *last_sge = NULL; 2102 2103 acc = wr->opcode >= IB_WR_RDMA_READ ? 2104 IB_ACCESS_LOCAL_WRITE : 0; 2105 for (i = 0; i < wr->num_sge; i++) { 2106 u32 length = wr->sg_list[i].length; 2107 2108 if (length == 0) 2109 continue; 2110 ret = rvt_lkey_ok(rkt, pd, &wqe->sg_list[j], last_sge, 2111 &wr->sg_list[i], acc); 2112 if (unlikely(ret < 0)) 2113 goto bail_inval_free; 2114 wqe->length += length; 2115 if (ret) 2116 last_sge = &wqe->sg_list[j]; 2117 j += ret; 2118 } 2119 wqe->wr.num_sge = j; 2120 } 2121 2122 /* 2123 * Calculate and set SWQE PSN values prior to handing it off 2124 * to the driver's check routine. This give the driver the 2125 * opportunity to adjust PSN values based on internal checks. 2126 */ 2127 log_pmtu = qp->log_pmtu; 2128 if (qp->allowed_ops == IB_OPCODE_UD) { 2129 struct rvt_ah *ah = rvt_get_swqe_ah(wqe); 2130 2131 log_pmtu = ah->log_pmtu; 2132 rdma_copy_ah_attr(wqe->ud_wr.attr, &ah->attr); 2133 } 2134 2135 if (rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL) { 2136 if (local_ops_delayed) 2137 atomic_inc(&qp->local_ops_pending); 2138 else 2139 wqe->wr.send_flags |= RVT_SEND_COMPLETION_ONLY; 2140 wqe->ssn = 0; 2141 wqe->psn = 0; 2142 wqe->lpsn = 0; 2143 } else { 2144 wqe->ssn = qp->s_ssn++; 2145 wqe->psn = qp->s_next_psn; 2146 wqe->lpsn = wqe->psn + 2147 (wqe->length ? 2148 ((wqe->length - 1) >> log_pmtu) : 2149 0); 2150 } 2151 2152 /* general part of wqe valid - allow for driver checks */ 2153 if (rdi->driver_f.setup_wqe) { 2154 ret = rdi->driver_f.setup_wqe(qp, wqe, call_send); 2155 if (ret < 0) 2156 goto bail_inval_free_ref; 2157 } 2158 2159 if (!(rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL)) 2160 qp->s_next_psn = wqe->lpsn + 1; 2161 2162 if (unlikely(reserved_op)) { 2163 wqe->wr.send_flags |= RVT_SEND_RESERVE_USED; 2164 rvt_qp_wqe_reserve(qp, wqe); 2165 } else { 2166 wqe->wr.send_flags &= ~RVT_SEND_RESERVE_USED; 2167 qp->s_avail--; 2168 } 2169 trace_rvt_post_one_wr(qp, wqe, wr->num_sge); 2170 smp_wmb(); /* see request builders */ 2171 qp->s_head = next; 2172 2173 return 0; 2174 2175 bail_inval_free_ref: 2176 if (qp->allowed_ops == IB_OPCODE_UD) 2177 rdma_destroy_ah_attr(wqe->ud_wr.attr); 2178 bail_inval_free: 2179 /* release mr holds */ 2180 while (j) { 2181 struct rvt_sge *sge = &wqe->sg_list[--j]; 2182 2183 rvt_put_mr(sge->mr); 2184 } 2185 return ret; 2186 } 2187 2188 /** 2189 * rvt_post_send - post a send on a QP 2190 * @ibqp: the QP to post the send on 2191 * @wr: the list of work requests to post 2192 * @bad_wr: the first bad WR is put here 2193 * 2194 * This may be called from interrupt context. 2195 * 2196 * Return: 0 on success else errno 2197 */ 2198 int rvt_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 2199 const struct ib_send_wr **bad_wr) 2200 { 2201 struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); 2202 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); 2203 unsigned long flags = 0; 2204 bool call_send; 2205 unsigned nreq = 0; 2206 int err = 0; 2207 2208 spin_lock_irqsave(&qp->s_hlock, flags); 2209 2210 /* 2211 * Ensure QP state is such that we can send. If not bail out early, 2212 * there is no need to do this every time we post a send. 2213 */ 2214 if (unlikely(!(ib_rvt_state_ops[qp->state] & RVT_POST_SEND_OK))) { 2215 spin_unlock_irqrestore(&qp->s_hlock, flags); 2216 return -EINVAL; 2217 } 2218 2219 /* 2220 * If the send queue is empty, and we only have a single WR then just go 2221 * ahead and kick the send engine into gear. Otherwise we will always 2222 * just schedule the send to happen later. 2223 */ 2224 call_send = qp->s_head == READ_ONCE(qp->s_last) && !wr->next; 2225 2226 for (; wr; wr = wr->next) { 2227 err = rvt_post_one_wr(qp, wr, &call_send); 2228 if (unlikely(err)) { 2229 *bad_wr = wr; 2230 goto bail; 2231 } 2232 nreq++; 2233 } 2234 bail: 2235 spin_unlock_irqrestore(&qp->s_hlock, flags); 2236 if (nreq) { 2237 /* 2238 * Only call do_send if there is exactly one packet, and the 2239 * driver said it was ok. 2240 */ 2241 if (nreq == 1 && call_send) 2242 rdi->driver_f.do_send(qp); 2243 else 2244 rdi->driver_f.schedule_send_no_lock(qp); 2245 } 2246 return err; 2247 } 2248 2249 /** 2250 * rvt_post_srq_receive - post a receive on a shared receive queue 2251 * @ibsrq: the SRQ to post the receive on 2252 * @wr: the list of work requests to post 2253 * @bad_wr: A pointer to the first WR to cause a problem is put here 2254 * 2255 * This may be called from interrupt context. 2256 * 2257 * Return: 0 on success else errno 2258 */ 2259 int rvt_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 2260 const struct ib_recv_wr **bad_wr) 2261 { 2262 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); 2263 struct rvt_krwq *wq; 2264 unsigned long flags; 2265 2266 for (; wr; wr = wr->next) { 2267 struct rvt_rwqe *wqe; 2268 u32 next; 2269 int i; 2270 2271 if ((unsigned)wr->num_sge > srq->rq.max_sge) { 2272 *bad_wr = wr; 2273 return -EINVAL; 2274 } 2275 2276 spin_lock_irqsave(&srq->rq.kwq->p_lock, flags); 2277 wq = srq->rq.kwq; 2278 next = wq->head + 1; 2279 if (next >= srq->rq.size) 2280 next = 0; 2281 if (next == READ_ONCE(wq->tail)) { 2282 spin_unlock_irqrestore(&srq->rq.kwq->p_lock, flags); 2283 *bad_wr = wr; 2284 return -ENOMEM; 2285 } 2286 2287 wqe = rvt_get_rwqe_ptr(&srq->rq, wq->head); 2288 wqe->wr_id = wr->wr_id; 2289 wqe->num_sge = wr->num_sge; 2290 for (i = 0; i < wr->num_sge; i++) { 2291 wqe->sg_list[i].addr = wr->sg_list[i].addr; 2292 wqe->sg_list[i].length = wr->sg_list[i].length; 2293 wqe->sg_list[i].lkey = wr->sg_list[i].lkey; 2294 } 2295 /* Make sure queue entry is written before the head index. */ 2296 smp_store_release(&wq->head, next); 2297 spin_unlock_irqrestore(&srq->rq.kwq->p_lock, flags); 2298 } 2299 return 0; 2300 } 2301 2302 /* 2303 * rvt used the internal kernel struct as part of its ABI, for now make sure 2304 * the kernel struct does not change layout. FIXME: rvt should never cast the 2305 * user struct to a kernel struct. 2306 */ 2307 static struct ib_sge *rvt_cast_sge(struct rvt_wqe_sge *sge) 2308 { 2309 BUILD_BUG_ON(offsetof(struct ib_sge, addr) != 2310 offsetof(struct rvt_wqe_sge, addr)); 2311 BUILD_BUG_ON(offsetof(struct ib_sge, length) != 2312 offsetof(struct rvt_wqe_sge, length)); 2313 BUILD_BUG_ON(offsetof(struct ib_sge, lkey) != 2314 offsetof(struct rvt_wqe_sge, lkey)); 2315 return (struct ib_sge *)sge; 2316 } 2317 2318 /* 2319 * Validate a RWQE and fill in the SGE state. 2320 * Return 1 if OK. 2321 */ 2322 static int init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe) 2323 { 2324 int i, j, ret; 2325 struct ib_wc wc; 2326 struct rvt_lkey_table *rkt; 2327 struct rvt_pd *pd; 2328 struct rvt_sge_state *ss; 2329 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 2330 2331 rkt = &rdi->lkey_table; 2332 pd = ibpd_to_rvtpd(qp->ibqp.srq ? qp->ibqp.srq->pd : qp->ibqp.pd); 2333 ss = &qp->r_sge; 2334 ss->sg_list = qp->r_sg_list; 2335 qp->r_len = 0; 2336 for (i = j = 0; i < wqe->num_sge; i++) { 2337 if (wqe->sg_list[i].length == 0) 2338 continue; 2339 /* Check LKEY */ 2340 ret = rvt_lkey_ok(rkt, pd, j ? &ss->sg_list[j - 1] : &ss->sge, 2341 NULL, rvt_cast_sge(&wqe->sg_list[i]), 2342 IB_ACCESS_LOCAL_WRITE); 2343 if (unlikely(ret <= 0)) 2344 goto bad_lkey; 2345 qp->r_len += wqe->sg_list[i].length; 2346 j++; 2347 } 2348 ss->num_sge = j; 2349 ss->total_len = qp->r_len; 2350 return 1; 2351 2352 bad_lkey: 2353 while (j) { 2354 struct rvt_sge *sge = --j ? &ss->sg_list[j - 1] : &ss->sge; 2355 2356 rvt_put_mr(sge->mr); 2357 } 2358 ss->num_sge = 0; 2359 memset(&wc, 0, sizeof(wc)); 2360 wc.wr_id = wqe->wr_id; 2361 wc.status = IB_WC_LOC_PROT_ERR; 2362 wc.opcode = IB_WC_RECV; 2363 wc.qp = &qp->ibqp; 2364 /* Signal solicited completion event. */ 2365 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1); 2366 return 0; 2367 } 2368 2369 /** 2370 * get_count - count numbers of request work queue entries 2371 * in circular buffer 2372 * @rq: data structure for request queue entry 2373 * @tail: tail indices of the circular buffer 2374 * @head: head indices of the circular buffer 2375 * 2376 * Return - total number of entries in the circular buffer 2377 */ 2378 static u32 get_count(struct rvt_rq *rq, u32 tail, u32 head) 2379 { 2380 u32 count; 2381 2382 count = head; 2383 2384 if (count >= rq->size) 2385 count = 0; 2386 if (count < tail) 2387 count += rq->size - tail; 2388 else 2389 count -= tail; 2390 2391 return count; 2392 } 2393 2394 /** 2395 * get_rvt_head - get head indices of the circular buffer 2396 * @rq: data structure for request queue entry 2397 * @ip: the QP 2398 * 2399 * Return - head index value 2400 */ 2401 static inline u32 get_rvt_head(struct rvt_rq *rq, void *ip) 2402 { 2403 u32 head; 2404 2405 if (ip) 2406 head = RDMA_READ_UAPI_ATOMIC(rq->wq->head); 2407 else 2408 head = rq->kwq->head; 2409 2410 return head; 2411 } 2412 2413 /** 2414 * rvt_get_rwqe - copy the next RWQE into the QP's RWQE 2415 * @qp: the QP 2416 * @wr_id_only: update qp->r_wr_id only, not qp->r_sge 2417 * 2418 * Return -1 if there is a local error, 0 if no RWQE is available, 2419 * otherwise return 1. 2420 * 2421 * Can be called from interrupt level. 2422 */ 2423 int rvt_get_rwqe(struct rvt_qp *qp, bool wr_id_only) 2424 { 2425 unsigned long flags; 2426 struct rvt_rq *rq; 2427 struct rvt_krwq *kwq = NULL; 2428 struct rvt_rwq *wq; 2429 struct rvt_srq *srq; 2430 struct rvt_rwqe *wqe; 2431 void (*handler)(struct ib_event *, void *); 2432 u32 tail; 2433 u32 head; 2434 int ret; 2435 void *ip = NULL; 2436 2437 if (qp->ibqp.srq) { 2438 srq = ibsrq_to_rvtsrq(qp->ibqp.srq); 2439 handler = srq->ibsrq.event_handler; 2440 rq = &srq->rq; 2441 ip = srq->ip; 2442 } else { 2443 srq = NULL; 2444 handler = NULL; 2445 rq = &qp->r_rq; 2446 ip = qp->ip; 2447 } 2448 2449 spin_lock_irqsave(&rq->kwq->c_lock, flags); 2450 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { 2451 ret = 0; 2452 goto unlock; 2453 } 2454 kwq = rq->kwq; 2455 if (ip) { 2456 wq = rq->wq; 2457 tail = RDMA_READ_UAPI_ATOMIC(wq->tail); 2458 } else { 2459 tail = kwq->tail; 2460 } 2461 2462 /* Validate tail before using it since it is user writable. */ 2463 if (tail >= rq->size) 2464 tail = 0; 2465 2466 if (kwq->count < RVT_RWQ_COUNT_THRESHOLD) { 2467 head = get_rvt_head(rq, ip); 2468 kwq->count = get_count(rq, tail, head); 2469 } 2470 if (unlikely(kwq->count == 0)) { 2471 ret = 0; 2472 goto unlock; 2473 } 2474 /* Make sure entry is read after the count is read. */ 2475 smp_rmb(); 2476 wqe = rvt_get_rwqe_ptr(rq, tail); 2477 /* 2478 * Even though we update the tail index in memory, the verbs 2479 * consumer is not supposed to post more entries until a 2480 * completion is generated. 2481 */ 2482 if (++tail >= rq->size) 2483 tail = 0; 2484 if (ip) 2485 RDMA_WRITE_UAPI_ATOMIC(wq->tail, tail); 2486 else 2487 kwq->tail = tail; 2488 if (!wr_id_only && !init_sge(qp, wqe)) { 2489 ret = -1; 2490 goto unlock; 2491 } 2492 qp->r_wr_id = wqe->wr_id; 2493 2494 kwq->count--; 2495 ret = 1; 2496 set_bit(RVT_R_WRID_VALID, &qp->r_aflags); 2497 if (handler) { 2498 /* 2499 * Validate head pointer value and compute 2500 * the number of remaining WQEs. 2501 */ 2502 if (kwq->count < srq->limit) { 2503 kwq->count = get_count(rq, tail, get_rvt_head(rq, ip)); 2504 if (kwq->count < srq->limit) { 2505 struct ib_event ev; 2506 2507 srq->limit = 0; 2508 spin_unlock_irqrestore(&rq->kwq->c_lock, flags); 2509 ev.device = qp->ibqp.device; 2510 ev.element.srq = qp->ibqp.srq; 2511 ev.event = IB_EVENT_SRQ_LIMIT_REACHED; 2512 handler(&ev, srq->ibsrq.srq_context); 2513 goto bail; 2514 } 2515 } 2516 } 2517 unlock: 2518 spin_unlock_irqrestore(&rq->kwq->c_lock, flags); 2519 bail: 2520 return ret; 2521 } 2522 EXPORT_SYMBOL(rvt_get_rwqe); 2523 2524 /** 2525 * qp_comm_est - handle trap with QP established 2526 * @qp: the QP 2527 */ 2528 void rvt_comm_est(struct rvt_qp *qp) 2529 { 2530 qp->r_flags |= RVT_R_COMM_EST; 2531 if (qp->ibqp.event_handler) { 2532 struct ib_event ev; 2533 2534 ev.device = qp->ibqp.device; 2535 ev.element.qp = &qp->ibqp; 2536 ev.event = IB_EVENT_COMM_EST; 2537 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 2538 } 2539 } 2540 EXPORT_SYMBOL(rvt_comm_est); 2541 2542 void rvt_rc_error(struct rvt_qp *qp, enum ib_wc_status err) 2543 { 2544 unsigned long flags; 2545 int lastwqe; 2546 2547 spin_lock_irqsave(&qp->s_lock, flags); 2548 lastwqe = rvt_error_qp(qp, err); 2549 spin_unlock_irqrestore(&qp->s_lock, flags); 2550 2551 if (lastwqe) { 2552 struct ib_event ev; 2553 2554 ev.device = qp->ibqp.device; 2555 ev.element.qp = &qp->ibqp; 2556 ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 2557 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 2558 } 2559 } 2560 EXPORT_SYMBOL(rvt_rc_error); 2561 2562 /* 2563 * rvt_rnr_tbl_to_usec - return index into ib_rvt_rnr_table 2564 * @index - the index 2565 * return usec from an index into ib_rvt_rnr_table 2566 */ 2567 unsigned long rvt_rnr_tbl_to_usec(u32 index) 2568 { 2569 return ib_rvt_rnr_table[(index & IB_AETH_CREDIT_MASK)]; 2570 } 2571 EXPORT_SYMBOL(rvt_rnr_tbl_to_usec); 2572 2573 static inline unsigned long rvt_aeth_to_usec(u32 aeth) 2574 { 2575 return ib_rvt_rnr_table[(aeth >> IB_AETH_CREDIT_SHIFT) & 2576 IB_AETH_CREDIT_MASK]; 2577 } 2578 2579 /* 2580 * rvt_add_retry_timer_ext - add/start a retry timer 2581 * @qp - the QP 2582 * @shift - timeout shift to wait for multiple packets 2583 * add a retry timer on the QP 2584 */ 2585 void rvt_add_retry_timer_ext(struct rvt_qp *qp, u8 shift) 2586 { 2587 struct ib_qp *ibqp = &qp->ibqp; 2588 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); 2589 2590 lockdep_assert_held(&qp->s_lock); 2591 qp->s_flags |= RVT_S_TIMER; 2592 /* 4.096 usec. * (1 << qp->timeout) */ 2593 qp->s_timer.expires = jiffies + rdi->busy_jiffies + 2594 (qp->timeout_jiffies << shift); 2595 add_timer(&qp->s_timer); 2596 } 2597 EXPORT_SYMBOL(rvt_add_retry_timer_ext); 2598 2599 /** 2600 * rvt_add_rnr_timer - add/start an rnr timer on the QP 2601 * @qp: the QP 2602 * @aeth: aeth of RNR timeout, simulated aeth for loopback 2603 */ 2604 void rvt_add_rnr_timer(struct rvt_qp *qp, u32 aeth) 2605 { 2606 u32 to; 2607 2608 lockdep_assert_held(&qp->s_lock); 2609 qp->s_flags |= RVT_S_WAIT_RNR; 2610 to = rvt_aeth_to_usec(aeth); 2611 trace_rvt_rnrnak_add(qp, to); 2612 hrtimer_start(&qp->s_rnr_timer, 2613 ns_to_ktime(1000 * to), HRTIMER_MODE_REL_PINNED); 2614 } 2615 EXPORT_SYMBOL(rvt_add_rnr_timer); 2616 2617 /** 2618 * rvt_stop_rc_timers - stop all timers 2619 * @qp: the QP 2620 * stop any pending timers 2621 */ 2622 void rvt_stop_rc_timers(struct rvt_qp *qp) 2623 { 2624 lockdep_assert_held(&qp->s_lock); 2625 /* Remove QP from all timers */ 2626 if (qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR)) { 2627 qp->s_flags &= ~(RVT_S_TIMER | RVT_S_WAIT_RNR); 2628 del_timer(&qp->s_timer); 2629 hrtimer_try_to_cancel(&qp->s_rnr_timer); 2630 } 2631 } 2632 EXPORT_SYMBOL(rvt_stop_rc_timers); 2633 2634 /** 2635 * rvt_stop_rnr_timer - stop an rnr timer 2636 * @qp - the QP 2637 * 2638 * stop an rnr timer and return if the timer 2639 * had been pending. 2640 */ 2641 static void rvt_stop_rnr_timer(struct rvt_qp *qp) 2642 { 2643 lockdep_assert_held(&qp->s_lock); 2644 /* Remove QP from rnr timer */ 2645 if (qp->s_flags & RVT_S_WAIT_RNR) { 2646 qp->s_flags &= ~RVT_S_WAIT_RNR; 2647 trace_rvt_rnrnak_stop(qp, 0); 2648 } 2649 } 2650 2651 /** 2652 * rvt_del_timers_sync - wait for any timeout routines to exit 2653 * @qp: the QP 2654 */ 2655 void rvt_del_timers_sync(struct rvt_qp *qp) 2656 { 2657 del_timer_sync(&qp->s_timer); 2658 hrtimer_cancel(&qp->s_rnr_timer); 2659 } 2660 EXPORT_SYMBOL(rvt_del_timers_sync); 2661 2662 /* 2663 * This is called from s_timer for missing responses. 2664 */ 2665 static void rvt_rc_timeout(struct timer_list *t) 2666 { 2667 struct rvt_qp *qp = from_timer(qp, t, s_timer); 2668 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 2669 unsigned long flags; 2670 2671 spin_lock_irqsave(&qp->r_lock, flags); 2672 spin_lock(&qp->s_lock); 2673 if (qp->s_flags & RVT_S_TIMER) { 2674 struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1]; 2675 2676 qp->s_flags &= ~RVT_S_TIMER; 2677 rvp->n_rc_timeouts++; 2678 del_timer(&qp->s_timer); 2679 trace_rvt_rc_timeout(qp, qp->s_last_psn + 1); 2680 if (rdi->driver_f.notify_restart_rc) 2681 rdi->driver_f.notify_restart_rc(qp, 2682 qp->s_last_psn + 1, 2683 1); 2684 rdi->driver_f.schedule_send(qp); 2685 } 2686 spin_unlock(&qp->s_lock); 2687 spin_unlock_irqrestore(&qp->r_lock, flags); 2688 } 2689 2690 /* 2691 * This is called from s_timer for RNR timeouts. 2692 */ 2693 enum hrtimer_restart rvt_rc_rnr_retry(struct hrtimer *t) 2694 { 2695 struct rvt_qp *qp = container_of(t, struct rvt_qp, s_rnr_timer); 2696 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 2697 unsigned long flags; 2698 2699 spin_lock_irqsave(&qp->s_lock, flags); 2700 rvt_stop_rnr_timer(qp); 2701 trace_rvt_rnrnak_timeout(qp, 0); 2702 rdi->driver_f.schedule_send(qp); 2703 spin_unlock_irqrestore(&qp->s_lock, flags); 2704 return HRTIMER_NORESTART; 2705 } 2706 EXPORT_SYMBOL(rvt_rc_rnr_retry); 2707 2708 /** 2709 * rvt_qp_iter_init - initial for QP iteration 2710 * @rdi: rvt devinfo 2711 * @v: u64 value 2712 * @cb: user-defined callback 2713 * 2714 * This returns an iterator suitable for iterating QPs 2715 * in the system. 2716 * 2717 * The @cb is a user-defined callback and @v is a 64-bit 2718 * value passed to and relevant for processing in the 2719 * @cb. An example use case would be to alter QP processing 2720 * based on criteria not part of the rvt_qp. 2721 * 2722 * Use cases that require memory allocation to succeed 2723 * must preallocate appropriately. 2724 * 2725 * Return: a pointer to an rvt_qp_iter or NULL 2726 */ 2727 struct rvt_qp_iter *rvt_qp_iter_init(struct rvt_dev_info *rdi, 2728 u64 v, 2729 void (*cb)(struct rvt_qp *qp, u64 v)) 2730 { 2731 struct rvt_qp_iter *i; 2732 2733 i = kzalloc(sizeof(*i), GFP_KERNEL); 2734 if (!i) 2735 return NULL; 2736 2737 i->rdi = rdi; 2738 /* number of special QPs (SMI/GSI) for device */ 2739 i->specials = rdi->ibdev.phys_port_cnt * 2; 2740 i->v = v; 2741 i->cb = cb; 2742 2743 return i; 2744 } 2745 EXPORT_SYMBOL(rvt_qp_iter_init); 2746 2747 /** 2748 * rvt_qp_iter_next - return the next QP in iter 2749 * @iter: the iterator 2750 * 2751 * Fine grained QP iterator suitable for use 2752 * with debugfs seq_file mechanisms. 2753 * 2754 * Updates iter->qp with the current QP when the return 2755 * value is 0. 2756 * 2757 * Return: 0 - iter->qp is valid 1 - no more QPs 2758 */ 2759 int rvt_qp_iter_next(struct rvt_qp_iter *iter) 2760 __must_hold(RCU) 2761 { 2762 int n = iter->n; 2763 int ret = 1; 2764 struct rvt_qp *pqp = iter->qp; 2765 struct rvt_qp *qp; 2766 struct rvt_dev_info *rdi = iter->rdi; 2767 2768 /* 2769 * The approach is to consider the special qps 2770 * as additional table entries before the 2771 * real hash table. Since the qp code sets 2772 * the qp->next hash link to NULL, this works just fine. 2773 * 2774 * iter->specials is 2 * # ports 2775 * 2776 * n = 0..iter->specials is the special qp indices 2777 * 2778 * n = iter->specials..rdi->qp_dev->qp_table_size+iter->specials are 2779 * the potential hash bucket entries 2780 * 2781 */ 2782 for (; n < rdi->qp_dev->qp_table_size + iter->specials; n++) { 2783 if (pqp) { 2784 qp = rcu_dereference(pqp->next); 2785 } else { 2786 if (n < iter->specials) { 2787 struct rvt_ibport *rvp; 2788 int pidx; 2789 2790 pidx = n % rdi->ibdev.phys_port_cnt; 2791 rvp = rdi->ports[pidx]; 2792 qp = rcu_dereference(rvp->qp[n & 1]); 2793 } else { 2794 qp = rcu_dereference( 2795 rdi->qp_dev->qp_table[ 2796 (n - iter->specials)]); 2797 } 2798 } 2799 pqp = qp; 2800 if (qp) { 2801 iter->qp = qp; 2802 iter->n = n; 2803 return 0; 2804 } 2805 } 2806 return ret; 2807 } 2808 EXPORT_SYMBOL(rvt_qp_iter_next); 2809 2810 /** 2811 * rvt_qp_iter - iterate all QPs 2812 * @rdi: rvt devinfo 2813 * @v: a 64-bit value 2814 * @cb: a callback 2815 * 2816 * This provides a way for iterating all QPs. 2817 * 2818 * The @cb is a user-defined callback and @v is a 64-bit 2819 * value passed to and relevant for processing in the 2820 * cb. An example use case would be to alter QP processing 2821 * based on criteria not part of the rvt_qp. 2822 * 2823 * The code has an internal iterator to simplify 2824 * non seq_file use cases. 2825 */ 2826 void rvt_qp_iter(struct rvt_dev_info *rdi, 2827 u64 v, 2828 void (*cb)(struct rvt_qp *qp, u64 v)) 2829 { 2830 int ret; 2831 struct rvt_qp_iter i = { 2832 .rdi = rdi, 2833 .specials = rdi->ibdev.phys_port_cnt * 2, 2834 .v = v, 2835 .cb = cb 2836 }; 2837 2838 rcu_read_lock(); 2839 do { 2840 ret = rvt_qp_iter_next(&i); 2841 if (!ret) { 2842 rvt_get_qp(i.qp); 2843 rcu_read_unlock(); 2844 i.cb(i.qp, i.v); 2845 rcu_read_lock(); 2846 rvt_put_qp(i.qp); 2847 } 2848 } while (!ret); 2849 rcu_read_unlock(); 2850 } 2851 EXPORT_SYMBOL(rvt_qp_iter); 2852 2853 /* 2854 * This should be called with s_lock held. 2855 */ 2856 void rvt_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe, 2857 enum ib_wc_status status) 2858 { 2859 u32 old_last, last; 2860 struct rvt_dev_info *rdi; 2861 2862 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_OR_FLUSH_SEND)) 2863 return; 2864 rdi = ib_to_rvt(qp->ibqp.device); 2865 2866 old_last = qp->s_last; 2867 trace_rvt_qp_send_completion(qp, wqe, old_last); 2868 last = rvt_qp_complete_swqe(qp, wqe, rdi->wc_opcode[wqe->wr.opcode], 2869 status); 2870 if (qp->s_acked == old_last) 2871 qp->s_acked = last; 2872 if (qp->s_cur == old_last) 2873 qp->s_cur = last; 2874 if (qp->s_tail == old_last) 2875 qp->s_tail = last; 2876 if (qp->state == IB_QPS_SQD && last == qp->s_cur) 2877 qp->s_draining = 0; 2878 } 2879 EXPORT_SYMBOL(rvt_send_complete); 2880 2881 /** 2882 * rvt_copy_sge - copy data to SGE memory 2883 * @qp: associated QP 2884 * @ss: the SGE state 2885 * @data: the data to copy 2886 * @length: the length of the data 2887 * @release: boolean to release MR 2888 * @copy_last: do a separate copy of the last 8 bytes 2889 */ 2890 void rvt_copy_sge(struct rvt_qp *qp, struct rvt_sge_state *ss, 2891 void *data, u32 length, 2892 bool release, bool copy_last) 2893 { 2894 struct rvt_sge *sge = &ss->sge; 2895 int i; 2896 bool in_last = false; 2897 bool cacheless_copy = false; 2898 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 2899 struct rvt_wss *wss = rdi->wss; 2900 unsigned int sge_copy_mode = rdi->dparms.sge_copy_mode; 2901 2902 if (sge_copy_mode == RVT_SGE_COPY_CACHELESS) { 2903 cacheless_copy = length >= PAGE_SIZE; 2904 } else if (sge_copy_mode == RVT_SGE_COPY_ADAPTIVE) { 2905 if (length >= PAGE_SIZE) { 2906 /* 2907 * NOTE: this *assumes*: 2908 * o The first vaddr is the dest. 2909 * o If multiple pages, then vaddr is sequential. 2910 */ 2911 wss_insert(wss, sge->vaddr); 2912 if (length >= (2 * PAGE_SIZE)) 2913 wss_insert(wss, (sge->vaddr + PAGE_SIZE)); 2914 2915 cacheless_copy = wss_exceeds_threshold(wss); 2916 } else { 2917 wss_advance_clean_counter(wss); 2918 } 2919 } 2920 2921 if (copy_last) { 2922 if (length > 8) { 2923 length -= 8; 2924 } else { 2925 copy_last = false; 2926 in_last = true; 2927 } 2928 } 2929 2930 again: 2931 while (length) { 2932 u32 len = rvt_get_sge_length(sge, length); 2933 2934 WARN_ON_ONCE(len == 0); 2935 if (unlikely(in_last)) { 2936 /* enforce byte transfer ordering */ 2937 for (i = 0; i < len; i++) 2938 ((u8 *)sge->vaddr)[i] = ((u8 *)data)[i]; 2939 } else if (cacheless_copy) { 2940 cacheless_memcpy(sge->vaddr, data, len); 2941 } else { 2942 memcpy(sge->vaddr, data, len); 2943 } 2944 rvt_update_sge(ss, len, release); 2945 data += len; 2946 length -= len; 2947 } 2948 2949 if (copy_last) { 2950 copy_last = false; 2951 in_last = true; 2952 length = 8; 2953 goto again; 2954 } 2955 } 2956 EXPORT_SYMBOL(rvt_copy_sge); 2957 2958 static enum ib_wc_status loopback_qp_drop(struct rvt_ibport *rvp, 2959 struct rvt_qp *sqp) 2960 { 2961 rvp->n_pkt_drops++; 2962 /* 2963 * For RC, the requester would timeout and retry so 2964 * shortcut the timeouts and just signal too many retries. 2965 */ 2966 return sqp->ibqp.qp_type == IB_QPT_RC ? 2967 IB_WC_RETRY_EXC_ERR : IB_WC_SUCCESS; 2968 } 2969 2970 /** 2971 * ruc_loopback - handle UC and RC loopback requests 2972 * @sqp: the sending QP 2973 * 2974 * This is called from rvt_do_send() to forward a WQE addressed to the same HFI 2975 * Note that although we are single threaded due to the send engine, we still 2976 * have to protect against post_send(). We don't have to worry about 2977 * receive interrupts since this is a connected protocol and all packets 2978 * will pass through here. 2979 */ 2980 void rvt_ruc_loopback(struct rvt_qp *sqp) 2981 { 2982 struct rvt_ibport *rvp = NULL; 2983 struct rvt_dev_info *rdi = ib_to_rvt(sqp->ibqp.device); 2984 struct rvt_qp *qp; 2985 struct rvt_swqe *wqe; 2986 struct rvt_sge *sge; 2987 unsigned long flags; 2988 struct ib_wc wc; 2989 u64 sdata; 2990 atomic64_t *maddr; 2991 enum ib_wc_status send_status; 2992 bool release; 2993 int ret; 2994 bool copy_last = false; 2995 int local_ops = 0; 2996 2997 rcu_read_lock(); 2998 rvp = rdi->ports[sqp->port_num - 1]; 2999 3000 /* 3001 * Note that we check the responder QP state after 3002 * checking the requester's state. 3003 */ 3004 3005 qp = rvt_lookup_qpn(ib_to_rvt(sqp->ibqp.device), rvp, 3006 sqp->remote_qpn); 3007 3008 spin_lock_irqsave(&sqp->s_lock, flags); 3009 3010 /* Return if we are already busy processing a work request. */ 3011 if ((sqp->s_flags & (RVT_S_BUSY | RVT_S_ANY_WAIT)) || 3012 !(ib_rvt_state_ops[sqp->state] & RVT_PROCESS_OR_FLUSH_SEND)) 3013 goto unlock; 3014 3015 sqp->s_flags |= RVT_S_BUSY; 3016 3017 again: 3018 if (sqp->s_last == READ_ONCE(sqp->s_head)) 3019 goto clr_busy; 3020 wqe = rvt_get_swqe_ptr(sqp, sqp->s_last); 3021 3022 /* Return if it is not OK to start a new work request. */ 3023 if (!(ib_rvt_state_ops[sqp->state] & RVT_PROCESS_NEXT_SEND_OK)) { 3024 if (!(ib_rvt_state_ops[sqp->state] & RVT_FLUSH_SEND)) 3025 goto clr_busy; 3026 /* We are in the error state, flush the work request. */ 3027 send_status = IB_WC_WR_FLUSH_ERR; 3028 goto flush_send; 3029 } 3030 3031 /* 3032 * We can rely on the entry not changing without the s_lock 3033 * being held until we update s_last. 3034 * We increment s_cur to indicate s_last is in progress. 3035 */ 3036 if (sqp->s_last == sqp->s_cur) { 3037 if (++sqp->s_cur >= sqp->s_size) 3038 sqp->s_cur = 0; 3039 } 3040 spin_unlock_irqrestore(&sqp->s_lock, flags); 3041 3042 if (!qp) { 3043 send_status = loopback_qp_drop(rvp, sqp); 3044 goto serr_no_r_lock; 3045 } 3046 spin_lock_irqsave(&qp->r_lock, flags); 3047 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) || 3048 qp->ibqp.qp_type != sqp->ibqp.qp_type) { 3049 send_status = loopback_qp_drop(rvp, sqp); 3050 goto serr; 3051 } 3052 3053 memset(&wc, 0, sizeof(wc)); 3054 send_status = IB_WC_SUCCESS; 3055 3056 release = true; 3057 sqp->s_sge.sge = wqe->sg_list[0]; 3058 sqp->s_sge.sg_list = wqe->sg_list + 1; 3059 sqp->s_sge.num_sge = wqe->wr.num_sge; 3060 sqp->s_len = wqe->length; 3061 switch (wqe->wr.opcode) { 3062 case IB_WR_REG_MR: 3063 goto send_comp; 3064 3065 case IB_WR_LOCAL_INV: 3066 if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) { 3067 if (rvt_invalidate_rkey(sqp, 3068 wqe->wr.ex.invalidate_rkey)) 3069 send_status = IB_WC_LOC_PROT_ERR; 3070 local_ops = 1; 3071 } 3072 goto send_comp; 3073 3074 case IB_WR_SEND_WITH_INV: 3075 case IB_WR_SEND_WITH_IMM: 3076 case IB_WR_SEND: 3077 ret = rvt_get_rwqe(qp, false); 3078 if (ret < 0) 3079 goto op_err; 3080 if (!ret) 3081 goto rnr_nak; 3082 if (wqe->length > qp->r_len) 3083 goto inv_err; 3084 switch (wqe->wr.opcode) { 3085 case IB_WR_SEND_WITH_INV: 3086 if (!rvt_invalidate_rkey(qp, 3087 wqe->wr.ex.invalidate_rkey)) { 3088 wc.wc_flags = IB_WC_WITH_INVALIDATE; 3089 wc.ex.invalidate_rkey = 3090 wqe->wr.ex.invalidate_rkey; 3091 } 3092 break; 3093 case IB_WR_SEND_WITH_IMM: 3094 wc.wc_flags = IB_WC_WITH_IMM; 3095 wc.ex.imm_data = wqe->wr.ex.imm_data; 3096 break; 3097 default: 3098 break; 3099 } 3100 break; 3101 3102 case IB_WR_RDMA_WRITE_WITH_IMM: 3103 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) 3104 goto inv_err; 3105 wc.wc_flags = IB_WC_WITH_IMM; 3106 wc.ex.imm_data = wqe->wr.ex.imm_data; 3107 ret = rvt_get_rwqe(qp, true); 3108 if (ret < 0) 3109 goto op_err; 3110 if (!ret) 3111 goto rnr_nak; 3112 /* skip copy_last set and qp_access_flags recheck */ 3113 goto do_write; 3114 case IB_WR_RDMA_WRITE: 3115 copy_last = rvt_is_user_qp(qp); 3116 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) 3117 goto inv_err; 3118 do_write: 3119 if (wqe->length == 0) 3120 break; 3121 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, wqe->length, 3122 wqe->rdma_wr.remote_addr, 3123 wqe->rdma_wr.rkey, 3124 IB_ACCESS_REMOTE_WRITE))) 3125 goto acc_err; 3126 qp->r_sge.sg_list = NULL; 3127 qp->r_sge.num_sge = 1; 3128 qp->r_sge.total_len = wqe->length; 3129 break; 3130 3131 case IB_WR_RDMA_READ: 3132 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) 3133 goto inv_err; 3134 if (unlikely(!rvt_rkey_ok(qp, &sqp->s_sge.sge, wqe->length, 3135 wqe->rdma_wr.remote_addr, 3136 wqe->rdma_wr.rkey, 3137 IB_ACCESS_REMOTE_READ))) 3138 goto acc_err; 3139 release = false; 3140 sqp->s_sge.sg_list = NULL; 3141 sqp->s_sge.num_sge = 1; 3142 qp->r_sge.sge = wqe->sg_list[0]; 3143 qp->r_sge.sg_list = wqe->sg_list + 1; 3144 qp->r_sge.num_sge = wqe->wr.num_sge; 3145 qp->r_sge.total_len = wqe->length; 3146 break; 3147 3148 case IB_WR_ATOMIC_CMP_AND_SWP: 3149 case IB_WR_ATOMIC_FETCH_AND_ADD: 3150 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) 3151 goto inv_err; 3152 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), 3153 wqe->atomic_wr.remote_addr, 3154 wqe->atomic_wr.rkey, 3155 IB_ACCESS_REMOTE_ATOMIC))) 3156 goto acc_err; 3157 /* Perform atomic OP and save result. */ 3158 maddr = (atomic64_t *)qp->r_sge.sge.vaddr; 3159 sdata = wqe->atomic_wr.compare_add; 3160 *(u64 *)sqp->s_sge.sge.vaddr = 3161 (wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ? 3162 (u64)atomic64_add_return(sdata, maddr) - sdata : 3163 (u64)cmpxchg((u64 *)qp->r_sge.sge.vaddr, 3164 sdata, wqe->atomic_wr.swap); 3165 rvt_put_mr(qp->r_sge.sge.mr); 3166 qp->r_sge.num_sge = 0; 3167 goto send_comp; 3168 3169 default: 3170 send_status = IB_WC_LOC_QP_OP_ERR; 3171 goto serr; 3172 } 3173 3174 sge = &sqp->s_sge.sge; 3175 while (sqp->s_len) { 3176 u32 len = rvt_get_sge_length(sge, sqp->s_len); 3177 3178 WARN_ON_ONCE(len == 0); 3179 rvt_copy_sge(qp, &qp->r_sge, sge->vaddr, 3180 len, release, copy_last); 3181 rvt_update_sge(&sqp->s_sge, len, !release); 3182 sqp->s_len -= len; 3183 } 3184 if (release) 3185 rvt_put_ss(&qp->r_sge); 3186 3187 if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) 3188 goto send_comp; 3189 3190 if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM) 3191 wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; 3192 else 3193 wc.opcode = IB_WC_RECV; 3194 wc.wr_id = qp->r_wr_id; 3195 wc.status = IB_WC_SUCCESS; 3196 wc.byte_len = wqe->length; 3197 wc.qp = &qp->ibqp; 3198 wc.src_qp = qp->remote_qpn; 3199 wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr) & U16_MAX; 3200 wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); 3201 wc.port_num = 1; 3202 /* Signal completion event if the solicited bit is set. */ 3203 rvt_recv_cq(qp, &wc, wqe->wr.send_flags & IB_SEND_SOLICITED); 3204 3205 send_comp: 3206 spin_unlock_irqrestore(&qp->r_lock, flags); 3207 spin_lock_irqsave(&sqp->s_lock, flags); 3208 rvp->n_loop_pkts++; 3209 flush_send: 3210 sqp->s_rnr_retry = sqp->s_rnr_retry_cnt; 3211 rvt_send_complete(sqp, wqe, send_status); 3212 if (local_ops) { 3213 atomic_dec(&sqp->local_ops_pending); 3214 local_ops = 0; 3215 } 3216 goto again; 3217 3218 rnr_nak: 3219 /* Handle RNR NAK */ 3220 if (qp->ibqp.qp_type == IB_QPT_UC) 3221 goto send_comp; 3222 rvp->n_rnr_naks++; 3223 /* 3224 * Note: we don't need the s_lock held since the BUSY flag 3225 * makes this single threaded. 3226 */ 3227 if (sqp->s_rnr_retry == 0) { 3228 send_status = IB_WC_RNR_RETRY_EXC_ERR; 3229 goto serr; 3230 } 3231 if (sqp->s_rnr_retry_cnt < 7) 3232 sqp->s_rnr_retry--; 3233 spin_unlock_irqrestore(&qp->r_lock, flags); 3234 spin_lock_irqsave(&sqp->s_lock, flags); 3235 if (!(ib_rvt_state_ops[sqp->state] & RVT_PROCESS_RECV_OK)) 3236 goto clr_busy; 3237 rvt_add_rnr_timer(sqp, qp->r_min_rnr_timer << 3238 IB_AETH_CREDIT_SHIFT); 3239 goto clr_busy; 3240 3241 op_err: 3242 send_status = IB_WC_REM_OP_ERR; 3243 wc.status = IB_WC_LOC_QP_OP_ERR; 3244 goto err; 3245 3246 inv_err: 3247 send_status = 3248 sqp->ibqp.qp_type == IB_QPT_RC ? 3249 IB_WC_REM_INV_REQ_ERR : 3250 IB_WC_SUCCESS; 3251 wc.status = IB_WC_LOC_QP_OP_ERR; 3252 goto err; 3253 3254 acc_err: 3255 send_status = IB_WC_REM_ACCESS_ERR; 3256 wc.status = IB_WC_LOC_PROT_ERR; 3257 err: 3258 /* responder goes to error state */ 3259 rvt_rc_error(qp, wc.status); 3260 3261 serr: 3262 spin_unlock_irqrestore(&qp->r_lock, flags); 3263 serr_no_r_lock: 3264 spin_lock_irqsave(&sqp->s_lock, flags); 3265 rvt_send_complete(sqp, wqe, send_status); 3266 if (sqp->ibqp.qp_type == IB_QPT_RC) { 3267 int lastwqe = rvt_error_qp(sqp, IB_WC_WR_FLUSH_ERR); 3268 3269 sqp->s_flags &= ~RVT_S_BUSY; 3270 spin_unlock_irqrestore(&sqp->s_lock, flags); 3271 if (lastwqe) { 3272 struct ib_event ev; 3273 3274 ev.device = sqp->ibqp.device; 3275 ev.element.qp = &sqp->ibqp; 3276 ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 3277 sqp->ibqp.event_handler(&ev, sqp->ibqp.qp_context); 3278 } 3279 goto done; 3280 } 3281 clr_busy: 3282 sqp->s_flags &= ~RVT_S_BUSY; 3283 unlock: 3284 spin_unlock_irqrestore(&sqp->s_lock, flags); 3285 done: 3286 rcu_read_unlock(); 3287 } 3288 EXPORT_SYMBOL(rvt_ruc_loopback); 3289