1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/pci.h> 35 #include <linux/platform_device.h> 36 #include <rdma/ib_addr.h> 37 #include <rdma/ib_umem.h> 38 #include <rdma/uverbs_ioctl.h> 39 #include "hns_roce_common.h" 40 #include "hns_roce_device.h" 41 #include "hns_roce_hem.h" 42 #include <rdma/hns-abi.h> 43 44 #define SQP_NUM (2 * HNS_ROCE_MAX_PORTS) 45 46 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) 47 { 48 struct device *dev = hr_dev->dev; 49 struct hns_roce_qp *qp; 50 51 xa_lock(&hr_dev->qp_table_xa); 52 qp = __hns_roce_qp_lookup(hr_dev, qpn); 53 if (qp) 54 atomic_inc(&qp->refcount); 55 xa_unlock(&hr_dev->qp_table_xa); 56 57 if (!qp) { 58 dev_warn(dev, "Async event for bogus QP %08x\n", qpn); 59 return; 60 } 61 62 qp->event(qp, (enum hns_roce_event)event_type); 63 64 if (atomic_dec_and_test(&qp->refcount)) 65 complete(&qp->free); 66 } 67 68 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, 69 enum hns_roce_event type) 70 { 71 struct ib_event event; 72 struct ib_qp *ibqp = &hr_qp->ibqp; 73 74 if (ibqp->event_handler) { 75 event.device = ibqp->device; 76 event.element.qp = ibqp; 77 switch (type) { 78 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 79 event.event = IB_EVENT_PATH_MIG; 80 break; 81 case HNS_ROCE_EVENT_TYPE_COMM_EST: 82 event.event = IB_EVENT_COMM_EST; 83 break; 84 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 85 event.event = IB_EVENT_SQ_DRAINED; 86 break; 87 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 88 event.event = IB_EVENT_QP_LAST_WQE_REACHED; 89 break; 90 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 91 event.event = IB_EVENT_QP_FATAL; 92 break; 93 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 94 event.event = IB_EVENT_PATH_MIG_ERR; 95 break; 96 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 97 event.event = IB_EVENT_QP_REQ_ERR; 98 break; 99 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 100 event.event = IB_EVENT_QP_ACCESS_ERR; 101 break; 102 default: 103 dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n", 104 type, hr_qp->qpn); 105 return; 106 } 107 ibqp->event_handler(&event, ibqp->qp_context); 108 } 109 } 110 111 static int hns_roce_reserve_range_qp(struct hns_roce_dev *hr_dev, int cnt, 112 int align, unsigned long *base) 113 { 114 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 115 116 return hns_roce_bitmap_alloc_range(&qp_table->bitmap, cnt, align, 117 base) ? 118 -ENOMEM : 119 0; 120 } 121 122 enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state) 123 { 124 switch (state) { 125 case IB_QPS_RESET: 126 return HNS_ROCE_QP_STATE_RST; 127 case IB_QPS_INIT: 128 return HNS_ROCE_QP_STATE_INIT; 129 case IB_QPS_RTR: 130 return HNS_ROCE_QP_STATE_RTR; 131 case IB_QPS_RTS: 132 return HNS_ROCE_QP_STATE_RTS; 133 case IB_QPS_SQD: 134 return HNS_ROCE_QP_STATE_SQD; 135 case IB_QPS_ERR: 136 return HNS_ROCE_QP_STATE_ERR; 137 default: 138 return HNS_ROCE_QP_NUM_STATE; 139 } 140 } 141 142 static int hns_roce_gsi_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, 143 struct hns_roce_qp *hr_qp) 144 { 145 struct xarray *xa = &hr_dev->qp_table_xa; 146 int ret; 147 148 if (!qpn) 149 return -EINVAL; 150 151 hr_qp->qpn = qpn; 152 atomic_set(&hr_qp->refcount, 1); 153 init_completion(&hr_qp->free); 154 155 ret = xa_err(xa_store_irq(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1), 156 hr_qp, GFP_KERNEL)); 157 if (ret) 158 dev_err(hr_dev->dev, "QPC xa_store failed\n"); 159 160 return ret; 161 } 162 163 static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, 164 struct hns_roce_qp *hr_qp) 165 { 166 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 167 struct device *dev = hr_dev->dev; 168 int ret; 169 170 if (!qpn) 171 return -EINVAL; 172 173 hr_qp->qpn = qpn; 174 175 /* Alloc memory for QPC */ 176 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn); 177 if (ret) { 178 dev_err(dev, "QPC table get failed\n"); 179 goto err_out; 180 } 181 182 /* Alloc memory for IRRL */ 183 ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 184 if (ret) { 185 dev_err(dev, "IRRL table get failed\n"); 186 goto err_put_qp; 187 } 188 189 if (hr_dev->caps.trrl_entry_sz) { 190 /* Alloc memory for TRRL */ 191 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table, 192 hr_qp->qpn); 193 if (ret) { 194 dev_err(dev, "TRRL table get failed\n"); 195 goto err_put_irrl; 196 } 197 } 198 199 if (hr_dev->caps.sccc_entry_sz) { 200 /* Alloc memory for SCC CTX */ 201 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table, 202 hr_qp->qpn); 203 if (ret) { 204 dev_err(dev, "SCC CTX table get failed\n"); 205 goto err_put_trrl; 206 } 207 } 208 209 ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp); 210 if (ret) 211 goto err_put_sccc; 212 213 return 0; 214 215 err_put_sccc: 216 if (hr_dev->caps.sccc_entry_sz) 217 hns_roce_table_put(hr_dev, &qp_table->sccc_table, 218 hr_qp->qpn); 219 220 err_put_trrl: 221 if (hr_dev->caps.trrl_entry_sz) 222 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 223 224 err_put_irrl: 225 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 226 227 err_put_qp: 228 hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); 229 230 err_out: 231 return ret; 232 } 233 234 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 235 { 236 struct xarray *xa = &hr_dev->qp_table_xa; 237 unsigned long flags; 238 239 xa_lock_irqsave(xa, flags); 240 __xa_erase(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1)); 241 xa_unlock_irqrestore(xa, flags); 242 } 243 244 void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 245 { 246 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 247 248 if (atomic_dec_and_test(&hr_qp->refcount)) 249 complete(&hr_qp->free); 250 wait_for_completion(&hr_qp->free); 251 252 if ((hr_qp->ibqp.qp_type) != IB_QPT_GSI) { 253 if (hr_dev->caps.trrl_entry_sz) 254 hns_roce_table_put(hr_dev, &qp_table->trrl_table, 255 hr_qp->qpn); 256 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 257 } 258 } 259 260 void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, 261 int cnt) 262 { 263 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 264 265 if (base_qpn < hr_dev->caps.reserved_qps) 266 return; 267 268 hns_roce_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt, BITMAP_RR); 269 } 270 271 static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev, 272 struct ib_qp_cap *cap, bool is_user, int has_rq, 273 struct hns_roce_qp *hr_qp) 274 { 275 struct device *dev = hr_dev->dev; 276 u32 max_cnt; 277 278 /* Check the validity of QP support capacity */ 279 if (cap->max_recv_wr > hr_dev->caps.max_wqes || 280 cap->max_recv_sge > hr_dev->caps.max_rq_sg) { 281 dev_err(dev, "RQ WR or sge error!max_recv_wr=%d max_recv_sge=%d\n", 282 cap->max_recv_wr, cap->max_recv_sge); 283 return -EINVAL; 284 } 285 286 /* If srq exist, set zero for relative number of rq */ 287 if (!has_rq) { 288 hr_qp->rq.wqe_cnt = 0; 289 hr_qp->rq.max_gs = 0; 290 cap->max_recv_wr = 0; 291 cap->max_recv_sge = 0; 292 } else { 293 if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge)) { 294 dev_err(dev, "user space no need config max_recv_wr max_recv_sge\n"); 295 return -EINVAL; 296 } 297 298 if (hr_dev->caps.min_wqes) 299 max_cnt = max(cap->max_recv_wr, hr_dev->caps.min_wqes); 300 else 301 max_cnt = cap->max_recv_wr; 302 303 hr_qp->rq.wqe_cnt = roundup_pow_of_two(max_cnt); 304 305 if ((u32)hr_qp->rq.wqe_cnt > hr_dev->caps.max_wqes) { 306 dev_err(dev, "while setting rq size, rq.wqe_cnt too large\n"); 307 return -EINVAL; 308 } 309 310 max_cnt = max(1U, cap->max_recv_sge); 311 hr_qp->rq.max_gs = roundup_pow_of_two(max_cnt); 312 if (hr_dev->caps.max_rq_sg <= 2) 313 hr_qp->rq.wqe_shift = 314 ilog2(hr_dev->caps.max_rq_desc_sz); 315 else 316 hr_qp->rq.wqe_shift = 317 ilog2(hr_dev->caps.max_rq_desc_sz 318 * hr_qp->rq.max_gs); 319 } 320 321 cap->max_recv_wr = hr_qp->rq.wqe_cnt; 322 cap->max_recv_sge = hr_qp->rq.max_gs; 323 324 return 0; 325 } 326 327 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev, 328 struct ib_qp_cap *cap, 329 struct hns_roce_ib_create_qp *ucmd) 330 { 331 u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz); 332 u8 max_sq_stride = ilog2(roundup_sq_stride); 333 334 /* Sanity check SQ size before proceeding */ 335 if (ucmd->log_sq_stride > max_sq_stride || 336 ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) { 337 ibdev_err(&hr_dev->ib_dev, "check SQ size error!\n"); 338 return -EINVAL; 339 } 340 341 if (cap->max_send_sge > hr_dev->caps.max_sq_sg) { 342 ibdev_err(&hr_dev->ib_dev, "SQ sge error! max_send_sge=%d\n", 343 cap->max_send_sge); 344 return -EINVAL; 345 } 346 347 return 0; 348 } 349 350 static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev, 351 struct ib_qp_cap *cap, 352 struct hns_roce_qp *hr_qp, 353 struct hns_roce_ib_create_qp *ucmd) 354 { 355 u32 ex_sge_num; 356 u32 page_size; 357 u32 max_cnt; 358 int ret; 359 360 if (check_shl_overflow(1, ucmd->log_sq_bb_count, &hr_qp->sq.wqe_cnt) || 361 hr_qp->sq.wqe_cnt > hr_dev->caps.max_wqes) 362 return -EINVAL; 363 364 ret = check_sq_size_with_integrity(hr_dev, cap, ucmd); 365 if (ret) { 366 ibdev_err(&hr_dev->ib_dev, "Sanity check sq size failed\n"); 367 return ret; 368 } 369 370 hr_qp->sq.wqe_shift = ucmd->log_sq_stride; 371 372 max_cnt = max(1U, cap->max_send_sge); 373 if (hr_dev->caps.max_sq_sg <= 2) 374 hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt); 375 else 376 hr_qp->sq.max_gs = max_cnt; 377 378 if (hr_qp->sq.max_gs > 2) 379 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt * 380 (hr_qp->sq.max_gs - 2)); 381 382 if ((hr_qp->sq.max_gs > 2) && (hr_dev->pci_dev->revision == 0x20)) { 383 if (hr_qp->sge.sge_cnt > hr_dev->caps.max_extend_sg) { 384 dev_err(hr_dev->dev, 385 "The extended sge cnt error! sge_cnt=%d\n", 386 hr_qp->sge.sge_cnt); 387 return -EINVAL; 388 } 389 } 390 391 hr_qp->sge.sge_shift = 4; 392 ex_sge_num = hr_qp->sge.sge_cnt; 393 394 /* Get buf size, SQ and RQ are aligned to page_szie */ 395 if (hr_dev->caps.max_sq_sg <= 2) { 396 hr_qp->buff_size = HNS_ROCE_ALIGN_UP((hr_qp->rq.wqe_cnt << 397 hr_qp->rq.wqe_shift), PAGE_SIZE) + 398 HNS_ROCE_ALIGN_UP((hr_qp->sq.wqe_cnt << 399 hr_qp->sq.wqe_shift), PAGE_SIZE); 400 401 hr_qp->sq.offset = 0; 402 hr_qp->rq.offset = HNS_ROCE_ALIGN_UP((hr_qp->sq.wqe_cnt << 403 hr_qp->sq.wqe_shift), PAGE_SIZE); 404 } else { 405 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); 406 hr_qp->sge.sge_cnt = ex_sge_num ? 407 max(page_size / (1 << hr_qp->sge.sge_shift), ex_sge_num) : 0; 408 hr_qp->buff_size = HNS_ROCE_ALIGN_UP((hr_qp->rq.wqe_cnt << 409 hr_qp->rq.wqe_shift), page_size) + 410 HNS_ROCE_ALIGN_UP((hr_qp->sge.sge_cnt << 411 hr_qp->sge.sge_shift), page_size) + 412 HNS_ROCE_ALIGN_UP((hr_qp->sq.wqe_cnt << 413 hr_qp->sq.wqe_shift), page_size); 414 415 hr_qp->sq.offset = 0; 416 if (ex_sge_num) { 417 hr_qp->sge.offset = HNS_ROCE_ALIGN_UP( 418 (hr_qp->sq.wqe_cnt << 419 hr_qp->sq.wqe_shift), 420 page_size); 421 hr_qp->rq.offset = hr_qp->sge.offset + 422 HNS_ROCE_ALIGN_UP((hr_qp->sge.sge_cnt << 423 hr_qp->sge.sge_shift), 424 page_size); 425 } else { 426 hr_qp->rq.offset = HNS_ROCE_ALIGN_UP( 427 (hr_qp->sq.wqe_cnt << 428 hr_qp->sq.wqe_shift), 429 page_size); 430 } 431 } 432 433 return 0; 434 } 435 436 static int split_wqe_buf_region(struct hns_roce_dev *hr_dev, 437 struct hns_roce_qp *hr_qp, 438 struct hns_roce_buf_region *regions, 439 int region_max, int page_shift) 440 { 441 int page_size = 1 << page_shift; 442 bool is_extend_sge; 443 int region_cnt = 0; 444 int buf_size; 445 int buf_cnt; 446 447 if (hr_qp->buff_size < 1 || region_max < 1) 448 return region_cnt; 449 450 if (hr_qp->sge.sge_cnt > 0) 451 is_extend_sge = true; 452 else 453 is_extend_sge = false; 454 455 /* sq region */ 456 if (is_extend_sge) 457 buf_size = hr_qp->sge.offset - hr_qp->sq.offset; 458 else 459 buf_size = hr_qp->rq.offset - hr_qp->sq.offset; 460 461 if (buf_size > 0 && region_cnt < region_max) { 462 buf_cnt = DIV_ROUND_UP(buf_size, page_size); 463 hns_roce_init_buf_region(®ions[region_cnt], 464 hr_dev->caps.wqe_sq_hop_num, 465 hr_qp->sq.offset / page_size, 466 buf_cnt); 467 region_cnt++; 468 } 469 470 /* sge region */ 471 if (is_extend_sge) { 472 buf_size = hr_qp->rq.offset - hr_qp->sge.offset; 473 if (buf_size > 0 && region_cnt < region_max) { 474 buf_cnt = DIV_ROUND_UP(buf_size, page_size); 475 hns_roce_init_buf_region(®ions[region_cnt], 476 hr_dev->caps.wqe_sge_hop_num, 477 hr_qp->sge.offset / page_size, 478 buf_cnt); 479 region_cnt++; 480 } 481 } 482 483 /* rq region */ 484 buf_size = hr_qp->buff_size - hr_qp->rq.offset; 485 if (buf_size > 0) { 486 buf_cnt = DIV_ROUND_UP(buf_size, page_size); 487 hns_roce_init_buf_region(®ions[region_cnt], 488 hr_dev->caps.wqe_rq_hop_num, 489 hr_qp->rq.offset / page_size, 490 buf_cnt); 491 region_cnt++; 492 } 493 494 return region_cnt; 495 } 496 497 static int calc_wqe_bt_page_shift(struct hns_roce_dev *hr_dev, 498 struct hns_roce_buf_region *regions, 499 int region_cnt) 500 { 501 int bt_pg_shift; 502 int ba_num; 503 int ret; 504 505 bt_pg_shift = PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz; 506 507 /* all root ba entries must in one bt page */ 508 do { 509 ba_num = (1 << bt_pg_shift) / BA_BYTE_LEN; 510 ret = hns_roce_hem_list_calc_root_ba(regions, region_cnt, 511 ba_num); 512 if (ret <= ba_num) 513 break; 514 515 bt_pg_shift++; 516 } while (ret > ba_num); 517 518 return bt_pg_shift - PAGE_SHIFT; 519 } 520 521 static int set_extend_sge_param(struct hns_roce_dev *hr_dev, 522 struct hns_roce_qp *hr_qp) 523 { 524 struct device *dev = hr_dev->dev; 525 526 if (hr_qp->sq.max_gs > 2) { 527 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt * 528 (hr_qp->sq.max_gs - 2)); 529 hr_qp->sge.sge_shift = 4; 530 } 531 532 /* ud sqwqe's sge use extend sge */ 533 if (hr_dev->caps.max_sq_sg > 2 && hr_qp->ibqp.qp_type == IB_QPT_GSI) { 534 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt * 535 hr_qp->sq.max_gs); 536 hr_qp->sge.sge_shift = 4; 537 } 538 539 if ((hr_qp->sq.max_gs > 2) && hr_dev->pci_dev->revision == 0x20) { 540 if (hr_qp->sge.sge_cnt > hr_dev->caps.max_extend_sg) { 541 dev_err(dev, "The extended sge cnt error! sge_cnt=%d\n", 542 hr_qp->sge.sge_cnt); 543 return -EINVAL; 544 } 545 } 546 547 return 0; 548 } 549 550 static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev, 551 struct ib_qp_cap *cap, 552 struct hns_roce_qp *hr_qp) 553 { 554 struct device *dev = hr_dev->dev; 555 u32 page_size; 556 u32 max_cnt; 557 int size; 558 int ret; 559 560 if (cap->max_send_wr > hr_dev->caps.max_wqes || 561 cap->max_send_sge > hr_dev->caps.max_sq_sg || 562 cap->max_inline_data > hr_dev->caps.max_sq_inline) { 563 dev_err(dev, "SQ WR or sge or inline data error!\n"); 564 return -EINVAL; 565 } 566 567 hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz); 568 569 if (hr_dev->caps.min_wqes) 570 max_cnt = max(cap->max_send_wr, hr_dev->caps.min_wqes); 571 else 572 max_cnt = cap->max_send_wr; 573 574 hr_qp->sq.wqe_cnt = roundup_pow_of_two(max_cnt); 575 if ((u32)hr_qp->sq.wqe_cnt > hr_dev->caps.max_wqes) { 576 dev_err(dev, "while setting kernel sq size, sq.wqe_cnt too large\n"); 577 return -EINVAL; 578 } 579 580 /* Get data_seg numbers */ 581 max_cnt = max(1U, cap->max_send_sge); 582 if (hr_dev->caps.max_sq_sg <= 2) 583 hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt); 584 else 585 hr_qp->sq.max_gs = max_cnt; 586 587 ret = set_extend_sge_param(hr_dev, hr_qp); 588 if (ret) { 589 dev_err(dev, "set extend sge parameters fail\n"); 590 return ret; 591 } 592 593 /* Get buf size, SQ and RQ are aligned to PAGE_SIZE */ 594 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); 595 hr_qp->sq.offset = 0; 596 size = HNS_ROCE_ALIGN_UP(hr_qp->sq.wqe_cnt << hr_qp->sq.wqe_shift, 597 page_size); 598 599 if (hr_dev->caps.max_sq_sg > 2 && hr_qp->sge.sge_cnt) { 600 hr_qp->sge.sge_cnt = max(page_size/(1 << hr_qp->sge.sge_shift), 601 (u32)hr_qp->sge.sge_cnt); 602 hr_qp->sge.offset = size; 603 size += HNS_ROCE_ALIGN_UP(hr_qp->sge.sge_cnt << 604 hr_qp->sge.sge_shift, page_size); 605 } 606 607 hr_qp->rq.offset = size; 608 size += HNS_ROCE_ALIGN_UP((hr_qp->rq.wqe_cnt << hr_qp->rq.wqe_shift), 609 page_size); 610 hr_qp->buff_size = size; 611 612 /* Get wr and sge number which send */ 613 cap->max_send_wr = hr_qp->sq.wqe_cnt; 614 cap->max_send_sge = hr_qp->sq.max_gs; 615 616 /* We don't support inline sends for kernel QPs (yet) */ 617 cap->max_inline_data = 0; 618 619 return 0; 620 } 621 622 static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr) 623 { 624 if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr) 625 return 0; 626 627 return 1; 628 } 629 630 static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr) 631 { 632 if (attr->qp_type == IB_QPT_XRC_INI || 633 attr->qp_type == IB_QPT_XRC_TGT || attr->srq || 634 !attr->cap.max_recv_wr) 635 return 0; 636 637 return 1; 638 } 639 640 static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp, 641 struct ib_qp_init_attr *init_attr) 642 { 643 u32 max_recv_sge = init_attr->cap.max_recv_sge; 644 struct hns_roce_rinl_wqe *wqe_list; 645 u32 wqe_cnt = hr_qp->rq.wqe_cnt; 646 int i; 647 648 /* allocate recv inline buf */ 649 wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe), 650 GFP_KERNEL); 651 652 if (!wqe_list) 653 goto err; 654 655 /* Allocate a continuous buffer for all inline sge we need */ 656 wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge * 657 sizeof(struct hns_roce_rinl_sge)), 658 GFP_KERNEL); 659 if (!wqe_list[0].sg_list) 660 goto err_wqe_list; 661 662 /* Assign buffers of sg_list to each inline wqe */ 663 for (i = 1; i < wqe_cnt; i++) 664 wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge]; 665 666 hr_qp->rq_inl_buf.wqe_list = wqe_list; 667 hr_qp->rq_inl_buf.wqe_cnt = wqe_cnt; 668 669 return 0; 670 671 err_wqe_list: 672 kfree(wqe_list); 673 674 err: 675 return -ENOMEM; 676 } 677 678 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp) 679 { 680 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list); 681 kfree(hr_qp->rq_inl_buf.wqe_list); 682 } 683 684 static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, 685 struct ib_pd *ib_pd, 686 struct ib_qp_init_attr *init_attr, 687 struct ib_udata *udata, unsigned long sqpn, 688 struct hns_roce_qp *hr_qp) 689 { 690 dma_addr_t *buf_list[ARRAY_SIZE(hr_qp->regions)] = { NULL }; 691 struct device *dev = hr_dev->dev; 692 struct hns_roce_ib_create_qp ucmd; 693 struct hns_roce_ib_create_qp_resp resp = {}; 694 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( 695 udata, struct hns_roce_ucontext, ibucontext); 696 struct hns_roce_buf_region *r; 697 unsigned long qpn = 0; 698 u32 page_shift; 699 int buf_count; 700 int ret; 701 int i; 702 703 mutex_init(&hr_qp->mutex); 704 spin_lock_init(&hr_qp->sq.lock); 705 spin_lock_init(&hr_qp->rq.lock); 706 707 hr_qp->state = IB_QPS_RESET; 708 709 hr_qp->ibqp.qp_type = init_attr->qp_type; 710 711 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) 712 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR; 713 else 714 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR; 715 716 ret = hns_roce_set_rq_size(hr_dev, &init_attr->cap, udata, 717 hns_roce_qp_has_rq(init_attr), hr_qp); 718 if (ret) { 719 dev_err(dev, "hns_roce_set_rq_size failed\n"); 720 goto err_out; 721 } 722 723 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && 724 hns_roce_qp_has_rq(init_attr)) { 725 ret = alloc_rq_inline_buf(hr_qp, init_attr); 726 if (ret) { 727 dev_err(dev, "allocate receive inline buffer failed\n"); 728 goto err_out; 729 } 730 } 731 732 page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; 733 if (udata) { 734 if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) { 735 dev_err(dev, "ib_copy_from_udata error for create qp\n"); 736 ret = -EFAULT; 737 goto err_alloc_rq_inline_buf; 738 } 739 740 ret = hns_roce_set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, 741 &ucmd); 742 if (ret) { 743 dev_err(dev, "hns_roce_set_user_sq_size error for create qp\n"); 744 goto err_alloc_rq_inline_buf; 745 } 746 747 hr_qp->umem = ib_umem_get(udata, ucmd.buf_addr, 748 hr_qp->buff_size, 0); 749 if (IS_ERR(hr_qp->umem)) { 750 dev_err(dev, "ib_umem_get error for create qp\n"); 751 ret = PTR_ERR(hr_qp->umem); 752 goto err_alloc_rq_inline_buf; 753 } 754 hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp, 755 hr_qp->regions, ARRAY_SIZE(hr_qp->regions), 756 page_shift); 757 ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list, 758 hr_qp->region_cnt); 759 if (ret) { 760 dev_err(dev, "alloc buf_list error for create qp\n"); 761 goto err_alloc_list; 762 } 763 764 for (i = 0; i < hr_qp->region_cnt; i++) { 765 r = &hr_qp->regions[i]; 766 buf_count = hns_roce_get_umem_bufs(hr_dev, 767 buf_list[i], r->count, r->offset, 768 hr_qp->umem, page_shift); 769 if (buf_count != r->count) { 770 dev_err(dev, 771 "get umem buf err, expect %d,ret %d.\n", 772 r->count, buf_count); 773 ret = -ENOBUFS; 774 goto err_get_bufs; 775 } 776 } 777 778 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) && 779 (udata->inlen >= sizeof(ucmd)) && 780 (udata->outlen >= sizeof(resp)) && 781 hns_roce_qp_has_sq(init_attr)) { 782 ret = hns_roce_db_map_user(uctx, udata, ucmd.sdb_addr, 783 &hr_qp->sdb); 784 if (ret) { 785 dev_err(dev, "sq record doorbell map failed!\n"); 786 goto err_get_bufs; 787 } 788 789 /* indicate kernel supports sq record db */ 790 resp.cap_flags |= HNS_ROCE_SUPPORT_SQ_RECORD_DB; 791 hr_qp->sdb_en = 1; 792 } 793 794 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && 795 (udata->outlen >= sizeof(resp)) && 796 hns_roce_qp_has_rq(init_attr)) { 797 ret = hns_roce_db_map_user(uctx, udata, ucmd.db_addr, 798 &hr_qp->rdb); 799 if (ret) { 800 dev_err(dev, "rq record doorbell map failed!\n"); 801 goto err_sq_dbmap; 802 } 803 804 /* indicate kernel supports rq record db */ 805 resp.cap_flags |= HNS_ROCE_SUPPORT_RQ_RECORD_DB; 806 hr_qp->rdb_en = 1; 807 } 808 } else { 809 if (init_attr->create_flags & 810 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) { 811 dev_err(dev, "init_attr->create_flags error!\n"); 812 ret = -EINVAL; 813 goto err_alloc_rq_inline_buf; 814 } 815 816 if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) { 817 dev_err(dev, "init_attr->create_flags error!\n"); 818 ret = -EINVAL; 819 goto err_alloc_rq_inline_buf; 820 } 821 822 /* Set SQ size */ 823 ret = hns_roce_set_kernel_sq_size(hr_dev, &init_attr->cap, 824 hr_qp); 825 if (ret) { 826 dev_err(dev, "hns_roce_set_kernel_sq_size error!\n"); 827 goto err_alloc_rq_inline_buf; 828 } 829 830 /* QP doorbell register address */ 831 hr_qp->sq.db_reg_l = hr_dev->reg_base + hr_dev->sdb_offset + 832 DB_REG_OFFSET * hr_dev->priv_uar.index; 833 hr_qp->rq.db_reg_l = hr_dev->reg_base + hr_dev->odb_offset + 834 DB_REG_OFFSET * hr_dev->priv_uar.index; 835 836 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && 837 hns_roce_qp_has_rq(init_attr)) { 838 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0); 839 if (ret) { 840 dev_err(dev, "rq record doorbell alloc failed!\n"); 841 goto err_alloc_rq_inline_buf; 842 } 843 *hr_qp->rdb.db_record = 0; 844 hr_qp->rdb_en = 1; 845 } 846 847 /* Allocate QP buf */ 848 if (hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, 849 (1 << page_shift) * 2, 850 &hr_qp->hr_buf, page_shift)) { 851 dev_err(dev, "hns_roce_buf_alloc error!\n"); 852 ret = -ENOMEM; 853 goto err_db; 854 } 855 hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp, 856 hr_qp->regions, ARRAY_SIZE(hr_qp->regions), 857 page_shift); 858 ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list, 859 hr_qp->region_cnt); 860 if (ret) { 861 dev_err(dev, "alloc buf_list error for create qp!\n"); 862 goto err_alloc_list; 863 } 864 865 for (i = 0; i < hr_qp->region_cnt; i++) { 866 r = &hr_qp->regions[i]; 867 buf_count = hns_roce_get_kmem_bufs(hr_dev, 868 buf_list[i], r->count, r->offset, 869 &hr_qp->hr_buf); 870 if (buf_count != r->count) { 871 dev_err(dev, 872 "get kmem buf err, expect %d,ret %d.\n", 873 r->count, buf_count); 874 ret = -ENOBUFS; 875 goto err_get_bufs; 876 } 877 } 878 879 hr_qp->sq.wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), 880 GFP_KERNEL); 881 if (ZERO_OR_NULL_PTR(hr_qp->sq.wrid)) { 882 ret = -ENOMEM; 883 goto err_get_bufs; 884 } 885 886 if (hr_qp->rq.wqe_cnt) { 887 hr_qp->rq.wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), 888 GFP_KERNEL); 889 if (ZERO_OR_NULL_PTR(hr_qp->rq.wrid)) { 890 ret = -ENOMEM; 891 goto err_sq_wrid; 892 } 893 } 894 } 895 896 if (sqpn) { 897 qpn = sqpn; 898 } else { 899 /* Get QPN */ 900 ret = hns_roce_reserve_range_qp(hr_dev, 1, 1, &qpn); 901 if (ret) { 902 dev_err(dev, "hns_roce_reserve_range_qp alloc qpn error\n"); 903 goto err_wrid; 904 } 905 } 906 907 hr_qp->wqe_bt_pg_shift = calc_wqe_bt_page_shift(hr_dev, hr_qp->regions, 908 hr_qp->region_cnt); 909 hns_roce_mtr_init(&hr_qp->mtr, PAGE_SHIFT + hr_qp->wqe_bt_pg_shift, 910 page_shift); 911 ret = hns_roce_mtr_attach(hr_dev, &hr_qp->mtr, buf_list, 912 hr_qp->regions, hr_qp->region_cnt); 913 if (ret) { 914 dev_err(dev, "mtr attach error for create qp\n"); 915 goto err_mtr; 916 } 917 918 if (init_attr->qp_type == IB_QPT_GSI && 919 hr_dev->hw_rev == HNS_ROCE_HW_VER1) { 920 /* In v1 engine, GSI QP context in RoCE engine's register */ 921 ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp); 922 if (ret) { 923 dev_err(dev, "hns_roce_qp_alloc failed!\n"); 924 goto err_qpn; 925 } 926 } else { 927 ret = hns_roce_qp_alloc(hr_dev, qpn, hr_qp); 928 if (ret) { 929 dev_err(dev, "hns_roce_qp_alloc failed!\n"); 930 goto err_qpn; 931 } 932 } 933 934 if (sqpn) 935 hr_qp->doorbell_qpn = 1; 936 else 937 hr_qp->doorbell_qpn = (u32)hr_qp->qpn; 938 939 if (udata) { 940 ret = ib_copy_to_udata(udata, &resp, 941 min(udata->outlen, sizeof(resp))); 942 if (ret) 943 goto err_qp; 944 } 945 946 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 947 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp); 948 if (ret) 949 goto err_qp; 950 } 951 952 hr_qp->event = hns_roce_ib_qp_event; 953 hns_roce_free_buf_list(buf_list, hr_qp->region_cnt); 954 955 return 0; 956 957 err_qp: 958 if (init_attr->qp_type == IB_QPT_GSI && 959 hr_dev->hw_rev == HNS_ROCE_HW_VER1) 960 hns_roce_qp_remove(hr_dev, hr_qp); 961 else 962 hns_roce_qp_free(hr_dev, hr_qp); 963 964 err_qpn: 965 if (!sqpn) 966 hns_roce_release_range_qp(hr_dev, qpn, 1); 967 968 err_mtr: 969 hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr); 970 971 err_wrid: 972 if (udata) { 973 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && 974 (udata->outlen >= sizeof(resp)) && 975 hns_roce_qp_has_rq(init_attr)) 976 hns_roce_db_unmap_user(uctx, &hr_qp->rdb); 977 } else { 978 if (hr_qp->rq.wqe_cnt) 979 kfree(hr_qp->rq.wrid); 980 } 981 982 err_sq_dbmap: 983 if (udata) 984 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) && 985 (udata->inlen >= sizeof(ucmd)) && 986 (udata->outlen >= sizeof(resp)) && 987 hns_roce_qp_has_sq(init_attr)) 988 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 989 990 err_sq_wrid: 991 if (!udata) 992 kfree(hr_qp->sq.wrid); 993 994 err_get_bufs: 995 hns_roce_free_buf_list(buf_list, hr_qp->region_cnt); 996 997 err_alloc_list: 998 if (!hr_qp->umem) 999 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); 1000 ib_umem_release(hr_qp->umem); 1001 1002 err_db: 1003 if (!udata && hns_roce_qp_has_rq(init_attr) && 1004 (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB)) 1005 hns_roce_free_db(hr_dev, &hr_qp->rdb); 1006 1007 err_alloc_rq_inline_buf: 1008 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && 1009 hns_roce_qp_has_rq(init_attr)) 1010 free_rq_inline_buf(hr_qp); 1011 1012 err_out: 1013 return ret; 1014 } 1015 1016 struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, 1017 struct ib_qp_init_attr *init_attr, 1018 struct ib_udata *udata) 1019 { 1020 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); 1021 struct ib_device *ibdev = &hr_dev->ib_dev; 1022 struct hns_roce_qp *hr_qp; 1023 int ret; 1024 1025 switch (init_attr->qp_type) { 1026 case IB_QPT_RC: { 1027 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); 1028 if (!hr_qp) 1029 return ERR_PTR(-ENOMEM); 1030 1031 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, 0, 1032 hr_qp); 1033 if (ret) { 1034 ibdev_err(ibdev, "Create QP 0x%06lx failed(%d)\n", 1035 hr_qp->qpn, ret); 1036 kfree(hr_qp); 1037 return ERR_PTR(ret); 1038 } 1039 1040 hr_qp->ibqp.qp_num = hr_qp->qpn; 1041 1042 break; 1043 } 1044 case IB_QPT_GSI: { 1045 /* Userspace is not allowed to create special QPs: */ 1046 if (udata) { 1047 ibdev_err(ibdev, "not support usr space GSI\n"); 1048 return ERR_PTR(-EINVAL); 1049 } 1050 1051 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); 1052 if (!hr_qp) 1053 return ERR_PTR(-ENOMEM); 1054 1055 hr_qp->port = init_attr->port_num - 1; 1056 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; 1057 1058 /* when hw version is v1, the sqpn is allocated */ 1059 if (hr_dev->caps.max_sq_sg <= 2) 1060 hr_qp->ibqp.qp_num = HNS_ROCE_MAX_PORTS + 1061 hr_dev->iboe.phy_port[hr_qp->port]; 1062 else 1063 hr_qp->ibqp.qp_num = 1; 1064 1065 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, 1066 hr_qp->ibqp.qp_num, hr_qp); 1067 if (ret) { 1068 ibdev_err(ibdev, "Create GSI QP failed!\n"); 1069 kfree(hr_qp); 1070 return ERR_PTR(ret); 1071 } 1072 1073 break; 1074 } 1075 default:{ 1076 ibdev_err(ibdev, "not support QP type %d\n", 1077 init_attr->qp_type); 1078 return ERR_PTR(-EINVAL); 1079 } 1080 } 1081 1082 return &hr_qp->ibqp; 1083 } 1084 1085 int to_hr_qp_type(int qp_type) 1086 { 1087 int transport_type; 1088 1089 if (qp_type == IB_QPT_RC) 1090 transport_type = SERV_TYPE_RC; 1091 else if (qp_type == IB_QPT_UC) 1092 transport_type = SERV_TYPE_UC; 1093 else if (qp_type == IB_QPT_UD) 1094 transport_type = SERV_TYPE_UD; 1095 else if (qp_type == IB_QPT_GSI) 1096 transport_type = SERV_TYPE_UD; 1097 else 1098 transport_type = -1; 1099 1100 return transport_type; 1101 } 1102 1103 static int check_mtu_validate(struct hns_roce_dev *hr_dev, 1104 struct hns_roce_qp *hr_qp, 1105 struct ib_qp_attr *attr, int attr_mask) 1106 { 1107 enum ib_mtu active_mtu; 1108 int p; 1109 1110 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1111 active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu); 1112 1113 if ((hr_dev->caps.max_mtu >= IB_MTU_2048 && 1114 attr->path_mtu > hr_dev->caps.max_mtu) || 1115 attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) { 1116 ibdev_err(&hr_dev->ib_dev, 1117 "attr path_mtu(%d)invalid while modify qp", 1118 attr->path_mtu); 1119 return -EINVAL; 1120 } 1121 1122 return 0; 1123 } 1124 1125 static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1126 int attr_mask) 1127 { 1128 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1129 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1130 int p; 1131 1132 if ((attr_mask & IB_QP_PORT) && 1133 (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) { 1134 ibdev_err(&hr_dev->ib_dev, 1135 "attr port_num invalid.attr->port_num=%d\n", 1136 attr->port_num); 1137 return -EINVAL; 1138 } 1139 1140 if (attr_mask & IB_QP_PKEY_INDEX) { 1141 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1142 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) { 1143 ibdev_err(&hr_dev->ib_dev, 1144 "attr pkey_index invalid.attr->pkey_index=%d\n", 1145 attr->pkey_index); 1146 return -EINVAL; 1147 } 1148 } 1149 1150 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && 1151 attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) { 1152 ibdev_err(&hr_dev->ib_dev, 1153 "attr max_rd_atomic invalid.attr->max_rd_atomic=%d\n", 1154 attr->max_rd_atomic); 1155 return -EINVAL; 1156 } 1157 1158 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && 1159 attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) { 1160 ibdev_err(&hr_dev->ib_dev, 1161 "attr max_dest_rd_atomic invalid.attr->max_dest_rd_atomic=%d\n", 1162 attr->max_dest_rd_atomic); 1163 return -EINVAL; 1164 } 1165 1166 if (attr_mask & IB_QP_PATH_MTU) 1167 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask); 1168 1169 return 0; 1170 } 1171 1172 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1173 int attr_mask, struct ib_udata *udata) 1174 { 1175 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1176 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1177 enum ib_qp_state cur_state, new_state; 1178 int ret = -EINVAL; 1179 1180 mutex_lock(&hr_qp->mutex); 1181 1182 cur_state = attr_mask & IB_QP_CUR_STATE ? 1183 attr->cur_qp_state : (enum ib_qp_state)hr_qp->state; 1184 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; 1185 1186 if (ibqp->uobject && 1187 (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) { 1188 if (hr_qp->sdb_en == 1) { 1189 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr); 1190 1191 if (hr_qp->rdb_en == 1) 1192 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr); 1193 } else { 1194 ibdev_warn(&hr_dev->ib_dev, 1195 "flush cqe is not supported in userspace!\n"); 1196 goto out; 1197 } 1198 } 1199 1200 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, 1201 attr_mask)) { 1202 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n"); 1203 goto out; 1204 } 1205 1206 ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask); 1207 if (ret) 1208 goto out; 1209 1210 if (cur_state == new_state && cur_state == IB_QPS_RESET) { 1211 if (hr_dev->caps.min_wqes) { 1212 ret = -EPERM; 1213 ibdev_err(&hr_dev->ib_dev, 1214 "cur_state=%d new_state=%d\n", cur_state, 1215 new_state); 1216 } else { 1217 ret = 0; 1218 } 1219 1220 goto out; 1221 } 1222 1223 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, 1224 new_state); 1225 1226 out: 1227 mutex_unlock(&hr_qp->mutex); 1228 1229 return ret; 1230 } 1231 1232 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) 1233 __acquires(&send_cq->lock) __acquires(&recv_cq->lock) 1234 { 1235 if (send_cq == recv_cq) { 1236 spin_lock_irq(&send_cq->lock); 1237 __acquire(&recv_cq->lock); 1238 } else if (send_cq->cqn < recv_cq->cqn) { 1239 spin_lock_irq(&send_cq->lock); 1240 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); 1241 } else { 1242 spin_lock_irq(&recv_cq->lock); 1243 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); 1244 } 1245 } 1246 1247 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, 1248 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock) 1249 __releases(&recv_cq->lock) 1250 { 1251 if (send_cq == recv_cq) { 1252 __release(&recv_cq->lock); 1253 spin_unlock_irq(&send_cq->lock); 1254 } else if (send_cq->cqn < recv_cq->cqn) { 1255 spin_unlock(&recv_cq->lock); 1256 spin_unlock_irq(&send_cq->lock); 1257 } else { 1258 spin_unlock(&send_cq->lock); 1259 spin_unlock_irq(&recv_cq->lock); 1260 } 1261 } 1262 1263 static void *get_wqe(struct hns_roce_qp *hr_qp, int offset) 1264 { 1265 1266 return hns_roce_buf_offset(&hr_qp->hr_buf, offset); 1267 } 1268 1269 void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n) 1270 { 1271 return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift)); 1272 } 1273 1274 void *get_send_wqe(struct hns_roce_qp *hr_qp, int n) 1275 { 1276 return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift)); 1277 } 1278 1279 void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n) 1280 { 1281 return hns_roce_buf_offset(&hr_qp->hr_buf, hr_qp->sge.offset + 1282 (n << hr_qp->sge.sge_shift)); 1283 } 1284 1285 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq, 1286 struct ib_cq *ib_cq) 1287 { 1288 struct hns_roce_cq *hr_cq; 1289 u32 cur; 1290 1291 cur = hr_wq->head - hr_wq->tail; 1292 if (likely(cur + nreq < hr_wq->wqe_cnt)) 1293 return false; 1294 1295 hr_cq = to_hr_cq(ib_cq); 1296 spin_lock(&hr_cq->lock); 1297 cur = hr_wq->head - hr_wq->tail; 1298 spin_unlock(&hr_cq->lock); 1299 1300 return cur + nreq >= hr_wq->wqe_cnt; 1301 } 1302 1303 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) 1304 { 1305 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 1306 int reserved_from_top = 0; 1307 int reserved_from_bot; 1308 int ret; 1309 1310 mutex_init(&qp_table->scc_mutex); 1311 xa_init(&hr_dev->qp_table_xa); 1312 1313 reserved_from_bot = hr_dev->caps.reserved_qps; 1314 1315 ret = hns_roce_bitmap_init(&qp_table->bitmap, hr_dev->caps.num_qps, 1316 hr_dev->caps.num_qps - 1, reserved_from_bot, 1317 reserved_from_top); 1318 if (ret) { 1319 dev_err(hr_dev->dev, "qp bitmap init failed!error=%d\n", 1320 ret); 1321 return ret; 1322 } 1323 1324 return 0; 1325 } 1326 1327 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) 1328 { 1329 hns_roce_bitmap_cleanup(&hr_dev->qp_table.bitmap); 1330 } 1331