1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/pci.h> 35 #include <linux/platform_device.h> 36 #include <rdma/ib_addr.h> 37 #include <rdma/ib_umem.h> 38 #include <rdma/uverbs_ioctl.h> 39 #include "hns_roce_common.h" 40 #include "hns_roce_device.h" 41 #include "hns_roce_hem.h" 42 #include <rdma/hns-abi.h> 43 44 #define SQP_NUM (2 * HNS_ROCE_MAX_PORTS) 45 46 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) 47 { 48 struct device *dev = hr_dev->dev; 49 struct hns_roce_qp *qp; 50 51 xa_lock(&hr_dev->qp_table_xa); 52 qp = __hns_roce_qp_lookup(hr_dev, qpn); 53 if (qp) 54 atomic_inc(&qp->refcount); 55 xa_unlock(&hr_dev->qp_table_xa); 56 57 if (!qp) { 58 dev_warn(dev, "Async event for bogus QP %08x\n", qpn); 59 return; 60 } 61 62 qp->event(qp, (enum hns_roce_event)event_type); 63 64 if (atomic_dec_and_test(&qp->refcount)) 65 complete(&qp->free); 66 } 67 68 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, 69 enum hns_roce_event type) 70 { 71 struct ib_event event; 72 struct ib_qp *ibqp = &hr_qp->ibqp; 73 74 if (ibqp->event_handler) { 75 event.device = ibqp->device; 76 event.element.qp = ibqp; 77 switch (type) { 78 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 79 event.event = IB_EVENT_PATH_MIG; 80 break; 81 case HNS_ROCE_EVENT_TYPE_COMM_EST: 82 event.event = IB_EVENT_COMM_EST; 83 break; 84 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 85 event.event = IB_EVENT_SQ_DRAINED; 86 break; 87 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 88 event.event = IB_EVENT_QP_LAST_WQE_REACHED; 89 break; 90 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 91 event.event = IB_EVENT_QP_FATAL; 92 break; 93 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 94 event.event = IB_EVENT_PATH_MIG_ERR; 95 break; 96 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 97 event.event = IB_EVENT_QP_REQ_ERR; 98 break; 99 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 100 event.event = IB_EVENT_QP_ACCESS_ERR; 101 break; 102 default: 103 dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n", 104 type, hr_qp->qpn); 105 return; 106 } 107 ibqp->event_handler(&event, ibqp->qp_context); 108 } 109 } 110 111 static int hns_roce_reserve_range_qp(struct hns_roce_dev *hr_dev, int cnt, 112 int align, unsigned long *base) 113 { 114 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 115 116 return hns_roce_bitmap_alloc_range(&qp_table->bitmap, cnt, align, 117 base) ? 118 -ENOMEM : 119 0; 120 } 121 122 enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state) 123 { 124 switch (state) { 125 case IB_QPS_RESET: 126 return HNS_ROCE_QP_STATE_RST; 127 case IB_QPS_INIT: 128 return HNS_ROCE_QP_STATE_INIT; 129 case IB_QPS_RTR: 130 return HNS_ROCE_QP_STATE_RTR; 131 case IB_QPS_RTS: 132 return HNS_ROCE_QP_STATE_RTS; 133 case IB_QPS_SQD: 134 return HNS_ROCE_QP_STATE_SQD; 135 case IB_QPS_ERR: 136 return HNS_ROCE_QP_STATE_ERR; 137 default: 138 return HNS_ROCE_QP_NUM_STATE; 139 } 140 } 141 142 static int hns_roce_gsi_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, 143 struct hns_roce_qp *hr_qp) 144 { 145 struct xarray *xa = &hr_dev->qp_table_xa; 146 int ret; 147 148 if (!qpn) 149 return -EINVAL; 150 151 hr_qp->qpn = qpn; 152 atomic_set(&hr_qp->refcount, 1); 153 init_completion(&hr_qp->free); 154 155 ret = xa_err(xa_store_irq(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1), 156 hr_qp, GFP_KERNEL)); 157 if (ret) 158 dev_err(hr_dev->dev, "QPC xa_store failed\n"); 159 160 return ret; 161 } 162 163 static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, 164 struct hns_roce_qp *hr_qp) 165 { 166 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 167 struct device *dev = hr_dev->dev; 168 int ret; 169 170 if (!qpn) 171 return -EINVAL; 172 173 hr_qp->qpn = qpn; 174 175 /* Alloc memory for QPC */ 176 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn); 177 if (ret) { 178 dev_err(dev, "QPC table get failed\n"); 179 goto err_out; 180 } 181 182 /* Alloc memory for IRRL */ 183 ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 184 if (ret) { 185 dev_err(dev, "IRRL table get failed\n"); 186 goto err_put_qp; 187 } 188 189 if (hr_dev->caps.trrl_entry_sz) { 190 /* Alloc memory for TRRL */ 191 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table, 192 hr_qp->qpn); 193 if (ret) { 194 dev_err(dev, "TRRL table get failed\n"); 195 goto err_put_irrl; 196 } 197 } 198 199 if (hr_dev->caps.sccc_entry_sz) { 200 /* Alloc memory for SCC CTX */ 201 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table, 202 hr_qp->qpn); 203 if (ret) { 204 dev_err(dev, "SCC CTX table get failed\n"); 205 goto err_put_trrl; 206 } 207 } 208 209 ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp); 210 if (ret) 211 goto err_put_sccc; 212 213 return 0; 214 215 err_put_sccc: 216 if (hr_dev->caps.sccc_entry_sz) 217 hns_roce_table_put(hr_dev, &qp_table->sccc_table, 218 hr_qp->qpn); 219 220 err_put_trrl: 221 if (hr_dev->caps.trrl_entry_sz) 222 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 223 224 err_put_irrl: 225 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 226 227 err_put_qp: 228 hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); 229 230 err_out: 231 return ret; 232 } 233 234 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 235 { 236 struct xarray *xa = &hr_dev->qp_table_xa; 237 unsigned long flags; 238 239 xa_lock_irqsave(xa, flags); 240 __xa_erase(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1)); 241 xa_unlock_irqrestore(xa, flags); 242 } 243 244 void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 245 { 246 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 247 248 if (atomic_dec_and_test(&hr_qp->refcount)) 249 complete(&hr_qp->free); 250 wait_for_completion(&hr_qp->free); 251 252 if ((hr_qp->ibqp.qp_type) != IB_QPT_GSI) { 253 if (hr_dev->caps.trrl_entry_sz) 254 hns_roce_table_put(hr_dev, &qp_table->trrl_table, 255 hr_qp->qpn); 256 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 257 } 258 } 259 260 void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, 261 int cnt) 262 { 263 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 264 265 if (base_qpn < hr_dev->caps.reserved_qps) 266 return; 267 268 hns_roce_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt, BITMAP_RR); 269 } 270 271 static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev, 272 struct ib_qp_cap *cap, bool is_user, int has_rq, 273 struct hns_roce_qp *hr_qp) 274 { 275 struct device *dev = hr_dev->dev; 276 u32 max_cnt; 277 278 /* Check the validity of QP support capacity */ 279 if (cap->max_recv_wr > hr_dev->caps.max_wqes || 280 cap->max_recv_sge > hr_dev->caps.max_rq_sg) { 281 dev_err(dev, "RQ WR or sge error!max_recv_wr=%d max_recv_sge=%d\n", 282 cap->max_recv_wr, cap->max_recv_sge); 283 return -EINVAL; 284 } 285 286 /* If srq exist, set zero for relative number of rq */ 287 if (!has_rq) { 288 hr_qp->rq.wqe_cnt = 0; 289 hr_qp->rq.max_gs = 0; 290 cap->max_recv_wr = 0; 291 cap->max_recv_sge = 0; 292 } else { 293 if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge)) { 294 dev_err(dev, "user space no need config max_recv_wr max_recv_sge\n"); 295 return -EINVAL; 296 } 297 298 if (hr_dev->caps.min_wqes) 299 max_cnt = max(cap->max_recv_wr, hr_dev->caps.min_wqes); 300 else 301 max_cnt = cap->max_recv_wr; 302 303 hr_qp->rq.wqe_cnt = roundup_pow_of_two(max_cnt); 304 305 if ((u32)hr_qp->rq.wqe_cnt > hr_dev->caps.max_wqes) { 306 dev_err(dev, "while setting rq size, rq.wqe_cnt too large\n"); 307 return -EINVAL; 308 } 309 310 max_cnt = max(1U, cap->max_recv_sge); 311 hr_qp->rq.max_gs = roundup_pow_of_two(max_cnt); 312 if (hr_dev->caps.max_rq_sg <= 2) 313 hr_qp->rq.wqe_shift = 314 ilog2(hr_dev->caps.max_rq_desc_sz); 315 else 316 hr_qp->rq.wqe_shift = 317 ilog2(hr_dev->caps.max_rq_desc_sz 318 * hr_qp->rq.max_gs); 319 } 320 321 cap->max_recv_wr = hr_qp->rq.wqe_cnt; 322 cap->max_recv_sge = hr_qp->rq.max_gs; 323 324 return 0; 325 } 326 327 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev, 328 struct ib_qp_cap *cap, 329 struct hns_roce_ib_create_qp *ucmd) 330 { 331 u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz); 332 u8 max_sq_stride = ilog2(roundup_sq_stride); 333 334 /* Sanity check SQ size before proceeding */ 335 if (ucmd->log_sq_stride > max_sq_stride || 336 ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) { 337 ibdev_err(&hr_dev->ib_dev, "check SQ size error!\n"); 338 return -EINVAL; 339 } 340 341 if (cap->max_send_sge > hr_dev->caps.max_sq_sg) { 342 ibdev_err(&hr_dev->ib_dev, "SQ sge error! max_send_sge=%d\n", 343 cap->max_send_sge); 344 return -EINVAL; 345 } 346 347 return 0; 348 } 349 350 static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev, 351 struct ib_qp_cap *cap, 352 struct hns_roce_qp *hr_qp, 353 struct hns_roce_ib_create_qp *ucmd) 354 { 355 u32 ex_sge_num; 356 u32 page_size; 357 u32 max_cnt; 358 int ret; 359 360 if (check_shl_overflow(1, ucmd->log_sq_bb_count, &hr_qp->sq.wqe_cnt) || 361 hr_qp->sq.wqe_cnt > hr_dev->caps.max_wqes) 362 return -EINVAL; 363 364 ret = check_sq_size_with_integrity(hr_dev, cap, ucmd); 365 if (ret) { 366 ibdev_err(&hr_dev->ib_dev, "Sanity check sq size failed\n"); 367 return ret; 368 } 369 370 hr_qp->sq.wqe_shift = ucmd->log_sq_stride; 371 372 max_cnt = max(1U, cap->max_send_sge); 373 if (hr_dev->caps.max_sq_sg <= 2) 374 hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt); 375 else 376 hr_qp->sq.max_gs = max_cnt; 377 378 if (hr_qp->sq.max_gs > 2) 379 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt * 380 (hr_qp->sq.max_gs - 2)); 381 382 if ((hr_qp->sq.max_gs > 2) && (hr_dev->pci_dev->revision == 0x20)) { 383 if (hr_qp->sge.sge_cnt > hr_dev->caps.max_extend_sg) { 384 dev_err(hr_dev->dev, 385 "The extended sge cnt error! sge_cnt=%d\n", 386 hr_qp->sge.sge_cnt); 387 return -EINVAL; 388 } 389 } 390 391 hr_qp->sge.sge_shift = 4; 392 ex_sge_num = hr_qp->sge.sge_cnt; 393 394 /* Get buf size, SQ and RQ are aligned to page_szie */ 395 if (hr_dev->caps.max_sq_sg <= 2) { 396 hr_qp->buff_size = round_up((hr_qp->rq.wqe_cnt << 397 hr_qp->rq.wqe_shift), PAGE_SIZE) + 398 round_up((hr_qp->sq.wqe_cnt << 399 hr_qp->sq.wqe_shift), PAGE_SIZE); 400 401 hr_qp->sq.offset = 0; 402 hr_qp->rq.offset = round_up((hr_qp->sq.wqe_cnt << 403 hr_qp->sq.wqe_shift), PAGE_SIZE); 404 } else { 405 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); 406 hr_qp->sge.sge_cnt = ex_sge_num ? 407 max(page_size / (1 << hr_qp->sge.sge_shift), ex_sge_num) : 0; 408 hr_qp->buff_size = round_up((hr_qp->rq.wqe_cnt << 409 hr_qp->rq.wqe_shift), page_size) + 410 round_up((hr_qp->sge.sge_cnt << 411 hr_qp->sge.sge_shift), page_size) + 412 round_up((hr_qp->sq.wqe_cnt << 413 hr_qp->sq.wqe_shift), page_size); 414 415 hr_qp->sq.offset = 0; 416 if (ex_sge_num) { 417 hr_qp->sge.offset = round_up((hr_qp->sq.wqe_cnt << 418 hr_qp->sq.wqe_shift), 419 page_size); 420 hr_qp->rq.offset = hr_qp->sge.offset + 421 round_up((hr_qp->sge.sge_cnt << 422 hr_qp->sge.sge_shift), 423 page_size); 424 } else { 425 hr_qp->rq.offset = round_up((hr_qp->sq.wqe_cnt << 426 hr_qp->sq.wqe_shift), 427 page_size); 428 } 429 } 430 431 return 0; 432 } 433 434 static int split_wqe_buf_region(struct hns_roce_dev *hr_dev, 435 struct hns_roce_qp *hr_qp, 436 struct hns_roce_buf_region *regions, 437 int region_max, int page_shift) 438 { 439 int page_size = 1 << page_shift; 440 bool is_extend_sge; 441 int region_cnt = 0; 442 int buf_size; 443 int buf_cnt; 444 445 if (hr_qp->buff_size < 1 || region_max < 1) 446 return region_cnt; 447 448 if (hr_qp->sge.sge_cnt > 0) 449 is_extend_sge = true; 450 else 451 is_extend_sge = false; 452 453 /* sq region */ 454 if (is_extend_sge) 455 buf_size = hr_qp->sge.offset - hr_qp->sq.offset; 456 else 457 buf_size = hr_qp->rq.offset - hr_qp->sq.offset; 458 459 if (buf_size > 0 && region_cnt < region_max) { 460 buf_cnt = DIV_ROUND_UP(buf_size, page_size); 461 hns_roce_init_buf_region(®ions[region_cnt], 462 hr_dev->caps.wqe_sq_hop_num, 463 hr_qp->sq.offset / page_size, 464 buf_cnt); 465 region_cnt++; 466 } 467 468 /* sge region */ 469 if (is_extend_sge) { 470 buf_size = hr_qp->rq.offset - hr_qp->sge.offset; 471 if (buf_size > 0 && region_cnt < region_max) { 472 buf_cnt = DIV_ROUND_UP(buf_size, page_size); 473 hns_roce_init_buf_region(®ions[region_cnt], 474 hr_dev->caps.wqe_sge_hop_num, 475 hr_qp->sge.offset / page_size, 476 buf_cnt); 477 region_cnt++; 478 } 479 } 480 481 /* rq region */ 482 buf_size = hr_qp->buff_size - hr_qp->rq.offset; 483 if (buf_size > 0) { 484 buf_cnt = DIV_ROUND_UP(buf_size, page_size); 485 hns_roce_init_buf_region(®ions[region_cnt], 486 hr_dev->caps.wqe_rq_hop_num, 487 hr_qp->rq.offset / page_size, 488 buf_cnt); 489 region_cnt++; 490 } 491 492 return region_cnt; 493 } 494 495 static int calc_wqe_bt_page_shift(struct hns_roce_dev *hr_dev, 496 struct hns_roce_buf_region *regions, 497 int region_cnt) 498 { 499 int bt_pg_shift; 500 int ba_num; 501 int ret; 502 503 bt_pg_shift = PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz; 504 505 /* all root ba entries must in one bt page */ 506 do { 507 ba_num = (1 << bt_pg_shift) / BA_BYTE_LEN; 508 ret = hns_roce_hem_list_calc_root_ba(regions, region_cnt, 509 ba_num); 510 if (ret <= ba_num) 511 break; 512 513 bt_pg_shift++; 514 } while (ret > ba_num); 515 516 return bt_pg_shift - PAGE_SHIFT; 517 } 518 519 static int set_extend_sge_param(struct hns_roce_dev *hr_dev, 520 struct hns_roce_qp *hr_qp) 521 { 522 struct device *dev = hr_dev->dev; 523 524 if (hr_qp->sq.max_gs > 2) { 525 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt * 526 (hr_qp->sq.max_gs - 2)); 527 hr_qp->sge.sge_shift = 4; 528 } 529 530 /* ud sqwqe's sge use extend sge */ 531 if (hr_dev->caps.max_sq_sg > 2 && hr_qp->ibqp.qp_type == IB_QPT_GSI) { 532 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt * 533 hr_qp->sq.max_gs); 534 hr_qp->sge.sge_shift = 4; 535 } 536 537 if ((hr_qp->sq.max_gs > 2) && hr_dev->pci_dev->revision == 0x20) { 538 if (hr_qp->sge.sge_cnt > hr_dev->caps.max_extend_sg) { 539 dev_err(dev, "The extended sge cnt error! sge_cnt=%d\n", 540 hr_qp->sge.sge_cnt); 541 return -EINVAL; 542 } 543 } 544 545 return 0; 546 } 547 548 static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev, 549 struct ib_qp_cap *cap, 550 struct hns_roce_qp *hr_qp) 551 { 552 struct device *dev = hr_dev->dev; 553 u32 page_size; 554 u32 max_cnt; 555 int size; 556 int ret; 557 558 if (cap->max_send_wr > hr_dev->caps.max_wqes || 559 cap->max_send_sge > hr_dev->caps.max_sq_sg || 560 cap->max_inline_data > hr_dev->caps.max_sq_inline) { 561 dev_err(dev, "SQ WR or sge or inline data error!\n"); 562 return -EINVAL; 563 } 564 565 hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz); 566 567 if (hr_dev->caps.min_wqes) 568 max_cnt = max(cap->max_send_wr, hr_dev->caps.min_wqes); 569 else 570 max_cnt = cap->max_send_wr; 571 572 hr_qp->sq.wqe_cnt = roundup_pow_of_two(max_cnt); 573 if ((u32)hr_qp->sq.wqe_cnt > hr_dev->caps.max_wqes) { 574 dev_err(dev, "while setting kernel sq size, sq.wqe_cnt too large\n"); 575 return -EINVAL; 576 } 577 578 /* Get data_seg numbers */ 579 max_cnt = max(1U, cap->max_send_sge); 580 if (hr_dev->caps.max_sq_sg <= 2) 581 hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt); 582 else 583 hr_qp->sq.max_gs = max_cnt; 584 585 ret = set_extend_sge_param(hr_dev, hr_qp); 586 if (ret) { 587 dev_err(dev, "set extend sge parameters fail\n"); 588 return ret; 589 } 590 591 /* Get buf size, SQ and RQ are aligned to PAGE_SIZE */ 592 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); 593 hr_qp->sq.offset = 0; 594 size = round_up(hr_qp->sq.wqe_cnt << hr_qp->sq.wqe_shift, page_size); 595 596 if (hr_dev->caps.max_sq_sg > 2 && hr_qp->sge.sge_cnt) { 597 hr_qp->sge.sge_cnt = max(page_size/(1 << hr_qp->sge.sge_shift), 598 (u32)hr_qp->sge.sge_cnt); 599 hr_qp->sge.offset = size; 600 size += round_up(hr_qp->sge.sge_cnt << hr_qp->sge.sge_shift, 601 page_size); 602 } 603 604 hr_qp->rq.offset = size; 605 size += round_up((hr_qp->rq.wqe_cnt << hr_qp->rq.wqe_shift), page_size); 606 hr_qp->buff_size = size; 607 608 /* Get wr and sge number which send */ 609 cap->max_send_wr = hr_qp->sq.wqe_cnt; 610 cap->max_send_sge = hr_qp->sq.max_gs; 611 612 /* We don't support inline sends for kernel QPs (yet) */ 613 cap->max_inline_data = 0; 614 615 return 0; 616 } 617 618 static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr) 619 { 620 if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr) 621 return 0; 622 623 return 1; 624 } 625 626 static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr) 627 { 628 if (attr->qp_type == IB_QPT_XRC_INI || 629 attr->qp_type == IB_QPT_XRC_TGT || attr->srq || 630 !attr->cap.max_recv_wr) 631 return 0; 632 633 return 1; 634 } 635 636 static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp, 637 struct ib_qp_init_attr *init_attr) 638 { 639 u32 max_recv_sge = init_attr->cap.max_recv_sge; 640 struct hns_roce_rinl_wqe *wqe_list; 641 u32 wqe_cnt = hr_qp->rq.wqe_cnt; 642 int i; 643 644 /* allocate recv inline buf */ 645 wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe), 646 GFP_KERNEL); 647 648 if (!wqe_list) 649 goto err; 650 651 /* Allocate a continuous buffer for all inline sge we need */ 652 wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge * 653 sizeof(struct hns_roce_rinl_sge)), 654 GFP_KERNEL); 655 if (!wqe_list[0].sg_list) 656 goto err_wqe_list; 657 658 /* Assign buffers of sg_list to each inline wqe */ 659 for (i = 1; i < wqe_cnt; i++) 660 wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge]; 661 662 hr_qp->rq_inl_buf.wqe_list = wqe_list; 663 hr_qp->rq_inl_buf.wqe_cnt = wqe_cnt; 664 665 return 0; 666 667 err_wqe_list: 668 kfree(wqe_list); 669 670 err: 671 return -ENOMEM; 672 } 673 674 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp) 675 { 676 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list); 677 kfree(hr_qp->rq_inl_buf.wqe_list); 678 } 679 680 static void add_qp_to_list(struct hns_roce_dev *hr_dev, 681 struct hns_roce_qp *hr_qp, 682 struct ib_cq *send_cq, struct ib_cq *recv_cq) 683 { 684 struct hns_roce_cq *hr_send_cq, *hr_recv_cq; 685 unsigned long flags; 686 687 hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL; 688 hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL; 689 690 spin_lock_irqsave(&hr_dev->qp_list_lock, flags); 691 hns_roce_lock_cqs(hr_send_cq, hr_recv_cq); 692 693 list_add_tail(&hr_qp->node, &hr_dev->qp_list); 694 if (hr_send_cq) 695 list_add_tail(&hr_qp->sq_node, &hr_send_cq->sq_list); 696 if (hr_recv_cq) 697 list_add_tail(&hr_qp->rq_node, &hr_recv_cq->rq_list); 698 699 hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq); 700 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); 701 } 702 703 static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, 704 struct ib_pd *ib_pd, 705 struct ib_qp_init_attr *init_attr, 706 struct ib_udata *udata, unsigned long sqpn, 707 struct hns_roce_qp *hr_qp) 708 { 709 dma_addr_t *buf_list[ARRAY_SIZE(hr_qp->regions)] = { NULL }; 710 struct device *dev = hr_dev->dev; 711 struct hns_roce_ib_create_qp ucmd; 712 struct hns_roce_ib_create_qp_resp resp = {}; 713 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( 714 udata, struct hns_roce_ucontext, ibucontext); 715 struct hns_roce_buf_region *r; 716 unsigned long qpn = 0; 717 u32 page_shift; 718 int buf_count; 719 int ret; 720 int i; 721 722 mutex_init(&hr_qp->mutex); 723 spin_lock_init(&hr_qp->sq.lock); 724 spin_lock_init(&hr_qp->rq.lock); 725 726 hr_qp->state = IB_QPS_RESET; 727 728 hr_qp->ibqp.qp_type = init_attr->qp_type; 729 730 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) 731 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR; 732 else 733 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR; 734 735 ret = hns_roce_set_rq_size(hr_dev, &init_attr->cap, udata, 736 hns_roce_qp_has_rq(init_attr), hr_qp); 737 if (ret) { 738 dev_err(dev, "hns_roce_set_rq_size failed\n"); 739 goto err_out; 740 } 741 742 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && 743 hns_roce_qp_has_rq(init_attr)) { 744 ret = alloc_rq_inline_buf(hr_qp, init_attr); 745 if (ret) { 746 dev_err(dev, "allocate receive inline buffer failed\n"); 747 goto err_out; 748 } 749 } 750 751 page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; 752 if (udata) { 753 if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) { 754 dev_err(dev, "ib_copy_from_udata error for create qp\n"); 755 ret = -EFAULT; 756 goto err_alloc_rq_inline_buf; 757 } 758 759 ret = hns_roce_set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, 760 &ucmd); 761 if (ret) { 762 dev_err(dev, "hns_roce_set_user_sq_size error for create qp\n"); 763 goto err_alloc_rq_inline_buf; 764 } 765 766 hr_qp->umem = ib_umem_get(ib_pd->device, ucmd.buf_addr, 767 hr_qp->buff_size, 0); 768 if (IS_ERR(hr_qp->umem)) { 769 dev_err(dev, "ib_umem_get error for create qp\n"); 770 ret = PTR_ERR(hr_qp->umem); 771 goto err_alloc_rq_inline_buf; 772 } 773 hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp, 774 hr_qp->regions, ARRAY_SIZE(hr_qp->regions), 775 page_shift); 776 ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list, 777 hr_qp->region_cnt); 778 if (ret) { 779 dev_err(dev, "alloc buf_list error for create qp\n"); 780 goto err_alloc_list; 781 } 782 783 for (i = 0; i < hr_qp->region_cnt; i++) { 784 r = &hr_qp->regions[i]; 785 buf_count = hns_roce_get_umem_bufs(hr_dev, 786 buf_list[i], r->count, r->offset, 787 hr_qp->umem, page_shift); 788 if (buf_count != r->count) { 789 dev_err(dev, 790 "get umem buf err, expect %d,ret %d.\n", 791 r->count, buf_count); 792 ret = -ENOBUFS; 793 goto err_get_bufs; 794 } 795 } 796 797 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) && 798 (udata->inlen >= sizeof(ucmd)) && 799 (udata->outlen >= sizeof(resp)) && 800 hns_roce_qp_has_sq(init_attr)) { 801 ret = hns_roce_db_map_user(uctx, udata, ucmd.sdb_addr, 802 &hr_qp->sdb); 803 if (ret) { 804 dev_err(dev, "sq record doorbell map failed!\n"); 805 goto err_get_bufs; 806 } 807 808 /* indicate kernel supports sq record db */ 809 resp.cap_flags |= HNS_ROCE_SUPPORT_SQ_RECORD_DB; 810 hr_qp->sdb_en = 1; 811 } 812 813 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && 814 (udata->outlen >= sizeof(resp)) && 815 hns_roce_qp_has_rq(init_attr)) { 816 ret = hns_roce_db_map_user(uctx, udata, ucmd.db_addr, 817 &hr_qp->rdb); 818 if (ret) { 819 dev_err(dev, "rq record doorbell map failed!\n"); 820 goto err_sq_dbmap; 821 } 822 823 /* indicate kernel supports rq record db */ 824 resp.cap_flags |= HNS_ROCE_SUPPORT_RQ_RECORD_DB; 825 hr_qp->rdb_en = 1; 826 } 827 } else { 828 if (init_attr->create_flags & 829 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) { 830 dev_err(dev, "init_attr->create_flags error!\n"); 831 ret = -EINVAL; 832 goto err_alloc_rq_inline_buf; 833 } 834 835 if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) { 836 dev_err(dev, "init_attr->create_flags error!\n"); 837 ret = -EINVAL; 838 goto err_alloc_rq_inline_buf; 839 } 840 841 /* Set SQ size */ 842 ret = hns_roce_set_kernel_sq_size(hr_dev, &init_attr->cap, 843 hr_qp); 844 if (ret) { 845 dev_err(dev, "hns_roce_set_kernel_sq_size error!\n"); 846 goto err_alloc_rq_inline_buf; 847 } 848 849 /* QP doorbell register address */ 850 hr_qp->sq.db_reg_l = hr_dev->reg_base + hr_dev->sdb_offset + 851 DB_REG_OFFSET * hr_dev->priv_uar.index; 852 hr_qp->rq.db_reg_l = hr_dev->reg_base + hr_dev->odb_offset + 853 DB_REG_OFFSET * hr_dev->priv_uar.index; 854 855 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && 856 hns_roce_qp_has_rq(init_attr)) { 857 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0); 858 if (ret) { 859 dev_err(dev, "rq record doorbell alloc failed!\n"); 860 goto err_alloc_rq_inline_buf; 861 } 862 *hr_qp->rdb.db_record = 0; 863 hr_qp->rdb_en = 1; 864 } 865 866 /* Allocate QP buf */ 867 if (hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, 868 (1 << page_shift) * 2, 869 &hr_qp->hr_buf, page_shift)) { 870 dev_err(dev, "hns_roce_buf_alloc error!\n"); 871 ret = -ENOMEM; 872 goto err_db; 873 } 874 hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp, 875 hr_qp->regions, ARRAY_SIZE(hr_qp->regions), 876 page_shift); 877 ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list, 878 hr_qp->region_cnt); 879 if (ret) { 880 dev_err(dev, "alloc buf_list error for create qp!\n"); 881 goto err_alloc_list; 882 } 883 884 for (i = 0; i < hr_qp->region_cnt; i++) { 885 r = &hr_qp->regions[i]; 886 buf_count = hns_roce_get_kmem_bufs(hr_dev, 887 buf_list[i], r->count, r->offset, 888 &hr_qp->hr_buf); 889 if (buf_count != r->count) { 890 dev_err(dev, 891 "get kmem buf err, expect %d,ret %d.\n", 892 r->count, buf_count); 893 ret = -ENOBUFS; 894 goto err_get_bufs; 895 } 896 } 897 898 hr_qp->sq.wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), 899 GFP_KERNEL); 900 if (ZERO_OR_NULL_PTR(hr_qp->sq.wrid)) { 901 ret = -ENOMEM; 902 goto err_get_bufs; 903 } 904 905 if (hr_qp->rq.wqe_cnt) { 906 hr_qp->rq.wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), 907 GFP_KERNEL); 908 if (ZERO_OR_NULL_PTR(hr_qp->rq.wrid)) { 909 ret = -ENOMEM; 910 goto err_sq_wrid; 911 } 912 } 913 } 914 915 if (sqpn) { 916 qpn = sqpn; 917 } else { 918 /* Get QPN */ 919 ret = hns_roce_reserve_range_qp(hr_dev, 1, 1, &qpn); 920 if (ret) { 921 dev_err(dev, "hns_roce_reserve_range_qp alloc qpn error\n"); 922 goto err_wrid; 923 } 924 } 925 926 hr_qp->wqe_bt_pg_shift = calc_wqe_bt_page_shift(hr_dev, hr_qp->regions, 927 hr_qp->region_cnt); 928 hns_roce_mtr_init(&hr_qp->mtr, PAGE_SHIFT + hr_qp->wqe_bt_pg_shift, 929 page_shift); 930 ret = hns_roce_mtr_attach(hr_dev, &hr_qp->mtr, buf_list, 931 hr_qp->regions, hr_qp->region_cnt); 932 if (ret) { 933 dev_err(dev, "mtr attach error for create qp\n"); 934 goto err_mtr; 935 } 936 937 if (init_attr->qp_type == IB_QPT_GSI && 938 hr_dev->hw_rev == HNS_ROCE_HW_VER1) { 939 /* In v1 engine, GSI QP context in RoCE engine's register */ 940 ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp); 941 if (ret) { 942 dev_err(dev, "hns_roce_qp_alloc failed!\n"); 943 goto err_qpn; 944 } 945 } else { 946 ret = hns_roce_qp_alloc(hr_dev, qpn, hr_qp); 947 if (ret) { 948 dev_err(dev, "hns_roce_qp_alloc failed!\n"); 949 goto err_qpn; 950 } 951 } 952 953 if (sqpn) 954 hr_qp->doorbell_qpn = 1; 955 else 956 hr_qp->doorbell_qpn = (u32)hr_qp->qpn; 957 958 if (udata) { 959 ret = ib_copy_to_udata(udata, &resp, 960 min(udata->outlen, sizeof(resp))); 961 if (ret) 962 goto err_qp; 963 } 964 965 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 966 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp); 967 if (ret) 968 goto err_qp; 969 } 970 971 hr_qp->event = hns_roce_ib_qp_event; 972 973 add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq, init_attr->recv_cq); 974 975 hns_roce_free_buf_list(buf_list, hr_qp->region_cnt); 976 977 return 0; 978 979 err_qp: 980 if (init_attr->qp_type == IB_QPT_GSI && 981 hr_dev->hw_rev == HNS_ROCE_HW_VER1) 982 hns_roce_qp_remove(hr_dev, hr_qp); 983 else 984 hns_roce_qp_free(hr_dev, hr_qp); 985 986 err_qpn: 987 if (!sqpn) 988 hns_roce_release_range_qp(hr_dev, qpn, 1); 989 990 err_mtr: 991 hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr); 992 993 err_wrid: 994 if (udata) { 995 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && 996 (udata->outlen >= sizeof(resp)) && 997 hns_roce_qp_has_rq(init_attr)) 998 hns_roce_db_unmap_user(uctx, &hr_qp->rdb); 999 } else { 1000 if (hr_qp->rq.wqe_cnt) 1001 kfree(hr_qp->rq.wrid); 1002 } 1003 1004 err_sq_dbmap: 1005 if (udata) 1006 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) && 1007 (udata->inlen >= sizeof(ucmd)) && 1008 (udata->outlen >= sizeof(resp)) && 1009 hns_roce_qp_has_sq(init_attr)) 1010 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 1011 1012 err_sq_wrid: 1013 if (!udata) 1014 kfree(hr_qp->sq.wrid); 1015 1016 err_get_bufs: 1017 hns_roce_free_buf_list(buf_list, hr_qp->region_cnt); 1018 1019 err_alloc_list: 1020 if (!hr_qp->umem) 1021 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); 1022 ib_umem_release(hr_qp->umem); 1023 1024 err_db: 1025 if (!udata && hns_roce_qp_has_rq(init_attr) && 1026 (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB)) 1027 hns_roce_free_db(hr_dev, &hr_qp->rdb); 1028 1029 err_alloc_rq_inline_buf: 1030 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && 1031 hns_roce_qp_has_rq(init_attr)) 1032 free_rq_inline_buf(hr_qp); 1033 1034 err_out: 1035 return ret; 1036 } 1037 1038 struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, 1039 struct ib_qp_init_attr *init_attr, 1040 struct ib_udata *udata) 1041 { 1042 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); 1043 struct ib_device *ibdev = &hr_dev->ib_dev; 1044 struct hns_roce_qp *hr_qp; 1045 int ret; 1046 1047 switch (init_attr->qp_type) { 1048 case IB_QPT_RC: { 1049 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); 1050 if (!hr_qp) 1051 return ERR_PTR(-ENOMEM); 1052 1053 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, 0, 1054 hr_qp); 1055 if (ret) { 1056 ibdev_err(ibdev, "Create QP 0x%06lx failed(%d)\n", 1057 hr_qp->qpn, ret); 1058 kfree(hr_qp); 1059 return ERR_PTR(ret); 1060 } 1061 1062 hr_qp->ibqp.qp_num = hr_qp->qpn; 1063 1064 break; 1065 } 1066 case IB_QPT_GSI: { 1067 /* Userspace is not allowed to create special QPs: */ 1068 if (udata) { 1069 ibdev_err(ibdev, "not support usr space GSI\n"); 1070 return ERR_PTR(-EINVAL); 1071 } 1072 1073 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); 1074 if (!hr_qp) 1075 return ERR_PTR(-ENOMEM); 1076 1077 hr_qp->port = init_attr->port_num - 1; 1078 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; 1079 1080 /* when hw version is v1, the sqpn is allocated */ 1081 if (hr_dev->caps.max_sq_sg <= 2) 1082 hr_qp->ibqp.qp_num = HNS_ROCE_MAX_PORTS + 1083 hr_dev->iboe.phy_port[hr_qp->port]; 1084 else 1085 hr_qp->ibqp.qp_num = 1; 1086 1087 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, 1088 hr_qp->ibqp.qp_num, hr_qp); 1089 if (ret) { 1090 ibdev_err(ibdev, "Create GSI QP failed!\n"); 1091 kfree(hr_qp); 1092 return ERR_PTR(ret); 1093 } 1094 1095 break; 1096 } 1097 default:{ 1098 ibdev_err(ibdev, "not support QP type %d\n", 1099 init_attr->qp_type); 1100 return ERR_PTR(-EINVAL); 1101 } 1102 } 1103 1104 return &hr_qp->ibqp; 1105 } 1106 1107 int to_hr_qp_type(int qp_type) 1108 { 1109 int transport_type; 1110 1111 if (qp_type == IB_QPT_RC) 1112 transport_type = SERV_TYPE_RC; 1113 else if (qp_type == IB_QPT_UC) 1114 transport_type = SERV_TYPE_UC; 1115 else if (qp_type == IB_QPT_UD) 1116 transport_type = SERV_TYPE_UD; 1117 else if (qp_type == IB_QPT_GSI) 1118 transport_type = SERV_TYPE_UD; 1119 else 1120 transport_type = -1; 1121 1122 return transport_type; 1123 } 1124 1125 static int check_mtu_validate(struct hns_roce_dev *hr_dev, 1126 struct hns_roce_qp *hr_qp, 1127 struct ib_qp_attr *attr, int attr_mask) 1128 { 1129 enum ib_mtu active_mtu; 1130 int p; 1131 1132 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1133 active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu); 1134 1135 if ((hr_dev->caps.max_mtu >= IB_MTU_2048 && 1136 attr->path_mtu > hr_dev->caps.max_mtu) || 1137 attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) { 1138 ibdev_err(&hr_dev->ib_dev, 1139 "attr path_mtu(%d)invalid while modify qp", 1140 attr->path_mtu); 1141 return -EINVAL; 1142 } 1143 1144 return 0; 1145 } 1146 1147 static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1148 int attr_mask) 1149 { 1150 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1151 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1152 int p; 1153 1154 if ((attr_mask & IB_QP_PORT) && 1155 (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) { 1156 ibdev_err(&hr_dev->ib_dev, 1157 "attr port_num invalid.attr->port_num=%d\n", 1158 attr->port_num); 1159 return -EINVAL; 1160 } 1161 1162 if (attr_mask & IB_QP_PKEY_INDEX) { 1163 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1164 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) { 1165 ibdev_err(&hr_dev->ib_dev, 1166 "attr pkey_index invalid.attr->pkey_index=%d\n", 1167 attr->pkey_index); 1168 return -EINVAL; 1169 } 1170 } 1171 1172 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && 1173 attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) { 1174 ibdev_err(&hr_dev->ib_dev, 1175 "attr max_rd_atomic invalid.attr->max_rd_atomic=%d\n", 1176 attr->max_rd_atomic); 1177 return -EINVAL; 1178 } 1179 1180 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && 1181 attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) { 1182 ibdev_err(&hr_dev->ib_dev, 1183 "attr max_dest_rd_atomic invalid.attr->max_dest_rd_atomic=%d\n", 1184 attr->max_dest_rd_atomic); 1185 return -EINVAL; 1186 } 1187 1188 if (attr_mask & IB_QP_PATH_MTU) 1189 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask); 1190 1191 return 0; 1192 } 1193 1194 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1195 int attr_mask, struct ib_udata *udata) 1196 { 1197 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1198 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1199 enum ib_qp_state cur_state, new_state; 1200 int ret = -EINVAL; 1201 1202 mutex_lock(&hr_qp->mutex); 1203 1204 cur_state = attr_mask & IB_QP_CUR_STATE ? 1205 attr->cur_qp_state : (enum ib_qp_state)hr_qp->state; 1206 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; 1207 1208 if (ibqp->uobject && 1209 (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) { 1210 if (hr_qp->sdb_en == 1) { 1211 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr); 1212 1213 if (hr_qp->rdb_en == 1) 1214 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr); 1215 } else { 1216 ibdev_warn(&hr_dev->ib_dev, 1217 "flush cqe is not supported in userspace!\n"); 1218 goto out; 1219 } 1220 } 1221 1222 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, 1223 attr_mask)) { 1224 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n"); 1225 goto out; 1226 } 1227 1228 ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask); 1229 if (ret) 1230 goto out; 1231 1232 if (cur_state == new_state && cur_state == IB_QPS_RESET) { 1233 if (hr_dev->caps.min_wqes) { 1234 ret = -EPERM; 1235 ibdev_err(&hr_dev->ib_dev, 1236 "cur_state=%d new_state=%d\n", cur_state, 1237 new_state); 1238 } else { 1239 ret = 0; 1240 } 1241 1242 goto out; 1243 } 1244 1245 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, 1246 new_state); 1247 1248 out: 1249 mutex_unlock(&hr_qp->mutex); 1250 1251 return ret; 1252 } 1253 1254 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) 1255 __acquires(&send_cq->lock) __acquires(&recv_cq->lock) 1256 { 1257 if (unlikely(send_cq == NULL && recv_cq == NULL)) { 1258 __acquire(&send_cq->lock); 1259 __acquire(&recv_cq->lock); 1260 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { 1261 spin_lock_irq(&send_cq->lock); 1262 __acquire(&recv_cq->lock); 1263 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { 1264 spin_lock_irq(&recv_cq->lock); 1265 __acquire(&send_cq->lock); 1266 } else if (send_cq == recv_cq) { 1267 spin_lock_irq(&send_cq->lock); 1268 __acquire(&recv_cq->lock); 1269 } else if (send_cq->cqn < recv_cq->cqn) { 1270 spin_lock_irq(&send_cq->lock); 1271 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); 1272 } else { 1273 spin_lock_irq(&recv_cq->lock); 1274 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); 1275 } 1276 } 1277 1278 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, 1279 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock) 1280 __releases(&recv_cq->lock) 1281 { 1282 if (unlikely(send_cq == NULL && recv_cq == NULL)) { 1283 __release(&recv_cq->lock); 1284 __release(&send_cq->lock); 1285 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { 1286 __release(&recv_cq->lock); 1287 spin_unlock(&send_cq->lock); 1288 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { 1289 __release(&send_cq->lock); 1290 spin_unlock(&recv_cq->lock); 1291 } else if (send_cq == recv_cq) { 1292 __release(&recv_cq->lock); 1293 spin_unlock_irq(&send_cq->lock); 1294 } else if (send_cq->cqn < recv_cq->cqn) { 1295 spin_unlock(&recv_cq->lock); 1296 spin_unlock_irq(&send_cq->lock); 1297 } else { 1298 spin_unlock(&send_cq->lock); 1299 spin_unlock_irq(&recv_cq->lock); 1300 } 1301 } 1302 1303 static void *get_wqe(struct hns_roce_qp *hr_qp, int offset) 1304 { 1305 1306 return hns_roce_buf_offset(&hr_qp->hr_buf, offset); 1307 } 1308 1309 void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n) 1310 { 1311 return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift)); 1312 } 1313 1314 void *get_send_wqe(struct hns_roce_qp *hr_qp, int n) 1315 { 1316 return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift)); 1317 } 1318 1319 void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n) 1320 { 1321 return hns_roce_buf_offset(&hr_qp->hr_buf, hr_qp->sge.offset + 1322 (n << hr_qp->sge.sge_shift)); 1323 } 1324 1325 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq, 1326 struct ib_cq *ib_cq) 1327 { 1328 struct hns_roce_cq *hr_cq; 1329 u32 cur; 1330 1331 cur = hr_wq->head - hr_wq->tail; 1332 if (likely(cur + nreq < hr_wq->wqe_cnt)) 1333 return false; 1334 1335 hr_cq = to_hr_cq(ib_cq); 1336 spin_lock(&hr_cq->lock); 1337 cur = hr_wq->head - hr_wq->tail; 1338 spin_unlock(&hr_cq->lock); 1339 1340 return cur + nreq >= hr_wq->wqe_cnt; 1341 } 1342 1343 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) 1344 { 1345 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 1346 int reserved_from_top = 0; 1347 int reserved_from_bot; 1348 int ret; 1349 1350 mutex_init(&qp_table->scc_mutex); 1351 xa_init(&hr_dev->qp_table_xa); 1352 1353 reserved_from_bot = hr_dev->caps.reserved_qps; 1354 1355 ret = hns_roce_bitmap_init(&qp_table->bitmap, hr_dev->caps.num_qps, 1356 hr_dev->caps.num_qps - 1, reserved_from_bot, 1357 reserved_from_top); 1358 if (ret) { 1359 dev_err(hr_dev->dev, "qp bitmap init failed!error=%d\n", 1360 ret); 1361 return ret; 1362 } 1363 1364 return 0; 1365 } 1366 1367 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) 1368 { 1369 hns_roce_bitmap_cleanup(&hr_dev->qp_table.bitmap); 1370 } 1371