1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/pci.h> 35 #include <linux/platform_device.h> 36 #include <rdma/ib_addr.h> 37 #include <rdma/ib_umem.h> 38 #include <rdma/uverbs_ioctl.h> 39 #include "hns_roce_common.h" 40 #include "hns_roce_device.h" 41 #include "hns_roce_hem.h" 42 43 static void flush_work_handle(struct work_struct *work) 44 { 45 struct hns_roce_work *flush_work = container_of(work, 46 struct hns_roce_work, work); 47 struct hns_roce_qp *hr_qp = container_of(flush_work, 48 struct hns_roce_qp, flush_work); 49 struct device *dev = flush_work->hr_dev->dev; 50 struct ib_qp_attr attr; 51 int attr_mask; 52 int ret; 53 54 attr_mask = IB_QP_STATE; 55 attr.qp_state = IB_QPS_ERR; 56 57 if (test_and_clear_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag)) { 58 ret = hns_roce_modify_qp(&hr_qp->ibqp, &attr, attr_mask, NULL); 59 if (ret) 60 dev_err(dev, "Modify QP to error state failed(%d) during CQE flush\n", 61 ret); 62 } 63 64 /* 65 * make sure we signal QP destroy leg that flush QP was completed 66 * so that it can safely proceed ahead now and destroy QP 67 */ 68 if (atomic_dec_and_test(&hr_qp->refcount)) 69 complete(&hr_qp->free); 70 } 71 72 void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 73 { 74 struct hns_roce_work *flush_work = &hr_qp->flush_work; 75 76 flush_work->hr_dev = hr_dev; 77 INIT_WORK(&flush_work->work, flush_work_handle); 78 atomic_inc(&hr_qp->refcount); 79 queue_work(hr_dev->irq_workq, &flush_work->work); 80 } 81 82 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) 83 { 84 struct device *dev = hr_dev->dev; 85 struct hns_roce_qp *qp; 86 87 xa_lock(&hr_dev->qp_table_xa); 88 qp = __hns_roce_qp_lookup(hr_dev, qpn); 89 if (qp) 90 atomic_inc(&qp->refcount); 91 xa_unlock(&hr_dev->qp_table_xa); 92 93 if (!qp) { 94 dev_warn(dev, "Async event for bogus QP %08x\n", qpn); 95 return; 96 } 97 98 if (hr_dev->hw_rev != HNS_ROCE_HW_VER1 && 99 (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR || 100 event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR || 101 event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR)) { 102 qp->state = IB_QPS_ERR; 103 if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag)) 104 init_flush_work(hr_dev, qp); 105 } 106 107 qp->event(qp, (enum hns_roce_event)event_type); 108 109 if (atomic_dec_and_test(&qp->refcount)) 110 complete(&qp->free); 111 } 112 113 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, 114 enum hns_roce_event type) 115 { 116 struct ib_qp *ibqp = &hr_qp->ibqp; 117 struct ib_event event; 118 119 if (ibqp->event_handler) { 120 event.device = ibqp->device; 121 event.element.qp = ibqp; 122 switch (type) { 123 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 124 event.event = IB_EVENT_PATH_MIG; 125 break; 126 case HNS_ROCE_EVENT_TYPE_COMM_EST: 127 event.event = IB_EVENT_COMM_EST; 128 break; 129 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 130 event.event = IB_EVENT_SQ_DRAINED; 131 break; 132 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 133 event.event = IB_EVENT_QP_LAST_WQE_REACHED; 134 break; 135 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 136 event.event = IB_EVENT_QP_FATAL; 137 break; 138 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 139 event.event = IB_EVENT_PATH_MIG_ERR; 140 break; 141 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 142 event.event = IB_EVENT_QP_REQ_ERR; 143 break; 144 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 145 event.event = IB_EVENT_QP_ACCESS_ERR; 146 break; 147 default: 148 dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n", 149 type, hr_qp->qpn); 150 return; 151 } 152 ibqp->event_handler(&event, ibqp->qp_context); 153 } 154 } 155 156 static u8 get_least_load_bankid_for_qp(struct hns_roce_bank *bank) 157 { 158 u32 least_load = bank[0].inuse; 159 u8 bankid = 0; 160 u32 bankcnt; 161 u8 i; 162 163 for (i = 1; i < HNS_ROCE_QP_BANK_NUM; i++) { 164 bankcnt = bank[i].inuse; 165 if (bankcnt < least_load) { 166 least_load = bankcnt; 167 bankid = i; 168 } 169 } 170 171 return bankid; 172 } 173 174 static int alloc_qpn_with_bankid(struct hns_roce_bank *bank, u8 bankid, 175 unsigned long *qpn) 176 { 177 int id; 178 179 id = ida_alloc_range(&bank->ida, bank->next, bank->max, GFP_KERNEL); 180 if (id < 0) { 181 id = ida_alloc_range(&bank->ida, bank->min, bank->max, 182 GFP_KERNEL); 183 if (id < 0) 184 return id; 185 } 186 187 /* the QPN should keep increasing until the max value is reached. */ 188 bank->next = (id + 1) > bank->max ? bank->min : id + 1; 189 190 /* the lower 3 bits is bankid */ 191 *qpn = (id << 3) | bankid; 192 193 return 0; 194 } 195 static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 196 { 197 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 198 unsigned long num = 0; 199 u8 bankid; 200 int ret; 201 202 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) { 203 /* when hw version is v1, the sqpn is allocated */ 204 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) 205 num = HNS_ROCE_MAX_PORTS + 206 hr_dev->iboe.phy_port[hr_qp->port]; 207 else 208 num = 1; 209 210 hr_qp->doorbell_qpn = 1; 211 } else { 212 spin_lock(&qp_table->bank_lock); 213 bankid = get_least_load_bankid_for_qp(qp_table->bank); 214 215 ret = alloc_qpn_with_bankid(&qp_table->bank[bankid], bankid, 216 &num); 217 if (ret) { 218 ibdev_err(&hr_dev->ib_dev, 219 "failed to alloc QPN, ret = %d\n", ret); 220 spin_unlock(&qp_table->bank_lock); 221 return ret; 222 } 223 224 qp_table->bank[bankid].inuse++; 225 spin_unlock(&qp_table->bank_lock); 226 227 hr_qp->doorbell_qpn = (u32)num; 228 } 229 230 hr_qp->qpn = num; 231 232 return 0; 233 } 234 235 enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state) 236 { 237 switch (state) { 238 case IB_QPS_RESET: 239 return HNS_ROCE_QP_STATE_RST; 240 case IB_QPS_INIT: 241 return HNS_ROCE_QP_STATE_INIT; 242 case IB_QPS_RTR: 243 return HNS_ROCE_QP_STATE_RTR; 244 case IB_QPS_RTS: 245 return HNS_ROCE_QP_STATE_RTS; 246 case IB_QPS_SQD: 247 return HNS_ROCE_QP_STATE_SQD; 248 case IB_QPS_ERR: 249 return HNS_ROCE_QP_STATE_ERR; 250 default: 251 return HNS_ROCE_QP_NUM_STATE; 252 } 253 } 254 255 static void add_qp_to_list(struct hns_roce_dev *hr_dev, 256 struct hns_roce_qp *hr_qp, 257 struct ib_cq *send_cq, struct ib_cq *recv_cq) 258 { 259 struct hns_roce_cq *hr_send_cq, *hr_recv_cq; 260 unsigned long flags; 261 262 hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL; 263 hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL; 264 265 spin_lock_irqsave(&hr_dev->qp_list_lock, flags); 266 hns_roce_lock_cqs(hr_send_cq, hr_recv_cq); 267 268 list_add_tail(&hr_qp->node, &hr_dev->qp_list); 269 if (hr_send_cq) 270 list_add_tail(&hr_qp->sq_node, &hr_send_cq->sq_list); 271 if (hr_recv_cq) 272 list_add_tail(&hr_qp->rq_node, &hr_recv_cq->rq_list); 273 274 hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq); 275 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); 276 } 277 278 static int hns_roce_qp_store(struct hns_roce_dev *hr_dev, 279 struct hns_roce_qp *hr_qp, 280 struct ib_qp_init_attr *init_attr) 281 { 282 struct xarray *xa = &hr_dev->qp_table_xa; 283 int ret; 284 285 if (!hr_qp->qpn) 286 return -EINVAL; 287 288 ret = xa_err(xa_store_irq(xa, hr_qp->qpn, hr_qp, GFP_KERNEL)); 289 if (ret) 290 dev_err(hr_dev->dev, "Failed to xa store for QPC\n"); 291 else 292 /* add QP to device's QP list for softwc */ 293 add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq, 294 init_attr->recv_cq); 295 296 return ret; 297 } 298 299 static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 300 { 301 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 302 struct device *dev = hr_dev->dev; 303 int ret; 304 305 if (!hr_qp->qpn) 306 return -EINVAL; 307 308 /* In v1 engine, GSI QP context is saved in the RoCE hw's register */ 309 if (hr_qp->ibqp.qp_type == IB_QPT_GSI && 310 hr_dev->hw_rev == HNS_ROCE_HW_VER1) 311 return 0; 312 313 /* Alloc memory for QPC */ 314 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn); 315 if (ret) { 316 dev_err(dev, "Failed to get QPC table\n"); 317 goto err_out; 318 } 319 320 /* Alloc memory for IRRL */ 321 ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 322 if (ret) { 323 dev_err(dev, "Failed to get IRRL table\n"); 324 goto err_put_qp; 325 } 326 327 if (hr_dev->caps.trrl_entry_sz) { 328 /* Alloc memory for TRRL */ 329 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table, 330 hr_qp->qpn); 331 if (ret) { 332 dev_err(dev, "Failed to get TRRL table\n"); 333 goto err_put_irrl; 334 } 335 } 336 337 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 338 /* Alloc memory for SCC CTX */ 339 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table, 340 hr_qp->qpn); 341 if (ret) { 342 dev_err(dev, "Failed to get SCC CTX table\n"); 343 goto err_put_trrl; 344 } 345 } 346 347 return 0; 348 349 err_put_trrl: 350 if (hr_dev->caps.trrl_entry_sz) 351 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 352 353 err_put_irrl: 354 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 355 356 err_put_qp: 357 hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); 358 359 err_out: 360 return ret; 361 } 362 363 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 364 { 365 struct xarray *xa = &hr_dev->qp_table_xa; 366 unsigned long flags; 367 368 list_del(&hr_qp->node); 369 list_del(&hr_qp->sq_node); 370 list_del(&hr_qp->rq_node); 371 372 xa_lock_irqsave(xa, flags); 373 __xa_erase(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1)); 374 xa_unlock_irqrestore(xa, flags); 375 } 376 377 static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 378 { 379 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 380 381 /* In v1 engine, GSI QP context is saved in the RoCE hw's register */ 382 if (hr_qp->ibqp.qp_type == IB_QPT_GSI && 383 hr_dev->hw_rev == HNS_ROCE_HW_VER1) 384 return; 385 386 if (hr_dev->caps.trrl_entry_sz) 387 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 388 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 389 } 390 391 static inline u8 get_qp_bankid(unsigned long qpn) 392 { 393 /* The lower 3 bits of QPN are used to hash to different banks */ 394 return (u8)(qpn & GENMASK(2, 0)); 395 } 396 397 static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 398 { 399 u8 bankid; 400 401 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) 402 return; 403 404 if (hr_qp->qpn < hr_dev->caps.reserved_qps) 405 return; 406 407 bankid = get_qp_bankid(hr_qp->qpn); 408 409 ida_free(&hr_dev->qp_table.bank[bankid].ida, hr_qp->qpn >> 3); 410 411 spin_lock(&hr_dev->qp_table.bank_lock); 412 hr_dev->qp_table.bank[bankid].inuse--; 413 spin_unlock(&hr_dev->qp_table.bank_lock); 414 } 415 416 static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, 417 struct hns_roce_qp *hr_qp, int has_rq) 418 { 419 u32 cnt; 420 421 /* If srq exist, set zero for relative number of rq */ 422 if (!has_rq) { 423 hr_qp->rq.wqe_cnt = 0; 424 hr_qp->rq.max_gs = 0; 425 hr_qp->rq_inl_buf.wqe_cnt = 0; 426 cap->max_recv_wr = 0; 427 cap->max_recv_sge = 0; 428 429 return 0; 430 } 431 432 /* Check the validity of QP support capacity */ 433 if (!cap->max_recv_wr || cap->max_recv_wr > hr_dev->caps.max_wqes || 434 cap->max_recv_sge > hr_dev->caps.max_rq_sg) { 435 ibdev_err(&hr_dev->ib_dev, "RQ config error, depth=%u, sge=%d\n", 436 cap->max_recv_wr, cap->max_recv_sge); 437 return -EINVAL; 438 } 439 440 cnt = roundup_pow_of_two(max(cap->max_recv_wr, hr_dev->caps.min_wqes)); 441 if (cnt > hr_dev->caps.max_wqes) { 442 ibdev_err(&hr_dev->ib_dev, "rq depth %u too large\n", 443 cap->max_recv_wr); 444 return -EINVAL; 445 } 446 447 hr_qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge)); 448 449 if (hr_dev->caps.max_rq_sg <= HNS_ROCE_SGE_IN_WQE) 450 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz); 451 else 452 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz * 453 hr_qp->rq.max_gs); 454 455 hr_qp->rq.wqe_cnt = cnt; 456 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) 457 hr_qp->rq_inl_buf.wqe_cnt = cnt; 458 else 459 hr_qp->rq_inl_buf.wqe_cnt = 0; 460 461 cap->max_recv_wr = cnt; 462 cap->max_recv_sge = hr_qp->rq.max_gs; 463 464 return 0; 465 } 466 467 static u32 get_wqe_ext_sge_cnt(struct hns_roce_qp *qp) 468 { 469 /* GSI/UD QP only has extended sge */ 470 if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) 471 return qp->sq.max_gs; 472 473 if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE) 474 return qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE; 475 476 return 0; 477 } 478 479 static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt, 480 struct hns_roce_qp *hr_qp, struct ib_qp_cap *cap) 481 { 482 u32 total_sge_cnt; 483 u32 wqe_sge_cnt; 484 485 hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT; 486 487 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) { 488 hr_qp->sq.max_gs = HNS_ROCE_SGE_IN_WQE; 489 return; 490 } 491 492 hr_qp->sq.max_gs = max(1U, cap->max_send_sge); 493 494 wqe_sge_cnt = get_wqe_ext_sge_cnt(hr_qp); 495 496 /* If the number of extended sge is not zero, they MUST use the 497 * space of HNS_HW_PAGE_SIZE at least. 498 */ 499 if (wqe_sge_cnt) { 500 total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt); 501 hr_qp->sge.sge_cnt = max(total_sge_cnt, 502 (u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE); 503 } 504 } 505 506 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev, 507 struct ib_qp_cap *cap, 508 struct hns_roce_ib_create_qp *ucmd) 509 { 510 u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz); 511 u8 max_sq_stride = ilog2(roundup_sq_stride); 512 513 /* Sanity check SQ size before proceeding */ 514 if (ucmd->log_sq_stride > max_sq_stride || 515 ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) { 516 ibdev_err(&hr_dev->ib_dev, "failed to check SQ stride size.\n"); 517 return -EINVAL; 518 } 519 520 if (cap->max_send_sge > hr_dev->caps.max_sq_sg) { 521 ibdev_err(&hr_dev->ib_dev, "failed to check SQ SGE size %u.\n", 522 cap->max_send_sge); 523 return -EINVAL; 524 } 525 526 return 0; 527 } 528 529 static int set_user_sq_size(struct hns_roce_dev *hr_dev, 530 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp, 531 struct hns_roce_ib_create_qp *ucmd) 532 { 533 struct ib_device *ibdev = &hr_dev->ib_dev; 534 u32 cnt = 0; 535 int ret; 536 537 if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) || 538 cnt > hr_dev->caps.max_wqes) 539 return -EINVAL; 540 541 ret = check_sq_size_with_integrity(hr_dev, cap, ucmd); 542 if (ret) { 543 ibdev_err(ibdev, "failed to check user SQ size, ret = %d.\n", 544 ret); 545 return ret; 546 } 547 548 set_ext_sge_param(hr_dev, cnt, hr_qp, cap); 549 550 hr_qp->sq.wqe_shift = ucmd->log_sq_stride; 551 hr_qp->sq.wqe_cnt = cnt; 552 553 return 0; 554 } 555 556 static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev, 557 struct hns_roce_qp *hr_qp, 558 struct hns_roce_buf_attr *buf_attr) 559 { 560 int buf_size; 561 int idx = 0; 562 563 hr_qp->buff_size = 0; 564 565 /* SQ WQE */ 566 hr_qp->sq.offset = 0; 567 buf_size = to_hr_hem_entries_size(hr_qp->sq.wqe_cnt, 568 hr_qp->sq.wqe_shift); 569 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 570 buf_attr->region[idx].size = buf_size; 571 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sq_hop_num; 572 idx++; 573 hr_qp->buff_size += buf_size; 574 } 575 576 /* extend SGE WQE in SQ */ 577 hr_qp->sge.offset = hr_qp->buff_size; 578 buf_size = to_hr_hem_entries_size(hr_qp->sge.sge_cnt, 579 hr_qp->sge.sge_shift); 580 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 581 buf_attr->region[idx].size = buf_size; 582 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sge_hop_num; 583 idx++; 584 hr_qp->buff_size += buf_size; 585 } 586 587 /* RQ WQE */ 588 hr_qp->rq.offset = hr_qp->buff_size; 589 buf_size = to_hr_hem_entries_size(hr_qp->rq.wqe_cnt, 590 hr_qp->rq.wqe_shift); 591 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 592 buf_attr->region[idx].size = buf_size; 593 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_rq_hop_num; 594 idx++; 595 hr_qp->buff_size += buf_size; 596 } 597 598 if (hr_qp->buff_size < 1) 599 return -EINVAL; 600 601 buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; 602 buf_attr->fixed_page = true; 603 buf_attr->region_count = idx; 604 605 return 0; 606 } 607 608 static int set_kernel_sq_size(struct hns_roce_dev *hr_dev, 609 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp) 610 { 611 struct ib_device *ibdev = &hr_dev->ib_dev; 612 u32 cnt; 613 614 if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes || 615 cap->max_send_sge > hr_dev->caps.max_sq_sg) { 616 ibdev_err(ibdev, 617 "failed to check SQ WR or SGE num, ret = %d.\n", 618 -EINVAL); 619 return -EINVAL; 620 } 621 622 cnt = roundup_pow_of_two(max(cap->max_send_wr, hr_dev->caps.min_wqes)); 623 if (cnt > hr_dev->caps.max_wqes) { 624 ibdev_err(ibdev, "failed to check WQE num, WQE num = %u.\n", 625 cnt); 626 return -EINVAL; 627 } 628 629 hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz); 630 hr_qp->sq.wqe_cnt = cnt; 631 632 set_ext_sge_param(hr_dev, cnt, hr_qp, cap); 633 634 /* sync the parameters of kernel QP to user's configuration */ 635 cap->max_send_wr = cnt; 636 cap->max_send_sge = hr_qp->sq.max_gs; 637 638 return 0; 639 } 640 641 static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr) 642 { 643 if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr) 644 return 0; 645 646 return 1; 647 } 648 649 static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr) 650 { 651 if (attr->qp_type == IB_QPT_XRC_INI || 652 attr->qp_type == IB_QPT_XRC_TGT || attr->srq || 653 !attr->cap.max_recv_wr) 654 return 0; 655 656 return 1; 657 } 658 659 static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp, 660 struct ib_qp_init_attr *init_attr) 661 { 662 u32 max_recv_sge = init_attr->cap.max_recv_sge; 663 u32 wqe_cnt = hr_qp->rq_inl_buf.wqe_cnt; 664 struct hns_roce_rinl_wqe *wqe_list; 665 int i; 666 667 /* allocate recv inline buf */ 668 wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe), 669 GFP_KERNEL); 670 671 if (!wqe_list) 672 goto err; 673 674 /* Allocate a continuous buffer for all inline sge we need */ 675 wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge * 676 sizeof(struct hns_roce_rinl_sge)), 677 GFP_KERNEL); 678 if (!wqe_list[0].sg_list) 679 goto err_wqe_list; 680 681 /* Assign buffers of sg_list to each inline wqe */ 682 for (i = 1; i < wqe_cnt; i++) 683 wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge]; 684 685 hr_qp->rq_inl_buf.wqe_list = wqe_list; 686 687 return 0; 688 689 err_wqe_list: 690 kfree(wqe_list); 691 692 err: 693 return -ENOMEM; 694 } 695 696 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp) 697 { 698 if (hr_qp->rq_inl_buf.wqe_list) 699 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list); 700 kfree(hr_qp->rq_inl_buf.wqe_list); 701 } 702 703 static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 704 struct ib_qp_init_attr *init_attr, 705 struct ib_udata *udata, unsigned long addr) 706 { 707 struct ib_device *ibdev = &hr_dev->ib_dev; 708 struct hns_roce_buf_attr buf_attr = {}; 709 int ret; 710 711 if (!udata && hr_qp->rq_inl_buf.wqe_cnt) { 712 ret = alloc_rq_inline_buf(hr_qp, init_attr); 713 if (ret) { 714 ibdev_err(ibdev, 715 "failed to alloc inline buf, ret = %d.\n", 716 ret); 717 return ret; 718 } 719 } else { 720 hr_qp->rq_inl_buf.wqe_list = NULL; 721 } 722 723 ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr); 724 if (ret) { 725 ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret); 726 goto err_inline; 727 } 728 ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr, 729 HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz, 730 udata, addr); 731 if (ret) { 732 ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret); 733 goto err_inline; 734 } 735 736 return 0; 737 err_inline: 738 free_rq_inline_buf(hr_qp); 739 740 return ret; 741 } 742 743 static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 744 { 745 hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr); 746 free_rq_inline_buf(hr_qp); 747 } 748 749 static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev, 750 struct ib_qp_init_attr *init_attr, 751 struct ib_udata *udata, 752 struct hns_roce_ib_create_qp_resp *resp, 753 struct hns_roce_ib_create_qp *ucmd) 754 { 755 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) && 756 udata->outlen >= offsetofend(typeof(*resp), cap_flags) && 757 hns_roce_qp_has_sq(init_attr) && 758 udata->inlen >= offsetofend(typeof(*ucmd), sdb_addr)); 759 } 760 761 static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev, 762 struct ib_qp_init_attr *init_attr, 763 struct ib_udata *udata, 764 struct hns_roce_ib_create_qp_resp *resp) 765 { 766 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && 767 udata->outlen >= offsetofend(typeof(*resp), cap_flags) && 768 hns_roce_qp_has_rq(init_attr)); 769 } 770 771 static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev, 772 struct ib_qp_init_attr *init_attr) 773 { 774 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && 775 hns_roce_qp_has_rq(init_attr)); 776 } 777 778 static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 779 struct ib_qp_init_attr *init_attr, 780 struct ib_udata *udata, 781 struct hns_roce_ib_create_qp *ucmd, 782 struct hns_roce_ib_create_qp_resp *resp) 783 { 784 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( 785 udata, struct hns_roce_ucontext, ibucontext); 786 struct ib_device *ibdev = &hr_dev->ib_dev; 787 int ret; 788 789 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SDI_MODE) 790 hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB; 791 792 if (udata) { 793 if (user_qp_has_sdb(hr_dev, init_attr, udata, resp, ucmd)) { 794 ret = hns_roce_db_map_user(uctx, udata, ucmd->sdb_addr, 795 &hr_qp->sdb); 796 if (ret) { 797 ibdev_err(ibdev, 798 "failed to map user SQ doorbell, ret = %d.\n", 799 ret); 800 goto err_out; 801 } 802 hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB; 803 resp->cap_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB; 804 } 805 806 if (user_qp_has_rdb(hr_dev, init_attr, udata, resp)) { 807 ret = hns_roce_db_map_user(uctx, udata, ucmd->db_addr, 808 &hr_qp->rdb); 809 if (ret) { 810 ibdev_err(ibdev, 811 "failed to map user RQ doorbell, ret = %d.\n", 812 ret); 813 goto err_sdb; 814 } 815 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; 816 resp->cap_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; 817 } 818 } else { 819 /* QP doorbell register address */ 820 hr_qp->sq.db_reg_l = hr_dev->reg_base + hr_dev->sdb_offset + 821 DB_REG_OFFSET * hr_dev->priv_uar.index; 822 hr_qp->rq.db_reg_l = hr_dev->reg_base + hr_dev->odb_offset + 823 DB_REG_OFFSET * hr_dev->priv_uar.index; 824 825 if (kernel_qp_has_rdb(hr_dev, init_attr)) { 826 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0); 827 if (ret) { 828 ibdev_err(ibdev, 829 "failed to alloc kernel RQ doorbell, ret = %d.\n", 830 ret); 831 goto err_out; 832 } 833 *hr_qp->rdb.db_record = 0; 834 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; 835 } 836 } 837 838 return 0; 839 err_sdb: 840 if (udata && hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) 841 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 842 err_out: 843 return ret; 844 } 845 846 static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 847 struct ib_udata *udata) 848 { 849 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( 850 udata, struct hns_roce_ucontext, ibucontext); 851 852 if (udata) { 853 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 854 hns_roce_db_unmap_user(uctx, &hr_qp->rdb); 855 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) 856 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 857 } else { 858 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 859 hns_roce_free_db(hr_dev, &hr_qp->rdb); 860 } 861 } 862 863 static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev, 864 struct hns_roce_qp *hr_qp) 865 { 866 struct ib_device *ibdev = &hr_dev->ib_dev; 867 u64 *sq_wrid = NULL; 868 u64 *rq_wrid = NULL; 869 int ret; 870 871 sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL); 872 if (ZERO_OR_NULL_PTR(sq_wrid)) { 873 ibdev_err(ibdev, "failed to alloc SQ wrid.\n"); 874 return -ENOMEM; 875 } 876 877 if (hr_qp->rq.wqe_cnt) { 878 rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL); 879 if (ZERO_OR_NULL_PTR(rq_wrid)) { 880 ibdev_err(ibdev, "failed to alloc RQ wrid.\n"); 881 ret = -ENOMEM; 882 goto err_sq; 883 } 884 } 885 886 hr_qp->sq.wrid = sq_wrid; 887 hr_qp->rq.wrid = rq_wrid; 888 return 0; 889 err_sq: 890 kfree(sq_wrid); 891 892 return ret; 893 } 894 895 static void free_kernel_wrid(struct hns_roce_qp *hr_qp) 896 { 897 kfree(hr_qp->rq.wrid); 898 kfree(hr_qp->sq.wrid); 899 } 900 901 static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 902 struct ib_qp_init_attr *init_attr, 903 struct ib_udata *udata, 904 struct hns_roce_ib_create_qp *ucmd) 905 { 906 struct ib_device *ibdev = &hr_dev->ib_dev; 907 int ret; 908 909 hr_qp->ibqp.qp_type = init_attr->qp_type; 910 911 if (init_attr->cap.max_inline_data > hr_dev->caps.max_sq_inline) 912 init_attr->cap.max_inline_data = hr_dev->caps.max_sq_inline; 913 914 hr_qp->max_inline_data = init_attr->cap.max_inline_data; 915 916 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) 917 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR; 918 else 919 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR; 920 921 ret = set_rq_size(hr_dev, &init_attr->cap, hr_qp, 922 hns_roce_qp_has_rq(init_attr)); 923 if (ret) { 924 ibdev_err(ibdev, "failed to set user RQ size, ret = %d.\n", 925 ret); 926 return ret; 927 } 928 929 if (udata) { 930 ret = ib_copy_from_udata(ucmd, udata, 931 min(udata->inlen, sizeof(*ucmd))); 932 if (ret) { 933 ibdev_err(ibdev, 934 "failed to copy QP ucmd, ret = %d\n", ret); 935 return ret; 936 } 937 938 ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd); 939 if (ret) 940 ibdev_err(ibdev, 941 "failed to set user SQ size, ret = %d.\n", 942 ret); 943 } else { 944 ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp); 945 if (ret) 946 ibdev_err(ibdev, 947 "failed to set kernel SQ size, ret = %d.\n", 948 ret); 949 } 950 951 return ret; 952 } 953 954 static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, 955 struct ib_pd *ib_pd, 956 struct ib_qp_init_attr *init_attr, 957 struct ib_udata *udata, 958 struct hns_roce_qp *hr_qp) 959 { 960 struct hns_roce_ib_create_qp_resp resp = {}; 961 struct ib_device *ibdev = &hr_dev->ib_dev; 962 struct hns_roce_ib_create_qp ucmd; 963 int ret; 964 965 mutex_init(&hr_qp->mutex); 966 spin_lock_init(&hr_qp->sq.lock); 967 spin_lock_init(&hr_qp->rq.lock); 968 969 hr_qp->state = IB_QPS_RESET; 970 hr_qp->flush_flag = 0; 971 972 if (init_attr->create_flags) 973 return -EOPNOTSUPP; 974 975 ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd); 976 if (ret) { 977 ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret); 978 return ret; 979 } 980 981 if (!udata) { 982 ret = alloc_kernel_wrid(hr_dev, hr_qp); 983 if (ret) { 984 ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n", 985 ret); 986 return ret; 987 } 988 } 989 990 ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp); 991 if (ret) { 992 ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n", 993 ret); 994 goto err_wrid; 995 } 996 997 ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr); 998 if (ret) { 999 ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret); 1000 goto err_db; 1001 } 1002 1003 ret = alloc_qpn(hr_dev, hr_qp); 1004 if (ret) { 1005 ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret); 1006 goto err_buf; 1007 } 1008 1009 ret = alloc_qpc(hr_dev, hr_qp); 1010 if (ret) { 1011 ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n", 1012 ret); 1013 goto err_qpn; 1014 } 1015 1016 ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr); 1017 if (ret) { 1018 ibdev_err(ibdev, "failed to store QP, ret = %d.\n", ret); 1019 goto err_qpc; 1020 } 1021 1022 if (udata) { 1023 ret = ib_copy_to_udata(udata, &resp, 1024 min(udata->outlen, sizeof(resp))); 1025 if (ret) { 1026 ibdev_err(ibdev, "copy qp resp failed!\n"); 1027 goto err_store; 1028 } 1029 } 1030 1031 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 1032 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp); 1033 if (ret) 1034 goto err_store; 1035 } 1036 1037 hr_qp->ibqp.qp_num = hr_qp->qpn; 1038 hr_qp->event = hns_roce_ib_qp_event; 1039 atomic_set(&hr_qp->refcount, 1); 1040 init_completion(&hr_qp->free); 1041 1042 return 0; 1043 1044 err_store: 1045 hns_roce_qp_remove(hr_dev, hr_qp); 1046 err_qpc: 1047 free_qpc(hr_dev, hr_qp); 1048 err_qpn: 1049 free_qpn(hr_dev, hr_qp); 1050 err_buf: 1051 free_qp_buf(hr_dev, hr_qp); 1052 err_db: 1053 free_qp_db(hr_dev, hr_qp, udata); 1054 err_wrid: 1055 free_kernel_wrid(hr_qp); 1056 return ret; 1057 } 1058 1059 void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 1060 struct ib_udata *udata) 1061 { 1062 if (atomic_dec_and_test(&hr_qp->refcount)) 1063 complete(&hr_qp->free); 1064 wait_for_completion(&hr_qp->free); 1065 1066 free_qpc(hr_dev, hr_qp); 1067 free_qpn(hr_dev, hr_qp); 1068 free_qp_buf(hr_dev, hr_qp); 1069 free_kernel_wrid(hr_qp); 1070 free_qp_db(hr_dev, hr_qp, udata); 1071 1072 kfree(hr_qp); 1073 } 1074 1075 static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type, 1076 bool is_user) 1077 { 1078 switch (type) { 1079 case IB_QPT_UD: 1080 if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 && 1081 is_user) 1082 goto out; 1083 fallthrough; 1084 case IB_QPT_RC: 1085 case IB_QPT_GSI: 1086 break; 1087 default: 1088 goto out; 1089 } 1090 1091 return 0; 1092 1093 out: 1094 ibdev_err(&hr_dev->ib_dev, "not support QP type %d\n", type); 1095 1096 return -EOPNOTSUPP; 1097 } 1098 1099 struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, 1100 struct ib_qp_init_attr *init_attr, 1101 struct ib_udata *udata) 1102 { 1103 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); 1104 struct ib_device *ibdev = &hr_dev->ib_dev; 1105 struct hns_roce_qp *hr_qp; 1106 int ret; 1107 1108 ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata); 1109 if (ret) 1110 return ERR_PTR(ret); 1111 1112 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); 1113 if (!hr_qp) 1114 return ERR_PTR(-ENOMEM); 1115 1116 if (init_attr->qp_type == IB_QPT_GSI) { 1117 hr_qp->port = init_attr->port_num - 1; 1118 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; 1119 } 1120 1121 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp); 1122 if (ret) { 1123 ibdev_err(ibdev, "Create QP type 0x%x failed(%d)\n", 1124 init_attr->qp_type, ret); 1125 1126 kfree(hr_qp); 1127 return ERR_PTR(ret); 1128 } 1129 1130 return &hr_qp->ibqp; 1131 } 1132 1133 int to_hr_qp_type(int qp_type) 1134 { 1135 int transport_type; 1136 1137 if (qp_type == IB_QPT_RC) 1138 transport_type = SERV_TYPE_RC; 1139 else if (qp_type == IB_QPT_UC) 1140 transport_type = SERV_TYPE_UC; 1141 else if (qp_type == IB_QPT_UD) 1142 transport_type = SERV_TYPE_UD; 1143 else if (qp_type == IB_QPT_GSI) 1144 transport_type = SERV_TYPE_UD; 1145 else 1146 transport_type = -1; 1147 1148 return transport_type; 1149 } 1150 1151 static int check_mtu_validate(struct hns_roce_dev *hr_dev, 1152 struct hns_roce_qp *hr_qp, 1153 struct ib_qp_attr *attr, int attr_mask) 1154 { 1155 enum ib_mtu active_mtu; 1156 int p; 1157 1158 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1159 active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu); 1160 1161 if ((hr_dev->caps.max_mtu >= IB_MTU_2048 && 1162 attr->path_mtu > hr_dev->caps.max_mtu) || 1163 attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) { 1164 ibdev_err(&hr_dev->ib_dev, 1165 "attr path_mtu(%d)invalid while modify qp", 1166 attr->path_mtu); 1167 return -EINVAL; 1168 } 1169 1170 return 0; 1171 } 1172 1173 static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1174 int attr_mask) 1175 { 1176 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1177 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1178 int p; 1179 1180 if ((attr_mask & IB_QP_PORT) && 1181 (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) { 1182 ibdev_err(&hr_dev->ib_dev, "invalid attr, port_num = %u.\n", 1183 attr->port_num); 1184 return -EINVAL; 1185 } 1186 1187 if (attr_mask & IB_QP_PKEY_INDEX) { 1188 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1189 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) { 1190 ibdev_err(&hr_dev->ib_dev, 1191 "invalid attr, pkey_index = %u.\n", 1192 attr->pkey_index); 1193 return -EINVAL; 1194 } 1195 } 1196 1197 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && 1198 attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) { 1199 ibdev_err(&hr_dev->ib_dev, 1200 "invalid attr, max_rd_atomic = %u.\n", 1201 attr->max_rd_atomic); 1202 return -EINVAL; 1203 } 1204 1205 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && 1206 attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) { 1207 ibdev_err(&hr_dev->ib_dev, 1208 "invalid attr, max_dest_rd_atomic = %u.\n", 1209 attr->max_dest_rd_atomic); 1210 return -EINVAL; 1211 } 1212 1213 if (attr_mask & IB_QP_PATH_MTU) 1214 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask); 1215 1216 return 0; 1217 } 1218 1219 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1220 int attr_mask, struct ib_udata *udata) 1221 { 1222 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1223 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1224 enum ib_qp_state cur_state, new_state; 1225 int ret = -EINVAL; 1226 1227 mutex_lock(&hr_qp->mutex); 1228 1229 if (attr_mask & IB_QP_CUR_STATE && attr->cur_qp_state != hr_qp->state) 1230 goto out; 1231 1232 cur_state = hr_qp->state; 1233 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; 1234 1235 if (ibqp->uobject && 1236 (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) { 1237 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) { 1238 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr); 1239 1240 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 1241 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr); 1242 } else { 1243 ibdev_warn(&hr_dev->ib_dev, 1244 "flush cqe is not supported in userspace!\n"); 1245 goto out; 1246 } 1247 } 1248 1249 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, 1250 attr_mask)) { 1251 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n"); 1252 goto out; 1253 } 1254 1255 ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask); 1256 if (ret) 1257 goto out; 1258 1259 if (cur_state == new_state && cur_state == IB_QPS_RESET) { 1260 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) { 1261 ret = -EPERM; 1262 ibdev_err(&hr_dev->ib_dev, 1263 "RST2RST state is not supported\n"); 1264 } else { 1265 ret = 0; 1266 } 1267 1268 goto out; 1269 } 1270 1271 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, 1272 new_state); 1273 1274 out: 1275 mutex_unlock(&hr_qp->mutex); 1276 1277 return ret; 1278 } 1279 1280 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) 1281 __acquires(&send_cq->lock) __acquires(&recv_cq->lock) 1282 { 1283 if (unlikely(send_cq == NULL && recv_cq == NULL)) { 1284 __acquire(&send_cq->lock); 1285 __acquire(&recv_cq->lock); 1286 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { 1287 spin_lock_irq(&send_cq->lock); 1288 __acquire(&recv_cq->lock); 1289 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { 1290 spin_lock_irq(&recv_cq->lock); 1291 __acquire(&send_cq->lock); 1292 } else if (send_cq == recv_cq) { 1293 spin_lock_irq(&send_cq->lock); 1294 __acquire(&recv_cq->lock); 1295 } else if (send_cq->cqn < recv_cq->cqn) { 1296 spin_lock_irq(&send_cq->lock); 1297 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); 1298 } else { 1299 spin_lock_irq(&recv_cq->lock); 1300 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); 1301 } 1302 } 1303 1304 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, 1305 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock) 1306 __releases(&recv_cq->lock) 1307 { 1308 if (unlikely(send_cq == NULL && recv_cq == NULL)) { 1309 __release(&recv_cq->lock); 1310 __release(&send_cq->lock); 1311 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { 1312 __release(&recv_cq->lock); 1313 spin_unlock(&send_cq->lock); 1314 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { 1315 __release(&send_cq->lock); 1316 spin_unlock(&recv_cq->lock); 1317 } else if (send_cq == recv_cq) { 1318 __release(&recv_cq->lock); 1319 spin_unlock_irq(&send_cq->lock); 1320 } else if (send_cq->cqn < recv_cq->cqn) { 1321 spin_unlock(&recv_cq->lock); 1322 spin_unlock_irq(&send_cq->lock); 1323 } else { 1324 spin_unlock(&send_cq->lock); 1325 spin_unlock_irq(&recv_cq->lock); 1326 } 1327 } 1328 1329 static inline void *get_wqe(struct hns_roce_qp *hr_qp, int offset) 1330 { 1331 return hns_roce_buf_offset(hr_qp->mtr.kmem, offset); 1332 } 1333 1334 void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n) 1335 { 1336 return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift)); 1337 } 1338 1339 void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n) 1340 { 1341 return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift)); 1342 } 1343 1344 void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n) 1345 { 1346 return get_wqe(hr_qp, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift)); 1347 } 1348 1349 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq, 1350 struct ib_cq *ib_cq) 1351 { 1352 struct hns_roce_cq *hr_cq; 1353 u32 cur; 1354 1355 cur = hr_wq->head - hr_wq->tail; 1356 if (likely(cur + nreq < hr_wq->wqe_cnt)) 1357 return false; 1358 1359 hr_cq = to_hr_cq(ib_cq); 1360 spin_lock(&hr_cq->lock); 1361 cur = hr_wq->head - hr_wq->tail; 1362 spin_unlock(&hr_cq->lock); 1363 1364 return cur + nreq >= hr_wq->wqe_cnt; 1365 } 1366 1367 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) 1368 { 1369 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 1370 unsigned int reserved_from_bot; 1371 unsigned int i; 1372 1373 mutex_init(&qp_table->scc_mutex); 1374 xa_init(&hr_dev->qp_table_xa); 1375 1376 reserved_from_bot = hr_dev->caps.reserved_qps; 1377 1378 for (i = 0; i < reserved_from_bot; i++) { 1379 hr_dev->qp_table.bank[get_qp_bankid(i)].inuse++; 1380 hr_dev->qp_table.bank[get_qp_bankid(i)].min++; 1381 } 1382 1383 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) { 1384 ida_init(&hr_dev->qp_table.bank[i].ida); 1385 hr_dev->qp_table.bank[i].max = hr_dev->caps.num_qps / 1386 HNS_ROCE_QP_BANK_NUM - 1; 1387 hr_dev->qp_table.bank[i].next = hr_dev->qp_table.bank[i].min; 1388 } 1389 1390 return 0; 1391 } 1392 1393 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) 1394 { 1395 int i; 1396 1397 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) 1398 ida_destroy(&hr_dev->qp_table.bank[i].ida); 1399 } 1400