1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/pci.h> 35 #include <rdma/ib_addr.h> 36 #include <rdma/ib_umem.h> 37 #include <rdma/uverbs_ioctl.h> 38 #include "hns_roce_common.h" 39 #include "hns_roce_device.h" 40 #include "hns_roce_hem.h" 41 42 static void flush_work_handle(struct work_struct *work) 43 { 44 struct hns_roce_work *flush_work = container_of(work, 45 struct hns_roce_work, work); 46 struct hns_roce_qp *hr_qp = container_of(flush_work, 47 struct hns_roce_qp, flush_work); 48 struct device *dev = flush_work->hr_dev->dev; 49 struct ib_qp_attr attr; 50 int attr_mask; 51 int ret; 52 53 attr_mask = IB_QP_STATE; 54 attr.qp_state = IB_QPS_ERR; 55 56 if (test_and_clear_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag)) { 57 ret = hns_roce_modify_qp(&hr_qp->ibqp, &attr, attr_mask, NULL); 58 if (ret) 59 dev_err(dev, "Modify QP to error state failed(%d) during CQE flush\n", 60 ret); 61 } 62 63 /* 64 * make sure we signal QP destroy leg that flush QP was completed 65 * so that it can safely proceed ahead now and destroy QP 66 */ 67 if (refcount_dec_and_test(&hr_qp->refcount)) 68 complete(&hr_qp->free); 69 } 70 71 void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 72 { 73 struct hns_roce_work *flush_work = &hr_qp->flush_work; 74 75 flush_work->hr_dev = hr_dev; 76 INIT_WORK(&flush_work->work, flush_work_handle); 77 refcount_inc(&hr_qp->refcount); 78 queue_work(hr_dev->irq_workq, &flush_work->work); 79 } 80 81 void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp) 82 { 83 /* 84 * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state 85 * gets into errored mode. Hence, as a workaround to this 86 * hardware limitation, driver needs to assist in flushing. But 87 * the flushing operation uses mailbox to convey the QP state to 88 * the hardware and which can sleep due to the mutex protection 89 * around the mailbox calls. Hence, use the deferred flush for 90 * now. 91 */ 92 if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag)) 93 init_flush_work(dev, qp); 94 } 95 96 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) 97 { 98 struct device *dev = hr_dev->dev; 99 struct hns_roce_qp *qp; 100 101 xa_lock(&hr_dev->qp_table_xa); 102 qp = __hns_roce_qp_lookup(hr_dev, qpn); 103 if (qp) 104 refcount_inc(&qp->refcount); 105 xa_unlock(&hr_dev->qp_table_xa); 106 107 if (!qp) { 108 dev_warn(dev, "Async event for bogus QP %08x\n", qpn); 109 return; 110 } 111 112 if (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR || 113 event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR || 114 event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR || 115 event_type == HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION || 116 event_type == HNS_ROCE_EVENT_TYPE_INVALID_XRCETH) { 117 qp->state = IB_QPS_ERR; 118 119 flush_cqe(hr_dev, qp); 120 } 121 122 qp->event(qp, (enum hns_roce_event)event_type); 123 124 if (refcount_dec_and_test(&qp->refcount)) 125 complete(&qp->free); 126 } 127 128 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, 129 enum hns_roce_event type) 130 { 131 struct ib_qp *ibqp = &hr_qp->ibqp; 132 struct ib_event event; 133 134 if (ibqp->event_handler) { 135 event.device = ibqp->device; 136 event.element.qp = ibqp; 137 switch (type) { 138 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 139 event.event = IB_EVENT_PATH_MIG; 140 break; 141 case HNS_ROCE_EVENT_TYPE_COMM_EST: 142 event.event = IB_EVENT_COMM_EST; 143 break; 144 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 145 event.event = IB_EVENT_SQ_DRAINED; 146 break; 147 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 148 event.event = IB_EVENT_QP_LAST_WQE_REACHED; 149 break; 150 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 151 event.event = IB_EVENT_QP_FATAL; 152 break; 153 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 154 event.event = IB_EVENT_PATH_MIG_ERR; 155 break; 156 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 157 event.event = IB_EVENT_QP_REQ_ERR; 158 break; 159 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 160 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION: 161 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH: 162 event.event = IB_EVENT_QP_ACCESS_ERR; 163 break; 164 default: 165 dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n", 166 type, hr_qp->qpn); 167 return; 168 } 169 ibqp->event_handler(&event, ibqp->qp_context); 170 } 171 } 172 173 static u8 get_least_load_bankid_for_qp(struct hns_roce_bank *bank) 174 { 175 u32 least_load = bank[0].inuse; 176 u8 bankid = 0; 177 u32 bankcnt; 178 u8 i; 179 180 for (i = 1; i < HNS_ROCE_QP_BANK_NUM; i++) { 181 bankcnt = bank[i].inuse; 182 if (bankcnt < least_load) { 183 least_load = bankcnt; 184 bankid = i; 185 } 186 } 187 188 return bankid; 189 } 190 191 static int alloc_qpn_with_bankid(struct hns_roce_bank *bank, u8 bankid, 192 unsigned long *qpn) 193 { 194 int id; 195 196 id = ida_alloc_range(&bank->ida, bank->next, bank->max, GFP_KERNEL); 197 if (id < 0) { 198 id = ida_alloc_range(&bank->ida, bank->min, bank->max, 199 GFP_KERNEL); 200 if (id < 0) 201 return id; 202 } 203 204 /* the QPN should keep increasing until the max value is reached. */ 205 bank->next = (id + 1) > bank->max ? bank->min : id + 1; 206 207 /* the lower 3 bits is bankid */ 208 *qpn = (id << 3) | bankid; 209 210 return 0; 211 } 212 static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 213 { 214 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 215 unsigned long num = 0; 216 u8 bankid; 217 int ret; 218 219 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) { 220 num = 1; 221 hr_qp->doorbell_qpn = 1; 222 } else { 223 mutex_lock(&qp_table->bank_mutex); 224 bankid = get_least_load_bankid_for_qp(qp_table->bank); 225 226 ret = alloc_qpn_with_bankid(&qp_table->bank[bankid], bankid, 227 &num); 228 if (ret) { 229 ibdev_err(&hr_dev->ib_dev, 230 "failed to alloc QPN, ret = %d\n", ret); 231 mutex_unlock(&qp_table->bank_mutex); 232 return ret; 233 } 234 235 qp_table->bank[bankid].inuse++; 236 mutex_unlock(&qp_table->bank_mutex); 237 238 hr_qp->doorbell_qpn = (u32)num; 239 } 240 241 hr_qp->qpn = num; 242 243 return 0; 244 } 245 246 enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state) 247 { 248 switch (state) { 249 case IB_QPS_RESET: 250 return HNS_ROCE_QP_STATE_RST; 251 case IB_QPS_INIT: 252 return HNS_ROCE_QP_STATE_INIT; 253 case IB_QPS_RTR: 254 return HNS_ROCE_QP_STATE_RTR; 255 case IB_QPS_RTS: 256 return HNS_ROCE_QP_STATE_RTS; 257 case IB_QPS_SQD: 258 return HNS_ROCE_QP_STATE_SQD; 259 case IB_QPS_ERR: 260 return HNS_ROCE_QP_STATE_ERR; 261 default: 262 return HNS_ROCE_QP_NUM_STATE; 263 } 264 } 265 266 static void add_qp_to_list(struct hns_roce_dev *hr_dev, 267 struct hns_roce_qp *hr_qp, 268 struct ib_cq *send_cq, struct ib_cq *recv_cq) 269 { 270 struct hns_roce_cq *hr_send_cq, *hr_recv_cq; 271 unsigned long flags; 272 273 hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL; 274 hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL; 275 276 spin_lock_irqsave(&hr_dev->qp_list_lock, flags); 277 hns_roce_lock_cqs(hr_send_cq, hr_recv_cq); 278 279 list_add_tail(&hr_qp->node, &hr_dev->qp_list); 280 if (hr_send_cq) 281 list_add_tail(&hr_qp->sq_node, &hr_send_cq->sq_list); 282 if (hr_recv_cq) 283 list_add_tail(&hr_qp->rq_node, &hr_recv_cq->rq_list); 284 285 hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq); 286 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); 287 } 288 289 static int hns_roce_qp_store(struct hns_roce_dev *hr_dev, 290 struct hns_roce_qp *hr_qp, 291 struct ib_qp_init_attr *init_attr) 292 { 293 struct xarray *xa = &hr_dev->qp_table_xa; 294 int ret; 295 296 if (!hr_qp->qpn) 297 return -EINVAL; 298 299 ret = xa_err(xa_store_irq(xa, hr_qp->qpn, hr_qp, GFP_KERNEL)); 300 if (ret) 301 dev_err(hr_dev->dev, "Failed to xa store for QPC\n"); 302 else 303 /* add QP to device's QP list for softwc */ 304 add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq, 305 init_attr->recv_cq); 306 307 return ret; 308 } 309 310 static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 311 { 312 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 313 struct device *dev = hr_dev->dev; 314 int ret; 315 316 if (!hr_qp->qpn) 317 return -EINVAL; 318 319 /* Alloc memory for QPC */ 320 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn); 321 if (ret) { 322 dev_err(dev, "Failed to get QPC table\n"); 323 goto err_out; 324 } 325 326 /* Alloc memory for IRRL */ 327 ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 328 if (ret) { 329 dev_err(dev, "Failed to get IRRL table\n"); 330 goto err_put_qp; 331 } 332 333 if (hr_dev->caps.trrl_entry_sz) { 334 /* Alloc memory for TRRL */ 335 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table, 336 hr_qp->qpn); 337 if (ret) { 338 dev_err(dev, "Failed to get TRRL table\n"); 339 goto err_put_irrl; 340 } 341 } 342 343 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 344 /* Alloc memory for SCC CTX */ 345 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table, 346 hr_qp->qpn); 347 if (ret) { 348 dev_err(dev, "Failed to get SCC CTX table\n"); 349 goto err_put_trrl; 350 } 351 } 352 353 return 0; 354 355 err_put_trrl: 356 if (hr_dev->caps.trrl_entry_sz) 357 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 358 359 err_put_irrl: 360 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 361 362 err_put_qp: 363 hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); 364 365 err_out: 366 return ret; 367 } 368 369 static void qp_user_mmap_entry_remove(struct hns_roce_qp *hr_qp) 370 { 371 rdma_user_mmap_entry_remove(&hr_qp->dwqe_mmap_entry->rdma_entry); 372 } 373 374 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 375 { 376 struct xarray *xa = &hr_dev->qp_table_xa; 377 unsigned long flags; 378 379 list_del(&hr_qp->node); 380 381 if (hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT) 382 list_del(&hr_qp->sq_node); 383 384 if (hr_qp->ibqp.qp_type != IB_QPT_XRC_INI && 385 hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT) 386 list_del(&hr_qp->rq_node); 387 388 xa_lock_irqsave(xa, flags); 389 __xa_erase(xa, hr_qp->qpn); 390 xa_unlock_irqrestore(xa, flags); 391 } 392 393 static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 394 { 395 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 396 397 if (hr_dev->caps.trrl_entry_sz) 398 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 399 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 400 } 401 402 static inline u8 get_qp_bankid(unsigned long qpn) 403 { 404 /* The lower 3 bits of QPN are used to hash to different banks */ 405 return (u8)(qpn & GENMASK(2, 0)); 406 } 407 408 static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 409 { 410 u8 bankid; 411 412 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) 413 return; 414 415 if (hr_qp->qpn < hr_dev->caps.reserved_qps) 416 return; 417 418 bankid = get_qp_bankid(hr_qp->qpn); 419 420 ida_free(&hr_dev->qp_table.bank[bankid].ida, hr_qp->qpn >> 3); 421 422 mutex_lock(&hr_dev->qp_table.bank_mutex); 423 hr_dev->qp_table.bank[bankid].inuse--; 424 mutex_unlock(&hr_dev->qp_table.bank_mutex); 425 } 426 427 static u32 proc_rq_sge(struct hns_roce_dev *dev, struct hns_roce_qp *hr_qp, 428 bool user) 429 { 430 u32 max_sge = dev->caps.max_rq_sg; 431 432 if (dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 433 return max_sge; 434 435 /* Reserve SGEs only for HIP08 in kernel; The userspace driver will 436 * calculate number of max_sge with reserved SGEs when allocating wqe 437 * buf, so there is no need to do this again in kernel. But the number 438 * may exceed the capacity of SGEs recorded in the firmware, so the 439 * kernel driver should just adapt the value accordingly. 440 */ 441 if (user) 442 max_sge = roundup_pow_of_two(max_sge + 1); 443 else 444 hr_qp->rq.rsv_sge = 1; 445 446 return max_sge; 447 } 448 449 static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, 450 struct hns_roce_qp *hr_qp, int has_rq, bool user) 451 { 452 u32 max_sge = proc_rq_sge(hr_dev, hr_qp, user); 453 u32 cnt; 454 455 /* If srq exist, set zero for relative number of rq */ 456 if (!has_rq) { 457 hr_qp->rq.wqe_cnt = 0; 458 hr_qp->rq.max_gs = 0; 459 hr_qp->rq_inl_buf.wqe_cnt = 0; 460 cap->max_recv_wr = 0; 461 cap->max_recv_sge = 0; 462 463 return 0; 464 } 465 466 /* Check the validity of QP support capacity */ 467 if (!cap->max_recv_wr || cap->max_recv_wr > hr_dev->caps.max_wqes || 468 cap->max_recv_sge > max_sge) { 469 ibdev_err(&hr_dev->ib_dev, 470 "RQ config error, depth = %u, sge = %u\n", 471 cap->max_recv_wr, cap->max_recv_sge); 472 return -EINVAL; 473 } 474 475 cnt = roundup_pow_of_two(max(cap->max_recv_wr, hr_dev->caps.min_wqes)); 476 if (cnt > hr_dev->caps.max_wqes) { 477 ibdev_err(&hr_dev->ib_dev, "rq depth %u too large\n", 478 cap->max_recv_wr); 479 return -EINVAL; 480 } 481 482 hr_qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge) + 483 hr_qp->rq.rsv_sge); 484 485 if (hr_dev->caps.max_rq_sg <= HNS_ROCE_SGE_IN_WQE) 486 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz); 487 else 488 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz * 489 hr_qp->rq.max_gs); 490 491 hr_qp->rq.wqe_cnt = cnt; 492 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE && 493 hr_qp->ibqp.qp_type != IB_QPT_UD && 494 hr_qp->ibqp.qp_type != IB_QPT_GSI) 495 hr_qp->rq_inl_buf.wqe_cnt = cnt; 496 else 497 hr_qp->rq_inl_buf.wqe_cnt = 0; 498 499 cap->max_recv_wr = cnt; 500 cap->max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge; 501 502 return 0; 503 } 504 505 static u32 get_wqe_ext_sge_cnt(struct hns_roce_qp *qp) 506 { 507 /* GSI/UD QP only has extended sge */ 508 if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) 509 return qp->sq.max_gs; 510 511 if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE) 512 return qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE; 513 514 return 0; 515 } 516 517 static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt, 518 struct hns_roce_qp *hr_qp, struct ib_qp_cap *cap) 519 { 520 u32 total_sge_cnt; 521 u32 wqe_sge_cnt; 522 523 hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT; 524 525 hr_qp->sq.max_gs = max(1U, cap->max_send_sge); 526 527 wqe_sge_cnt = get_wqe_ext_sge_cnt(hr_qp); 528 529 /* If the number of extended sge is not zero, they MUST use the 530 * space of HNS_HW_PAGE_SIZE at least. 531 */ 532 if (wqe_sge_cnt) { 533 total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt); 534 hr_qp->sge.sge_cnt = max(total_sge_cnt, 535 (u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE); 536 } 537 } 538 539 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev, 540 struct ib_qp_cap *cap, 541 struct hns_roce_ib_create_qp *ucmd) 542 { 543 u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz); 544 u8 max_sq_stride = ilog2(roundup_sq_stride); 545 546 /* Sanity check SQ size before proceeding */ 547 if (ucmd->log_sq_stride > max_sq_stride || 548 ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) { 549 ibdev_err(&hr_dev->ib_dev, "failed to check SQ stride size.\n"); 550 return -EINVAL; 551 } 552 553 if (cap->max_send_sge > hr_dev->caps.max_sq_sg) { 554 ibdev_err(&hr_dev->ib_dev, "failed to check SQ SGE size %u.\n", 555 cap->max_send_sge); 556 return -EINVAL; 557 } 558 559 return 0; 560 } 561 562 static int set_user_sq_size(struct hns_roce_dev *hr_dev, 563 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp, 564 struct hns_roce_ib_create_qp *ucmd) 565 { 566 struct ib_device *ibdev = &hr_dev->ib_dev; 567 u32 cnt = 0; 568 int ret; 569 570 if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) || 571 cnt > hr_dev->caps.max_wqes) 572 return -EINVAL; 573 574 ret = check_sq_size_with_integrity(hr_dev, cap, ucmd); 575 if (ret) { 576 ibdev_err(ibdev, "failed to check user SQ size, ret = %d.\n", 577 ret); 578 return ret; 579 } 580 581 set_ext_sge_param(hr_dev, cnt, hr_qp, cap); 582 583 hr_qp->sq.wqe_shift = ucmd->log_sq_stride; 584 hr_qp->sq.wqe_cnt = cnt; 585 586 return 0; 587 } 588 589 static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev, 590 struct hns_roce_qp *hr_qp, 591 struct hns_roce_buf_attr *buf_attr) 592 { 593 int buf_size; 594 int idx = 0; 595 596 hr_qp->buff_size = 0; 597 598 /* SQ WQE */ 599 hr_qp->sq.offset = 0; 600 buf_size = to_hr_hem_entries_size(hr_qp->sq.wqe_cnt, 601 hr_qp->sq.wqe_shift); 602 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 603 buf_attr->region[idx].size = buf_size; 604 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sq_hop_num; 605 idx++; 606 hr_qp->buff_size += buf_size; 607 } 608 609 /* extend SGE WQE in SQ */ 610 hr_qp->sge.offset = hr_qp->buff_size; 611 buf_size = to_hr_hem_entries_size(hr_qp->sge.sge_cnt, 612 hr_qp->sge.sge_shift); 613 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 614 buf_attr->region[idx].size = buf_size; 615 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sge_hop_num; 616 idx++; 617 hr_qp->buff_size += buf_size; 618 } 619 620 /* RQ WQE */ 621 hr_qp->rq.offset = hr_qp->buff_size; 622 buf_size = to_hr_hem_entries_size(hr_qp->rq.wqe_cnt, 623 hr_qp->rq.wqe_shift); 624 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 625 buf_attr->region[idx].size = buf_size; 626 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_rq_hop_num; 627 idx++; 628 hr_qp->buff_size += buf_size; 629 } 630 631 if (hr_qp->buff_size < 1) 632 return -EINVAL; 633 634 buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; 635 buf_attr->region_count = idx; 636 637 return 0; 638 } 639 640 static int set_kernel_sq_size(struct hns_roce_dev *hr_dev, 641 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp) 642 { 643 struct ib_device *ibdev = &hr_dev->ib_dev; 644 u32 cnt; 645 646 if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes || 647 cap->max_send_sge > hr_dev->caps.max_sq_sg) { 648 ibdev_err(ibdev, "failed to check SQ WR or SGE num.\n"); 649 return -EINVAL; 650 } 651 652 cnt = roundup_pow_of_two(max(cap->max_send_wr, hr_dev->caps.min_wqes)); 653 if (cnt > hr_dev->caps.max_wqes) { 654 ibdev_err(ibdev, "failed to check WQE num, WQE num = %u.\n", 655 cnt); 656 return -EINVAL; 657 } 658 659 hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz); 660 hr_qp->sq.wqe_cnt = cnt; 661 662 set_ext_sge_param(hr_dev, cnt, hr_qp, cap); 663 664 /* sync the parameters of kernel QP to user's configuration */ 665 cap->max_send_wr = cnt; 666 cap->max_send_sge = hr_qp->sq.max_gs; 667 668 return 0; 669 } 670 671 static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr) 672 { 673 if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr) 674 return 0; 675 676 return 1; 677 } 678 679 static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr) 680 { 681 if (attr->qp_type == IB_QPT_XRC_INI || 682 attr->qp_type == IB_QPT_XRC_TGT || attr->srq || 683 !attr->cap.max_recv_wr) 684 return 0; 685 686 return 1; 687 } 688 689 static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp, 690 struct ib_qp_init_attr *init_attr) 691 { 692 u32 max_recv_sge = init_attr->cap.max_recv_sge; 693 u32 wqe_cnt = hr_qp->rq_inl_buf.wqe_cnt; 694 struct hns_roce_rinl_wqe *wqe_list; 695 int i; 696 697 /* allocate recv inline buf */ 698 wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe), 699 GFP_KERNEL); 700 if (!wqe_list) 701 goto err; 702 703 /* Allocate a continuous buffer for all inline sge we need */ 704 wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge * 705 sizeof(struct hns_roce_rinl_sge)), 706 GFP_KERNEL); 707 if (!wqe_list[0].sg_list) 708 goto err_wqe_list; 709 710 /* Assign buffers of sg_list to each inline wqe */ 711 for (i = 1; i < wqe_cnt; i++) 712 wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge]; 713 714 hr_qp->rq_inl_buf.wqe_list = wqe_list; 715 716 return 0; 717 718 err_wqe_list: 719 kfree(wqe_list); 720 721 err: 722 return -ENOMEM; 723 } 724 725 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp) 726 { 727 if (hr_qp->rq_inl_buf.wqe_list) 728 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list); 729 kfree(hr_qp->rq_inl_buf.wqe_list); 730 } 731 732 static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 733 struct ib_qp_init_attr *init_attr, 734 struct ib_udata *udata, unsigned long addr) 735 { 736 struct ib_device *ibdev = &hr_dev->ib_dev; 737 struct hns_roce_buf_attr buf_attr = {}; 738 int ret; 739 740 if (!udata && hr_qp->rq_inl_buf.wqe_cnt) { 741 ret = alloc_rq_inline_buf(hr_qp, init_attr); 742 if (ret) { 743 ibdev_err(ibdev, 744 "failed to alloc inline buf, ret = %d.\n", 745 ret); 746 return ret; 747 } 748 } else { 749 hr_qp->rq_inl_buf.wqe_list = NULL; 750 } 751 752 ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr); 753 if (ret) { 754 ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret); 755 goto err_inline; 756 } 757 ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr, 758 PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz, 759 udata, addr); 760 if (ret) { 761 ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret); 762 goto err_inline; 763 } 764 765 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DIRECT_WQE) 766 hr_qp->en_flags |= HNS_ROCE_QP_CAP_DIRECT_WQE; 767 768 return 0; 769 770 err_inline: 771 free_rq_inline_buf(hr_qp); 772 773 return ret; 774 } 775 776 static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 777 { 778 hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr); 779 free_rq_inline_buf(hr_qp); 780 } 781 782 static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev, 783 struct ib_qp_init_attr *init_attr, 784 struct ib_udata *udata, 785 struct hns_roce_ib_create_qp_resp *resp, 786 struct hns_roce_ib_create_qp *ucmd) 787 { 788 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 789 udata->outlen >= offsetofend(typeof(*resp), cap_flags) && 790 hns_roce_qp_has_sq(init_attr) && 791 udata->inlen >= offsetofend(typeof(*ucmd), sdb_addr)); 792 } 793 794 static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev, 795 struct ib_qp_init_attr *init_attr, 796 struct ib_udata *udata, 797 struct hns_roce_ib_create_qp_resp *resp) 798 { 799 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 800 udata->outlen >= offsetofend(typeof(*resp), cap_flags) && 801 hns_roce_qp_has_rq(init_attr)); 802 } 803 804 static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev, 805 struct ib_qp_init_attr *init_attr) 806 { 807 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 808 hns_roce_qp_has_rq(init_attr)); 809 } 810 811 static int qp_mmap_entry(struct hns_roce_qp *hr_qp, 812 struct hns_roce_dev *hr_dev, 813 struct ib_udata *udata, 814 struct hns_roce_ib_create_qp_resp *resp) 815 { 816 struct hns_roce_ucontext *uctx = 817 rdma_udata_to_drv_context(udata, 818 struct hns_roce_ucontext, ibucontext); 819 struct rdma_user_mmap_entry *rdma_entry; 820 u64 address; 821 822 address = hr_dev->dwqe_page + hr_qp->qpn * HNS_ROCE_DWQE_SIZE; 823 824 hr_qp->dwqe_mmap_entry = 825 hns_roce_user_mmap_entry_insert(&uctx->ibucontext, address, 826 HNS_ROCE_DWQE_SIZE, 827 HNS_ROCE_MMAP_TYPE_DWQE); 828 829 if (!hr_qp->dwqe_mmap_entry) { 830 ibdev_err(&hr_dev->ib_dev, "failed to get dwqe mmap entry.\n"); 831 return -ENOMEM; 832 } 833 834 rdma_entry = &hr_qp->dwqe_mmap_entry->rdma_entry; 835 resp->dwqe_mmap_key = rdma_user_mmap_get_offset(rdma_entry); 836 837 return 0; 838 } 839 840 static int alloc_user_qp_db(struct hns_roce_dev *hr_dev, 841 struct hns_roce_qp *hr_qp, 842 struct ib_qp_init_attr *init_attr, 843 struct ib_udata *udata, 844 struct hns_roce_ib_create_qp *ucmd, 845 struct hns_roce_ib_create_qp_resp *resp) 846 { 847 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata, 848 struct hns_roce_ucontext, ibucontext); 849 struct ib_device *ibdev = &hr_dev->ib_dev; 850 int ret; 851 852 if (user_qp_has_sdb(hr_dev, init_attr, udata, resp, ucmd)) { 853 ret = hns_roce_db_map_user(uctx, ucmd->sdb_addr, &hr_qp->sdb); 854 if (ret) { 855 ibdev_err(ibdev, 856 "failed to map user SQ doorbell, ret = %d.\n", 857 ret); 858 goto err_out; 859 } 860 hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB; 861 } 862 863 if (user_qp_has_rdb(hr_dev, init_attr, udata, resp)) { 864 ret = hns_roce_db_map_user(uctx, ucmd->db_addr, &hr_qp->rdb); 865 if (ret) { 866 ibdev_err(ibdev, 867 "failed to map user RQ doorbell, ret = %d.\n", 868 ret); 869 goto err_sdb; 870 } 871 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; 872 } 873 874 return 0; 875 876 err_sdb: 877 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) 878 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 879 err_out: 880 return ret; 881 } 882 883 static int alloc_kernel_qp_db(struct hns_roce_dev *hr_dev, 884 struct hns_roce_qp *hr_qp, 885 struct ib_qp_init_attr *init_attr) 886 { 887 struct ib_device *ibdev = &hr_dev->ib_dev; 888 int ret; 889 890 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 891 hr_qp->sq.db_reg = hr_dev->mem_base + 892 HNS_ROCE_DWQE_SIZE * hr_qp->qpn; 893 else 894 hr_qp->sq.db_reg = hr_dev->reg_base + hr_dev->sdb_offset + 895 DB_REG_OFFSET * hr_dev->priv_uar.index; 896 897 hr_qp->rq.db_reg = hr_dev->reg_base + hr_dev->odb_offset + 898 DB_REG_OFFSET * hr_dev->priv_uar.index; 899 900 if (kernel_qp_has_rdb(hr_dev, init_attr)) { 901 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0); 902 if (ret) { 903 ibdev_err(ibdev, 904 "failed to alloc kernel RQ doorbell, ret = %d.\n", 905 ret); 906 return ret; 907 } 908 *hr_qp->rdb.db_record = 0; 909 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; 910 } 911 912 return 0; 913 } 914 915 static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 916 struct ib_qp_init_attr *init_attr, 917 struct ib_udata *udata, 918 struct hns_roce_ib_create_qp *ucmd, 919 struct hns_roce_ib_create_qp_resp *resp) 920 { 921 int ret; 922 923 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SDI_MODE) 924 hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB; 925 926 if (udata) { 927 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) { 928 ret = qp_mmap_entry(hr_qp, hr_dev, udata, resp); 929 if (ret) 930 return ret; 931 } 932 933 ret = alloc_user_qp_db(hr_dev, hr_qp, init_attr, udata, ucmd, 934 resp); 935 if (ret) 936 goto err_remove_qp; 937 } else { 938 ret = alloc_kernel_qp_db(hr_dev, hr_qp, init_attr); 939 if (ret) 940 return ret; 941 } 942 943 return 0; 944 945 err_remove_qp: 946 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) 947 qp_user_mmap_entry_remove(hr_qp); 948 949 return ret; 950 } 951 952 static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 953 struct ib_udata *udata) 954 { 955 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( 956 udata, struct hns_roce_ucontext, ibucontext); 957 958 if (udata) { 959 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 960 hns_roce_db_unmap_user(uctx, &hr_qp->rdb); 961 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) 962 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 963 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) 964 qp_user_mmap_entry_remove(hr_qp); 965 } else { 966 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 967 hns_roce_free_db(hr_dev, &hr_qp->rdb); 968 } 969 } 970 971 static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev, 972 struct hns_roce_qp *hr_qp) 973 { 974 struct ib_device *ibdev = &hr_dev->ib_dev; 975 u64 *sq_wrid = NULL; 976 u64 *rq_wrid = NULL; 977 int ret; 978 979 sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL); 980 if (ZERO_OR_NULL_PTR(sq_wrid)) { 981 ibdev_err(ibdev, "failed to alloc SQ wrid.\n"); 982 return -ENOMEM; 983 } 984 985 if (hr_qp->rq.wqe_cnt) { 986 rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL); 987 if (ZERO_OR_NULL_PTR(rq_wrid)) { 988 ibdev_err(ibdev, "failed to alloc RQ wrid.\n"); 989 ret = -ENOMEM; 990 goto err_sq; 991 } 992 } 993 994 hr_qp->sq.wrid = sq_wrid; 995 hr_qp->rq.wrid = rq_wrid; 996 return 0; 997 err_sq: 998 kfree(sq_wrid); 999 1000 return ret; 1001 } 1002 1003 static void free_kernel_wrid(struct hns_roce_qp *hr_qp) 1004 { 1005 kfree(hr_qp->rq.wrid); 1006 kfree(hr_qp->sq.wrid); 1007 } 1008 1009 static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 1010 struct ib_qp_init_attr *init_attr, 1011 struct ib_udata *udata, 1012 struct hns_roce_ib_create_qp *ucmd) 1013 { 1014 struct ib_device *ibdev = &hr_dev->ib_dev; 1015 int ret; 1016 1017 if (init_attr->cap.max_inline_data > hr_dev->caps.max_sq_inline) 1018 init_attr->cap.max_inline_data = hr_dev->caps.max_sq_inline; 1019 1020 hr_qp->max_inline_data = init_attr->cap.max_inline_data; 1021 1022 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) 1023 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR; 1024 else 1025 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR; 1026 1027 ret = set_rq_size(hr_dev, &init_attr->cap, hr_qp, 1028 hns_roce_qp_has_rq(init_attr), !!udata); 1029 if (ret) { 1030 ibdev_err(ibdev, "failed to set user RQ size, ret = %d.\n", 1031 ret); 1032 return ret; 1033 } 1034 1035 if (udata) { 1036 ret = ib_copy_from_udata(ucmd, udata, 1037 min(udata->inlen, sizeof(*ucmd))); 1038 if (ret) { 1039 ibdev_err(ibdev, 1040 "failed to copy QP ucmd, ret = %d\n", ret); 1041 return ret; 1042 } 1043 1044 ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd); 1045 if (ret) 1046 ibdev_err(ibdev, 1047 "failed to set user SQ size, ret = %d.\n", 1048 ret); 1049 } else { 1050 ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp); 1051 if (ret) 1052 ibdev_err(ibdev, 1053 "failed to set kernel SQ size, ret = %d.\n", 1054 ret); 1055 } 1056 1057 return ret; 1058 } 1059 1060 static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, 1061 struct ib_pd *ib_pd, 1062 struct ib_qp_init_attr *init_attr, 1063 struct ib_udata *udata, 1064 struct hns_roce_qp *hr_qp) 1065 { 1066 struct hns_roce_ib_create_qp_resp resp = {}; 1067 struct ib_device *ibdev = &hr_dev->ib_dev; 1068 struct hns_roce_ib_create_qp ucmd; 1069 int ret; 1070 1071 mutex_init(&hr_qp->mutex); 1072 spin_lock_init(&hr_qp->sq.lock); 1073 spin_lock_init(&hr_qp->rq.lock); 1074 1075 hr_qp->state = IB_QPS_RESET; 1076 hr_qp->flush_flag = 0; 1077 1078 if (init_attr->create_flags) 1079 return -EOPNOTSUPP; 1080 1081 ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd); 1082 if (ret) { 1083 ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret); 1084 return ret; 1085 } 1086 1087 if (!udata) { 1088 ret = alloc_kernel_wrid(hr_dev, hr_qp); 1089 if (ret) { 1090 ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n", 1091 ret); 1092 return ret; 1093 } 1094 } 1095 1096 ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr); 1097 if (ret) { 1098 ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret); 1099 goto err_buf; 1100 } 1101 1102 ret = alloc_qpn(hr_dev, hr_qp); 1103 if (ret) { 1104 ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret); 1105 goto err_qpn; 1106 } 1107 1108 ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp); 1109 if (ret) { 1110 ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n", 1111 ret); 1112 goto err_db; 1113 } 1114 1115 ret = alloc_qpc(hr_dev, hr_qp); 1116 if (ret) { 1117 ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n", 1118 ret); 1119 goto err_qpc; 1120 } 1121 1122 ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr); 1123 if (ret) { 1124 ibdev_err(ibdev, "failed to store QP, ret = %d.\n", ret); 1125 goto err_store; 1126 } 1127 1128 if (udata) { 1129 resp.cap_flags = hr_qp->en_flags; 1130 ret = ib_copy_to_udata(udata, &resp, 1131 min(udata->outlen, sizeof(resp))); 1132 if (ret) { 1133 ibdev_err(ibdev, "copy qp resp failed!\n"); 1134 goto err_store; 1135 } 1136 } 1137 1138 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 1139 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp); 1140 if (ret) 1141 goto err_flow_ctrl; 1142 } 1143 1144 hr_qp->ibqp.qp_num = hr_qp->qpn; 1145 hr_qp->event = hns_roce_ib_qp_event; 1146 refcount_set(&hr_qp->refcount, 1); 1147 init_completion(&hr_qp->free); 1148 1149 return 0; 1150 1151 err_flow_ctrl: 1152 hns_roce_qp_remove(hr_dev, hr_qp); 1153 err_store: 1154 free_qpc(hr_dev, hr_qp); 1155 err_qpc: 1156 free_qp_db(hr_dev, hr_qp, udata); 1157 err_db: 1158 free_qpn(hr_dev, hr_qp); 1159 err_qpn: 1160 free_qp_buf(hr_dev, hr_qp); 1161 err_buf: 1162 free_kernel_wrid(hr_qp); 1163 return ret; 1164 } 1165 1166 void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 1167 struct ib_udata *udata) 1168 { 1169 if (refcount_dec_and_test(&hr_qp->refcount)) 1170 complete(&hr_qp->free); 1171 wait_for_completion(&hr_qp->free); 1172 1173 free_qpc(hr_dev, hr_qp); 1174 free_qpn(hr_dev, hr_qp); 1175 free_qp_buf(hr_dev, hr_qp); 1176 free_kernel_wrid(hr_qp); 1177 free_qp_db(hr_dev, hr_qp, udata); 1178 } 1179 1180 static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type, 1181 bool is_user) 1182 { 1183 switch (type) { 1184 case IB_QPT_XRC_INI: 1185 case IB_QPT_XRC_TGT: 1186 if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC)) 1187 goto out; 1188 break; 1189 case IB_QPT_UD: 1190 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 && 1191 is_user) 1192 goto out; 1193 break; 1194 case IB_QPT_RC: 1195 case IB_QPT_GSI: 1196 break; 1197 default: 1198 goto out; 1199 } 1200 1201 return 0; 1202 1203 out: 1204 ibdev_err(&hr_dev->ib_dev, "not support QP type %d\n", type); 1205 1206 return -EOPNOTSUPP; 1207 } 1208 1209 int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, 1210 struct ib_udata *udata) 1211 { 1212 struct ib_device *ibdev = qp->device; 1213 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); 1214 struct hns_roce_qp *hr_qp = to_hr_qp(qp); 1215 struct ib_pd *pd = qp->pd; 1216 int ret; 1217 1218 ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata); 1219 if (ret) 1220 return ret; 1221 1222 if (init_attr->qp_type == IB_QPT_XRC_TGT) 1223 hr_qp->xrcdn = to_hr_xrcd(init_attr->xrcd)->xrcdn; 1224 1225 if (init_attr->qp_type == IB_QPT_GSI) { 1226 hr_qp->port = init_attr->port_num - 1; 1227 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; 1228 } 1229 1230 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp); 1231 if (ret) 1232 ibdev_err(ibdev, "Create QP type 0x%x failed(%d)\n", 1233 init_attr->qp_type, ret); 1234 1235 return ret; 1236 } 1237 1238 int to_hr_qp_type(int qp_type) 1239 { 1240 switch (qp_type) { 1241 case IB_QPT_RC: 1242 return SERV_TYPE_RC; 1243 case IB_QPT_UD: 1244 case IB_QPT_GSI: 1245 return SERV_TYPE_UD; 1246 case IB_QPT_XRC_INI: 1247 case IB_QPT_XRC_TGT: 1248 return SERV_TYPE_XRC; 1249 default: 1250 return -1; 1251 } 1252 } 1253 1254 static int check_mtu_validate(struct hns_roce_dev *hr_dev, 1255 struct hns_roce_qp *hr_qp, 1256 struct ib_qp_attr *attr, int attr_mask) 1257 { 1258 enum ib_mtu active_mtu; 1259 int p; 1260 1261 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1262 active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu); 1263 1264 if ((hr_dev->caps.max_mtu >= IB_MTU_2048 && 1265 attr->path_mtu > hr_dev->caps.max_mtu) || 1266 attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) { 1267 ibdev_err(&hr_dev->ib_dev, 1268 "attr path_mtu(%d)invalid while modify qp", 1269 attr->path_mtu); 1270 return -EINVAL; 1271 } 1272 1273 return 0; 1274 } 1275 1276 static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1277 int attr_mask) 1278 { 1279 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1280 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1281 int p; 1282 1283 if ((attr_mask & IB_QP_PORT) && 1284 (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) { 1285 ibdev_err(&hr_dev->ib_dev, "invalid attr, port_num = %u.\n", 1286 attr->port_num); 1287 return -EINVAL; 1288 } 1289 1290 if (attr_mask & IB_QP_PKEY_INDEX) { 1291 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1292 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) { 1293 ibdev_err(&hr_dev->ib_dev, 1294 "invalid attr, pkey_index = %u.\n", 1295 attr->pkey_index); 1296 return -EINVAL; 1297 } 1298 } 1299 1300 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && 1301 attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) { 1302 ibdev_err(&hr_dev->ib_dev, 1303 "invalid attr, max_rd_atomic = %u.\n", 1304 attr->max_rd_atomic); 1305 return -EINVAL; 1306 } 1307 1308 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && 1309 attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) { 1310 ibdev_err(&hr_dev->ib_dev, 1311 "invalid attr, max_dest_rd_atomic = %u.\n", 1312 attr->max_dest_rd_atomic); 1313 return -EINVAL; 1314 } 1315 1316 if (attr_mask & IB_QP_PATH_MTU) 1317 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask); 1318 1319 return 0; 1320 } 1321 1322 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1323 int attr_mask, struct ib_udata *udata) 1324 { 1325 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1326 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1327 enum ib_qp_state cur_state, new_state; 1328 int ret = -EINVAL; 1329 1330 mutex_lock(&hr_qp->mutex); 1331 1332 if (attr_mask & IB_QP_CUR_STATE && attr->cur_qp_state != hr_qp->state) 1333 goto out; 1334 1335 cur_state = hr_qp->state; 1336 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; 1337 1338 if (ibqp->uobject && 1339 (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) { 1340 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) { 1341 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr); 1342 1343 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 1344 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr); 1345 } else { 1346 ibdev_warn(&hr_dev->ib_dev, 1347 "flush cqe is not supported in userspace!\n"); 1348 goto out; 1349 } 1350 } 1351 1352 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, 1353 attr_mask)) { 1354 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n"); 1355 goto out; 1356 } 1357 1358 ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask); 1359 if (ret) 1360 goto out; 1361 1362 if (cur_state == new_state && cur_state == IB_QPS_RESET) 1363 goto out; 1364 1365 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, 1366 new_state); 1367 1368 out: 1369 mutex_unlock(&hr_qp->mutex); 1370 1371 return ret; 1372 } 1373 1374 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) 1375 __acquires(&send_cq->lock) __acquires(&recv_cq->lock) 1376 { 1377 if (unlikely(send_cq == NULL && recv_cq == NULL)) { 1378 __acquire(&send_cq->lock); 1379 __acquire(&recv_cq->lock); 1380 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { 1381 spin_lock_irq(&send_cq->lock); 1382 __acquire(&recv_cq->lock); 1383 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { 1384 spin_lock_irq(&recv_cq->lock); 1385 __acquire(&send_cq->lock); 1386 } else if (send_cq == recv_cq) { 1387 spin_lock_irq(&send_cq->lock); 1388 __acquire(&recv_cq->lock); 1389 } else if (send_cq->cqn < recv_cq->cqn) { 1390 spin_lock_irq(&send_cq->lock); 1391 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); 1392 } else { 1393 spin_lock_irq(&recv_cq->lock); 1394 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); 1395 } 1396 } 1397 1398 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, 1399 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock) 1400 __releases(&recv_cq->lock) 1401 { 1402 if (unlikely(send_cq == NULL && recv_cq == NULL)) { 1403 __release(&recv_cq->lock); 1404 __release(&send_cq->lock); 1405 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { 1406 __release(&recv_cq->lock); 1407 spin_unlock(&send_cq->lock); 1408 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { 1409 __release(&send_cq->lock); 1410 spin_unlock(&recv_cq->lock); 1411 } else if (send_cq == recv_cq) { 1412 __release(&recv_cq->lock); 1413 spin_unlock_irq(&send_cq->lock); 1414 } else if (send_cq->cqn < recv_cq->cqn) { 1415 spin_unlock(&recv_cq->lock); 1416 spin_unlock_irq(&send_cq->lock); 1417 } else { 1418 spin_unlock(&send_cq->lock); 1419 spin_unlock_irq(&recv_cq->lock); 1420 } 1421 } 1422 1423 static inline void *get_wqe(struct hns_roce_qp *hr_qp, u32 offset) 1424 { 1425 return hns_roce_buf_offset(hr_qp->mtr.kmem, offset); 1426 } 1427 1428 void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n) 1429 { 1430 return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift)); 1431 } 1432 1433 void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n) 1434 { 1435 return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift)); 1436 } 1437 1438 void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n) 1439 { 1440 return get_wqe(hr_qp, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift)); 1441 } 1442 1443 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq, 1444 struct ib_cq *ib_cq) 1445 { 1446 struct hns_roce_cq *hr_cq; 1447 u32 cur; 1448 1449 cur = hr_wq->head - hr_wq->tail; 1450 if (likely(cur + nreq < hr_wq->wqe_cnt)) 1451 return false; 1452 1453 hr_cq = to_hr_cq(ib_cq); 1454 spin_lock(&hr_cq->lock); 1455 cur = hr_wq->head - hr_wq->tail; 1456 spin_unlock(&hr_cq->lock); 1457 1458 return cur + nreq >= hr_wq->wqe_cnt; 1459 } 1460 1461 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) 1462 { 1463 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 1464 unsigned int reserved_from_bot; 1465 unsigned int i; 1466 1467 qp_table->idx_table.spare_idx = kcalloc(hr_dev->caps.num_qps, 1468 sizeof(u32), GFP_KERNEL); 1469 if (!qp_table->idx_table.spare_idx) 1470 return -ENOMEM; 1471 1472 mutex_init(&qp_table->scc_mutex); 1473 mutex_init(&qp_table->bank_mutex); 1474 xa_init(&hr_dev->qp_table_xa); 1475 1476 reserved_from_bot = hr_dev->caps.reserved_qps; 1477 1478 for (i = 0; i < reserved_from_bot; i++) { 1479 hr_dev->qp_table.bank[get_qp_bankid(i)].inuse++; 1480 hr_dev->qp_table.bank[get_qp_bankid(i)].min++; 1481 } 1482 1483 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) { 1484 ida_init(&hr_dev->qp_table.bank[i].ida); 1485 hr_dev->qp_table.bank[i].max = hr_dev->caps.num_qps / 1486 HNS_ROCE_QP_BANK_NUM - 1; 1487 hr_dev->qp_table.bank[i].next = hr_dev->qp_table.bank[i].min; 1488 } 1489 1490 return 0; 1491 } 1492 1493 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) 1494 { 1495 int i; 1496 1497 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) 1498 ida_destroy(&hr_dev->qp_table.bank[i].ida); 1499 kfree(hr_dev->qp_table.idx_table.spare_idx); 1500 } 1501