1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/pci.h> 35 #include <linux/platform_device.h> 36 #include <rdma/ib_addr.h> 37 #include <rdma/ib_umem.h> 38 #include <rdma/uverbs_ioctl.h> 39 #include "hns_roce_common.h" 40 #include "hns_roce_device.h" 41 #include "hns_roce_hem.h" 42 43 static void flush_work_handle(struct work_struct *work) 44 { 45 struct hns_roce_work *flush_work = container_of(work, 46 struct hns_roce_work, work); 47 struct hns_roce_qp *hr_qp = container_of(flush_work, 48 struct hns_roce_qp, flush_work); 49 struct device *dev = flush_work->hr_dev->dev; 50 struct ib_qp_attr attr; 51 int attr_mask; 52 int ret; 53 54 attr_mask = IB_QP_STATE; 55 attr.qp_state = IB_QPS_ERR; 56 57 if (test_and_clear_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag)) { 58 ret = hns_roce_modify_qp(&hr_qp->ibqp, &attr, attr_mask, NULL); 59 if (ret) 60 dev_err(dev, "Modify QP to error state failed(%d) during CQE flush\n", 61 ret); 62 } 63 64 /* 65 * make sure we signal QP destroy leg that flush QP was completed 66 * so that it can safely proceed ahead now and destroy QP 67 */ 68 if (refcount_dec_and_test(&hr_qp->refcount)) 69 complete(&hr_qp->free); 70 } 71 72 void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 73 { 74 struct hns_roce_work *flush_work = &hr_qp->flush_work; 75 76 flush_work->hr_dev = hr_dev; 77 INIT_WORK(&flush_work->work, flush_work_handle); 78 refcount_inc(&hr_qp->refcount); 79 queue_work(hr_dev->irq_workq, &flush_work->work); 80 } 81 82 void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp) 83 { 84 /* 85 * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state 86 * gets into errored mode. Hence, as a workaround to this 87 * hardware limitation, driver needs to assist in flushing. But 88 * the flushing operation uses mailbox to convey the QP state to 89 * the hardware and which can sleep due to the mutex protection 90 * around the mailbox calls. Hence, use the deferred flush for 91 * now. 92 */ 93 if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag)) 94 init_flush_work(dev, qp); 95 } 96 97 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) 98 { 99 struct device *dev = hr_dev->dev; 100 struct hns_roce_qp *qp; 101 102 xa_lock(&hr_dev->qp_table_xa); 103 qp = __hns_roce_qp_lookup(hr_dev, qpn); 104 if (qp) 105 refcount_inc(&qp->refcount); 106 xa_unlock(&hr_dev->qp_table_xa); 107 108 if (!qp) { 109 dev_warn(dev, "Async event for bogus QP %08x\n", qpn); 110 return; 111 } 112 113 if (hr_dev->hw_rev != HNS_ROCE_HW_VER1 && 114 (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR || 115 event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR || 116 event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR || 117 event_type == HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION || 118 event_type == HNS_ROCE_EVENT_TYPE_INVALID_XRCETH)) { 119 qp->state = IB_QPS_ERR; 120 121 flush_cqe(hr_dev, qp); 122 } 123 124 qp->event(qp, (enum hns_roce_event)event_type); 125 126 if (refcount_dec_and_test(&qp->refcount)) 127 complete(&qp->free); 128 } 129 130 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, 131 enum hns_roce_event type) 132 { 133 struct ib_qp *ibqp = &hr_qp->ibqp; 134 struct ib_event event; 135 136 if (ibqp->event_handler) { 137 event.device = ibqp->device; 138 event.element.qp = ibqp; 139 switch (type) { 140 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 141 event.event = IB_EVENT_PATH_MIG; 142 break; 143 case HNS_ROCE_EVENT_TYPE_COMM_EST: 144 event.event = IB_EVENT_COMM_EST; 145 break; 146 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 147 event.event = IB_EVENT_SQ_DRAINED; 148 break; 149 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 150 event.event = IB_EVENT_QP_LAST_WQE_REACHED; 151 break; 152 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 153 event.event = IB_EVENT_QP_FATAL; 154 break; 155 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 156 event.event = IB_EVENT_PATH_MIG_ERR; 157 break; 158 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 159 event.event = IB_EVENT_QP_REQ_ERR; 160 break; 161 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 162 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION: 163 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH: 164 event.event = IB_EVENT_QP_ACCESS_ERR; 165 break; 166 default: 167 dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n", 168 type, hr_qp->qpn); 169 return; 170 } 171 ibqp->event_handler(&event, ibqp->qp_context); 172 } 173 } 174 175 static u8 get_least_load_bankid_for_qp(struct hns_roce_bank *bank) 176 { 177 u32 least_load = bank[0].inuse; 178 u8 bankid = 0; 179 u32 bankcnt; 180 u8 i; 181 182 for (i = 1; i < HNS_ROCE_QP_BANK_NUM; i++) { 183 bankcnt = bank[i].inuse; 184 if (bankcnt < least_load) { 185 least_load = bankcnt; 186 bankid = i; 187 } 188 } 189 190 return bankid; 191 } 192 193 static int alloc_qpn_with_bankid(struct hns_roce_bank *bank, u8 bankid, 194 unsigned long *qpn) 195 { 196 int id; 197 198 id = ida_alloc_range(&bank->ida, bank->next, bank->max, GFP_KERNEL); 199 if (id < 0) { 200 id = ida_alloc_range(&bank->ida, bank->min, bank->max, 201 GFP_KERNEL); 202 if (id < 0) 203 return id; 204 } 205 206 /* the QPN should keep increasing until the max value is reached. */ 207 bank->next = (id + 1) > bank->max ? bank->min : id + 1; 208 209 /* the lower 3 bits is bankid */ 210 *qpn = (id << 3) | bankid; 211 212 return 0; 213 } 214 static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 215 { 216 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 217 unsigned long num = 0; 218 u8 bankid; 219 int ret; 220 221 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) { 222 /* when hw version is v1, the sqpn is allocated */ 223 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) 224 num = HNS_ROCE_MAX_PORTS + 225 hr_dev->iboe.phy_port[hr_qp->port]; 226 else 227 num = 1; 228 229 hr_qp->doorbell_qpn = 1; 230 } else { 231 mutex_lock(&qp_table->bank_mutex); 232 bankid = get_least_load_bankid_for_qp(qp_table->bank); 233 234 ret = alloc_qpn_with_bankid(&qp_table->bank[bankid], bankid, 235 &num); 236 if (ret) { 237 ibdev_err(&hr_dev->ib_dev, 238 "failed to alloc QPN, ret = %d\n", ret); 239 mutex_unlock(&qp_table->bank_mutex); 240 return ret; 241 } 242 243 qp_table->bank[bankid].inuse++; 244 mutex_unlock(&qp_table->bank_mutex); 245 246 hr_qp->doorbell_qpn = (u32)num; 247 } 248 249 hr_qp->qpn = num; 250 251 return 0; 252 } 253 254 enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state) 255 { 256 switch (state) { 257 case IB_QPS_RESET: 258 return HNS_ROCE_QP_STATE_RST; 259 case IB_QPS_INIT: 260 return HNS_ROCE_QP_STATE_INIT; 261 case IB_QPS_RTR: 262 return HNS_ROCE_QP_STATE_RTR; 263 case IB_QPS_RTS: 264 return HNS_ROCE_QP_STATE_RTS; 265 case IB_QPS_SQD: 266 return HNS_ROCE_QP_STATE_SQD; 267 case IB_QPS_ERR: 268 return HNS_ROCE_QP_STATE_ERR; 269 default: 270 return HNS_ROCE_QP_NUM_STATE; 271 } 272 } 273 274 static void add_qp_to_list(struct hns_roce_dev *hr_dev, 275 struct hns_roce_qp *hr_qp, 276 struct ib_cq *send_cq, struct ib_cq *recv_cq) 277 { 278 struct hns_roce_cq *hr_send_cq, *hr_recv_cq; 279 unsigned long flags; 280 281 hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL; 282 hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL; 283 284 spin_lock_irqsave(&hr_dev->qp_list_lock, flags); 285 hns_roce_lock_cqs(hr_send_cq, hr_recv_cq); 286 287 list_add_tail(&hr_qp->node, &hr_dev->qp_list); 288 if (hr_send_cq) 289 list_add_tail(&hr_qp->sq_node, &hr_send_cq->sq_list); 290 if (hr_recv_cq) 291 list_add_tail(&hr_qp->rq_node, &hr_recv_cq->rq_list); 292 293 hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq); 294 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); 295 } 296 297 static int hns_roce_qp_store(struct hns_roce_dev *hr_dev, 298 struct hns_roce_qp *hr_qp, 299 struct ib_qp_init_attr *init_attr) 300 { 301 struct xarray *xa = &hr_dev->qp_table_xa; 302 int ret; 303 304 if (!hr_qp->qpn) 305 return -EINVAL; 306 307 ret = xa_err(xa_store_irq(xa, hr_qp->qpn, hr_qp, GFP_KERNEL)); 308 if (ret) 309 dev_err(hr_dev->dev, "Failed to xa store for QPC\n"); 310 else 311 /* add QP to device's QP list for softwc */ 312 add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq, 313 init_attr->recv_cq); 314 315 return ret; 316 } 317 318 static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 319 { 320 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 321 struct device *dev = hr_dev->dev; 322 int ret; 323 324 if (!hr_qp->qpn) 325 return -EINVAL; 326 327 /* In v1 engine, GSI QP context is saved in the RoCE hw's register */ 328 if (hr_qp->ibqp.qp_type == IB_QPT_GSI && 329 hr_dev->hw_rev == HNS_ROCE_HW_VER1) 330 return 0; 331 332 /* Alloc memory for QPC */ 333 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn); 334 if (ret) { 335 dev_err(dev, "Failed to get QPC table\n"); 336 goto err_out; 337 } 338 339 /* Alloc memory for IRRL */ 340 ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 341 if (ret) { 342 dev_err(dev, "Failed to get IRRL table\n"); 343 goto err_put_qp; 344 } 345 346 if (hr_dev->caps.trrl_entry_sz) { 347 /* Alloc memory for TRRL */ 348 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table, 349 hr_qp->qpn); 350 if (ret) { 351 dev_err(dev, "Failed to get TRRL table\n"); 352 goto err_put_irrl; 353 } 354 } 355 356 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 357 /* Alloc memory for SCC CTX */ 358 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table, 359 hr_qp->qpn); 360 if (ret) { 361 dev_err(dev, "Failed to get SCC CTX table\n"); 362 goto err_put_trrl; 363 } 364 } 365 366 return 0; 367 368 err_put_trrl: 369 if (hr_dev->caps.trrl_entry_sz) 370 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 371 372 err_put_irrl: 373 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 374 375 err_put_qp: 376 hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); 377 378 err_out: 379 return ret; 380 } 381 382 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 383 { 384 struct xarray *xa = &hr_dev->qp_table_xa; 385 unsigned long flags; 386 387 list_del(&hr_qp->node); 388 389 if (hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT) 390 list_del(&hr_qp->sq_node); 391 392 if (hr_qp->ibqp.qp_type != IB_QPT_XRC_INI && 393 hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT) 394 list_del(&hr_qp->rq_node); 395 396 xa_lock_irqsave(xa, flags); 397 __xa_erase(xa, hr_qp->qpn); 398 xa_unlock_irqrestore(xa, flags); 399 } 400 401 static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 402 { 403 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 404 405 /* In v1 engine, GSI QP context is saved in the RoCE hw's register */ 406 if (hr_qp->ibqp.qp_type == IB_QPT_GSI && 407 hr_dev->hw_rev == HNS_ROCE_HW_VER1) 408 return; 409 410 if (hr_dev->caps.trrl_entry_sz) 411 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 412 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 413 } 414 415 static inline u8 get_qp_bankid(unsigned long qpn) 416 { 417 /* The lower 3 bits of QPN are used to hash to different banks */ 418 return (u8)(qpn & GENMASK(2, 0)); 419 } 420 421 static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 422 { 423 u8 bankid; 424 425 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) 426 return; 427 428 if (hr_qp->qpn < hr_dev->caps.reserved_qps) 429 return; 430 431 bankid = get_qp_bankid(hr_qp->qpn); 432 433 ida_free(&hr_dev->qp_table.bank[bankid].ida, hr_qp->qpn >> 3); 434 435 mutex_lock(&hr_dev->qp_table.bank_mutex); 436 hr_dev->qp_table.bank[bankid].inuse--; 437 mutex_unlock(&hr_dev->qp_table.bank_mutex); 438 } 439 440 static u32 proc_rq_sge(struct hns_roce_dev *dev, struct hns_roce_qp *hr_qp, 441 bool user) 442 { 443 u32 max_sge = dev->caps.max_rq_sg; 444 445 if (dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 446 return max_sge; 447 448 /* Reserve SGEs only for HIP08 in kernel; The userspace driver will 449 * calculate number of max_sge with reserved SGEs when allocating wqe 450 * buf, so there is no need to do this again in kernel. But the number 451 * may exceed the capacity of SGEs recorded in the firmware, so the 452 * kernel driver should just adapt the value accordingly. 453 */ 454 if (user) 455 max_sge = roundup_pow_of_two(max_sge + 1); 456 else 457 hr_qp->rq.rsv_sge = 1; 458 459 return max_sge; 460 } 461 462 static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, 463 struct hns_roce_qp *hr_qp, int has_rq, bool user) 464 { 465 u32 max_sge = proc_rq_sge(hr_dev, hr_qp, user); 466 u32 cnt; 467 468 /* If srq exist, set zero for relative number of rq */ 469 if (!has_rq) { 470 hr_qp->rq.wqe_cnt = 0; 471 hr_qp->rq.max_gs = 0; 472 hr_qp->rq_inl_buf.wqe_cnt = 0; 473 cap->max_recv_wr = 0; 474 cap->max_recv_sge = 0; 475 476 return 0; 477 } 478 479 /* Check the validity of QP support capacity */ 480 if (!cap->max_recv_wr || cap->max_recv_wr > hr_dev->caps.max_wqes || 481 cap->max_recv_sge > max_sge) { 482 ibdev_err(&hr_dev->ib_dev, 483 "RQ config error, depth = %u, sge = %u\n", 484 cap->max_recv_wr, cap->max_recv_sge); 485 return -EINVAL; 486 } 487 488 cnt = roundup_pow_of_two(max(cap->max_recv_wr, hr_dev->caps.min_wqes)); 489 if (cnt > hr_dev->caps.max_wqes) { 490 ibdev_err(&hr_dev->ib_dev, "rq depth %u too large\n", 491 cap->max_recv_wr); 492 return -EINVAL; 493 } 494 495 hr_qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge) + 496 hr_qp->rq.rsv_sge); 497 498 if (hr_dev->caps.max_rq_sg <= HNS_ROCE_SGE_IN_WQE) 499 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz); 500 else 501 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz * 502 hr_qp->rq.max_gs); 503 504 hr_qp->rq.wqe_cnt = cnt; 505 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE && 506 hr_qp->ibqp.qp_type != IB_QPT_UD && 507 hr_qp->ibqp.qp_type != IB_QPT_GSI) 508 hr_qp->rq_inl_buf.wqe_cnt = cnt; 509 else 510 hr_qp->rq_inl_buf.wqe_cnt = 0; 511 512 cap->max_recv_wr = cnt; 513 cap->max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge; 514 515 return 0; 516 } 517 518 static u32 get_wqe_ext_sge_cnt(struct hns_roce_qp *qp) 519 { 520 /* GSI/UD QP only has extended sge */ 521 if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) 522 return qp->sq.max_gs; 523 524 if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE) 525 return qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE; 526 527 return 0; 528 } 529 530 static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt, 531 struct hns_roce_qp *hr_qp, struct ib_qp_cap *cap) 532 { 533 u32 total_sge_cnt; 534 u32 wqe_sge_cnt; 535 536 hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT; 537 538 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) { 539 hr_qp->sq.max_gs = HNS_ROCE_SGE_IN_WQE; 540 return; 541 } 542 543 hr_qp->sq.max_gs = max(1U, cap->max_send_sge); 544 545 wqe_sge_cnt = get_wqe_ext_sge_cnt(hr_qp); 546 547 /* If the number of extended sge is not zero, they MUST use the 548 * space of HNS_HW_PAGE_SIZE at least. 549 */ 550 if (wqe_sge_cnt) { 551 total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt); 552 hr_qp->sge.sge_cnt = max(total_sge_cnt, 553 (u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE); 554 } 555 } 556 557 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev, 558 struct ib_qp_cap *cap, 559 struct hns_roce_ib_create_qp *ucmd) 560 { 561 u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz); 562 u8 max_sq_stride = ilog2(roundup_sq_stride); 563 564 /* Sanity check SQ size before proceeding */ 565 if (ucmd->log_sq_stride > max_sq_stride || 566 ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) { 567 ibdev_err(&hr_dev->ib_dev, "failed to check SQ stride size.\n"); 568 return -EINVAL; 569 } 570 571 if (cap->max_send_sge > hr_dev->caps.max_sq_sg) { 572 ibdev_err(&hr_dev->ib_dev, "failed to check SQ SGE size %u.\n", 573 cap->max_send_sge); 574 return -EINVAL; 575 } 576 577 return 0; 578 } 579 580 static int set_user_sq_size(struct hns_roce_dev *hr_dev, 581 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp, 582 struct hns_roce_ib_create_qp *ucmd) 583 { 584 struct ib_device *ibdev = &hr_dev->ib_dev; 585 u32 cnt = 0; 586 int ret; 587 588 if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) || 589 cnt > hr_dev->caps.max_wqes) 590 return -EINVAL; 591 592 ret = check_sq_size_with_integrity(hr_dev, cap, ucmd); 593 if (ret) { 594 ibdev_err(ibdev, "failed to check user SQ size, ret = %d.\n", 595 ret); 596 return ret; 597 } 598 599 set_ext_sge_param(hr_dev, cnt, hr_qp, cap); 600 601 hr_qp->sq.wqe_shift = ucmd->log_sq_stride; 602 hr_qp->sq.wqe_cnt = cnt; 603 604 return 0; 605 } 606 607 static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev, 608 struct hns_roce_qp *hr_qp, 609 struct hns_roce_buf_attr *buf_attr) 610 { 611 int buf_size; 612 int idx = 0; 613 614 hr_qp->buff_size = 0; 615 616 /* SQ WQE */ 617 hr_qp->sq.offset = 0; 618 buf_size = to_hr_hem_entries_size(hr_qp->sq.wqe_cnt, 619 hr_qp->sq.wqe_shift); 620 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 621 buf_attr->region[idx].size = buf_size; 622 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sq_hop_num; 623 idx++; 624 hr_qp->buff_size += buf_size; 625 } 626 627 /* extend SGE WQE in SQ */ 628 hr_qp->sge.offset = hr_qp->buff_size; 629 buf_size = to_hr_hem_entries_size(hr_qp->sge.sge_cnt, 630 hr_qp->sge.sge_shift); 631 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 632 buf_attr->region[idx].size = buf_size; 633 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sge_hop_num; 634 idx++; 635 hr_qp->buff_size += buf_size; 636 } 637 638 /* RQ WQE */ 639 hr_qp->rq.offset = hr_qp->buff_size; 640 buf_size = to_hr_hem_entries_size(hr_qp->rq.wqe_cnt, 641 hr_qp->rq.wqe_shift); 642 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 643 buf_attr->region[idx].size = buf_size; 644 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_rq_hop_num; 645 idx++; 646 hr_qp->buff_size += buf_size; 647 } 648 649 if (hr_qp->buff_size < 1) 650 return -EINVAL; 651 652 buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; 653 buf_attr->region_count = idx; 654 655 return 0; 656 } 657 658 static int set_kernel_sq_size(struct hns_roce_dev *hr_dev, 659 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp) 660 { 661 struct ib_device *ibdev = &hr_dev->ib_dev; 662 u32 cnt; 663 664 if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes || 665 cap->max_send_sge > hr_dev->caps.max_sq_sg) { 666 ibdev_err(ibdev, "failed to check SQ WR or SGE num.\n"); 667 return -EINVAL; 668 } 669 670 cnt = roundup_pow_of_two(max(cap->max_send_wr, hr_dev->caps.min_wqes)); 671 if (cnt > hr_dev->caps.max_wqes) { 672 ibdev_err(ibdev, "failed to check WQE num, WQE num = %u.\n", 673 cnt); 674 return -EINVAL; 675 } 676 677 hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz); 678 hr_qp->sq.wqe_cnt = cnt; 679 680 set_ext_sge_param(hr_dev, cnt, hr_qp, cap); 681 682 /* sync the parameters of kernel QP to user's configuration */ 683 cap->max_send_wr = cnt; 684 cap->max_send_sge = hr_qp->sq.max_gs; 685 686 return 0; 687 } 688 689 static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr) 690 { 691 if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr) 692 return 0; 693 694 return 1; 695 } 696 697 static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr) 698 { 699 if (attr->qp_type == IB_QPT_XRC_INI || 700 attr->qp_type == IB_QPT_XRC_TGT || attr->srq || 701 !attr->cap.max_recv_wr) 702 return 0; 703 704 return 1; 705 } 706 707 static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp, 708 struct ib_qp_init_attr *init_attr) 709 { 710 u32 max_recv_sge = init_attr->cap.max_recv_sge; 711 u32 wqe_cnt = hr_qp->rq_inl_buf.wqe_cnt; 712 struct hns_roce_rinl_wqe *wqe_list; 713 int i; 714 715 /* allocate recv inline buf */ 716 wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe), 717 GFP_KERNEL); 718 if (!wqe_list) 719 goto err; 720 721 /* Allocate a continuous buffer for all inline sge we need */ 722 wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge * 723 sizeof(struct hns_roce_rinl_sge)), 724 GFP_KERNEL); 725 if (!wqe_list[0].sg_list) 726 goto err_wqe_list; 727 728 /* Assign buffers of sg_list to each inline wqe */ 729 for (i = 1; i < wqe_cnt; i++) 730 wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge]; 731 732 hr_qp->rq_inl_buf.wqe_list = wqe_list; 733 734 return 0; 735 736 err_wqe_list: 737 kfree(wqe_list); 738 739 err: 740 return -ENOMEM; 741 } 742 743 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp) 744 { 745 if (hr_qp->rq_inl_buf.wqe_list) 746 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list); 747 kfree(hr_qp->rq_inl_buf.wqe_list); 748 } 749 750 static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 751 struct ib_qp_init_attr *init_attr, 752 struct ib_udata *udata, unsigned long addr) 753 { 754 struct ib_device *ibdev = &hr_dev->ib_dev; 755 struct hns_roce_buf_attr buf_attr = {}; 756 int ret; 757 758 if (!udata && hr_qp->rq_inl_buf.wqe_cnt) { 759 ret = alloc_rq_inline_buf(hr_qp, init_attr); 760 if (ret) { 761 ibdev_err(ibdev, 762 "failed to alloc inline buf, ret = %d.\n", 763 ret); 764 return ret; 765 } 766 } else { 767 hr_qp->rq_inl_buf.wqe_list = NULL; 768 } 769 770 ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr); 771 if (ret) { 772 ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret); 773 goto err_inline; 774 } 775 ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr, 776 PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz, 777 udata, addr); 778 if (ret) { 779 ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret); 780 goto err_inline; 781 } 782 783 return 0; 784 err_inline: 785 free_rq_inline_buf(hr_qp); 786 787 return ret; 788 } 789 790 static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 791 { 792 hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr); 793 free_rq_inline_buf(hr_qp); 794 } 795 796 static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev, 797 struct ib_qp_init_attr *init_attr, 798 struct ib_udata *udata, 799 struct hns_roce_ib_create_qp_resp *resp, 800 struct hns_roce_ib_create_qp *ucmd) 801 { 802 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 803 udata->outlen >= offsetofend(typeof(*resp), cap_flags) && 804 hns_roce_qp_has_sq(init_attr) && 805 udata->inlen >= offsetofend(typeof(*ucmd), sdb_addr)); 806 } 807 808 static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev, 809 struct ib_qp_init_attr *init_attr, 810 struct ib_udata *udata, 811 struct hns_roce_ib_create_qp_resp *resp) 812 { 813 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 814 udata->outlen >= offsetofend(typeof(*resp), cap_flags) && 815 hns_roce_qp_has_rq(init_attr)); 816 } 817 818 static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev, 819 struct ib_qp_init_attr *init_attr) 820 { 821 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 822 hns_roce_qp_has_rq(init_attr)); 823 } 824 825 static int alloc_user_qp_db(struct hns_roce_dev *hr_dev, 826 struct hns_roce_qp *hr_qp, 827 struct ib_qp_init_attr *init_attr, 828 struct ib_udata *udata, 829 struct hns_roce_ib_create_qp *ucmd, 830 struct hns_roce_ib_create_qp_resp *resp) 831 { 832 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata, 833 struct hns_roce_ucontext, ibucontext); 834 struct ib_device *ibdev = &hr_dev->ib_dev; 835 int ret; 836 837 if (user_qp_has_sdb(hr_dev, init_attr, udata, resp, ucmd)) { 838 ret = hns_roce_db_map_user(uctx, ucmd->sdb_addr, &hr_qp->sdb); 839 if (ret) { 840 ibdev_err(ibdev, 841 "failed to map user SQ doorbell, ret = %d.\n", 842 ret); 843 goto err_out; 844 } 845 hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB; 846 } 847 848 if (user_qp_has_rdb(hr_dev, init_attr, udata, resp)) { 849 ret = hns_roce_db_map_user(uctx, ucmd->db_addr, &hr_qp->rdb); 850 if (ret) { 851 ibdev_err(ibdev, 852 "failed to map user RQ doorbell, ret = %d.\n", 853 ret); 854 goto err_sdb; 855 } 856 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; 857 } 858 859 return 0; 860 861 err_sdb: 862 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) 863 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 864 err_out: 865 return ret; 866 } 867 868 static int alloc_kernel_qp_db(struct hns_roce_dev *hr_dev, 869 struct hns_roce_qp *hr_qp, 870 struct ib_qp_init_attr *init_attr) 871 { 872 struct ib_device *ibdev = &hr_dev->ib_dev; 873 int ret; 874 875 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 876 hr_qp->sq.db_reg = hr_dev->mem_base + 877 HNS_ROCE_DWQE_SIZE * hr_qp->qpn; 878 else 879 hr_qp->sq.db_reg = hr_dev->reg_base + hr_dev->sdb_offset + 880 DB_REG_OFFSET * hr_dev->priv_uar.index; 881 882 hr_qp->rq.db_reg = hr_dev->reg_base + hr_dev->odb_offset + 883 DB_REG_OFFSET * hr_dev->priv_uar.index; 884 885 if (kernel_qp_has_rdb(hr_dev, init_attr)) { 886 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0); 887 if (ret) { 888 ibdev_err(ibdev, 889 "failed to alloc kernel RQ doorbell, ret = %d.\n", 890 ret); 891 return ret; 892 } 893 *hr_qp->rdb.db_record = 0; 894 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; 895 } 896 897 return 0; 898 } 899 900 static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 901 struct ib_qp_init_attr *init_attr, 902 struct ib_udata *udata, 903 struct hns_roce_ib_create_qp *ucmd, 904 struct hns_roce_ib_create_qp_resp *resp) 905 { 906 int ret; 907 908 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SDI_MODE) 909 hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB; 910 911 if (udata) { 912 ret = alloc_user_qp_db(hr_dev, hr_qp, init_attr, udata, ucmd, 913 resp); 914 if (ret) 915 return ret; 916 } else { 917 ret = alloc_kernel_qp_db(hr_dev, hr_qp, init_attr); 918 if (ret) 919 return ret; 920 } 921 922 return 0; 923 } 924 925 static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 926 struct ib_udata *udata) 927 { 928 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( 929 udata, struct hns_roce_ucontext, ibucontext); 930 931 if (udata) { 932 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 933 hns_roce_db_unmap_user(uctx, &hr_qp->rdb); 934 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) 935 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 936 } else { 937 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 938 hns_roce_free_db(hr_dev, &hr_qp->rdb); 939 } 940 } 941 942 static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev, 943 struct hns_roce_qp *hr_qp) 944 { 945 struct ib_device *ibdev = &hr_dev->ib_dev; 946 u64 *sq_wrid = NULL; 947 u64 *rq_wrid = NULL; 948 int ret; 949 950 sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL); 951 if (ZERO_OR_NULL_PTR(sq_wrid)) { 952 ibdev_err(ibdev, "failed to alloc SQ wrid.\n"); 953 return -ENOMEM; 954 } 955 956 if (hr_qp->rq.wqe_cnt) { 957 rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL); 958 if (ZERO_OR_NULL_PTR(rq_wrid)) { 959 ibdev_err(ibdev, "failed to alloc RQ wrid.\n"); 960 ret = -ENOMEM; 961 goto err_sq; 962 } 963 } 964 965 hr_qp->sq.wrid = sq_wrid; 966 hr_qp->rq.wrid = rq_wrid; 967 return 0; 968 err_sq: 969 kfree(sq_wrid); 970 971 return ret; 972 } 973 974 static void free_kernel_wrid(struct hns_roce_qp *hr_qp) 975 { 976 kfree(hr_qp->rq.wrid); 977 kfree(hr_qp->sq.wrid); 978 } 979 980 static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 981 struct ib_qp_init_attr *init_attr, 982 struct ib_udata *udata, 983 struct hns_roce_ib_create_qp *ucmd) 984 { 985 struct ib_device *ibdev = &hr_dev->ib_dev; 986 int ret; 987 988 if (init_attr->cap.max_inline_data > hr_dev->caps.max_sq_inline) 989 init_attr->cap.max_inline_data = hr_dev->caps.max_sq_inline; 990 991 hr_qp->max_inline_data = init_attr->cap.max_inline_data; 992 993 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) 994 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR; 995 else 996 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR; 997 998 ret = set_rq_size(hr_dev, &init_attr->cap, hr_qp, 999 hns_roce_qp_has_rq(init_attr), !!udata); 1000 if (ret) { 1001 ibdev_err(ibdev, "failed to set user RQ size, ret = %d.\n", 1002 ret); 1003 return ret; 1004 } 1005 1006 if (udata) { 1007 ret = ib_copy_from_udata(ucmd, udata, 1008 min(udata->inlen, sizeof(*ucmd))); 1009 if (ret) { 1010 ibdev_err(ibdev, 1011 "failed to copy QP ucmd, ret = %d\n", ret); 1012 return ret; 1013 } 1014 1015 ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd); 1016 if (ret) 1017 ibdev_err(ibdev, 1018 "failed to set user SQ size, ret = %d.\n", 1019 ret); 1020 } else { 1021 ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp); 1022 if (ret) 1023 ibdev_err(ibdev, 1024 "failed to set kernel SQ size, ret = %d.\n", 1025 ret); 1026 } 1027 1028 return ret; 1029 } 1030 1031 static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, 1032 struct ib_pd *ib_pd, 1033 struct ib_qp_init_attr *init_attr, 1034 struct ib_udata *udata, 1035 struct hns_roce_qp *hr_qp) 1036 { 1037 struct hns_roce_ib_create_qp_resp resp = {}; 1038 struct ib_device *ibdev = &hr_dev->ib_dev; 1039 struct hns_roce_ib_create_qp ucmd; 1040 int ret; 1041 1042 mutex_init(&hr_qp->mutex); 1043 spin_lock_init(&hr_qp->sq.lock); 1044 spin_lock_init(&hr_qp->rq.lock); 1045 1046 hr_qp->state = IB_QPS_RESET; 1047 hr_qp->flush_flag = 0; 1048 1049 if (init_attr->create_flags) 1050 return -EOPNOTSUPP; 1051 1052 ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd); 1053 if (ret) { 1054 ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret); 1055 return ret; 1056 } 1057 1058 if (!udata) { 1059 ret = alloc_kernel_wrid(hr_dev, hr_qp); 1060 if (ret) { 1061 ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n", 1062 ret); 1063 return ret; 1064 } 1065 } 1066 1067 ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr); 1068 if (ret) { 1069 ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret); 1070 goto err_buf; 1071 } 1072 1073 ret = alloc_qpn(hr_dev, hr_qp); 1074 if (ret) { 1075 ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret); 1076 goto err_qpn; 1077 } 1078 1079 ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp); 1080 if (ret) { 1081 ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n", 1082 ret); 1083 goto err_db; 1084 } 1085 1086 ret = alloc_qpc(hr_dev, hr_qp); 1087 if (ret) { 1088 ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n", 1089 ret); 1090 goto err_qpc; 1091 } 1092 1093 ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr); 1094 if (ret) { 1095 ibdev_err(ibdev, "failed to store QP, ret = %d.\n", ret); 1096 goto err_store; 1097 } 1098 1099 if (udata) { 1100 resp.cap_flags = hr_qp->en_flags; 1101 ret = ib_copy_to_udata(udata, &resp, 1102 min(udata->outlen, sizeof(resp))); 1103 if (ret) { 1104 ibdev_err(ibdev, "copy qp resp failed!\n"); 1105 goto err_store; 1106 } 1107 } 1108 1109 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 1110 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp); 1111 if (ret) 1112 goto err_flow_ctrl; 1113 } 1114 1115 hr_qp->ibqp.qp_num = hr_qp->qpn; 1116 hr_qp->event = hns_roce_ib_qp_event; 1117 refcount_set(&hr_qp->refcount, 1); 1118 init_completion(&hr_qp->free); 1119 1120 return 0; 1121 1122 err_flow_ctrl: 1123 hns_roce_qp_remove(hr_dev, hr_qp); 1124 err_store: 1125 free_qpc(hr_dev, hr_qp); 1126 err_qpc: 1127 free_qp_db(hr_dev, hr_qp, udata); 1128 err_db: 1129 free_qpn(hr_dev, hr_qp); 1130 err_qpn: 1131 free_qp_buf(hr_dev, hr_qp); 1132 err_buf: 1133 free_kernel_wrid(hr_qp); 1134 return ret; 1135 } 1136 1137 void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 1138 struct ib_udata *udata) 1139 { 1140 if (refcount_dec_and_test(&hr_qp->refcount)) 1141 complete(&hr_qp->free); 1142 wait_for_completion(&hr_qp->free); 1143 1144 free_qpc(hr_dev, hr_qp); 1145 free_qpn(hr_dev, hr_qp); 1146 free_qp_buf(hr_dev, hr_qp); 1147 free_kernel_wrid(hr_qp); 1148 free_qp_db(hr_dev, hr_qp, udata); 1149 } 1150 1151 static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type, 1152 bool is_user) 1153 { 1154 switch (type) { 1155 case IB_QPT_XRC_INI: 1156 case IB_QPT_XRC_TGT: 1157 if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC)) 1158 goto out; 1159 break; 1160 case IB_QPT_UD: 1161 if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 && 1162 is_user) 1163 goto out; 1164 break; 1165 case IB_QPT_RC: 1166 case IB_QPT_GSI: 1167 break; 1168 default: 1169 goto out; 1170 } 1171 1172 return 0; 1173 1174 out: 1175 ibdev_err(&hr_dev->ib_dev, "not support QP type %d\n", type); 1176 1177 return -EOPNOTSUPP; 1178 } 1179 1180 int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, 1181 struct ib_udata *udata) 1182 { 1183 struct ib_device *ibdev = qp->device; 1184 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); 1185 struct hns_roce_qp *hr_qp = to_hr_qp(qp); 1186 struct ib_pd *pd = qp->pd; 1187 int ret; 1188 1189 ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata); 1190 if (ret) 1191 return ret; 1192 1193 if (init_attr->qp_type == IB_QPT_XRC_TGT) 1194 hr_qp->xrcdn = to_hr_xrcd(init_attr->xrcd)->xrcdn; 1195 1196 if (init_attr->qp_type == IB_QPT_GSI) { 1197 hr_qp->port = init_attr->port_num - 1; 1198 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; 1199 } 1200 1201 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp); 1202 if (ret) 1203 ibdev_err(ibdev, "Create QP type 0x%x failed(%d)\n", 1204 init_attr->qp_type, ret); 1205 1206 return ret; 1207 } 1208 1209 int to_hr_qp_type(int qp_type) 1210 { 1211 switch (qp_type) { 1212 case IB_QPT_RC: 1213 return SERV_TYPE_RC; 1214 case IB_QPT_UD: 1215 case IB_QPT_GSI: 1216 return SERV_TYPE_UD; 1217 case IB_QPT_XRC_INI: 1218 case IB_QPT_XRC_TGT: 1219 return SERV_TYPE_XRC; 1220 default: 1221 return -1; 1222 } 1223 } 1224 1225 static int check_mtu_validate(struct hns_roce_dev *hr_dev, 1226 struct hns_roce_qp *hr_qp, 1227 struct ib_qp_attr *attr, int attr_mask) 1228 { 1229 enum ib_mtu active_mtu; 1230 int p; 1231 1232 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1233 active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu); 1234 1235 if ((hr_dev->caps.max_mtu >= IB_MTU_2048 && 1236 attr->path_mtu > hr_dev->caps.max_mtu) || 1237 attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) { 1238 ibdev_err(&hr_dev->ib_dev, 1239 "attr path_mtu(%d)invalid while modify qp", 1240 attr->path_mtu); 1241 return -EINVAL; 1242 } 1243 1244 return 0; 1245 } 1246 1247 static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1248 int attr_mask) 1249 { 1250 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1251 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1252 int p; 1253 1254 if ((attr_mask & IB_QP_PORT) && 1255 (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) { 1256 ibdev_err(&hr_dev->ib_dev, "invalid attr, port_num = %u.\n", 1257 attr->port_num); 1258 return -EINVAL; 1259 } 1260 1261 if (attr_mask & IB_QP_PKEY_INDEX) { 1262 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1263 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) { 1264 ibdev_err(&hr_dev->ib_dev, 1265 "invalid attr, pkey_index = %u.\n", 1266 attr->pkey_index); 1267 return -EINVAL; 1268 } 1269 } 1270 1271 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && 1272 attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) { 1273 ibdev_err(&hr_dev->ib_dev, 1274 "invalid attr, max_rd_atomic = %u.\n", 1275 attr->max_rd_atomic); 1276 return -EINVAL; 1277 } 1278 1279 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && 1280 attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) { 1281 ibdev_err(&hr_dev->ib_dev, 1282 "invalid attr, max_dest_rd_atomic = %u.\n", 1283 attr->max_dest_rd_atomic); 1284 return -EINVAL; 1285 } 1286 1287 if (attr_mask & IB_QP_PATH_MTU) 1288 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask); 1289 1290 return 0; 1291 } 1292 1293 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1294 int attr_mask, struct ib_udata *udata) 1295 { 1296 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1297 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1298 enum ib_qp_state cur_state, new_state; 1299 int ret = -EINVAL; 1300 1301 mutex_lock(&hr_qp->mutex); 1302 1303 if (attr_mask & IB_QP_CUR_STATE && attr->cur_qp_state != hr_qp->state) 1304 goto out; 1305 1306 cur_state = hr_qp->state; 1307 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; 1308 1309 if (ibqp->uobject && 1310 (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) { 1311 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) { 1312 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr); 1313 1314 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 1315 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr); 1316 } else { 1317 ibdev_warn(&hr_dev->ib_dev, 1318 "flush cqe is not supported in userspace!\n"); 1319 goto out; 1320 } 1321 } 1322 1323 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, 1324 attr_mask)) { 1325 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n"); 1326 goto out; 1327 } 1328 1329 ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask); 1330 if (ret) 1331 goto out; 1332 1333 if (cur_state == new_state && cur_state == IB_QPS_RESET) 1334 goto out; 1335 1336 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, 1337 new_state); 1338 1339 out: 1340 mutex_unlock(&hr_qp->mutex); 1341 1342 return ret; 1343 } 1344 1345 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) 1346 __acquires(&send_cq->lock) __acquires(&recv_cq->lock) 1347 { 1348 if (unlikely(send_cq == NULL && recv_cq == NULL)) { 1349 __acquire(&send_cq->lock); 1350 __acquire(&recv_cq->lock); 1351 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { 1352 spin_lock_irq(&send_cq->lock); 1353 __acquire(&recv_cq->lock); 1354 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { 1355 spin_lock_irq(&recv_cq->lock); 1356 __acquire(&send_cq->lock); 1357 } else if (send_cq == recv_cq) { 1358 spin_lock_irq(&send_cq->lock); 1359 __acquire(&recv_cq->lock); 1360 } else if (send_cq->cqn < recv_cq->cqn) { 1361 spin_lock_irq(&send_cq->lock); 1362 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); 1363 } else { 1364 spin_lock_irq(&recv_cq->lock); 1365 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); 1366 } 1367 } 1368 1369 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, 1370 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock) 1371 __releases(&recv_cq->lock) 1372 { 1373 if (unlikely(send_cq == NULL && recv_cq == NULL)) { 1374 __release(&recv_cq->lock); 1375 __release(&send_cq->lock); 1376 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { 1377 __release(&recv_cq->lock); 1378 spin_unlock(&send_cq->lock); 1379 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { 1380 __release(&send_cq->lock); 1381 spin_unlock(&recv_cq->lock); 1382 } else if (send_cq == recv_cq) { 1383 __release(&recv_cq->lock); 1384 spin_unlock_irq(&send_cq->lock); 1385 } else if (send_cq->cqn < recv_cq->cqn) { 1386 spin_unlock(&recv_cq->lock); 1387 spin_unlock_irq(&send_cq->lock); 1388 } else { 1389 spin_unlock(&send_cq->lock); 1390 spin_unlock_irq(&recv_cq->lock); 1391 } 1392 } 1393 1394 static inline void *get_wqe(struct hns_roce_qp *hr_qp, int offset) 1395 { 1396 return hns_roce_buf_offset(hr_qp->mtr.kmem, offset); 1397 } 1398 1399 void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n) 1400 { 1401 return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift)); 1402 } 1403 1404 void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n) 1405 { 1406 return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift)); 1407 } 1408 1409 void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n) 1410 { 1411 return get_wqe(hr_qp, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift)); 1412 } 1413 1414 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq, 1415 struct ib_cq *ib_cq) 1416 { 1417 struct hns_roce_cq *hr_cq; 1418 u32 cur; 1419 1420 cur = hr_wq->head - hr_wq->tail; 1421 if (likely(cur + nreq < hr_wq->wqe_cnt)) 1422 return false; 1423 1424 hr_cq = to_hr_cq(ib_cq); 1425 spin_lock(&hr_cq->lock); 1426 cur = hr_wq->head - hr_wq->tail; 1427 spin_unlock(&hr_cq->lock); 1428 1429 return cur + nreq >= hr_wq->wqe_cnt; 1430 } 1431 1432 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) 1433 { 1434 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 1435 unsigned int reserved_from_bot; 1436 unsigned int i; 1437 1438 qp_table->idx_table.spare_idx = kcalloc(hr_dev->caps.num_qps, 1439 sizeof(u32), GFP_KERNEL); 1440 if (!qp_table->idx_table.spare_idx) 1441 return -ENOMEM; 1442 1443 mutex_init(&qp_table->scc_mutex); 1444 mutex_init(&qp_table->bank_mutex); 1445 xa_init(&hr_dev->qp_table_xa); 1446 1447 reserved_from_bot = hr_dev->caps.reserved_qps; 1448 1449 for (i = 0; i < reserved_from_bot; i++) { 1450 hr_dev->qp_table.bank[get_qp_bankid(i)].inuse++; 1451 hr_dev->qp_table.bank[get_qp_bankid(i)].min++; 1452 } 1453 1454 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) { 1455 ida_init(&hr_dev->qp_table.bank[i].ida); 1456 hr_dev->qp_table.bank[i].max = hr_dev->caps.num_qps / 1457 HNS_ROCE_QP_BANK_NUM - 1; 1458 hr_dev->qp_table.bank[i].next = hr_dev->qp_table.bank[i].min; 1459 } 1460 1461 return 0; 1462 } 1463 1464 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) 1465 { 1466 int i; 1467 1468 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) 1469 ida_destroy(&hr_dev->qp_table.bank[i].ida); 1470 kfree(hr_dev->qp_table.idx_table.spare_idx); 1471 } 1472