1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/pci.h> 35 #include <linux/platform_device.h> 36 #include <rdma/ib_addr.h> 37 #include <rdma/ib_umem.h> 38 #include <rdma/uverbs_ioctl.h> 39 #include "hns_roce_common.h" 40 #include "hns_roce_device.h" 41 #include "hns_roce_hem.h" 42 43 static void flush_work_handle(struct work_struct *work) 44 { 45 struct hns_roce_work *flush_work = container_of(work, 46 struct hns_roce_work, work); 47 struct hns_roce_qp *hr_qp = container_of(flush_work, 48 struct hns_roce_qp, flush_work); 49 struct device *dev = flush_work->hr_dev->dev; 50 struct ib_qp_attr attr; 51 int attr_mask; 52 int ret; 53 54 attr_mask = IB_QP_STATE; 55 attr.qp_state = IB_QPS_ERR; 56 57 if (test_and_clear_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag)) { 58 ret = hns_roce_modify_qp(&hr_qp->ibqp, &attr, attr_mask, NULL); 59 if (ret) 60 dev_err(dev, "Modify QP to error state failed(%d) during CQE flush\n", 61 ret); 62 } 63 64 /* 65 * make sure we signal QP destroy leg that flush QP was completed 66 * so that it can safely proceed ahead now and destroy QP 67 */ 68 if (refcount_dec_and_test(&hr_qp->refcount)) 69 complete(&hr_qp->free); 70 } 71 72 void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 73 { 74 struct hns_roce_work *flush_work = &hr_qp->flush_work; 75 76 flush_work->hr_dev = hr_dev; 77 INIT_WORK(&flush_work->work, flush_work_handle); 78 refcount_inc(&hr_qp->refcount); 79 queue_work(hr_dev->irq_workq, &flush_work->work); 80 } 81 82 void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp) 83 { 84 /* 85 * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state 86 * gets into errored mode. Hence, as a workaround to this 87 * hardware limitation, driver needs to assist in flushing. But 88 * the flushing operation uses mailbox to convey the QP state to 89 * the hardware and which can sleep due to the mutex protection 90 * around the mailbox calls. Hence, use the deferred flush for 91 * now. 92 */ 93 if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag)) 94 init_flush_work(dev, qp); 95 } 96 97 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) 98 { 99 struct device *dev = hr_dev->dev; 100 struct hns_roce_qp *qp; 101 102 xa_lock(&hr_dev->qp_table_xa); 103 qp = __hns_roce_qp_lookup(hr_dev, qpn); 104 if (qp) 105 refcount_inc(&qp->refcount); 106 xa_unlock(&hr_dev->qp_table_xa); 107 108 if (!qp) { 109 dev_warn(dev, "Async event for bogus QP %08x\n", qpn); 110 return; 111 } 112 113 if (hr_dev->hw_rev != HNS_ROCE_HW_VER1 && 114 (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR || 115 event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR || 116 event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR || 117 event_type == HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION || 118 event_type == HNS_ROCE_EVENT_TYPE_INVALID_XRCETH)) { 119 qp->state = IB_QPS_ERR; 120 121 flush_cqe(hr_dev, qp); 122 } 123 124 qp->event(qp, (enum hns_roce_event)event_type); 125 126 if (refcount_dec_and_test(&qp->refcount)) 127 complete(&qp->free); 128 } 129 130 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, 131 enum hns_roce_event type) 132 { 133 struct ib_qp *ibqp = &hr_qp->ibqp; 134 struct ib_event event; 135 136 if (ibqp->event_handler) { 137 event.device = ibqp->device; 138 event.element.qp = ibqp; 139 switch (type) { 140 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 141 event.event = IB_EVENT_PATH_MIG; 142 break; 143 case HNS_ROCE_EVENT_TYPE_COMM_EST: 144 event.event = IB_EVENT_COMM_EST; 145 break; 146 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 147 event.event = IB_EVENT_SQ_DRAINED; 148 break; 149 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 150 event.event = IB_EVENT_QP_LAST_WQE_REACHED; 151 break; 152 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 153 event.event = IB_EVENT_QP_FATAL; 154 break; 155 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 156 event.event = IB_EVENT_PATH_MIG_ERR; 157 break; 158 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 159 event.event = IB_EVENT_QP_REQ_ERR; 160 break; 161 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 162 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION: 163 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH: 164 event.event = IB_EVENT_QP_ACCESS_ERR; 165 break; 166 default: 167 dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n", 168 type, hr_qp->qpn); 169 return; 170 } 171 ibqp->event_handler(&event, ibqp->qp_context); 172 } 173 } 174 175 static u8 get_least_load_bankid_for_qp(struct hns_roce_bank *bank) 176 { 177 u32 least_load = bank[0].inuse; 178 u8 bankid = 0; 179 u32 bankcnt; 180 u8 i; 181 182 for (i = 1; i < HNS_ROCE_QP_BANK_NUM; i++) { 183 bankcnt = bank[i].inuse; 184 if (bankcnt < least_load) { 185 least_load = bankcnt; 186 bankid = i; 187 } 188 } 189 190 return bankid; 191 } 192 193 static int alloc_qpn_with_bankid(struct hns_roce_bank *bank, u8 bankid, 194 unsigned long *qpn) 195 { 196 int id; 197 198 id = ida_alloc_range(&bank->ida, bank->next, bank->max, GFP_KERNEL); 199 if (id < 0) { 200 id = ida_alloc_range(&bank->ida, bank->min, bank->max, 201 GFP_KERNEL); 202 if (id < 0) 203 return id; 204 } 205 206 /* the QPN should keep increasing until the max value is reached. */ 207 bank->next = (id + 1) > bank->max ? bank->min : id + 1; 208 209 /* the lower 3 bits is bankid */ 210 *qpn = (id << 3) | bankid; 211 212 return 0; 213 } 214 static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 215 { 216 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 217 unsigned long num = 0; 218 u8 bankid; 219 int ret; 220 221 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) { 222 /* when hw version is v1, the sqpn is allocated */ 223 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) 224 num = HNS_ROCE_MAX_PORTS + 225 hr_dev->iboe.phy_port[hr_qp->port]; 226 else 227 num = 1; 228 229 hr_qp->doorbell_qpn = 1; 230 } else { 231 mutex_lock(&qp_table->bank_mutex); 232 bankid = get_least_load_bankid_for_qp(qp_table->bank); 233 234 ret = alloc_qpn_with_bankid(&qp_table->bank[bankid], bankid, 235 &num); 236 if (ret) { 237 ibdev_err(&hr_dev->ib_dev, 238 "failed to alloc QPN, ret = %d\n", ret); 239 mutex_unlock(&qp_table->bank_mutex); 240 return ret; 241 } 242 243 qp_table->bank[bankid].inuse++; 244 mutex_unlock(&qp_table->bank_mutex); 245 246 hr_qp->doorbell_qpn = (u32)num; 247 } 248 249 hr_qp->qpn = num; 250 251 return 0; 252 } 253 254 enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state) 255 { 256 switch (state) { 257 case IB_QPS_RESET: 258 return HNS_ROCE_QP_STATE_RST; 259 case IB_QPS_INIT: 260 return HNS_ROCE_QP_STATE_INIT; 261 case IB_QPS_RTR: 262 return HNS_ROCE_QP_STATE_RTR; 263 case IB_QPS_RTS: 264 return HNS_ROCE_QP_STATE_RTS; 265 case IB_QPS_SQD: 266 return HNS_ROCE_QP_STATE_SQD; 267 case IB_QPS_ERR: 268 return HNS_ROCE_QP_STATE_ERR; 269 default: 270 return HNS_ROCE_QP_NUM_STATE; 271 } 272 } 273 274 static void add_qp_to_list(struct hns_roce_dev *hr_dev, 275 struct hns_roce_qp *hr_qp, 276 struct ib_cq *send_cq, struct ib_cq *recv_cq) 277 { 278 struct hns_roce_cq *hr_send_cq, *hr_recv_cq; 279 unsigned long flags; 280 281 hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL; 282 hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL; 283 284 spin_lock_irqsave(&hr_dev->qp_list_lock, flags); 285 hns_roce_lock_cqs(hr_send_cq, hr_recv_cq); 286 287 list_add_tail(&hr_qp->node, &hr_dev->qp_list); 288 if (hr_send_cq) 289 list_add_tail(&hr_qp->sq_node, &hr_send_cq->sq_list); 290 if (hr_recv_cq) 291 list_add_tail(&hr_qp->rq_node, &hr_recv_cq->rq_list); 292 293 hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq); 294 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); 295 } 296 297 static int hns_roce_qp_store(struct hns_roce_dev *hr_dev, 298 struct hns_roce_qp *hr_qp, 299 struct ib_qp_init_attr *init_attr) 300 { 301 struct xarray *xa = &hr_dev->qp_table_xa; 302 int ret; 303 304 if (!hr_qp->qpn) 305 return -EINVAL; 306 307 ret = xa_err(xa_store_irq(xa, hr_qp->qpn, hr_qp, GFP_KERNEL)); 308 if (ret) 309 dev_err(hr_dev->dev, "Failed to xa store for QPC\n"); 310 else 311 /* add QP to device's QP list for softwc */ 312 add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq, 313 init_attr->recv_cq); 314 315 return ret; 316 } 317 318 static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 319 { 320 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 321 struct device *dev = hr_dev->dev; 322 int ret; 323 324 if (!hr_qp->qpn) 325 return -EINVAL; 326 327 /* In v1 engine, GSI QP context is saved in the RoCE hw's register */ 328 if (hr_qp->ibqp.qp_type == IB_QPT_GSI && 329 hr_dev->hw_rev == HNS_ROCE_HW_VER1) 330 return 0; 331 332 /* Alloc memory for QPC */ 333 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn); 334 if (ret) { 335 dev_err(dev, "Failed to get QPC table\n"); 336 goto err_out; 337 } 338 339 /* Alloc memory for IRRL */ 340 ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 341 if (ret) { 342 dev_err(dev, "Failed to get IRRL table\n"); 343 goto err_put_qp; 344 } 345 346 if (hr_dev->caps.trrl_entry_sz) { 347 /* Alloc memory for TRRL */ 348 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table, 349 hr_qp->qpn); 350 if (ret) { 351 dev_err(dev, "Failed to get TRRL table\n"); 352 goto err_put_irrl; 353 } 354 } 355 356 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 357 /* Alloc memory for SCC CTX */ 358 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table, 359 hr_qp->qpn); 360 if (ret) { 361 dev_err(dev, "Failed to get SCC CTX table\n"); 362 goto err_put_trrl; 363 } 364 } 365 366 return 0; 367 368 err_put_trrl: 369 if (hr_dev->caps.trrl_entry_sz) 370 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 371 372 err_put_irrl: 373 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 374 375 err_put_qp: 376 hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); 377 378 err_out: 379 return ret; 380 } 381 382 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 383 { 384 struct xarray *xa = &hr_dev->qp_table_xa; 385 unsigned long flags; 386 387 list_del(&hr_qp->node); 388 389 if (hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT) 390 list_del(&hr_qp->sq_node); 391 392 if (hr_qp->ibqp.qp_type != IB_QPT_XRC_INI && 393 hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT) 394 list_del(&hr_qp->rq_node); 395 396 xa_lock_irqsave(xa, flags); 397 __xa_erase(xa, hr_qp->qpn); 398 xa_unlock_irqrestore(xa, flags); 399 } 400 401 static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 402 { 403 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 404 405 /* In v1 engine, GSI QP context is saved in the RoCE hw's register */ 406 if (hr_qp->ibqp.qp_type == IB_QPT_GSI && 407 hr_dev->hw_rev == HNS_ROCE_HW_VER1) 408 return; 409 410 if (hr_dev->caps.trrl_entry_sz) 411 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 412 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 413 } 414 415 static inline u8 get_qp_bankid(unsigned long qpn) 416 { 417 /* The lower 3 bits of QPN are used to hash to different banks */ 418 return (u8)(qpn & GENMASK(2, 0)); 419 } 420 421 static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 422 { 423 u8 bankid; 424 425 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) 426 return; 427 428 if (hr_qp->qpn < hr_dev->caps.reserved_qps) 429 return; 430 431 bankid = get_qp_bankid(hr_qp->qpn); 432 433 ida_free(&hr_dev->qp_table.bank[bankid].ida, hr_qp->qpn >> 3); 434 435 mutex_lock(&hr_dev->qp_table.bank_mutex); 436 hr_dev->qp_table.bank[bankid].inuse--; 437 mutex_unlock(&hr_dev->qp_table.bank_mutex); 438 } 439 440 static u32 proc_rq_sge(struct hns_roce_dev *dev, struct hns_roce_qp *hr_qp, 441 bool user) 442 { 443 u32 max_sge = dev->caps.max_rq_sg; 444 445 if (dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 446 return max_sge; 447 448 /* Reserve SGEs only for HIP08 in kernel; The userspace driver will 449 * calculate number of max_sge with reserved SGEs when allocating wqe 450 * buf, so there is no need to do this again in kernel. But the number 451 * may exceed the capacity of SGEs recorded in the firmware, so the 452 * kernel driver should just adapt the value accordingly. 453 */ 454 if (user) 455 max_sge = roundup_pow_of_two(max_sge + 1); 456 else 457 hr_qp->rq.rsv_sge = 1; 458 459 return max_sge; 460 } 461 462 static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, 463 struct hns_roce_qp *hr_qp, int has_rq, bool user) 464 { 465 u32 max_sge = proc_rq_sge(hr_dev, hr_qp, user); 466 u32 cnt; 467 468 /* If srq exist, set zero for relative number of rq */ 469 if (!has_rq) { 470 hr_qp->rq.wqe_cnt = 0; 471 hr_qp->rq.max_gs = 0; 472 hr_qp->rq_inl_buf.wqe_cnt = 0; 473 cap->max_recv_wr = 0; 474 cap->max_recv_sge = 0; 475 476 return 0; 477 } 478 479 /* Check the validity of QP support capacity */ 480 if (!cap->max_recv_wr || cap->max_recv_wr > hr_dev->caps.max_wqes || 481 cap->max_recv_sge > max_sge) { 482 ibdev_err(&hr_dev->ib_dev, 483 "RQ config error, depth = %u, sge = %u\n", 484 cap->max_recv_wr, cap->max_recv_sge); 485 return -EINVAL; 486 } 487 488 cnt = roundup_pow_of_two(max(cap->max_recv_wr, hr_dev->caps.min_wqes)); 489 if (cnt > hr_dev->caps.max_wqes) { 490 ibdev_err(&hr_dev->ib_dev, "rq depth %u too large\n", 491 cap->max_recv_wr); 492 return -EINVAL; 493 } 494 495 hr_qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge) + 496 hr_qp->rq.rsv_sge); 497 498 if (hr_dev->caps.max_rq_sg <= HNS_ROCE_SGE_IN_WQE) 499 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz); 500 else 501 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz * 502 hr_qp->rq.max_gs); 503 504 hr_qp->rq.wqe_cnt = cnt; 505 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE && 506 hr_qp->ibqp.qp_type != IB_QPT_UD && 507 hr_qp->ibqp.qp_type != IB_QPT_GSI) 508 hr_qp->rq_inl_buf.wqe_cnt = cnt; 509 else 510 hr_qp->rq_inl_buf.wqe_cnt = 0; 511 512 cap->max_recv_wr = cnt; 513 cap->max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge; 514 515 return 0; 516 } 517 518 static u32 get_wqe_ext_sge_cnt(struct hns_roce_qp *qp) 519 { 520 /* GSI/UD QP only has extended sge */ 521 if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) 522 return qp->sq.max_gs; 523 524 if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE) 525 return qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE; 526 527 return 0; 528 } 529 530 static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt, 531 struct hns_roce_qp *hr_qp, struct ib_qp_cap *cap) 532 { 533 u32 total_sge_cnt; 534 u32 wqe_sge_cnt; 535 536 hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT; 537 538 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) { 539 hr_qp->sq.max_gs = HNS_ROCE_SGE_IN_WQE; 540 return; 541 } 542 543 hr_qp->sq.max_gs = max(1U, cap->max_send_sge); 544 545 wqe_sge_cnt = get_wqe_ext_sge_cnt(hr_qp); 546 547 /* If the number of extended sge is not zero, they MUST use the 548 * space of HNS_HW_PAGE_SIZE at least. 549 */ 550 if (wqe_sge_cnt) { 551 total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt); 552 hr_qp->sge.sge_cnt = max(total_sge_cnt, 553 (u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE); 554 } 555 } 556 557 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev, 558 struct ib_qp_cap *cap, 559 struct hns_roce_ib_create_qp *ucmd) 560 { 561 u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz); 562 u8 max_sq_stride = ilog2(roundup_sq_stride); 563 564 /* Sanity check SQ size before proceeding */ 565 if (ucmd->log_sq_stride > max_sq_stride || 566 ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) { 567 ibdev_err(&hr_dev->ib_dev, "failed to check SQ stride size.\n"); 568 return -EINVAL; 569 } 570 571 if (cap->max_send_sge > hr_dev->caps.max_sq_sg) { 572 ibdev_err(&hr_dev->ib_dev, "failed to check SQ SGE size %u.\n", 573 cap->max_send_sge); 574 return -EINVAL; 575 } 576 577 return 0; 578 } 579 580 static int set_user_sq_size(struct hns_roce_dev *hr_dev, 581 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp, 582 struct hns_roce_ib_create_qp *ucmd) 583 { 584 struct ib_device *ibdev = &hr_dev->ib_dev; 585 u32 cnt = 0; 586 int ret; 587 588 if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) || 589 cnt > hr_dev->caps.max_wqes) 590 return -EINVAL; 591 592 ret = check_sq_size_with_integrity(hr_dev, cap, ucmd); 593 if (ret) { 594 ibdev_err(ibdev, "failed to check user SQ size, ret = %d.\n", 595 ret); 596 return ret; 597 } 598 599 set_ext_sge_param(hr_dev, cnt, hr_qp, cap); 600 601 hr_qp->sq.wqe_shift = ucmd->log_sq_stride; 602 hr_qp->sq.wqe_cnt = cnt; 603 604 return 0; 605 } 606 607 static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev, 608 struct hns_roce_qp *hr_qp, 609 struct hns_roce_buf_attr *buf_attr) 610 { 611 int buf_size; 612 int idx = 0; 613 614 hr_qp->buff_size = 0; 615 616 /* SQ WQE */ 617 hr_qp->sq.offset = 0; 618 buf_size = to_hr_hem_entries_size(hr_qp->sq.wqe_cnt, 619 hr_qp->sq.wqe_shift); 620 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 621 buf_attr->region[idx].size = buf_size; 622 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sq_hop_num; 623 idx++; 624 hr_qp->buff_size += buf_size; 625 } 626 627 /* extend SGE WQE in SQ */ 628 hr_qp->sge.offset = hr_qp->buff_size; 629 buf_size = to_hr_hem_entries_size(hr_qp->sge.sge_cnt, 630 hr_qp->sge.sge_shift); 631 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 632 buf_attr->region[idx].size = buf_size; 633 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sge_hop_num; 634 idx++; 635 hr_qp->buff_size += buf_size; 636 } 637 638 /* RQ WQE */ 639 hr_qp->rq.offset = hr_qp->buff_size; 640 buf_size = to_hr_hem_entries_size(hr_qp->rq.wqe_cnt, 641 hr_qp->rq.wqe_shift); 642 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 643 buf_attr->region[idx].size = buf_size; 644 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_rq_hop_num; 645 idx++; 646 hr_qp->buff_size += buf_size; 647 } 648 649 if (hr_qp->buff_size < 1) 650 return -EINVAL; 651 652 buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; 653 buf_attr->region_count = idx; 654 655 return 0; 656 } 657 658 static int set_kernel_sq_size(struct hns_roce_dev *hr_dev, 659 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp) 660 { 661 struct ib_device *ibdev = &hr_dev->ib_dev; 662 u32 cnt; 663 664 if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes || 665 cap->max_send_sge > hr_dev->caps.max_sq_sg) { 666 ibdev_err(ibdev, "failed to check SQ WR or SGE num.\n"); 667 return -EINVAL; 668 } 669 670 cnt = roundup_pow_of_two(max(cap->max_send_wr, hr_dev->caps.min_wqes)); 671 if (cnt > hr_dev->caps.max_wqes) { 672 ibdev_err(ibdev, "failed to check WQE num, WQE num = %u.\n", 673 cnt); 674 return -EINVAL; 675 } 676 677 hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz); 678 hr_qp->sq.wqe_cnt = cnt; 679 680 set_ext_sge_param(hr_dev, cnt, hr_qp, cap); 681 682 /* sync the parameters of kernel QP to user's configuration */ 683 cap->max_send_wr = cnt; 684 cap->max_send_sge = hr_qp->sq.max_gs; 685 686 return 0; 687 } 688 689 static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr) 690 { 691 if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr) 692 return 0; 693 694 return 1; 695 } 696 697 static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr) 698 { 699 if (attr->qp_type == IB_QPT_XRC_INI || 700 attr->qp_type == IB_QPT_XRC_TGT || attr->srq || 701 !attr->cap.max_recv_wr) 702 return 0; 703 704 return 1; 705 } 706 707 static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp, 708 struct ib_qp_init_attr *init_attr) 709 { 710 u32 max_recv_sge = init_attr->cap.max_recv_sge; 711 u32 wqe_cnt = hr_qp->rq_inl_buf.wqe_cnt; 712 struct hns_roce_rinl_wqe *wqe_list; 713 int i; 714 715 /* allocate recv inline buf */ 716 wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe), 717 GFP_KERNEL); 718 719 if (!wqe_list) 720 goto err; 721 722 /* Allocate a continuous buffer for all inline sge we need */ 723 wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge * 724 sizeof(struct hns_roce_rinl_sge)), 725 GFP_KERNEL); 726 if (!wqe_list[0].sg_list) 727 goto err_wqe_list; 728 729 /* Assign buffers of sg_list to each inline wqe */ 730 for (i = 1; i < wqe_cnt; i++) 731 wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge]; 732 733 hr_qp->rq_inl_buf.wqe_list = wqe_list; 734 735 return 0; 736 737 err_wqe_list: 738 kfree(wqe_list); 739 740 err: 741 return -ENOMEM; 742 } 743 744 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp) 745 { 746 if (hr_qp->rq_inl_buf.wqe_list) 747 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list); 748 kfree(hr_qp->rq_inl_buf.wqe_list); 749 } 750 751 static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 752 struct ib_qp_init_attr *init_attr, 753 struct ib_udata *udata, unsigned long addr) 754 { 755 struct ib_device *ibdev = &hr_dev->ib_dev; 756 struct hns_roce_buf_attr buf_attr = {}; 757 int ret; 758 759 if (!udata && hr_qp->rq_inl_buf.wqe_cnt) { 760 ret = alloc_rq_inline_buf(hr_qp, init_attr); 761 if (ret) { 762 ibdev_err(ibdev, 763 "failed to alloc inline buf, ret = %d.\n", 764 ret); 765 return ret; 766 } 767 } else { 768 hr_qp->rq_inl_buf.wqe_list = NULL; 769 } 770 771 ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr); 772 if (ret) { 773 ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret); 774 goto err_inline; 775 } 776 ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr, 777 PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz, 778 udata, addr); 779 if (ret) { 780 ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret); 781 goto err_inline; 782 } 783 784 return 0; 785 err_inline: 786 free_rq_inline_buf(hr_qp); 787 788 return ret; 789 } 790 791 static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 792 { 793 hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr); 794 free_rq_inline_buf(hr_qp); 795 } 796 797 static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev, 798 struct ib_qp_init_attr *init_attr, 799 struct ib_udata *udata, 800 struct hns_roce_ib_create_qp_resp *resp, 801 struct hns_roce_ib_create_qp *ucmd) 802 { 803 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 804 udata->outlen >= offsetofend(typeof(*resp), cap_flags) && 805 hns_roce_qp_has_sq(init_attr) && 806 udata->inlen >= offsetofend(typeof(*ucmd), sdb_addr)); 807 } 808 809 static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev, 810 struct ib_qp_init_attr *init_attr, 811 struct ib_udata *udata, 812 struct hns_roce_ib_create_qp_resp *resp) 813 { 814 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 815 udata->outlen >= offsetofend(typeof(*resp), cap_flags) && 816 hns_roce_qp_has_rq(init_attr)); 817 } 818 819 static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev, 820 struct ib_qp_init_attr *init_attr) 821 { 822 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 823 hns_roce_qp_has_rq(init_attr)); 824 } 825 826 static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 827 struct ib_qp_init_attr *init_attr, 828 struct ib_udata *udata, 829 struct hns_roce_ib_create_qp *ucmd, 830 struct hns_roce_ib_create_qp_resp *resp) 831 { 832 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( 833 udata, struct hns_roce_ucontext, ibucontext); 834 struct ib_device *ibdev = &hr_dev->ib_dev; 835 int ret; 836 837 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SDI_MODE) 838 hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB; 839 840 if (udata) { 841 if (user_qp_has_sdb(hr_dev, init_attr, udata, resp, ucmd)) { 842 ret = hns_roce_db_map_user(uctx, ucmd->sdb_addr, 843 &hr_qp->sdb); 844 if (ret) { 845 ibdev_err(ibdev, 846 "failed to map user SQ doorbell, ret = %d.\n", 847 ret); 848 goto err_out; 849 } 850 hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB; 851 resp->cap_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB; 852 } 853 854 if (user_qp_has_rdb(hr_dev, init_attr, udata, resp)) { 855 ret = hns_roce_db_map_user(uctx, ucmd->db_addr, 856 &hr_qp->rdb); 857 if (ret) { 858 ibdev_err(ibdev, 859 "failed to map user RQ doorbell, ret = %d.\n", 860 ret); 861 goto err_sdb; 862 } 863 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; 864 resp->cap_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; 865 } 866 } else { 867 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 868 hr_qp->sq.db_reg = hr_dev->mem_base + 869 HNS_ROCE_DWQE_SIZE * hr_qp->qpn; 870 else 871 hr_qp->sq.db_reg = 872 hr_dev->reg_base + hr_dev->sdb_offset + 873 DB_REG_OFFSET * hr_dev->priv_uar.index; 874 875 hr_qp->rq.db_reg = hr_dev->reg_base + hr_dev->odb_offset + 876 DB_REG_OFFSET * hr_dev->priv_uar.index; 877 878 if (kernel_qp_has_rdb(hr_dev, init_attr)) { 879 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0); 880 if (ret) { 881 ibdev_err(ibdev, 882 "failed to alloc kernel RQ doorbell, ret = %d.\n", 883 ret); 884 goto err_out; 885 } 886 *hr_qp->rdb.db_record = 0; 887 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; 888 } 889 } 890 891 return 0; 892 err_sdb: 893 if (udata && hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) 894 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 895 err_out: 896 return ret; 897 } 898 899 static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 900 struct ib_udata *udata) 901 { 902 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( 903 udata, struct hns_roce_ucontext, ibucontext); 904 905 if (udata) { 906 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 907 hns_roce_db_unmap_user(uctx, &hr_qp->rdb); 908 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) 909 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 910 } else { 911 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 912 hns_roce_free_db(hr_dev, &hr_qp->rdb); 913 } 914 } 915 916 static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev, 917 struct hns_roce_qp *hr_qp) 918 { 919 struct ib_device *ibdev = &hr_dev->ib_dev; 920 u64 *sq_wrid = NULL; 921 u64 *rq_wrid = NULL; 922 int ret; 923 924 sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL); 925 if (ZERO_OR_NULL_PTR(sq_wrid)) { 926 ibdev_err(ibdev, "failed to alloc SQ wrid.\n"); 927 return -ENOMEM; 928 } 929 930 if (hr_qp->rq.wqe_cnt) { 931 rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL); 932 if (ZERO_OR_NULL_PTR(rq_wrid)) { 933 ibdev_err(ibdev, "failed to alloc RQ wrid.\n"); 934 ret = -ENOMEM; 935 goto err_sq; 936 } 937 } 938 939 hr_qp->sq.wrid = sq_wrid; 940 hr_qp->rq.wrid = rq_wrid; 941 return 0; 942 err_sq: 943 kfree(sq_wrid); 944 945 return ret; 946 } 947 948 static void free_kernel_wrid(struct hns_roce_qp *hr_qp) 949 { 950 kfree(hr_qp->rq.wrid); 951 kfree(hr_qp->sq.wrid); 952 } 953 954 static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 955 struct ib_qp_init_attr *init_attr, 956 struct ib_udata *udata, 957 struct hns_roce_ib_create_qp *ucmd) 958 { 959 struct ib_device *ibdev = &hr_dev->ib_dev; 960 int ret; 961 962 hr_qp->ibqp.qp_type = init_attr->qp_type; 963 964 if (init_attr->cap.max_inline_data > hr_dev->caps.max_sq_inline) 965 init_attr->cap.max_inline_data = hr_dev->caps.max_sq_inline; 966 967 hr_qp->max_inline_data = init_attr->cap.max_inline_data; 968 969 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) 970 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR; 971 else 972 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR; 973 974 ret = set_rq_size(hr_dev, &init_attr->cap, hr_qp, 975 hns_roce_qp_has_rq(init_attr), !!udata); 976 if (ret) { 977 ibdev_err(ibdev, "failed to set user RQ size, ret = %d.\n", 978 ret); 979 return ret; 980 } 981 982 if (udata) { 983 ret = ib_copy_from_udata(ucmd, udata, 984 min(udata->inlen, sizeof(*ucmd))); 985 if (ret) { 986 ibdev_err(ibdev, 987 "failed to copy QP ucmd, ret = %d\n", ret); 988 return ret; 989 } 990 991 ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd); 992 if (ret) 993 ibdev_err(ibdev, 994 "failed to set user SQ size, ret = %d.\n", 995 ret); 996 } else { 997 ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp); 998 if (ret) 999 ibdev_err(ibdev, 1000 "failed to set kernel SQ size, ret = %d.\n", 1001 ret); 1002 } 1003 1004 return ret; 1005 } 1006 1007 static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, 1008 struct ib_pd *ib_pd, 1009 struct ib_qp_init_attr *init_attr, 1010 struct ib_udata *udata, 1011 struct hns_roce_qp *hr_qp) 1012 { 1013 struct hns_roce_ib_create_qp_resp resp = {}; 1014 struct ib_device *ibdev = &hr_dev->ib_dev; 1015 struct hns_roce_ib_create_qp ucmd; 1016 int ret; 1017 1018 mutex_init(&hr_qp->mutex); 1019 spin_lock_init(&hr_qp->sq.lock); 1020 spin_lock_init(&hr_qp->rq.lock); 1021 1022 hr_qp->state = IB_QPS_RESET; 1023 hr_qp->flush_flag = 0; 1024 1025 if (init_attr->create_flags) 1026 return -EOPNOTSUPP; 1027 1028 ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd); 1029 if (ret) { 1030 ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret); 1031 return ret; 1032 } 1033 1034 if (!udata) { 1035 ret = alloc_kernel_wrid(hr_dev, hr_qp); 1036 if (ret) { 1037 ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n", 1038 ret); 1039 return ret; 1040 } 1041 } 1042 1043 ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr); 1044 if (ret) { 1045 ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret); 1046 goto err_buf; 1047 } 1048 1049 ret = alloc_qpn(hr_dev, hr_qp); 1050 if (ret) { 1051 ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret); 1052 goto err_qpn; 1053 } 1054 1055 ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp); 1056 if (ret) { 1057 ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n", 1058 ret); 1059 goto err_db; 1060 } 1061 1062 ret = alloc_qpc(hr_dev, hr_qp); 1063 if (ret) { 1064 ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n", 1065 ret); 1066 goto err_qpc; 1067 } 1068 1069 ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr); 1070 if (ret) { 1071 ibdev_err(ibdev, "failed to store QP, ret = %d.\n", ret); 1072 goto err_store; 1073 } 1074 1075 if (udata) { 1076 ret = ib_copy_to_udata(udata, &resp, 1077 min(udata->outlen, sizeof(resp))); 1078 if (ret) { 1079 ibdev_err(ibdev, "copy qp resp failed!\n"); 1080 goto err_store; 1081 } 1082 } 1083 1084 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 1085 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp); 1086 if (ret) 1087 goto err_flow_ctrl; 1088 } 1089 1090 hr_qp->ibqp.qp_num = hr_qp->qpn; 1091 hr_qp->event = hns_roce_ib_qp_event; 1092 refcount_set(&hr_qp->refcount, 1); 1093 init_completion(&hr_qp->free); 1094 1095 return 0; 1096 1097 err_flow_ctrl: 1098 hns_roce_qp_remove(hr_dev, hr_qp); 1099 err_store: 1100 free_qpc(hr_dev, hr_qp); 1101 err_qpc: 1102 free_qp_db(hr_dev, hr_qp, udata); 1103 err_db: 1104 free_qpn(hr_dev, hr_qp); 1105 err_qpn: 1106 free_qp_buf(hr_dev, hr_qp); 1107 err_buf: 1108 free_kernel_wrid(hr_qp); 1109 return ret; 1110 } 1111 1112 void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 1113 struct ib_udata *udata) 1114 { 1115 if (refcount_dec_and_test(&hr_qp->refcount)) 1116 complete(&hr_qp->free); 1117 wait_for_completion(&hr_qp->free); 1118 1119 free_qpc(hr_dev, hr_qp); 1120 free_qpn(hr_dev, hr_qp); 1121 free_qp_buf(hr_dev, hr_qp); 1122 free_kernel_wrid(hr_qp); 1123 free_qp_db(hr_dev, hr_qp, udata); 1124 1125 kfree(hr_qp); 1126 } 1127 1128 static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type, 1129 bool is_user) 1130 { 1131 switch (type) { 1132 case IB_QPT_XRC_INI: 1133 case IB_QPT_XRC_TGT: 1134 if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC)) 1135 goto out; 1136 break; 1137 case IB_QPT_UD: 1138 if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 && 1139 is_user) 1140 goto out; 1141 break; 1142 case IB_QPT_RC: 1143 case IB_QPT_GSI: 1144 break; 1145 default: 1146 goto out; 1147 } 1148 1149 return 0; 1150 1151 out: 1152 ibdev_err(&hr_dev->ib_dev, "not support QP type %d\n", type); 1153 1154 return -EOPNOTSUPP; 1155 } 1156 1157 struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, 1158 struct ib_qp_init_attr *init_attr, 1159 struct ib_udata *udata) 1160 { 1161 struct ib_device *ibdev = pd ? pd->device : init_attr->xrcd->device; 1162 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); 1163 struct hns_roce_qp *hr_qp; 1164 int ret; 1165 1166 ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata); 1167 if (ret) 1168 return ERR_PTR(ret); 1169 1170 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); 1171 if (!hr_qp) 1172 return ERR_PTR(-ENOMEM); 1173 1174 if (init_attr->qp_type == IB_QPT_XRC_INI) 1175 init_attr->recv_cq = NULL; 1176 1177 if (init_attr->qp_type == IB_QPT_XRC_TGT) { 1178 hr_qp->xrcdn = to_hr_xrcd(init_attr->xrcd)->xrcdn; 1179 init_attr->recv_cq = NULL; 1180 init_attr->send_cq = NULL; 1181 } 1182 1183 if (init_attr->qp_type == IB_QPT_GSI) { 1184 hr_qp->port = init_attr->port_num - 1; 1185 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; 1186 } 1187 1188 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp); 1189 if (ret) { 1190 ibdev_err(ibdev, "Create QP type 0x%x failed(%d)\n", 1191 init_attr->qp_type, ret); 1192 1193 kfree(hr_qp); 1194 return ERR_PTR(ret); 1195 } 1196 1197 return &hr_qp->ibqp; 1198 } 1199 1200 int to_hr_qp_type(int qp_type) 1201 { 1202 switch (qp_type) { 1203 case IB_QPT_RC: 1204 return SERV_TYPE_RC; 1205 case IB_QPT_UD: 1206 case IB_QPT_GSI: 1207 return SERV_TYPE_UD; 1208 case IB_QPT_XRC_INI: 1209 case IB_QPT_XRC_TGT: 1210 return SERV_TYPE_XRC; 1211 default: 1212 return -1; 1213 } 1214 } 1215 1216 static int check_mtu_validate(struct hns_roce_dev *hr_dev, 1217 struct hns_roce_qp *hr_qp, 1218 struct ib_qp_attr *attr, int attr_mask) 1219 { 1220 enum ib_mtu active_mtu; 1221 int p; 1222 1223 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1224 active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu); 1225 1226 if ((hr_dev->caps.max_mtu >= IB_MTU_2048 && 1227 attr->path_mtu > hr_dev->caps.max_mtu) || 1228 attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) { 1229 ibdev_err(&hr_dev->ib_dev, 1230 "attr path_mtu(%d)invalid while modify qp", 1231 attr->path_mtu); 1232 return -EINVAL; 1233 } 1234 1235 return 0; 1236 } 1237 1238 static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1239 int attr_mask) 1240 { 1241 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1242 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1243 int p; 1244 1245 if ((attr_mask & IB_QP_PORT) && 1246 (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) { 1247 ibdev_err(&hr_dev->ib_dev, "invalid attr, port_num = %u.\n", 1248 attr->port_num); 1249 return -EINVAL; 1250 } 1251 1252 if (attr_mask & IB_QP_PKEY_INDEX) { 1253 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1254 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) { 1255 ibdev_err(&hr_dev->ib_dev, 1256 "invalid attr, pkey_index = %u.\n", 1257 attr->pkey_index); 1258 return -EINVAL; 1259 } 1260 } 1261 1262 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && 1263 attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) { 1264 ibdev_err(&hr_dev->ib_dev, 1265 "invalid attr, max_rd_atomic = %u.\n", 1266 attr->max_rd_atomic); 1267 return -EINVAL; 1268 } 1269 1270 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && 1271 attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) { 1272 ibdev_err(&hr_dev->ib_dev, 1273 "invalid attr, max_dest_rd_atomic = %u.\n", 1274 attr->max_dest_rd_atomic); 1275 return -EINVAL; 1276 } 1277 1278 if (attr_mask & IB_QP_PATH_MTU) 1279 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask); 1280 1281 return 0; 1282 } 1283 1284 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1285 int attr_mask, struct ib_udata *udata) 1286 { 1287 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1288 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1289 enum ib_qp_state cur_state, new_state; 1290 int ret = -EINVAL; 1291 1292 mutex_lock(&hr_qp->mutex); 1293 1294 if (attr_mask & IB_QP_CUR_STATE && attr->cur_qp_state != hr_qp->state) 1295 goto out; 1296 1297 cur_state = hr_qp->state; 1298 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; 1299 1300 if (ibqp->uobject && 1301 (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) { 1302 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) { 1303 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr); 1304 1305 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 1306 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr); 1307 } else { 1308 ibdev_warn(&hr_dev->ib_dev, 1309 "flush cqe is not supported in userspace!\n"); 1310 goto out; 1311 } 1312 } 1313 1314 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, 1315 attr_mask)) { 1316 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n"); 1317 goto out; 1318 } 1319 1320 ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask); 1321 if (ret) 1322 goto out; 1323 1324 if (cur_state == new_state && cur_state == IB_QPS_RESET) { 1325 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) { 1326 ret = -EPERM; 1327 ibdev_err(&hr_dev->ib_dev, 1328 "RST2RST state is not supported\n"); 1329 } else { 1330 ret = 0; 1331 } 1332 1333 goto out; 1334 } 1335 1336 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, 1337 new_state); 1338 1339 out: 1340 mutex_unlock(&hr_qp->mutex); 1341 1342 return ret; 1343 } 1344 1345 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) 1346 __acquires(&send_cq->lock) __acquires(&recv_cq->lock) 1347 { 1348 if (unlikely(send_cq == NULL && recv_cq == NULL)) { 1349 __acquire(&send_cq->lock); 1350 __acquire(&recv_cq->lock); 1351 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { 1352 spin_lock_irq(&send_cq->lock); 1353 __acquire(&recv_cq->lock); 1354 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { 1355 spin_lock_irq(&recv_cq->lock); 1356 __acquire(&send_cq->lock); 1357 } else if (send_cq == recv_cq) { 1358 spin_lock_irq(&send_cq->lock); 1359 __acquire(&recv_cq->lock); 1360 } else if (send_cq->cqn < recv_cq->cqn) { 1361 spin_lock_irq(&send_cq->lock); 1362 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); 1363 } else { 1364 spin_lock_irq(&recv_cq->lock); 1365 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); 1366 } 1367 } 1368 1369 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, 1370 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock) 1371 __releases(&recv_cq->lock) 1372 { 1373 if (unlikely(send_cq == NULL && recv_cq == NULL)) { 1374 __release(&recv_cq->lock); 1375 __release(&send_cq->lock); 1376 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { 1377 __release(&recv_cq->lock); 1378 spin_unlock(&send_cq->lock); 1379 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { 1380 __release(&send_cq->lock); 1381 spin_unlock(&recv_cq->lock); 1382 } else if (send_cq == recv_cq) { 1383 __release(&recv_cq->lock); 1384 spin_unlock_irq(&send_cq->lock); 1385 } else if (send_cq->cqn < recv_cq->cqn) { 1386 spin_unlock(&recv_cq->lock); 1387 spin_unlock_irq(&send_cq->lock); 1388 } else { 1389 spin_unlock(&send_cq->lock); 1390 spin_unlock_irq(&recv_cq->lock); 1391 } 1392 } 1393 1394 static inline void *get_wqe(struct hns_roce_qp *hr_qp, int offset) 1395 { 1396 return hns_roce_buf_offset(hr_qp->mtr.kmem, offset); 1397 } 1398 1399 void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n) 1400 { 1401 return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift)); 1402 } 1403 1404 void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n) 1405 { 1406 return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift)); 1407 } 1408 1409 void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n) 1410 { 1411 return get_wqe(hr_qp, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift)); 1412 } 1413 1414 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq, 1415 struct ib_cq *ib_cq) 1416 { 1417 struct hns_roce_cq *hr_cq; 1418 u32 cur; 1419 1420 cur = hr_wq->head - hr_wq->tail; 1421 if (likely(cur + nreq < hr_wq->wqe_cnt)) 1422 return false; 1423 1424 hr_cq = to_hr_cq(ib_cq); 1425 spin_lock(&hr_cq->lock); 1426 cur = hr_wq->head - hr_wq->tail; 1427 spin_unlock(&hr_cq->lock); 1428 1429 return cur + nreq >= hr_wq->wqe_cnt; 1430 } 1431 1432 void hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) 1433 { 1434 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 1435 unsigned int reserved_from_bot; 1436 unsigned int i; 1437 1438 mutex_init(&qp_table->scc_mutex); 1439 mutex_init(&qp_table->bank_mutex); 1440 xa_init(&hr_dev->qp_table_xa); 1441 1442 reserved_from_bot = hr_dev->caps.reserved_qps; 1443 1444 for (i = 0; i < reserved_from_bot; i++) { 1445 hr_dev->qp_table.bank[get_qp_bankid(i)].inuse++; 1446 hr_dev->qp_table.bank[get_qp_bankid(i)].min++; 1447 } 1448 1449 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) { 1450 ida_init(&hr_dev->qp_table.bank[i].ida); 1451 hr_dev->qp_table.bank[i].max = hr_dev->caps.num_qps / 1452 HNS_ROCE_QP_BANK_NUM - 1; 1453 hr_dev->qp_table.bank[i].next = hr_dev->qp_table.bank[i].min; 1454 } 1455 } 1456 1457 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) 1458 { 1459 int i; 1460 1461 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) 1462 ida_destroy(&hr_dev->qp_table.bank[i].ida); 1463 } 1464