1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/dma-mapping.h> 33 #include <linux/crc32.h> 34 #include <net/ip.h> 35 #include <net/ipv6.h> 36 #include <net/udp.h> 37 #include <linux/iommu.h> 38 39 #include <rdma/ib_verbs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/iw_cm.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/ib_addr.h> 44 #include <rdma/ib_cache.h> 45 46 #include <linux/qed/common_hsi.h> 47 #include "qedr_hsi_rdma.h" 48 #include <linux/qed/qed_if.h> 49 #include "qedr.h" 50 #include "verbs.h" 51 #include <rdma/qedr-abi.h> 52 #include "qedr_roce_cm.h" 53 54 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 55 56 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, 57 size_t len) 58 { 59 size_t min_len = min_t(size_t, len, udata->outlen); 60 61 return ib_copy_to_udata(udata, src, min_len); 62 } 63 64 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 65 { 66 if (index > QEDR_ROCE_PKEY_TABLE_LEN) 67 return -EINVAL; 68 69 *pkey = QEDR_ROCE_PKEY_DEFAULT; 70 return 0; 71 } 72 73 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port, 74 int index, union ib_gid *sgid) 75 { 76 struct qedr_dev *dev = get_qedr_dev(ibdev); 77 78 memset(sgid->raw, 0, sizeof(sgid->raw)); 79 ether_addr_copy(sgid->raw, dev->ndev->dev_addr); 80 81 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index, 82 sgid->global.interface_id, sgid->global.subnet_prefix); 83 84 return 0; 85 } 86 87 int qedr_query_gid(struct ib_device *ibdev, u8 port, int index, 88 union ib_gid *sgid) 89 { 90 struct qedr_dev *dev = get_qedr_dev(ibdev); 91 int rc = 0; 92 93 if (!rdma_cap_roce_gid_table(ibdev, port)) 94 return -ENODEV; 95 96 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL); 97 if (rc == -EAGAIN) { 98 memcpy(sgid, &zgid, sizeof(*sgid)); 99 return 0; 100 } 101 102 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index, 103 sgid->global.interface_id, sgid->global.subnet_prefix); 104 105 return rc; 106 } 107 108 int qedr_add_gid(struct ib_device *device, u8 port_num, 109 unsigned int index, const union ib_gid *gid, 110 const struct ib_gid_attr *attr, void **context) 111 { 112 if (!rdma_cap_roce_gid_table(device, port_num)) 113 return -EINVAL; 114 115 if (port_num > QEDR_MAX_PORT) 116 return -EINVAL; 117 118 if (!context) 119 return -EINVAL; 120 121 return 0; 122 } 123 124 int qedr_del_gid(struct ib_device *device, u8 port_num, 125 unsigned int index, void **context) 126 { 127 if (!rdma_cap_roce_gid_table(device, port_num)) 128 return -EINVAL; 129 130 if (port_num > QEDR_MAX_PORT) 131 return -EINVAL; 132 133 if (!context) 134 return -EINVAL; 135 136 return 0; 137 } 138 139 int qedr_query_device(struct ib_device *ibdev, 140 struct ib_device_attr *attr, struct ib_udata *udata) 141 { 142 struct qedr_dev *dev = get_qedr_dev(ibdev); 143 struct qedr_device_attr *qattr = &dev->attr; 144 145 if (!dev->rdma_ctx) { 146 DP_ERR(dev, 147 "qedr_query_device called with invalid params rdma_ctx=%p\n", 148 dev->rdma_ctx); 149 return -EINVAL; 150 } 151 152 memset(attr, 0, sizeof(*attr)); 153 154 attr->fw_ver = qattr->fw_ver; 155 attr->sys_image_guid = qattr->sys_image_guid; 156 attr->max_mr_size = qattr->max_mr_size; 157 attr->page_size_cap = qattr->page_size_caps; 158 attr->vendor_id = qattr->vendor_id; 159 attr->vendor_part_id = qattr->vendor_part_id; 160 attr->hw_ver = qattr->hw_ver; 161 attr->max_qp = qattr->max_qp; 162 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe); 163 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 164 IB_DEVICE_RC_RNR_NAK_GEN | 165 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS; 166 167 attr->max_sge = qattr->max_sge; 168 attr->max_sge_rd = qattr->max_sge; 169 attr->max_cq = qattr->max_cq; 170 attr->max_cqe = qattr->max_cqe; 171 attr->max_mr = qattr->max_mr; 172 attr->max_mw = qattr->max_mw; 173 attr->max_pd = qattr->max_pd; 174 attr->atomic_cap = dev->atomic_cap; 175 attr->max_fmr = qattr->max_fmr; 176 attr->max_map_per_fmr = 16; 177 attr->max_qp_init_rd_atom = 178 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 179 attr->max_qp_rd_atom = 180 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 181 attr->max_qp_init_rd_atom); 182 183 attr->max_srq = qattr->max_srq; 184 attr->max_srq_sge = qattr->max_srq_sge; 185 attr->max_srq_wr = qattr->max_srq_wr; 186 187 attr->local_ca_ack_delay = qattr->dev_ack_delay; 188 attr->max_fast_reg_page_list_len = qattr->max_mr / 8; 189 attr->max_pkeys = QEDR_ROCE_PKEY_MAX; 190 attr->max_ah = qattr->max_ah; 191 192 return 0; 193 } 194 195 #define QEDR_SPEED_SDR (1) 196 #define QEDR_SPEED_DDR (2) 197 #define QEDR_SPEED_QDR (4) 198 #define QEDR_SPEED_FDR10 (8) 199 #define QEDR_SPEED_FDR (16) 200 #define QEDR_SPEED_EDR (32) 201 202 static inline void get_link_speed_and_width(int speed, u8 *ib_speed, 203 u8 *ib_width) 204 { 205 switch (speed) { 206 case 1000: 207 *ib_speed = QEDR_SPEED_SDR; 208 *ib_width = IB_WIDTH_1X; 209 break; 210 case 10000: 211 *ib_speed = QEDR_SPEED_QDR; 212 *ib_width = IB_WIDTH_1X; 213 break; 214 215 case 20000: 216 *ib_speed = QEDR_SPEED_DDR; 217 *ib_width = IB_WIDTH_4X; 218 break; 219 220 case 25000: 221 *ib_speed = QEDR_SPEED_EDR; 222 *ib_width = IB_WIDTH_1X; 223 break; 224 225 case 40000: 226 *ib_speed = QEDR_SPEED_QDR; 227 *ib_width = IB_WIDTH_4X; 228 break; 229 230 case 50000: 231 *ib_speed = QEDR_SPEED_QDR; 232 *ib_width = IB_WIDTH_4X; 233 break; 234 235 case 100000: 236 *ib_speed = QEDR_SPEED_EDR; 237 *ib_width = IB_WIDTH_4X; 238 break; 239 240 default: 241 /* Unsupported */ 242 *ib_speed = QEDR_SPEED_SDR; 243 *ib_width = IB_WIDTH_1X; 244 } 245 } 246 247 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr) 248 { 249 struct qedr_dev *dev; 250 struct qed_rdma_port *rdma_port; 251 252 dev = get_qedr_dev(ibdev); 253 if (port > 1) { 254 DP_ERR(dev, "invalid_port=0x%x\n", port); 255 return -EINVAL; 256 } 257 258 if (!dev->rdma_ctx) { 259 DP_ERR(dev, "rdma_ctx is NULL\n"); 260 return -EINVAL; 261 } 262 263 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx); 264 265 /* *attr being zeroed by the caller, avoid zeroing it here */ 266 if (rdma_port->port_state == QED_RDMA_PORT_UP) { 267 attr->state = IB_PORT_ACTIVE; 268 attr->phys_state = 5; 269 } else { 270 attr->state = IB_PORT_DOWN; 271 attr->phys_state = 3; 272 } 273 attr->max_mtu = IB_MTU_4096; 274 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); 275 attr->lid = 0; 276 attr->lmc = 0; 277 attr->sm_lid = 0; 278 attr->sm_sl = 0; 279 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS; 280 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 281 attr->gid_tbl_len = 1; 282 attr->pkey_tbl_len = 1; 283 } else { 284 attr->gid_tbl_len = QEDR_MAX_SGID; 285 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; 286 } 287 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 288 attr->qkey_viol_cntr = 0; 289 get_link_speed_and_width(rdma_port->link_speed, 290 &attr->active_speed, &attr->active_width); 291 attr->max_msg_sz = rdma_port->max_msg_size; 292 attr->max_vl_num = 4; 293 294 return 0; 295 } 296 297 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask, 298 struct ib_port_modify *props) 299 { 300 struct qedr_dev *dev; 301 302 dev = get_qedr_dev(ibdev); 303 if (port > 1) { 304 DP_ERR(dev, "invalid_port=0x%x\n", port); 305 return -EINVAL; 306 } 307 308 return 0; 309 } 310 311 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr, 312 unsigned long len) 313 { 314 struct qedr_mm *mm; 315 316 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 317 if (!mm) 318 return -ENOMEM; 319 320 mm->key.phy_addr = phy_addr; 321 /* This function might be called with a length which is not a multiple 322 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel 323 * forces this granularity by increasing the requested size if needed. 324 * When qedr_mmap is called, it will search the list with the updated 325 * length as a key. To prevent search failures, the length is rounded up 326 * in advance to PAGE_SIZE. 327 */ 328 mm->key.len = roundup(len, PAGE_SIZE); 329 INIT_LIST_HEAD(&mm->entry); 330 331 mutex_lock(&uctx->mm_list_lock); 332 list_add(&mm->entry, &uctx->mm_head); 333 mutex_unlock(&uctx->mm_list_lock); 334 335 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 336 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n", 337 (unsigned long long)mm->key.phy_addr, 338 (unsigned long)mm->key.len, uctx); 339 340 return 0; 341 } 342 343 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr, 344 unsigned long len) 345 { 346 bool found = false; 347 struct qedr_mm *mm; 348 349 mutex_lock(&uctx->mm_list_lock); 350 list_for_each_entry(mm, &uctx->mm_head, entry) { 351 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 352 continue; 353 354 found = true; 355 break; 356 } 357 mutex_unlock(&uctx->mm_list_lock); 358 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 359 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n", 360 mm->key.phy_addr, mm->key.len, uctx, found); 361 362 return found; 363 } 364 365 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev, 366 struct ib_udata *udata) 367 { 368 int rc; 369 struct qedr_ucontext *ctx; 370 struct qedr_alloc_ucontext_resp uresp; 371 struct qedr_dev *dev = get_qedr_dev(ibdev); 372 struct qed_rdma_add_user_out_params oparams; 373 374 if (!udata) 375 return ERR_PTR(-EFAULT); 376 377 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 378 if (!ctx) 379 return ERR_PTR(-ENOMEM); 380 381 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams); 382 if (rc) { 383 DP_ERR(dev, 384 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n", 385 rc); 386 goto err; 387 } 388 389 ctx->dpi = oparams.dpi; 390 ctx->dpi_addr = oparams.dpi_addr; 391 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 392 ctx->dpi_size = oparams.dpi_size; 393 INIT_LIST_HEAD(&ctx->mm_head); 394 mutex_init(&ctx->mm_list_lock); 395 396 memset(&uresp, 0, sizeof(uresp)); 397 398 uresp.dpm_enabled = dev->user_dpm_enabled; 399 uresp.wids_enabled = 1; 400 uresp.wid_count = oparams.wid_count; 401 uresp.db_pa = ctx->dpi_phys_addr; 402 uresp.db_size = ctx->dpi_size; 403 uresp.max_send_wr = dev->attr.max_sqe; 404 uresp.max_recv_wr = dev->attr.max_rqe; 405 uresp.max_srq_wr = dev->attr.max_srq_wr; 406 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE; 407 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE; 408 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 409 uresp.max_cqes = QEDR_MAX_CQES; 410 411 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 412 if (rc) 413 goto err; 414 415 ctx->dev = dev; 416 417 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size); 418 if (rc) 419 goto err; 420 421 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n", 422 &ctx->ibucontext); 423 return &ctx->ibucontext; 424 425 err: 426 kfree(ctx); 427 return ERR_PTR(rc); 428 } 429 430 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx) 431 { 432 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx); 433 struct qedr_mm *mm, *tmp; 434 int status = 0; 435 436 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n", 437 uctx); 438 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi); 439 440 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 441 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 442 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n", 443 mm->key.phy_addr, mm->key.len, uctx); 444 list_del(&mm->entry); 445 kfree(mm); 446 } 447 448 kfree(uctx); 449 return status; 450 } 451 452 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 453 { 454 struct qedr_ucontext *ucontext = get_qedr_ucontext(context); 455 struct qedr_dev *dev = get_qedr_dev(context->device); 456 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 457 u64 unmapped_db = dev->db_phys_addr; 458 unsigned long len = (vma->vm_end - vma->vm_start); 459 int rc = 0; 460 bool found; 461 462 DP_DEBUG(dev, QEDR_MSG_INIT, 463 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n", 464 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len); 465 if (vma->vm_start & (PAGE_SIZE - 1)) { 466 DP_ERR(dev, "Vma_start not page aligned = %ld\n", 467 vma->vm_start); 468 return -EINVAL; 469 } 470 471 found = qedr_search_mmap(ucontext, vm_page, len); 472 if (!found) { 473 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n", 474 vma->vm_pgoff); 475 return -EINVAL; 476 } 477 478 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n"); 479 480 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + 481 dev->db_size))) { 482 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n"); 483 if (vma->vm_flags & VM_READ) { 484 DP_ERR(dev, "Trying to map doorbell bar for read\n"); 485 return -EPERM; 486 } 487 488 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 489 490 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 491 PAGE_SIZE, vma->vm_page_prot); 492 } else { 493 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n"); 494 rc = remap_pfn_range(vma, vma->vm_start, 495 vma->vm_pgoff, len, vma->vm_page_prot); 496 } 497 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc); 498 return rc; 499 } 500 501 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev, 502 struct ib_ucontext *context, struct ib_udata *udata) 503 { 504 struct qedr_dev *dev = get_qedr_dev(ibdev); 505 struct qedr_pd *pd; 506 u16 pd_id; 507 int rc; 508 509 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n", 510 (udata && context) ? "User Lib" : "Kernel"); 511 512 if (!dev->rdma_ctx) { 513 DP_ERR(dev, "invalid RDMA context\n"); 514 return ERR_PTR(-EINVAL); 515 } 516 517 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 518 if (!pd) 519 return ERR_PTR(-ENOMEM); 520 521 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id); 522 if (rc) 523 goto err; 524 525 pd->pd_id = pd_id; 526 527 if (udata && context) { 528 struct qedr_alloc_pd_uresp uresp; 529 530 uresp.pd_id = pd_id; 531 532 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 533 if (rc) { 534 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); 535 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); 536 goto err; 537 } 538 539 pd->uctx = get_qedr_ucontext(context); 540 pd->uctx->pd = pd; 541 } 542 543 return &pd->ibpd; 544 545 err: 546 kfree(pd); 547 return ERR_PTR(rc); 548 } 549 550 int qedr_dealloc_pd(struct ib_pd *ibpd) 551 { 552 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 553 struct qedr_pd *pd = get_qedr_pd(ibpd); 554 555 if (!pd) { 556 pr_err("Invalid PD received in dealloc_pd\n"); 557 return -EINVAL; 558 } 559 560 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id); 561 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id); 562 563 kfree(pd); 564 565 return 0; 566 } 567 568 static void qedr_free_pbl(struct qedr_dev *dev, 569 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) 570 { 571 struct pci_dev *pdev = dev->pdev; 572 int i; 573 574 for (i = 0; i < pbl_info->num_pbls; i++) { 575 if (!pbl[i].va) 576 continue; 577 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 578 pbl[i].va, pbl[i].pa); 579 } 580 581 kfree(pbl); 582 } 583 584 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024) 585 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024) 586 587 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 588 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 589 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE) 590 591 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev, 592 struct qedr_pbl_info *pbl_info, 593 gfp_t flags) 594 { 595 struct pci_dev *pdev = dev->pdev; 596 struct qedr_pbl *pbl_table; 597 dma_addr_t *pbl_main_tbl; 598 dma_addr_t pa; 599 void *va; 600 int i; 601 602 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags); 603 if (!pbl_table) 604 return ERR_PTR(-ENOMEM); 605 606 for (i = 0; i < pbl_info->num_pbls; i++) { 607 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, 608 &pa, flags); 609 if (!va) 610 goto err; 611 612 memset(va, 0, pbl_info->pbl_size); 613 pbl_table[i].va = va; 614 pbl_table[i].pa = pa; 615 } 616 617 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 618 * the first one with physical pointers to all of the rest 619 */ 620 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 621 for (i = 0; i < pbl_info->num_pbls - 1; i++) 622 pbl_main_tbl[i] = pbl_table[i + 1].pa; 623 624 return pbl_table; 625 626 err: 627 for (i--; i >= 0; i--) 628 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 629 pbl_table[i].va, pbl_table[i].pa); 630 631 qedr_free_pbl(dev, pbl_info, pbl_table); 632 633 return ERR_PTR(-ENOMEM); 634 } 635 636 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev, 637 struct qedr_pbl_info *pbl_info, 638 u32 num_pbes, int two_layer_capable) 639 { 640 u32 pbl_capacity; 641 u32 pbl_size; 642 u32 num_pbls; 643 644 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 645 if (num_pbes > MAX_PBES_TWO_LAYER) { 646 DP_ERR(dev, "prepare pbl table: too many pages %d\n", 647 num_pbes); 648 return -EINVAL; 649 } 650 651 /* calculate required pbl page size */ 652 pbl_size = MIN_FW_PBL_PAGE_SIZE; 653 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 654 NUM_PBES_ON_PAGE(pbl_size); 655 656 while (pbl_capacity < num_pbes) { 657 pbl_size *= 2; 658 pbl_capacity = pbl_size / sizeof(u64); 659 pbl_capacity = pbl_capacity * pbl_capacity; 660 } 661 662 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 663 num_pbls++; /* One for the layer0 ( points to the pbls) */ 664 pbl_info->two_layered = true; 665 } else { 666 /* One layered PBL */ 667 num_pbls = 1; 668 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, 669 roundup_pow_of_two((num_pbes * sizeof(u64)))); 670 pbl_info->two_layered = false; 671 } 672 673 pbl_info->num_pbls = num_pbls; 674 pbl_info->pbl_size = pbl_size; 675 pbl_info->num_pbes = num_pbes; 676 677 DP_DEBUG(dev, QEDR_MSG_MR, 678 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n", 679 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 680 681 return 0; 682 } 683 684 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, 685 struct qedr_pbl *pbl, 686 struct qedr_pbl_info *pbl_info, u32 pg_shift) 687 { 688 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 689 u32 fw_pg_cnt, fw_pg_per_umem_pg; 690 struct qedr_pbl *pbl_tbl; 691 struct scatterlist *sg; 692 struct regpair *pbe; 693 u64 pg_addr; 694 int entry; 695 696 if (!pbl_info->num_pbes) 697 return; 698 699 /* If we have a two layered pbl, the first pbl points to the rest 700 * of the pbls and the first entry lays on the second pbl in the table 701 */ 702 if (pbl_info->two_layered) 703 pbl_tbl = &pbl[1]; 704 else 705 pbl_tbl = pbl; 706 707 pbe = (struct regpair *)pbl_tbl->va; 708 if (!pbe) { 709 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n"); 710 return; 711 } 712 713 pbe_cnt = 0; 714 715 shift = umem->page_shift; 716 717 fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift); 718 719 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 720 pages = sg_dma_len(sg) >> shift; 721 pg_addr = sg_dma_address(sg); 722 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 723 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) { 724 pbe->lo = cpu_to_le32(pg_addr); 725 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); 726 727 pg_addr += BIT(pg_shift); 728 pbe_cnt++; 729 total_num_pbes++; 730 pbe++; 731 732 if (total_num_pbes == pbl_info->num_pbes) 733 return; 734 735 /* If the given pbl is full storing the pbes, 736 * move to next pbl. 737 */ 738 if (pbe_cnt == 739 (pbl_info->pbl_size / sizeof(u64))) { 740 pbl_tbl++; 741 pbe = (struct regpair *)pbl_tbl->va; 742 pbe_cnt = 0; 743 } 744 745 fw_pg_cnt++; 746 } 747 } 748 } 749 } 750 751 static int qedr_copy_cq_uresp(struct qedr_dev *dev, 752 struct qedr_cq *cq, struct ib_udata *udata) 753 { 754 struct qedr_create_cq_uresp uresp; 755 int rc; 756 757 memset(&uresp, 0, sizeof(uresp)); 758 759 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 760 uresp.icid = cq->icid; 761 762 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 763 if (rc) 764 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); 765 766 return rc; 767 } 768 769 static void consume_cqe(struct qedr_cq *cq) 770 { 771 if (cq->latest_cqe == cq->toggle_cqe) 772 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 773 774 cq->latest_cqe = qed_chain_consume(&cq->pbl); 775 } 776 777 static inline int qedr_align_cq_entries(int entries) 778 { 779 u64 size, aligned_size; 780 781 /* We allocate an extra entry that we don't report to the FW. */ 782 size = (entries + 1) * QEDR_CQE_SIZE; 783 aligned_size = ALIGN(size, PAGE_SIZE); 784 785 return aligned_size / QEDR_CQE_SIZE; 786 } 787 788 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx, 789 struct qedr_dev *dev, 790 struct qedr_userq *q, 791 u64 buf_addr, size_t buf_len, 792 int access, int dmasync, 793 int alloc_and_init) 794 { 795 u32 fw_pages; 796 int rc; 797 798 q->buf_addr = buf_addr; 799 q->buf_len = buf_len; 800 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync); 801 if (IS_ERR(q->umem)) { 802 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", 803 PTR_ERR(q->umem)); 804 return PTR_ERR(q->umem); 805 } 806 807 fw_pages = ib_umem_page_count(q->umem) << 808 (q->umem->page_shift - FW_PAGE_SHIFT); 809 810 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); 811 if (rc) 812 goto err0; 813 814 if (alloc_and_init) { 815 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 816 if (IS_ERR(q->pbl_tbl)) { 817 rc = PTR_ERR(q->pbl_tbl); 818 goto err0; 819 } 820 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info, 821 FW_PAGE_SHIFT); 822 } else { 823 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); 824 if (!q->pbl_tbl) { 825 rc = -ENOMEM; 826 goto err0; 827 } 828 } 829 830 return 0; 831 832 err0: 833 ib_umem_release(q->umem); 834 q->umem = NULL; 835 836 return rc; 837 } 838 839 static inline void qedr_init_cq_params(struct qedr_cq *cq, 840 struct qedr_ucontext *ctx, 841 struct qedr_dev *dev, int vector, 842 int chain_entries, int page_cnt, 843 u64 pbl_ptr, 844 struct qed_rdma_create_cq_in_params 845 *params) 846 { 847 memset(params, 0, sizeof(*params)); 848 params->cq_handle_hi = upper_32_bits((uintptr_t)cq); 849 params->cq_handle_lo = lower_32_bits((uintptr_t)cq); 850 params->cnq_id = vector; 851 params->cq_size = chain_entries - 1; 852 params->dpi = (ctx) ? ctx->dpi : dev->dpi; 853 params->pbl_num_pages = page_cnt; 854 params->pbl_ptr = pbl_ptr; 855 params->pbl_two_level = 0; 856 } 857 858 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags) 859 { 860 /* Flush data before signalling doorbell */ 861 wmb(); 862 cq->db.data.agg_flags = flags; 863 cq->db.data.value = cpu_to_le32(cons); 864 writeq(cq->db.raw, cq->db_addr); 865 866 /* Make sure write would stick */ 867 mmiowb(); 868 } 869 870 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 871 { 872 struct qedr_cq *cq = get_qedr_cq(ibcq); 873 unsigned long sflags; 874 struct qedr_dev *dev; 875 876 dev = get_qedr_dev(ibcq->device); 877 878 if (cq->destroyed) { 879 DP_ERR(dev, 880 "warning: arm was invoked after destroy for cq %p (icid=%d)\n", 881 cq, cq->icid); 882 return -EINVAL; 883 } 884 885 886 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 887 return 0; 888 889 spin_lock_irqsave(&cq->cq_lock, sflags); 890 891 cq->arm_flags = 0; 892 893 if (flags & IB_CQ_SOLICITED) 894 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 895 896 if (flags & IB_CQ_NEXT_COMP) 897 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 898 899 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 900 901 spin_unlock_irqrestore(&cq->cq_lock, sflags); 902 903 return 0; 904 } 905 906 struct ib_cq *qedr_create_cq(struct ib_device *ibdev, 907 const struct ib_cq_init_attr *attr, 908 struct ib_ucontext *ib_ctx, struct ib_udata *udata) 909 { 910 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx); 911 struct qed_rdma_destroy_cq_out_params destroy_oparams; 912 struct qed_rdma_destroy_cq_in_params destroy_iparams; 913 struct qedr_dev *dev = get_qedr_dev(ibdev); 914 struct qed_rdma_create_cq_in_params params; 915 struct qedr_create_cq_ureq ureq; 916 int vector = attr->comp_vector; 917 int entries = attr->cqe; 918 struct qedr_cq *cq; 919 int chain_entries; 920 int page_cnt; 921 u64 pbl_ptr; 922 u16 icid; 923 int rc; 924 925 DP_DEBUG(dev, QEDR_MSG_INIT, 926 "create_cq: called from %s. entries=%d, vector=%d\n", 927 udata ? "User Lib" : "Kernel", entries, vector); 928 929 if (entries > QEDR_MAX_CQES) { 930 DP_ERR(dev, 931 "create cq: the number of entries %d is too high. Must be equal or below %d.\n", 932 entries, QEDR_MAX_CQES); 933 return ERR_PTR(-EINVAL); 934 } 935 936 chain_entries = qedr_align_cq_entries(entries); 937 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); 938 939 cq = kzalloc(sizeof(*cq), GFP_KERNEL); 940 if (!cq) 941 return ERR_PTR(-ENOMEM); 942 943 if (udata) { 944 memset(&ureq, 0, sizeof(ureq)); 945 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { 946 DP_ERR(dev, 947 "create cq: problem copying data from user space\n"); 948 goto err0; 949 } 950 951 if (!ureq.len) { 952 DP_ERR(dev, 953 "create cq: cannot create a cq with 0 entries\n"); 954 goto err0; 955 } 956 957 cq->cq_type = QEDR_CQ_TYPE_USER; 958 959 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, 960 ureq.len, IB_ACCESS_LOCAL_WRITE, 961 1, 1); 962 if (rc) 963 goto err0; 964 965 pbl_ptr = cq->q.pbl_tbl->pa; 966 page_cnt = cq->q.pbl_info.num_pbes; 967 968 cq->ibcq.cqe = chain_entries; 969 } else { 970 cq->cq_type = QEDR_CQ_TYPE_KERNEL; 971 972 rc = dev->ops->common->chain_alloc(dev->cdev, 973 QED_CHAIN_USE_TO_CONSUME, 974 QED_CHAIN_MODE_PBL, 975 QED_CHAIN_CNT_TYPE_U32, 976 chain_entries, 977 sizeof(union rdma_cqe), 978 &cq->pbl, NULL); 979 if (rc) 980 goto err1; 981 982 page_cnt = qed_chain_get_page_cnt(&cq->pbl); 983 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); 984 cq->ibcq.cqe = cq->pbl.capacity; 985 } 986 987 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, 988 pbl_ptr, ¶ms); 989 990 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 991 if (rc) 992 goto err2; 993 994 cq->icid = icid; 995 cq->sig = QEDR_CQ_MAGIC_NUMBER; 996 spin_lock_init(&cq->cq_lock); 997 998 if (ib_ctx) { 999 rc = qedr_copy_cq_uresp(dev, cq, udata); 1000 if (rc) 1001 goto err3; 1002 } else { 1003 /* Generate doorbell address. */ 1004 cq->db_addr = dev->db_addr + 1005 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 1006 cq->db.data.icid = cq->icid; 1007 cq->db.data.params = DB_AGG_CMD_SET << 1008 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 1009 1010 /* point to the very last element, passing it we will toggle */ 1011 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); 1012 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 1013 cq->latest_cqe = NULL; 1014 consume_cqe(cq); 1015 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 1016 } 1017 1018 DP_DEBUG(dev, QEDR_MSG_CQ, 1019 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", 1020 cq->icid, cq, params.cq_size); 1021 1022 return &cq->ibcq; 1023 1024 err3: 1025 destroy_iparams.icid = cq->icid; 1026 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, 1027 &destroy_oparams); 1028 err2: 1029 if (udata) 1030 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1031 else 1032 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1033 err1: 1034 if (udata) 1035 ib_umem_release(cq->q.umem); 1036 err0: 1037 kfree(cq); 1038 return ERR_PTR(-EINVAL); 1039 } 1040 1041 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 1042 { 1043 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1044 struct qedr_cq *cq = get_qedr_cq(ibcq); 1045 1046 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq); 1047 1048 return 0; 1049 } 1050 1051 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) 1052 #define QEDR_DESTROY_CQ_ITER_DURATION (10) 1053 1054 int qedr_destroy_cq(struct ib_cq *ibcq) 1055 { 1056 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1057 struct qed_rdma_destroy_cq_out_params oparams; 1058 struct qed_rdma_destroy_cq_in_params iparams; 1059 struct qedr_cq *cq = get_qedr_cq(ibcq); 1060 int iter; 1061 int rc; 1062 1063 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); 1064 1065 cq->destroyed = 1; 1066 1067 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 1068 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 1069 goto done; 1070 1071 iparams.icid = cq->icid; 1072 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 1073 if (rc) 1074 return rc; 1075 1076 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1077 1078 if (ibcq->uobject && ibcq->uobject->context) { 1079 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1080 ib_umem_release(cq->q.umem); 1081 } 1082 1083 /* We don't want the IRQ handler to handle a non-existing CQ so we 1084 * wait until all CNQ interrupts, if any, are received. This will always 1085 * happen and will always happen very fast. If not, then a serious error 1086 * has occured. That is why we can use a long delay. 1087 * We spin for a short time so we don’t lose time on context switching 1088 * in case all the completions are handled in that span. Otherwise 1089 * we sleep for a while and check again. Since the CNQ may be 1090 * associated with (only) the current CPU we use msleep to allow the 1091 * current CPU to be freed. 1092 * The CNQ notification is increased in qedr_irq_handler(). 1093 */ 1094 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1095 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1096 udelay(QEDR_DESTROY_CQ_ITER_DURATION); 1097 iter--; 1098 } 1099 1100 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1101 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1102 msleep(QEDR_DESTROY_CQ_ITER_DURATION); 1103 iter--; 1104 } 1105 1106 if (oparams.num_cq_notif != cq->cnq_notif) 1107 goto err; 1108 1109 /* Note that we don't need to have explicit code to wait for the 1110 * completion of the event handler because it is invoked from the EQ. 1111 * Since the destroy CQ ramrod has also been received on the EQ we can 1112 * be certain that there's no event handler in process. 1113 */ 1114 done: 1115 cq->sig = ~cq->sig; 1116 1117 kfree(cq); 1118 1119 return 0; 1120 1121 err: 1122 DP_ERR(dev, 1123 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n", 1124 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif); 1125 1126 return -EINVAL; 1127 } 1128 1129 static inline int get_gid_info_from_table(struct ib_qp *ibqp, 1130 struct ib_qp_attr *attr, 1131 int attr_mask, 1132 struct qed_rdma_modify_qp_in_params 1133 *qp_params) 1134 { 1135 enum rdma_network_type nw_type; 1136 struct ib_gid_attr gid_attr; 1137 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1138 union ib_gid gid; 1139 u32 ipv4_addr; 1140 int rc = 0; 1141 int i; 1142 1143 rc = ib_get_cached_gid(ibqp->device, 1144 rdma_ah_get_port_num(&attr->ah_attr), 1145 grh->sgid_index, &gid, &gid_attr); 1146 if (rc) 1147 return rc; 1148 1149 if (!memcmp(&gid, &zgid, sizeof(gid))) 1150 return -ENOENT; 1151 1152 if (gid_attr.ndev) { 1153 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev); 1154 1155 dev_put(gid_attr.ndev); 1156 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid); 1157 switch (nw_type) { 1158 case RDMA_NETWORK_IPV6: 1159 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0], 1160 sizeof(qp_params->sgid)); 1161 memcpy(&qp_params->dgid.bytes[0], 1162 &grh->dgid, 1163 sizeof(qp_params->dgid)); 1164 qp_params->roce_mode = ROCE_V2_IPV6; 1165 SET_FIELD(qp_params->modify_flags, 1166 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1167 break; 1168 case RDMA_NETWORK_IB: 1169 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0], 1170 sizeof(qp_params->sgid)); 1171 memcpy(&qp_params->dgid.bytes[0], 1172 &grh->dgid, 1173 sizeof(qp_params->dgid)); 1174 qp_params->roce_mode = ROCE_V1; 1175 break; 1176 case RDMA_NETWORK_IPV4: 1177 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid)); 1178 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid)); 1179 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw); 1180 qp_params->sgid.ipv4_addr = ipv4_addr; 1181 ipv4_addr = 1182 qedr_get_ipv4_from_gid(grh->dgid.raw); 1183 qp_params->dgid.ipv4_addr = ipv4_addr; 1184 SET_FIELD(qp_params->modify_flags, 1185 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1186 qp_params->roce_mode = ROCE_V2_IPV4; 1187 break; 1188 } 1189 } 1190 1191 for (i = 0; i < 4; i++) { 1192 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 1193 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 1194 } 1195 1196 if (qp_params->vlan_id >= VLAN_CFI_MASK) 1197 qp_params->vlan_id = 0; 1198 1199 return 0; 1200 } 1201 1202 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, 1203 struct ib_qp_init_attr *attrs) 1204 { 1205 struct qedr_device_attr *qattr = &dev->attr; 1206 1207 /* QP0... attrs->qp_type == IB_QPT_GSI */ 1208 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { 1209 DP_DEBUG(dev, QEDR_MSG_QP, 1210 "create qp: unsupported qp type=0x%x requested\n", 1211 attrs->qp_type); 1212 return -EINVAL; 1213 } 1214 1215 if (attrs->cap.max_send_wr > qattr->max_sqe) { 1216 DP_ERR(dev, 1217 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n", 1218 attrs->cap.max_send_wr, qattr->max_sqe); 1219 return -EINVAL; 1220 } 1221 1222 if (attrs->cap.max_inline_data > qattr->max_inline) { 1223 DP_ERR(dev, 1224 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n", 1225 attrs->cap.max_inline_data, qattr->max_inline); 1226 return -EINVAL; 1227 } 1228 1229 if (attrs->cap.max_send_sge > qattr->max_sge) { 1230 DP_ERR(dev, 1231 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n", 1232 attrs->cap.max_send_sge, qattr->max_sge); 1233 return -EINVAL; 1234 } 1235 1236 if (attrs->cap.max_recv_sge > qattr->max_sge) { 1237 DP_ERR(dev, 1238 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n", 1239 attrs->cap.max_recv_sge, qattr->max_sge); 1240 return -EINVAL; 1241 } 1242 1243 /* Unprivileged user space cannot create special QP */ 1244 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) { 1245 DP_ERR(dev, 1246 "create qp: userspace can't create special QPs of type=0x%x\n", 1247 attrs->qp_type); 1248 return -EINVAL; 1249 } 1250 1251 return 0; 1252 } 1253 1254 static void qedr_copy_rq_uresp(struct qedr_dev *dev, 1255 struct qedr_create_qp_uresp *uresp, 1256 struct qedr_qp *qp) 1257 { 1258 /* iWARP requires two doorbells per RQ. */ 1259 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1260 uresp->rq_db_offset = 1261 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1262 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1263 } else { 1264 uresp->rq_db_offset = 1265 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1266 } 1267 1268 uresp->rq_icid = qp->icid; 1269 } 1270 1271 static void qedr_copy_sq_uresp(struct qedr_dev *dev, 1272 struct qedr_create_qp_uresp *uresp, 1273 struct qedr_qp *qp) 1274 { 1275 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1276 1277 /* iWARP uses the same cid for rq and sq */ 1278 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1279 uresp->sq_icid = qp->icid; 1280 else 1281 uresp->sq_icid = qp->icid + 1; 1282 } 1283 1284 static int qedr_copy_qp_uresp(struct qedr_dev *dev, 1285 struct qedr_qp *qp, struct ib_udata *udata) 1286 { 1287 struct qedr_create_qp_uresp uresp; 1288 int rc; 1289 1290 memset(&uresp, 0, sizeof(uresp)); 1291 qedr_copy_sq_uresp(dev, &uresp, qp); 1292 qedr_copy_rq_uresp(dev, &uresp, qp); 1293 1294 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 1295 uresp.qp_id = qp->qp_id; 1296 1297 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1298 if (rc) 1299 DP_ERR(dev, 1300 "create qp: failed a copy to user space with qp icid=0x%x.\n", 1301 qp->icid); 1302 1303 return rc; 1304 } 1305 1306 static void qedr_set_common_qp_params(struct qedr_dev *dev, 1307 struct qedr_qp *qp, 1308 struct qedr_pd *pd, 1309 struct ib_qp_init_attr *attrs) 1310 { 1311 spin_lock_init(&qp->q_lock); 1312 atomic_set(&qp->refcnt, 1); 1313 qp->pd = pd; 1314 qp->qp_type = attrs->qp_type; 1315 qp->max_inline_data = attrs->cap.max_inline_data; 1316 qp->sq.max_sges = attrs->cap.max_send_sge; 1317 qp->state = QED_ROCE_QP_STATE_RESET; 1318 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1319 qp->sq_cq = get_qedr_cq(attrs->send_cq); 1320 qp->rq_cq = get_qedr_cq(attrs->recv_cq); 1321 qp->dev = dev; 1322 qp->rq.max_sges = attrs->cap.max_recv_sge; 1323 1324 DP_DEBUG(dev, QEDR_MSG_QP, 1325 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 1326 qp->rq.max_sges, qp->rq_cq->icid); 1327 DP_DEBUG(dev, QEDR_MSG_QP, 1328 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n", 1329 pd->pd_id, qp->qp_type, qp->max_inline_data, 1330 qp->state, qp->signaled, (attrs->srq) ? 1 : 0); 1331 DP_DEBUG(dev, QEDR_MSG_QP, 1332 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1333 qp->sq.max_sges, qp->sq_cq->icid); 1334 } 1335 1336 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1337 { 1338 qp->sq.db = dev->db_addr + 1339 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1340 qp->sq.db_data.data.icid = qp->icid + 1; 1341 qp->rq.db = dev->db_addr + 1342 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1343 qp->rq.db_data.data.icid = qp->icid; 1344 } 1345 1346 static inline void 1347 qedr_init_common_qp_in_params(struct qedr_dev *dev, 1348 struct qedr_pd *pd, 1349 struct qedr_qp *qp, 1350 struct ib_qp_init_attr *attrs, 1351 bool fmr_and_reserved_lkey, 1352 struct qed_rdma_create_qp_in_params *params) 1353 { 1354 /* QP handle to be written in an async event */ 1355 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp); 1356 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp); 1357 1358 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 1359 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 1360 params->pd = pd->pd_id; 1361 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 1362 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; 1363 params->stats_queue = 0; 1364 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1365 params->srq_id = 0; 1366 params->use_srq = false; 1367 } 1368 1369 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) 1370 { 1371 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. " 1372 "qp=%p. " 1373 "sq_addr=0x%llx, " 1374 "sq_len=%zd, " 1375 "rq_addr=0x%llx, " 1376 "rq_len=%zd" 1377 "\n", 1378 qp, 1379 qp->usq.buf_addr, 1380 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len); 1381 } 1382 1383 static int qedr_idr_add(struct qedr_dev *dev, void *ptr, u32 id) 1384 { 1385 int rc; 1386 1387 if (!rdma_protocol_iwarp(&dev->ibdev, 1)) 1388 return 0; 1389 1390 idr_preload(GFP_KERNEL); 1391 spin_lock_irq(&dev->idr_lock); 1392 1393 rc = idr_alloc(&dev->qpidr, ptr, id, id + 1, GFP_ATOMIC); 1394 1395 spin_unlock_irq(&dev->idr_lock); 1396 idr_preload_end(); 1397 1398 return rc < 0 ? rc : 0; 1399 } 1400 1401 static void qedr_idr_remove(struct qedr_dev *dev, u32 id) 1402 { 1403 if (!rdma_protocol_iwarp(&dev->ibdev, 1)) 1404 return; 1405 1406 spin_lock_irq(&dev->idr_lock); 1407 idr_remove(&dev->qpidr, id); 1408 spin_unlock_irq(&dev->idr_lock); 1409 } 1410 1411 static inline void 1412 qedr_iwarp_populate_user_qp(struct qedr_dev *dev, 1413 struct qedr_qp *qp, 1414 struct qed_rdma_create_qp_out_params *out_params) 1415 { 1416 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 1417 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 1418 1419 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 1420 &qp->usq.pbl_info, FW_PAGE_SHIFT); 1421 1422 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 1423 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 1424 1425 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 1426 &qp->urq.pbl_info, FW_PAGE_SHIFT); 1427 } 1428 1429 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp) 1430 { 1431 if (qp->usq.umem) 1432 ib_umem_release(qp->usq.umem); 1433 qp->usq.umem = NULL; 1434 1435 if (qp->urq.umem) 1436 ib_umem_release(qp->urq.umem); 1437 qp->urq.umem = NULL; 1438 } 1439 1440 static int qedr_create_user_qp(struct qedr_dev *dev, 1441 struct qedr_qp *qp, 1442 struct ib_pd *ibpd, 1443 struct ib_udata *udata, 1444 struct ib_qp_init_attr *attrs) 1445 { 1446 struct qed_rdma_create_qp_in_params in_params; 1447 struct qed_rdma_create_qp_out_params out_params; 1448 struct qedr_pd *pd = get_qedr_pd(ibpd); 1449 struct ib_ucontext *ib_ctx = NULL; 1450 struct qedr_create_qp_ureq ureq; 1451 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1); 1452 int rc = -EINVAL; 1453 1454 ib_ctx = ibpd->uobject->context; 1455 1456 memset(&ureq, 0, sizeof(ureq)); 1457 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq)); 1458 if (rc) { 1459 DP_ERR(dev, "Problem copying data from user space\n"); 1460 return rc; 1461 } 1462 1463 /* SQ - read access only (0), dma sync not required (0) */ 1464 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr, 1465 ureq.sq_len, 0, 0, alloc_and_init); 1466 if (rc) 1467 return rc; 1468 1469 /* RQ - read access only (0), dma sync not required (0) */ 1470 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr, 1471 ureq.rq_len, 0, 0, alloc_and_init); 1472 if (rc) 1473 return rc; 1474 1475 memset(&in_params, 0, sizeof(in_params)); 1476 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 1477 in_params.qp_handle_lo = ureq.qp_handle_lo; 1478 in_params.qp_handle_hi = ureq.qp_handle_hi; 1479 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 1480 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 1481 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 1482 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 1483 1484 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1485 &in_params, &out_params); 1486 1487 if (!qp->qed_qp) { 1488 rc = -ENOMEM; 1489 goto err1; 1490 } 1491 1492 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1493 qedr_iwarp_populate_user_qp(dev, qp, &out_params); 1494 1495 qp->qp_id = out_params.qp_id; 1496 qp->icid = out_params.icid; 1497 1498 rc = qedr_copy_qp_uresp(dev, qp, udata); 1499 if (rc) 1500 goto err; 1501 1502 qedr_qp_user_print(dev, qp); 1503 1504 return 0; 1505 err: 1506 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1507 if (rc) 1508 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc); 1509 1510 err1: 1511 qedr_cleanup_user(dev, qp); 1512 return rc; 1513 } 1514 1515 static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1516 { 1517 qp->sq.db = dev->db_addr + 1518 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1519 qp->sq.db_data.data.icid = qp->icid; 1520 1521 qp->rq.db = dev->db_addr + 1522 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1523 qp->rq.db_data.data.icid = qp->icid; 1524 qp->rq.iwarp_db2 = dev->db_addr + 1525 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1526 qp->rq.iwarp_db2_data.data.icid = qp->icid; 1527 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 1528 } 1529 1530 static int 1531 qedr_roce_create_kernel_qp(struct qedr_dev *dev, 1532 struct qedr_qp *qp, 1533 struct qed_rdma_create_qp_in_params *in_params, 1534 u32 n_sq_elems, u32 n_rq_elems) 1535 { 1536 struct qed_rdma_create_qp_out_params out_params; 1537 int rc; 1538 1539 rc = dev->ops->common->chain_alloc(dev->cdev, 1540 QED_CHAIN_USE_TO_PRODUCE, 1541 QED_CHAIN_MODE_PBL, 1542 QED_CHAIN_CNT_TYPE_U32, 1543 n_sq_elems, 1544 QEDR_SQE_ELEMENT_SIZE, 1545 &qp->sq.pbl, NULL); 1546 1547 if (rc) 1548 return rc; 1549 1550 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl); 1551 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl); 1552 1553 rc = dev->ops->common->chain_alloc(dev->cdev, 1554 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1555 QED_CHAIN_MODE_PBL, 1556 QED_CHAIN_CNT_TYPE_U32, 1557 n_rq_elems, 1558 QEDR_RQE_ELEMENT_SIZE, 1559 &qp->rq.pbl, NULL); 1560 if (rc) 1561 return rc; 1562 1563 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl); 1564 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl); 1565 1566 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1567 in_params, &out_params); 1568 1569 if (!qp->qed_qp) 1570 return -EINVAL; 1571 1572 qp->qp_id = out_params.qp_id; 1573 qp->icid = out_params.icid; 1574 1575 qedr_set_roce_db_info(dev, qp); 1576 return rc; 1577 } 1578 1579 static int 1580 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev, 1581 struct qedr_qp *qp, 1582 struct qed_rdma_create_qp_in_params *in_params, 1583 u32 n_sq_elems, u32 n_rq_elems) 1584 { 1585 struct qed_rdma_create_qp_out_params out_params; 1586 struct qed_chain_ext_pbl ext_pbl; 1587 int rc; 1588 1589 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems, 1590 QEDR_SQE_ELEMENT_SIZE, 1591 QED_CHAIN_MODE_PBL); 1592 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems, 1593 QEDR_RQE_ELEMENT_SIZE, 1594 QED_CHAIN_MODE_PBL); 1595 1596 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1597 in_params, &out_params); 1598 1599 if (!qp->qed_qp) 1600 return -EINVAL; 1601 1602 /* Now we allocate the chain */ 1603 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt; 1604 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys; 1605 1606 rc = dev->ops->common->chain_alloc(dev->cdev, 1607 QED_CHAIN_USE_TO_PRODUCE, 1608 QED_CHAIN_MODE_PBL, 1609 QED_CHAIN_CNT_TYPE_U32, 1610 n_sq_elems, 1611 QEDR_SQE_ELEMENT_SIZE, 1612 &qp->sq.pbl, &ext_pbl); 1613 1614 if (rc) 1615 goto err; 1616 1617 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt; 1618 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys; 1619 1620 rc = dev->ops->common->chain_alloc(dev->cdev, 1621 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1622 QED_CHAIN_MODE_PBL, 1623 QED_CHAIN_CNT_TYPE_U32, 1624 n_rq_elems, 1625 QEDR_RQE_ELEMENT_SIZE, 1626 &qp->rq.pbl, &ext_pbl); 1627 1628 if (rc) 1629 goto err; 1630 1631 qp->qp_id = out_params.qp_id; 1632 qp->icid = out_params.icid; 1633 1634 qedr_set_iwarp_db_info(dev, qp); 1635 return rc; 1636 1637 err: 1638 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1639 1640 return rc; 1641 } 1642 1643 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) 1644 { 1645 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl); 1646 kfree(qp->wqe_wr_id); 1647 1648 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl); 1649 kfree(qp->rqe_wr_id); 1650 } 1651 1652 static int qedr_create_kernel_qp(struct qedr_dev *dev, 1653 struct qedr_qp *qp, 1654 struct ib_pd *ibpd, 1655 struct ib_qp_init_attr *attrs) 1656 { 1657 struct qed_rdma_create_qp_in_params in_params; 1658 struct qedr_pd *pd = get_qedr_pd(ibpd); 1659 int rc = -EINVAL; 1660 u32 n_rq_elems; 1661 u32 n_sq_elems; 1662 u32 n_sq_entries; 1663 1664 memset(&in_params, 0, sizeof(in_params)); 1665 1666 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in 1667 * the ring. The ring should allow at least a single WR, even if the 1668 * user requested none, due to allocation issues. 1669 * We should add an extra WR since the prod and cons indices of 1670 * wqe_wr_id are managed in such a way that the WQ is considered full 1671 * when (prod+1)%max_wr==cons. We currently don't do that because we 1672 * double the number of entries due an iSER issue that pushes far more 1673 * WRs than indicated. If we decline its ib_post_send() then we get 1674 * error prints in the dmesg we'd like to avoid. 1675 */ 1676 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 1677 dev->attr.max_sqe); 1678 1679 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id), 1680 GFP_KERNEL); 1681 if (!qp->wqe_wr_id) { 1682 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n"); 1683 return -ENOMEM; 1684 } 1685 1686 /* QP handle to be written in CQE */ 1687 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp); 1688 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp); 1689 1690 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in 1691 * the ring. There ring should allow at least a single WR, even if the 1692 * user requested none, due to allocation issues. 1693 */ 1694 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1); 1695 1696 /* Allocate driver internal RQ array */ 1697 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id), 1698 GFP_KERNEL); 1699 if (!qp->rqe_wr_id) { 1700 DP_ERR(dev, 1701 "create qp: failed RQ shadow memory allocation\n"); 1702 kfree(qp->wqe_wr_id); 1703 return -ENOMEM; 1704 } 1705 1706 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 1707 1708 n_sq_entries = attrs->cap.max_send_wr; 1709 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe); 1710 n_sq_entries = max_t(u32, n_sq_entries, 1); 1711 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE; 1712 1713 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE; 1714 1715 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1716 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params, 1717 n_sq_elems, n_rq_elems); 1718 else 1719 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params, 1720 n_sq_elems, n_rq_elems); 1721 if (rc) 1722 qedr_cleanup_kernel(dev, qp); 1723 1724 return rc; 1725 } 1726 1727 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, 1728 struct ib_qp_init_attr *attrs, 1729 struct ib_udata *udata) 1730 { 1731 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 1732 struct qedr_pd *pd = get_qedr_pd(ibpd); 1733 struct qedr_qp *qp; 1734 struct ib_qp *ibqp; 1735 int rc = 0; 1736 1737 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", 1738 udata ? "user library" : "kernel", pd); 1739 1740 rc = qedr_check_qp_attrs(ibpd, dev, attrs); 1741 if (rc) 1742 return ERR_PTR(rc); 1743 1744 if (attrs->srq) 1745 return ERR_PTR(-EINVAL); 1746 1747 DP_DEBUG(dev, QEDR_MSG_QP, 1748 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 1749 udata ? "user library" : "kernel", attrs->event_handler, pd, 1750 get_qedr_cq(attrs->send_cq), 1751 get_qedr_cq(attrs->send_cq)->icid, 1752 get_qedr_cq(attrs->recv_cq), 1753 get_qedr_cq(attrs->recv_cq)->icid); 1754 1755 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1756 if (!qp) { 1757 DP_ERR(dev, "create qp: failed allocating memory\n"); 1758 return ERR_PTR(-ENOMEM); 1759 } 1760 1761 qedr_set_common_qp_params(dev, qp, pd, attrs); 1762 1763 if (attrs->qp_type == IB_QPT_GSI) { 1764 ibqp = qedr_create_gsi_qp(dev, attrs, qp); 1765 if (IS_ERR(ibqp)) 1766 kfree(qp); 1767 return ibqp; 1768 } 1769 1770 if (udata) 1771 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); 1772 else 1773 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); 1774 1775 if (rc) 1776 goto err; 1777 1778 qp->ibqp.qp_num = qp->qp_id; 1779 1780 rc = qedr_idr_add(dev, qp, qp->qp_id); 1781 if (rc) 1782 goto err; 1783 1784 return &qp->ibqp; 1785 1786 err: 1787 kfree(qp); 1788 1789 return ERR_PTR(-EFAULT); 1790 } 1791 1792 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) 1793 { 1794 switch (qp_state) { 1795 case QED_ROCE_QP_STATE_RESET: 1796 return IB_QPS_RESET; 1797 case QED_ROCE_QP_STATE_INIT: 1798 return IB_QPS_INIT; 1799 case QED_ROCE_QP_STATE_RTR: 1800 return IB_QPS_RTR; 1801 case QED_ROCE_QP_STATE_RTS: 1802 return IB_QPS_RTS; 1803 case QED_ROCE_QP_STATE_SQD: 1804 return IB_QPS_SQD; 1805 case QED_ROCE_QP_STATE_ERR: 1806 return IB_QPS_ERR; 1807 case QED_ROCE_QP_STATE_SQE: 1808 return IB_QPS_SQE; 1809 } 1810 return IB_QPS_ERR; 1811 } 1812 1813 static enum qed_roce_qp_state qedr_get_state_from_ibqp( 1814 enum ib_qp_state qp_state) 1815 { 1816 switch (qp_state) { 1817 case IB_QPS_RESET: 1818 return QED_ROCE_QP_STATE_RESET; 1819 case IB_QPS_INIT: 1820 return QED_ROCE_QP_STATE_INIT; 1821 case IB_QPS_RTR: 1822 return QED_ROCE_QP_STATE_RTR; 1823 case IB_QPS_RTS: 1824 return QED_ROCE_QP_STATE_RTS; 1825 case IB_QPS_SQD: 1826 return QED_ROCE_QP_STATE_SQD; 1827 case IB_QPS_ERR: 1828 return QED_ROCE_QP_STATE_ERR; 1829 default: 1830 return QED_ROCE_QP_STATE_ERR; 1831 } 1832 } 1833 1834 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) 1835 { 1836 qed_chain_reset(&qph->pbl); 1837 qph->prod = 0; 1838 qph->cons = 0; 1839 qph->wqe_cons = 0; 1840 qph->db_data.data.value = cpu_to_le16(0); 1841 } 1842 1843 static int qedr_update_qp_state(struct qedr_dev *dev, 1844 struct qedr_qp *qp, 1845 enum qed_roce_qp_state new_state) 1846 { 1847 int status = 0; 1848 1849 if (new_state == qp->state) 1850 return 0; 1851 1852 switch (qp->state) { 1853 case QED_ROCE_QP_STATE_RESET: 1854 switch (new_state) { 1855 case QED_ROCE_QP_STATE_INIT: 1856 qp->prev_wqe_size = 0; 1857 qedr_reset_qp_hwq_info(&qp->sq); 1858 qedr_reset_qp_hwq_info(&qp->rq); 1859 break; 1860 default: 1861 status = -EINVAL; 1862 break; 1863 }; 1864 break; 1865 case QED_ROCE_QP_STATE_INIT: 1866 switch (new_state) { 1867 case QED_ROCE_QP_STATE_RTR: 1868 /* Update doorbell (in case post_recv was 1869 * done before move to RTR) 1870 */ 1871 1872 if (rdma_protocol_roce(&dev->ibdev, 1)) { 1873 wmb(); 1874 writel(qp->rq.db_data.raw, qp->rq.db); 1875 /* Make sure write takes effect */ 1876 mmiowb(); 1877 } 1878 break; 1879 case QED_ROCE_QP_STATE_ERR: 1880 break; 1881 default: 1882 /* Invalid state change. */ 1883 status = -EINVAL; 1884 break; 1885 }; 1886 break; 1887 case QED_ROCE_QP_STATE_RTR: 1888 /* RTR->XXX */ 1889 switch (new_state) { 1890 case QED_ROCE_QP_STATE_RTS: 1891 break; 1892 case QED_ROCE_QP_STATE_ERR: 1893 break; 1894 default: 1895 /* Invalid state change. */ 1896 status = -EINVAL; 1897 break; 1898 }; 1899 break; 1900 case QED_ROCE_QP_STATE_RTS: 1901 /* RTS->XXX */ 1902 switch (new_state) { 1903 case QED_ROCE_QP_STATE_SQD: 1904 break; 1905 case QED_ROCE_QP_STATE_ERR: 1906 break; 1907 default: 1908 /* Invalid state change. */ 1909 status = -EINVAL; 1910 break; 1911 }; 1912 break; 1913 case QED_ROCE_QP_STATE_SQD: 1914 /* SQD->XXX */ 1915 switch (new_state) { 1916 case QED_ROCE_QP_STATE_RTS: 1917 case QED_ROCE_QP_STATE_ERR: 1918 break; 1919 default: 1920 /* Invalid state change. */ 1921 status = -EINVAL; 1922 break; 1923 }; 1924 break; 1925 case QED_ROCE_QP_STATE_ERR: 1926 /* ERR->XXX */ 1927 switch (new_state) { 1928 case QED_ROCE_QP_STATE_RESET: 1929 if ((qp->rq.prod != qp->rq.cons) || 1930 (qp->sq.prod != qp->sq.cons)) { 1931 DP_NOTICE(dev, 1932 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n", 1933 qp->rq.prod, qp->rq.cons, qp->sq.prod, 1934 qp->sq.cons); 1935 status = -EINVAL; 1936 } 1937 break; 1938 default: 1939 status = -EINVAL; 1940 break; 1941 }; 1942 break; 1943 default: 1944 status = -EINVAL; 1945 break; 1946 }; 1947 1948 return status; 1949 } 1950 1951 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1952 int attr_mask, struct ib_udata *udata) 1953 { 1954 struct qedr_qp *qp = get_qedr_qp(ibqp); 1955 struct qed_rdma_modify_qp_in_params qp_params = { 0 }; 1956 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); 1957 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1958 enum ib_qp_state old_qp_state, new_qp_state; 1959 int rc = 0; 1960 1961 DP_DEBUG(dev, QEDR_MSG_QP, 1962 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask, 1963 attr->qp_state); 1964 1965 old_qp_state = qedr_get_ibqp_state(qp->state); 1966 if (attr_mask & IB_QP_STATE) 1967 new_qp_state = attr->qp_state; 1968 else 1969 new_qp_state = old_qp_state; 1970 1971 if (rdma_protocol_roce(&dev->ibdev, 1)) { 1972 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state, 1973 ibqp->qp_type, attr_mask, 1974 IB_LINK_LAYER_ETHERNET)) { 1975 DP_ERR(dev, 1976 "modify qp: invalid attribute mask=0x%x specified for\n" 1977 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n", 1978 attr_mask, qp->qp_id, ibqp->qp_type, 1979 old_qp_state, new_qp_state); 1980 rc = -EINVAL; 1981 goto err; 1982 } 1983 } 1984 1985 /* Translate the masks... */ 1986 if (attr_mask & IB_QP_STATE) { 1987 SET_FIELD(qp_params.modify_flags, 1988 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 1989 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state); 1990 } 1991 1992 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 1993 qp_params.sqd_async = true; 1994 1995 if (attr_mask & IB_QP_PKEY_INDEX) { 1996 SET_FIELD(qp_params.modify_flags, 1997 QED_ROCE_MODIFY_QP_VALID_PKEY, 1); 1998 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) { 1999 rc = -EINVAL; 2000 goto err; 2001 } 2002 2003 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT; 2004 } 2005 2006 if (attr_mask & IB_QP_QKEY) 2007 qp->qkey = attr->qkey; 2008 2009 if (attr_mask & IB_QP_ACCESS_FLAGS) { 2010 SET_FIELD(qp_params.modify_flags, 2011 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 2012 qp_params.incoming_rdma_read_en = attr->qp_access_flags & 2013 IB_ACCESS_REMOTE_READ; 2014 qp_params.incoming_rdma_write_en = attr->qp_access_flags & 2015 IB_ACCESS_REMOTE_WRITE; 2016 qp_params.incoming_atomic_en = attr->qp_access_flags & 2017 IB_ACCESS_REMOTE_ATOMIC; 2018 } 2019 2020 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 2021 if (attr_mask & IB_QP_PATH_MTU) { 2022 if (attr->path_mtu < IB_MTU_256 || 2023 attr->path_mtu > IB_MTU_4096) { 2024 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n"); 2025 rc = -EINVAL; 2026 goto err; 2027 } 2028 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 2029 ib_mtu_enum_to_int(iboe_get_mtu 2030 (dev->ndev->mtu))); 2031 } 2032 2033 if (!qp->mtu) { 2034 qp->mtu = 2035 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2036 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu); 2037 } 2038 2039 SET_FIELD(qp_params.modify_flags, 2040 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1); 2041 2042 qp_params.traffic_class_tos = grh->traffic_class; 2043 qp_params.flow_label = grh->flow_label; 2044 qp_params.hop_limit_ttl = grh->hop_limit; 2045 2046 qp->sgid_idx = grh->sgid_index; 2047 2048 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params); 2049 if (rc) { 2050 DP_ERR(dev, 2051 "modify qp: problems with GID index %d (rc=%d)\n", 2052 grh->sgid_index, rc); 2053 return rc; 2054 } 2055 2056 rc = qedr_get_dmac(dev, &attr->ah_attr, 2057 qp_params.remote_mac_addr); 2058 if (rc) 2059 return rc; 2060 2061 qp_params.use_local_mac = true; 2062 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr); 2063 2064 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n", 2065 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 2066 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 2067 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n", 2068 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 2069 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 2070 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n", 2071 qp_params.remote_mac_addr); 2072 2073 qp_params.mtu = qp->mtu; 2074 qp_params.lb_indication = false; 2075 } 2076 2077 if (!qp_params.mtu) { 2078 /* Stay with current MTU */ 2079 if (qp->mtu) 2080 qp_params.mtu = qp->mtu; 2081 else 2082 qp_params.mtu = 2083 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2084 } 2085 2086 if (attr_mask & IB_QP_TIMEOUT) { 2087 SET_FIELD(qp_params.modify_flags, 2088 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 2089 2090 qp_params.ack_timeout = attr->timeout; 2091 if (attr->timeout) { 2092 u32 temp; 2093 2094 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000; 2095 /* FW requires [msec] */ 2096 qp_params.ack_timeout = temp; 2097 } else { 2098 /* Infinite */ 2099 qp_params.ack_timeout = 0; 2100 } 2101 } 2102 if (attr_mask & IB_QP_RETRY_CNT) { 2103 SET_FIELD(qp_params.modify_flags, 2104 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 2105 qp_params.retry_cnt = attr->retry_cnt; 2106 } 2107 2108 if (attr_mask & IB_QP_RNR_RETRY) { 2109 SET_FIELD(qp_params.modify_flags, 2110 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1); 2111 qp_params.rnr_retry_cnt = attr->rnr_retry; 2112 } 2113 2114 if (attr_mask & IB_QP_RQ_PSN) { 2115 SET_FIELD(qp_params.modify_flags, 2116 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1); 2117 qp_params.rq_psn = attr->rq_psn; 2118 qp->rq_psn = attr->rq_psn; 2119 } 2120 2121 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 2122 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) { 2123 rc = -EINVAL; 2124 DP_ERR(dev, 2125 "unsupported max_rd_atomic=%d, supported=%d\n", 2126 attr->max_rd_atomic, 2127 dev->attr.max_qp_req_rd_atomic_resc); 2128 goto err; 2129 } 2130 2131 SET_FIELD(qp_params.modify_flags, 2132 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1); 2133 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 2134 } 2135 2136 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 2137 SET_FIELD(qp_params.modify_flags, 2138 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1); 2139 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 2140 } 2141 2142 if (attr_mask & IB_QP_SQ_PSN) { 2143 SET_FIELD(qp_params.modify_flags, 2144 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1); 2145 qp_params.sq_psn = attr->sq_psn; 2146 qp->sq_psn = attr->sq_psn; 2147 } 2148 2149 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 2150 if (attr->max_dest_rd_atomic > 2151 dev->attr.max_qp_resp_rd_atomic_resc) { 2152 DP_ERR(dev, 2153 "unsupported max_dest_rd_atomic=%d, supported=%d\n", 2154 attr->max_dest_rd_atomic, 2155 dev->attr.max_qp_resp_rd_atomic_resc); 2156 2157 rc = -EINVAL; 2158 goto err; 2159 } 2160 2161 SET_FIELD(qp_params.modify_flags, 2162 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1); 2163 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 2164 } 2165 2166 if (attr_mask & IB_QP_DEST_QPN) { 2167 SET_FIELD(qp_params.modify_flags, 2168 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1); 2169 2170 qp_params.dest_qp = attr->dest_qp_num; 2171 qp->dest_qp_num = attr->dest_qp_num; 2172 } 2173 2174 if (qp->qp_type != IB_QPT_GSI) 2175 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx, 2176 qp->qed_qp, &qp_params); 2177 2178 if (attr_mask & IB_QP_STATE) { 2179 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 2180 rc = qedr_update_qp_state(dev, qp, qp_params.new_state); 2181 qp->state = qp_params.new_state; 2182 } 2183 2184 err: 2185 return rc; 2186 } 2187 2188 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params) 2189 { 2190 int ib_qp_acc_flags = 0; 2191 2192 if (params->incoming_rdma_write_en) 2193 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 2194 if (params->incoming_rdma_read_en) 2195 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 2196 if (params->incoming_atomic_en) 2197 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 2198 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 2199 return ib_qp_acc_flags; 2200 } 2201 2202 int qedr_query_qp(struct ib_qp *ibqp, 2203 struct ib_qp_attr *qp_attr, 2204 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 2205 { 2206 struct qed_rdma_query_qp_out_params params; 2207 struct qedr_qp *qp = get_qedr_qp(ibqp); 2208 struct qedr_dev *dev = qp->dev; 2209 int rc = 0; 2210 2211 memset(¶ms, 0, sizeof(params)); 2212 2213 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms); 2214 if (rc) 2215 goto err; 2216 2217 memset(qp_attr, 0, sizeof(*qp_attr)); 2218 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 2219 2220 qp_attr->qp_state = qedr_get_ibqp_state(params.state); 2221 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state); 2222 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu); 2223 qp_attr->path_mig_state = IB_MIG_MIGRATED; 2224 qp_attr->rq_psn = params.rq_psn; 2225 qp_attr->sq_psn = params.sq_psn; 2226 qp_attr->dest_qp_num = params.dest_qp; 2227 2228 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms); 2229 2230 qp_attr->cap.max_send_wr = qp->sq.max_wr; 2231 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 2232 qp_attr->cap.max_send_sge = qp->sq.max_sges; 2233 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 2234 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; 2235 qp_init_attr->cap = qp_attr->cap; 2236 2237 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2238 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 2239 params.flow_label, qp->sgid_idx, 2240 params.hop_limit_ttl, params.traffic_class_tos); 2241 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); 2242 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 2243 rdma_ah_set_sl(&qp_attr->ah_attr, 0); 2244 qp_attr->timeout = params.timeout; 2245 qp_attr->rnr_retry = params.rnr_retry; 2246 qp_attr->retry_cnt = params.retry_cnt; 2247 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 2248 qp_attr->pkey_index = params.pkey_index; 2249 qp_attr->port_num = 1; 2250 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 2251 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 2252 qp_attr->alt_pkey_index = 0; 2253 qp_attr->alt_port_num = 0; 2254 qp_attr->alt_timeout = 0; 2255 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 2256 2257 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0; 2258 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 2259 qp_attr->max_rd_atomic = params.max_rd_atomic; 2260 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0; 2261 2262 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n", 2263 qp_attr->cap.max_inline_data); 2264 2265 err: 2266 return rc; 2267 } 2268 2269 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp) 2270 { 2271 int rc = 0; 2272 2273 if (qp->qp_type != IB_QPT_GSI) { 2274 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2275 if (rc) 2276 return rc; 2277 } 2278 2279 if (qp->ibqp.uobject && qp->ibqp.uobject->context) 2280 qedr_cleanup_user(dev, qp); 2281 else 2282 qedr_cleanup_kernel(dev, qp); 2283 2284 return 0; 2285 } 2286 2287 int qedr_destroy_qp(struct ib_qp *ibqp) 2288 { 2289 struct qedr_qp *qp = get_qedr_qp(ibqp); 2290 struct qedr_dev *dev = qp->dev; 2291 struct ib_qp_attr attr; 2292 int attr_mask = 0; 2293 int rc = 0; 2294 2295 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n", 2296 qp, qp->qp_type); 2297 2298 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2299 if ((qp->state != QED_ROCE_QP_STATE_RESET) && 2300 (qp->state != QED_ROCE_QP_STATE_ERR) && 2301 (qp->state != QED_ROCE_QP_STATE_INIT)) { 2302 2303 attr.qp_state = IB_QPS_ERR; 2304 attr_mask |= IB_QP_STATE; 2305 2306 /* Change the QP state to ERROR */ 2307 qedr_modify_qp(ibqp, &attr, attr_mask, NULL); 2308 } 2309 } else { 2310 /* Wait for the connect/accept to complete */ 2311 if (qp->ep) { 2312 int wait_count = 1; 2313 2314 while (qp->ep->during_connect) { 2315 DP_DEBUG(dev, QEDR_MSG_QP, 2316 "Still in during connect/accept\n"); 2317 2318 msleep(100); 2319 if (wait_count++ > 200) { 2320 DP_NOTICE(dev, 2321 "during connect timeout\n"); 2322 break; 2323 } 2324 } 2325 } 2326 } 2327 2328 if (qp->qp_type == IB_QPT_GSI) 2329 qedr_destroy_gsi_qp(dev); 2330 2331 qedr_free_qp_resources(dev, qp); 2332 2333 if (atomic_dec_and_test(&qp->refcnt)) { 2334 qedr_idr_remove(dev, qp->qp_id); 2335 kfree(qp); 2336 } 2337 return rc; 2338 } 2339 2340 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr, 2341 struct ib_udata *udata) 2342 { 2343 struct qedr_ah *ah; 2344 2345 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 2346 if (!ah) 2347 return ERR_PTR(-ENOMEM); 2348 2349 ah->attr = *attr; 2350 2351 return &ah->ibah; 2352 } 2353 2354 int qedr_destroy_ah(struct ib_ah *ibah) 2355 { 2356 struct qedr_ah *ah = get_qedr_ah(ibah); 2357 2358 kfree(ah); 2359 return 0; 2360 } 2361 2362 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) 2363 { 2364 struct qedr_pbl *pbl, *tmp; 2365 2366 if (info->pbl_table) 2367 list_add_tail(&info->pbl_table->list_entry, 2368 &info->free_pbl_list); 2369 2370 if (!list_empty(&info->inuse_pbl_list)) 2371 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 2372 2373 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 2374 list_del(&pbl->list_entry); 2375 qedr_free_pbl(dev, &info->pbl_info, pbl); 2376 } 2377 } 2378 2379 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info, 2380 size_t page_list_len, bool two_layered) 2381 { 2382 struct qedr_pbl *tmp; 2383 int rc; 2384 2385 INIT_LIST_HEAD(&info->free_pbl_list); 2386 INIT_LIST_HEAD(&info->inuse_pbl_list); 2387 2388 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info, 2389 page_list_len, two_layered); 2390 if (rc) 2391 goto done; 2392 2393 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2394 if (IS_ERR(info->pbl_table)) { 2395 rc = PTR_ERR(info->pbl_table); 2396 goto done; 2397 } 2398 2399 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n", 2400 &info->pbl_table->pa); 2401 2402 /* in usual case we use 2 PBLs, so we add one to free 2403 * list and allocating another one 2404 */ 2405 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2406 if (IS_ERR(tmp)) { 2407 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n"); 2408 goto done; 2409 } 2410 2411 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 2412 2413 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa); 2414 2415 done: 2416 if (rc) 2417 free_mr_info(dev, info); 2418 2419 return rc; 2420 } 2421 2422 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 2423 u64 usr_addr, int acc, struct ib_udata *udata) 2424 { 2425 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2426 struct qedr_mr *mr; 2427 struct qedr_pd *pd; 2428 int rc = -ENOMEM; 2429 2430 pd = get_qedr_pd(ibpd); 2431 DP_DEBUG(dev, QEDR_MSG_MR, 2432 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 2433 pd->pd_id, start, len, usr_addr, acc); 2434 2435 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 2436 return ERR_PTR(-EINVAL); 2437 2438 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2439 if (!mr) 2440 return ERR_PTR(rc); 2441 2442 mr->type = QEDR_MR_USER; 2443 2444 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0); 2445 if (IS_ERR(mr->umem)) { 2446 rc = -EFAULT; 2447 goto err0; 2448 } 2449 2450 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); 2451 if (rc) 2452 goto err1; 2453 2454 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 2455 &mr->info.pbl_info, mr->umem->page_shift); 2456 2457 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2458 if (rc) { 2459 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2460 goto err1; 2461 } 2462 2463 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2464 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2465 mr->hw_mr.key = 0; 2466 mr->hw_mr.pd = pd->pd_id; 2467 mr->hw_mr.local_read = 1; 2468 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2469 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2470 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2471 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2472 mr->hw_mr.mw_bind = false; 2473 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 2474 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2475 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2476 mr->hw_mr.page_size_log = mr->umem->page_shift; 2477 mr->hw_mr.fbo = ib_umem_offset(mr->umem); 2478 mr->hw_mr.length = len; 2479 mr->hw_mr.vaddr = usr_addr; 2480 mr->hw_mr.zbva = false; 2481 mr->hw_mr.phy_mr = false; 2482 mr->hw_mr.dma_mr = false; 2483 2484 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2485 if (rc) { 2486 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2487 goto err2; 2488 } 2489 2490 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2491 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2492 mr->hw_mr.remote_atomic) 2493 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2494 2495 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n", 2496 mr->ibmr.lkey); 2497 return &mr->ibmr; 2498 2499 err2: 2500 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2501 err1: 2502 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2503 err0: 2504 kfree(mr); 2505 return ERR_PTR(rc); 2506 } 2507 2508 int qedr_dereg_mr(struct ib_mr *ib_mr) 2509 { 2510 struct qedr_mr *mr = get_qedr_mr(ib_mr); 2511 struct qedr_dev *dev = get_qedr_dev(ib_mr->device); 2512 int rc = 0; 2513 2514 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid); 2515 if (rc) 2516 return rc; 2517 2518 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2519 2520 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR)) 2521 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2522 2523 /* it could be user registered memory. */ 2524 if (mr->umem) 2525 ib_umem_release(mr->umem); 2526 2527 kfree(mr); 2528 2529 return rc; 2530 } 2531 2532 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, 2533 int max_page_list_len) 2534 { 2535 struct qedr_pd *pd = get_qedr_pd(ibpd); 2536 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2537 struct qedr_mr *mr; 2538 int rc = -ENOMEM; 2539 2540 DP_DEBUG(dev, QEDR_MSG_MR, 2541 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id, 2542 max_page_list_len); 2543 2544 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2545 if (!mr) 2546 return ERR_PTR(rc); 2547 2548 mr->dev = dev; 2549 mr->type = QEDR_MR_FRMR; 2550 2551 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1); 2552 if (rc) 2553 goto err0; 2554 2555 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2556 if (rc) { 2557 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2558 goto err0; 2559 } 2560 2561 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2562 mr->hw_mr.tid_type = QED_RDMA_TID_FMR; 2563 mr->hw_mr.key = 0; 2564 mr->hw_mr.pd = pd->pd_id; 2565 mr->hw_mr.local_read = 1; 2566 mr->hw_mr.local_write = 0; 2567 mr->hw_mr.remote_read = 0; 2568 mr->hw_mr.remote_write = 0; 2569 mr->hw_mr.remote_atomic = 0; 2570 mr->hw_mr.mw_bind = false; 2571 mr->hw_mr.pbl_ptr = 0; 2572 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2573 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2574 mr->hw_mr.fbo = 0; 2575 mr->hw_mr.length = 0; 2576 mr->hw_mr.vaddr = 0; 2577 mr->hw_mr.zbva = false; 2578 mr->hw_mr.phy_mr = true; 2579 mr->hw_mr.dma_mr = false; 2580 2581 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2582 if (rc) { 2583 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2584 goto err1; 2585 } 2586 2587 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2588 mr->ibmr.rkey = mr->ibmr.lkey; 2589 2590 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey); 2591 return mr; 2592 2593 err1: 2594 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2595 err0: 2596 kfree(mr); 2597 return ERR_PTR(rc); 2598 } 2599 2600 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, 2601 enum ib_mr_type mr_type, u32 max_num_sg) 2602 { 2603 struct qedr_mr *mr; 2604 2605 if (mr_type != IB_MR_TYPE_MEM_REG) 2606 return ERR_PTR(-EINVAL); 2607 2608 mr = __qedr_alloc_mr(ibpd, max_num_sg); 2609 2610 if (IS_ERR(mr)) 2611 return ERR_PTR(-EINVAL); 2612 2613 return &mr->ibmr; 2614 } 2615 2616 static int qedr_set_page(struct ib_mr *ibmr, u64 addr) 2617 { 2618 struct qedr_mr *mr = get_qedr_mr(ibmr); 2619 struct qedr_pbl *pbl_table; 2620 struct regpair *pbe; 2621 u32 pbes_in_page; 2622 2623 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 2624 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages); 2625 return -ENOMEM; 2626 } 2627 2628 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n", 2629 mr->npages, addr); 2630 2631 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 2632 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 2633 pbe = (struct regpair *)pbl_table->va; 2634 pbe += mr->npages % pbes_in_page; 2635 pbe->lo = cpu_to_le32((u32)addr); 2636 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 2637 2638 mr->npages++; 2639 2640 return 0; 2641 } 2642 2643 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info) 2644 { 2645 int work = info->completed - info->completed_handled - 1; 2646 2647 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work); 2648 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 2649 struct qedr_pbl *pbl; 2650 2651 /* Free all the page list that are possible to be freed 2652 * (all the ones that were invalidated), under the assumption 2653 * that if an FMR was completed successfully that means that 2654 * if there was an invalidate operation before it also ended 2655 */ 2656 pbl = list_first_entry(&info->inuse_pbl_list, 2657 struct qedr_pbl, list_entry); 2658 list_move_tail(&pbl->list_entry, &info->free_pbl_list); 2659 info->completed_handled++; 2660 } 2661 } 2662 2663 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 2664 int sg_nents, unsigned int *sg_offset) 2665 { 2666 struct qedr_mr *mr = get_qedr_mr(ibmr); 2667 2668 mr->npages = 0; 2669 2670 handle_completed_mrs(mr->dev, &mr->info); 2671 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page); 2672 } 2673 2674 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc) 2675 { 2676 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2677 struct qedr_pd *pd = get_qedr_pd(ibpd); 2678 struct qedr_mr *mr; 2679 int rc; 2680 2681 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2682 if (!mr) 2683 return ERR_PTR(-ENOMEM); 2684 2685 mr->type = QEDR_MR_DMA; 2686 2687 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2688 if (rc) { 2689 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2690 goto err1; 2691 } 2692 2693 /* index only, 18 bit long, lkey = itid << 8 | key */ 2694 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2695 mr->hw_mr.pd = pd->pd_id; 2696 mr->hw_mr.local_read = 1; 2697 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2698 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2699 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2700 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2701 mr->hw_mr.dma_mr = true; 2702 2703 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2704 if (rc) { 2705 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2706 goto err2; 2707 } 2708 2709 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2710 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2711 mr->hw_mr.remote_atomic) 2712 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2713 2714 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey); 2715 return &mr->ibmr; 2716 2717 err2: 2718 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2719 err1: 2720 kfree(mr); 2721 return ERR_PTR(rc); 2722 } 2723 2724 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq) 2725 { 2726 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 2727 } 2728 2729 static int sge_data_len(struct ib_sge *sg_list, int num_sge) 2730 { 2731 int i, len = 0; 2732 2733 for (i = 0; i < num_sge; i++) 2734 len += sg_list[i].length; 2735 2736 return len; 2737 } 2738 2739 static void swap_wqe_data64(u64 *p) 2740 { 2741 int i; 2742 2743 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 2744 *p = cpu_to_be64(cpu_to_le64(*p)); 2745 } 2746 2747 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev, 2748 struct qedr_qp *qp, u8 *wqe_size, 2749 struct ib_send_wr *wr, 2750 struct ib_send_wr **bad_wr, u8 *bits, 2751 u8 bit) 2752 { 2753 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 2754 char *seg_prt, *wqe; 2755 int i, seg_siz; 2756 2757 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 2758 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size); 2759 *bad_wr = wr; 2760 return 0; 2761 } 2762 2763 if (!data_size) 2764 return data_size; 2765 2766 *bits |= bit; 2767 2768 seg_prt = NULL; 2769 wqe = NULL; 2770 seg_siz = 0; 2771 2772 /* Copy data inline */ 2773 for (i = 0; i < wr->num_sge; i++) { 2774 u32 len = wr->sg_list[i].length; 2775 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 2776 2777 while (len > 0) { 2778 u32 cur; 2779 2780 /* New segment required */ 2781 if (!seg_siz) { 2782 wqe = (char *)qed_chain_produce(&qp->sq.pbl); 2783 seg_prt = wqe; 2784 seg_siz = sizeof(struct rdma_sq_common_wqe); 2785 (*wqe_size)++; 2786 } 2787 2788 /* Calculate currently allowed length */ 2789 cur = min_t(u32, len, seg_siz); 2790 memcpy(seg_prt, src, cur); 2791 2792 /* Update segment variables */ 2793 seg_prt += cur; 2794 seg_siz -= cur; 2795 2796 /* Update sge variables */ 2797 src += cur; 2798 len -= cur; 2799 2800 /* Swap fully-completed segments */ 2801 if (!seg_siz) 2802 swap_wqe_data64((u64 *)wqe); 2803 } 2804 } 2805 2806 /* swap last not completed segment */ 2807 if (seg_siz) 2808 swap_wqe_data64((u64 *)wqe); 2809 2810 return data_size; 2811 } 2812 2813 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 2814 do { \ 2815 DMA_REGPAIR_LE(sge->addr, vaddr); \ 2816 (sge)->length = cpu_to_le32(vlength); \ 2817 (sge)->flags = cpu_to_le32(vflags); \ 2818 } while (0) 2819 2820 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 2821 do { \ 2822 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \ 2823 (hdr)->num_sges = num_sge; \ 2824 } while (0) 2825 2826 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 2827 do { \ 2828 DMA_REGPAIR_LE(sge->addr, vaddr); \ 2829 (sge)->length = cpu_to_le32(vlength); \ 2830 (sge)->l_key = cpu_to_le32(vlkey); \ 2831 } while (0) 2832 2833 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, 2834 struct ib_send_wr *wr) 2835 { 2836 u32 data_size = 0; 2837 int i; 2838 2839 for (i = 0; i < wr->num_sge; i++) { 2840 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl); 2841 2842 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr); 2843 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 2844 sge->length = cpu_to_le32(wr->sg_list[i].length); 2845 data_size += wr->sg_list[i].length; 2846 } 2847 2848 if (wqe_size) 2849 *wqe_size += wr->num_sge; 2850 2851 return data_size; 2852 } 2853 2854 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev, 2855 struct qedr_qp *qp, 2856 struct rdma_sq_rdma_wqe_1st *rwqe, 2857 struct rdma_sq_rdma_wqe_2nd *rwqe2, 2858 struct ib_send_wr *wr, 2859 struct ib_send_wr **bad_wr) 2860 { 2861 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 2862 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr); 2863 2864 if (wr->send_flags & IB_SEND_INLINE && 2865 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 2866 wr->opcode == IB_WR_RDMA_WRITE)) { 2867 u8 flags = 0; 2868 2869 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 2870 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, 2871 bad_wr, &rwqe->flags, flags); 2872 } 2873 2874 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr); 2875 } 2876 2877 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev, 2878 struct qedr_qp *qp, 2879 struct rdma_sq_send_wqe_1st *swqe, 2880 struct rdma_sq_send_wqe_2st *swqe2, 2881 struct ib_send_wr *wr, 2882 struct ib_send_wr **bad_wr) 2883 { 2884 memset(swqe2, 0, sizeof(*swqe2)); 2885 if (wr->send_flags & IB_SEND_INLINE) { 2886 u8 flags = 0; 2887 2888 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 2889 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, 2890 bad_wr, &swqe->flags, flags); 2891 } 2892 2893 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr); 2894 } 2895 2896 static int qedr_prepare_reg(struct qedr_qp *qp, 2897 struct rdma_sq_fmr_wqe_1st *fwqe1, 2898 struct ib_reg_wr *wr) 2899 { 2900 struct qedr_mr *mr = get_qedr_mr(wr->mr); 2901 struct rdma_sq_fmr_wqe_2nd *fwqe2; 2902 2903 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl); 2904 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 2905 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 2906 fwqe1->l_key = wr->key; 2907 2908 fwqe2->access_ctrl = 0; 2909 2910 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 2911 !!(wr->access & IB_ACCESS_REMOTE_READ)); 2912 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 2913 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 2914 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 2915 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 2916 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 2917 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 2918 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 2919 fwqe2->fmr_ctrl = 0; 2920 2921 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 2922 ilog2(mr->ibmr.page_size) - 12); 2923 2924 fwqe2->length_hi = 0; 2925 fwqe2->length_lo = mr->ibmr.length; 2926 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 2927 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 2928 2929 qp->wqe_wr_id[qp->sq.prod].mr = mr; 2930 2931 return 0; 2932 } 2933 2934 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 2935 { 2936 switch (opcode) { 2937 case IB_WR_RDMA_WRITE: 2938 case IB_WR_RDMA_WRITE_WITH_IMM: 2939 return IB_WC_RDMA_WRITE; 2940 case IB_WR_SEND_WITH_IMM: 2941 case IB_WR_SEND: 2942 case IB_WR_SEND_WITH_INV: 2943 return IB_WC_SEND; 2944 case IB_WR_RDMA_READ: 2945 case IB_WR_RDMA_READ_WITH_INV: 2946 return IB_WC_RDMA_READ; 2947 case IB_WR_ATOMIC_CMP_AND_SWP: 2948 return IB_WC_COMP_SWAP; 2949 case IB_WR_ATOMIC_FETCH_AND_ADD: 2950 return IB_WC_FETCH_ADD; 2951 case IB_WR_REG_MR: 2952 return IB_WC_REG_MR; 2953 case IB_WR_LOCAL_INV: 2954 return IB_WC_LOCAL_INV; 2955 default: 2956 return IB_WC_SEND; 2957 } 2958 } 2959 2960 static inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr) 2961 { 2962 int wq_is_full, err_wr, pbl_is_full; 2963 struct qedr_dev *dev = qp->dev; 2964 2965 /* prevent SQ overflow and/or processing of a bad WR */ 2966 err_wr = wr->num_sge > qp->sq.max_sges; 2967 wq_is_full = qedr_wq_is_full(&qp->sq); 2968 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) < 2969 QEDR_MAX_SQE_ELEMENTS_PER_SQE; 2970 if (wq_is_full || err_wr || pbl_is_full) { 2971 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) { 2972 DP_ERR(dev, 2973 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n", 2974 qp); 2975 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL; 2976 } 2977 2978 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) { 2979 DP_ERR(dev, 2980 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n", 2981 qp); 2982 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR; 2983 } 2984 2985 if (pbl_is_full && 2986 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) { 2987 DP_ERR(dev, 2988 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n", 2989 qp); 2990 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL; 2991 } 2992 return false; 2993 } 2994 return true; 2995 } 2996 2997 static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 2998 struct ib_send_wr **bad_wr) 2999 { 3000 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3001 struct qedr_qp *qp = get_qedr_qp(ibqp); 3002 struct rdma_sq_atomic_wqe_1st *awqe1; 3003 struct rdma_sq_atomic_wqe_2nd *awqe2; 3004 struct rdma_sq_atomic_wqe_3rd *awqe3; 3005 struct rdma_sq_send_wqe_2st *swqe2; 3006 struct rdma_sq_local_inv_wqe *iwqe; 3007 struct rdma_sq_rdma_wqe_2nd *rwqe2; 3008 struct rdma_sq_send_wqe_1st *swqe; 3009 struct rdma_sq_rdma_wqe_1st *rwqe; 3010 struct rdma_sq_fmr_wqe_1st *fwqe1; 3011 struct rdma_sq_common_wqe *wqe; 3012 u32 length; 3013 int rc = 0; 3014 bool comp; 3015 3016 if (!qedr_can_post_send(qp, wr)) { 3017 *bad_wr = wr; 3018 return -ENOMEM; 3019 } 3020 3021 wqe = qed_chain_produce(&qp->sq.pbl); 3022 qp->wqe_wr_id[qp->sq.prod].signaled = 3023 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 3024 3025 wqe->flags = 0; 3026 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, 3027 !!(wr->send_flags & IB_SEND_SOLICITED)); 3028 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled; 3029 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 3030 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, 3031 !!(wr->send_flags & IB_SEND_FENCE)); 3032 wqe->prev_wqe_size = qp->prev_wqe_size; 3033 3034 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode); 3035 3036 switch (wr->opcode) { 3037 case IB_WR_SEND_WITH_IMM: 3038 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 3039 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3040 swqe->wqe_size = 2; 3041 swqe2 = qed_chain_produce(&qp->sq.pbl); 3042 3043 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data); 3044 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3045 wr, bad_wr); 3046 swqe->length = cpu_to_le32(length); 3047 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3048 qp->prev_wqe_size = swqe->wqe_size; 3049 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3050 break; 3051 case IB_WR_SEND: 3052 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 3053 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3054 3055 swqe->wqe_size = 2; 3056 swqe2 = qed_chain_produce(&qp->sq.pbl); 3057 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3058 wr, bad_wr); 3059 swqe->length = cpu_to_le32(length); 3060 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3061 qp->prev_wqe_size = swqe->wqe_size; 3062 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3063 break; 3064 case IB_WR_SEND_WITH_INV: 3065 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 3066 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3067 swqe2 = qed_chain_produce(&qp->sq.pbl); 3068 swqe->wqe_size = 2; 3069 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey); 3070 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3071 wr, bad_wr); 3072 swqe->length = cpu_to_le32(length); 3073 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3074 qp->prev_wqe_size = swqe->wqe_size; 3075 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3076 break; 3077 3078 case IB_WR_RDMA_WRITE_WITH_IMM: 3079 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 3080 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3081 3082 rwqe->wqe_size = 2; 3083 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 3084 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3085 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3086 wr, bad_wr); 3087 rwqe->length = cpu_to_le32(length); 3088 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3089 qp->prev_wqe_size = rwqe->wqe_size; 3090 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3091 break; 3092 case IB_WR_RDMA_WRITE: 3093 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 3094 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3095 3096 rwqe->wqe_size = 2; 3097 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3098 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3099 wr, bad_wr); 3100 rwqe->length = cpu_to_le32(length); 3101 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3102 qp->prev_wqe_size = rwqe->wqe_size; 3103 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3104 break; 3105 case IB_WR_RDMA_READ_WITH_INV: 3106 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1); 3107 /* fallthrough -- same is identical to RDMA READ */ 3108 3109 case IB_WR_RDMA_READ: 3110 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 3111 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3112 3113 rwqe->wqe_size = 2; 3114 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3115 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3116 wr, bad_wr); 3117 rwqe->length = cpu_to_le32(length); 3118 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3119 qp->prev_wqe_size = rwqe->wqe_size; 3120 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3121 break; 3122 3123 case IB_WR_ATOMIC_CMP_AND_SWP: 3124 case IB_WR_ATOMIC_FETCH_AND_ADD: 3125 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe; 3126 awqe1->wqe_size = 4; 3127 3128 awqe2 = qed_chain_produce(&qp->sq.pbl); 3129 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr); 3130 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 3131 3132 awqe3 = qed_chain_produce(&qp->sq.pbl); 3133 3134 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 3135 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 3136 DMA_REGPAIR_LE(awqe3->swap_data, 3137 atomic_wr(wr)->compare_add); 3138 } else { 3139 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 3140 DMA_REGPAIR_LE(awqe3->swap_data, 3141 atomic_wr(wr)->swap); 3142 DMA_REGPAIR_LE(awqe3->cmp_data, 3143 atomic_wr(wr)->compare_add); 3144 } 3145 3146 qedr_prepare_sq_sges(qp, NULL, wr); 3147 3148 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size; 3149 qp->prev_wqe_size = awqe1->wqe_size; 3150 break; 3151 3152 case IB_WR_LOCAL_INV: 3153 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 3154 iwqe->wqe_size = 1; 3155 3156 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 3157 iwqe->inv_l_key = wr->ex.invalidate_rkey; 3158 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size; 3159 qp->prev_wqe_size = iwqe->wqe_size; 3160 break; 3161 case IB_WR_REG_MR: 3162 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n"); 3163 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 3164 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 3165 fwqe1->wqe_size = 2; 3166 3167 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr)); 3168 if (rc) { 3169 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc); 3170 *bad_wr = wr; 3171 break; 3172 } 3173 3174 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 3175 qp->prev_wqe_size = fwqe1->wqe_size; 3176 break; 3177 default: 3178 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode); 3179 rc = -EINVAL; 3180 *bad_wr = wr; 3181 break; 3182 } 3183 3184 if (*bad_wr) { 3185 u16 value; 3186 3187 /* Restore prod to its position before 3188 * this WR was processed 3189 */ 3190 value = le16_to_cpu(qp->sq.db_data.data.value); 3191 qed_chain_set_prod(&qp->sq.pbl, value, wqe); 3192 3193 /* Restore prev_wqe_size */ 3194 qp->prev_wqe_size = wqe->prev_wqe_size; 3195 rc = -EINVAL; 3196 DP_ERR(dev, "POST SEND FAILED\n"); 3197 } 3198 3199 return rc; 3200 } 3201 3202 int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 3203 struct ib_send_wr **bad_wr) 3204 { 3205 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3206 struct qedr_qp *qp = get_qedr_qp(ibqp); 3207 unsigned long flags; 3208 int rc = 0; 3209 3210 *bad_wr = NULL; 3211 3212 if (qp->qp_type == IB_QPT_GSI) 3213 return qedr_gsi_post_send(ibqp, wr, bad_wr); 3214 3215 spin_lock_irqsave(&qp->q_lock, flags); 3216 3217 if (rdma_protocol_roce(&dev->ibdev, 1)) { 3218 if ((qp->state != QED_ROCE_QP_STATE_RTS) && 3219 (qp->state != QED_ROCE_QP_STATE_ERR) && 3220 (qp->state != QED_ROCE_QP_STATE_SQD)) { 3221 spin_unlock_irqrestore(&qp->q_lock, flags); 3222 *bad_wr = wr; 3223 DP_DEBUG(dev, QEDR_MSG_CQ, 3224 "QP in wrong state! QP icid=0x%x state %d\n", 3225 qp->icid, qp->state); 3226 return -EINVAL; 3227 } 3228 } 3229 3230 while (wr) { 3231 rc = __qedr_post_send(ibqp, wr, bad_wr); 3232 if (rc) 3233 break; 3234 3235 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 3236 3237 qedr_inc_sw_prod(&qp->sq); 3238 3239 qp->sq.db_data.data.value++; 3240 3241 wr = wr->next; 3242 } 3243 3244 /* Trigger doorbell 3245 * If there was a failure in the first WR then it will be triggered in 3246 * vane. However this is not harmful (as long as the producer value is 3247 * unchanged). For performance reasons we avoid checking for this 3248 * redundant doorbell. 3249 */ 3250 wmb(); 3251 writel(qp->sq.db_data.raw, qp->sq.db); 3252 3253 /* Make sure write sticks */ 3254 mmiowb(); 3255 3256 spin_unlock_irqrestore(&qp->q_lock, flags); 3257 3258 return rc; 3259 } 3260 3261 int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, 3262 struct ib_recv_wr **bad_wr) 3263 { 3264 struct qedr_qp *qp = get_qedr_qp(ibqp); 3265 struct qedr_dev *dev = qp->dev; 3266 unsigned long flags; 3267 int status = 0; 3268 3269 if (qp->qp_type == IB_QPT_GSI) 3270 return qedr_gsi_post_recv(ibqp, wr, bad_wr); 3271 3272 spin_lock_irqsave(&qp->q_lock, flags); 3273 3274 if (qp->state == QED_ROCE_QP_STATE_RESET) { 3275 spin_unlock_irqrestore(&qp->q_lock, flags); 3276 *bad_wr = wr; 3277 return -EINVAL; 3278 } 3279 3280 while (wr) { 3281 int i; 3282 3283 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) < 3284 QEDR_MAX_RQE_ELEMENTS_PER_RQE || 3285 wr->num_sge > qp->rq.max_sges) { 3286 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n", 3287 qed_chain_get_elem_left_u32(&qp->rq.pbl), 3288 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge, 3289 qp->rq.max_sges); 3290 status = -ENOMEM; 3291 *bad_wr = wr; 3292 break; 3293 } 3294 for (i = 0; i < wr->num_sge; i++) { 3295 u32 flags = 0; 3296 struct rdma_rq_sge *rqe = 3297 qed_chain_produce(&qp->rq.pbl); 3298 3299 /* First one must include the number 3300 * of SGE in the list 3301 */ 3302 if (!i) 3303 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 3304 wr->num_sge); 3305 3306 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 3307 wr->sg_list[i].lkey); 3308 3309 RQ_SGE_SET(rqe, wr->sg_list[i].addr, 3310 wr->sg_list[i].length, flags); 3311 } 3312 3313 /* Special case of no sges. FW requires between 1-4 sges... 3314 * in this case we need to post 1 sge with length zero. this is 3315 * because rdma write with immediate consumes an RQ. 3316 */ 3317 if (!wr->num_sge) { 3318 u32 flags = 0; 3319 struct rdma_rq_sge *rqe = 3320 qed_chain_produce(&qp->rq.pbl); 3321 3322 /* First one must include the number 3323 * of SGE in the list 3324 */ 3325 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0); 3326 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 3327 3328 RQ_SGE_SET(rqe, 0, 0, flags); 3329 i = 1; 3330 } 3331 3332 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 3333 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 3334 3335 qedr_inc_sw_prod(&qp->rq); 3336 3337 /* Flush all the writes before signalling doorbell */ 3338 wmb(); 3339 3340 qp->rq.db_data.data.value++; 3341 3342 writel(qp->rq.db_data.raw, qp->rq.db); 3343 3344 /* Make sure write sticks */ 3345 mmiowb(); 3346 3347 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 3348 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 3349 mmiowb(); /* for second doorbell */ 3350 } 3351 3352 wr = wr->next; 3353 } 3354 3355 spin_unlock_irqrestore(&qp->q_lock, flags); 3356 3357 return status; 3358 } 3359 3360 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe) 3361 { 3362 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3363 3364 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) == 3365 cq->pbl_toggle; 3366 } 3367 3368 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe) 3369 { 3370 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3371 struct qedr_qp *qp; 3372 3373 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi, 3374 resp_cqe->qp_handle.lo, 3375 u64); 3376 return qp; 3377 } 3378 3379 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe) 3380 { 3381 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3382 3383 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 3384 } 3385 3386 /* Return latest CQE (needs processing) */ 3387 static union rdma_cqe *get_cqe(struct qedr_cq *cq) 3388 { 3389 return cq->latest_cqe; 3390 } 3391 3392 /* In fmr we need to increase the number of fmr completed counter for the fmr 3393 * algorithm determining whether we can free a pbl or not. 3394 * we need to perform this whether the work request was signaled or not. for 3395 * this purpose we call this function from the condition that checks if a wr 3396 * should be skipped, to make sure we don't miss it ( possibly this fmr 3397 * operation was not signalted) 3398 */ 3399 static inline void qedr_chk_if_fmr(struct qedr_qp *qp) 3400 { 3401 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 3402 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3403 } 3404 3405 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, 3406 struct qedr_cq *cq, int num_entries, 3407 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, 3408 int force) 3409 { 3410 u16 cnt = 0; 3411 3412 while (num_entries && qp->sq.wqe_cons != hw_cons) { 3413 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 3414 qedr_chk_if_fmr(qp); 3415 /* skip WC */ 3416 goto next_cqe; 3417 } 3418 3419 /* fill WC */ 3420 wc->status = status; 3421 wc->vendor_err = 0; 3422 wc->wc_flags = 0; 3423 wc->src_qp = qp->id; 3424 wc->qp = &qp->ibqp; 3425 3426 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 3427 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 3428 3429 switch (wc->opcode) { 3430 case IB_WC_RDMA_WRITE: 3431 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3432 break; 3433 case IB_WC_COMP_SWAP: 3434 case IB_WC_FETCH_ADD: 3435 wc->byte_len = 8; 3436 break; 3437 case IB_WC_REG_MR: 3438 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3439 break; 3440 case IB_WC_RDMA_READ: 3441 case IB_WC_SEND: 3442 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3443 break; 3444 default: 3445 break; 3446 } 3447 3448 num_entries--; 3449 wc++; 3450 cnt++; 3451 next_cqe: 3452 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 3453 qed_chain_consume(&qp->sq.pbl); 3454 qedr_inc_sw_cons(&qp->sq); 3455 } 3456 3457 return cnt; 3458 } 3459 3460 static int qedr_poll_cq_req(struct qedr_dev *dev, 3461 struct qedr_qp *qp, struct qedr_cq *cq, 3462 int num_entries, struct ib_wc *wc, 3463 struct rdma_cqe_requester *req) 3464 { 3465 int cnt = 0; 3466 3467 switch (req->status) { 3468 case RDMA_CQE_REQ_STS_OK: 3469 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3470 IB_WC_SUCCESS, 0); 3471 break; 3472 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 3473 if (qp->state != QED_ROCE_QP_STATE_ERR) 3474 DP_ERR(dev, 3475 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3476 cq->icid, qp->icid); 3477 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3478 IB_WC_WR_FLUSH_ERR, 1); 3479 break; 3480 default: 3481 /* process all WQE before the cosumer */ 3482 qp->state = QED_ROCE_QP_STATE_ERR; 3483 cnt = process_req(dev, qp, cq, num_entries, wc, 3484 req->sq_cons - 1, IB_WC_SUCCESS, 0); 3485 wc += cnt; 3486 /* if we have extra WC fill it with actual error info */ 3487 if (cnt < num_entries) { 3488 enum ib_wc_status wc_status; 3489 3490 switch (req->status) { 3491 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 3492 DP_ERR(dev, 3493 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3494 cq->icid, qp->icid); 3495 wc_status = IB_WC_BAD_RESP_ERR; 3496 break; 3497 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 3498 DP_ERR(dev, 3499 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3500 cq->icid, qp->icid); 3501 wc_status = IB_WC_LOC_LEN_ERR; 3502 break; 3503 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 3504 DP_ERR(dev, 3505 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3506 cq->icid, qp->icid); 3507 wc_status = IB_WC_LOC_QP_OP_ERR; 3508 break; 3509 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 3510 DP_ERR(dev, 3511 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3512 cq->icid, qp->icid); 3513 wc_status = IB_WC_LOC_PROT_ERR; 3514 break; 3515 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 3516 DP_ERR(dev, 3517 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3518 cq->icid, qp->icid); 3519 wc_status = IB_WC_MW_BIND_ERR; 3520 break; 3521 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 3522 DP_ERR(dev, 3523 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3524 cq->icid, qp->icid); 3525 wc_status = IB_WC_REM_INV_REQ_ERR; 3526 break; 3527 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 3528 DP_ERR(dev, 3529 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3530 cq->icid, qp->icid); 3531 wc_status = IB_WC_REM_ACCESS_ERR; 3532 break; 3533 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 3534 DP_ERR(dev, 3535 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3536 cq->icid, qp->icid); 3537 wc_status = IB_WC_REM_OP_ERR; 3538 break; 3539 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 3540 DP_ERR(dev, 3541 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3542 cq->icid, qp->icid); 3543 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 3544 break; 3545 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 3546 DP_ERR(dev, 3547 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3548 cq->icid, qp->icid); 3549 wc_status = IB_WC_RETRY_EXC_ERR; 3550 break; 3551 default: 3552 DP_ERR(dev, 3553 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3554 cq->icid, qp->icid); 3555 wc_status = IB_WC_GENERAL_ERR; 3556 } 3557 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 3558 wc_status, 1); 3559 } 3560 } 3561 3562 return cnt; 3563 } 3564 3565 static inline int qedr_cqe_resp_status_to_ib(u8 status) 3566 { 3567 switch (status) { 3568 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 3569 return IB_WC_LOC_ACCESS_ERR; 3570 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 3571 return IB_WC_LOC_LEN_ERR; 3572 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 3573 return IB_WC_LOC_QP_OP_ERR; 3574 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 3575 return IB_WC_LOC_PROT_ERR; 3576 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 3577 return IB_WC_MW_BIND_ERR; 3578 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 3579 return IB_WC_REM_INV_RD_REQ_ERR; 3580 case RDMA_CQE_RESP_STS_OK: 3581 return IB_WC_SUCCESS; 3582 default: 3583 return IB_WC_GENERAL_ERR; 3584 } 3585 } 3586 3587 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp, 3588 struct ib_wc *wc) 3589 { 3590 wc->status = IB_WC_SUCCESS; 3591 wc->byte_len = le32_to_cpu(resp->length); 3592 3593 if (resp->flags & QEDR_RESP_IMM) { 3594 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 3595 wc->wc_flags |= IB_WC_WITH_IMM; 3596 3597 if (resp->flags & QEDR_RESP_RDMA) 3598 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 3599 3600 if (resp->flags & QEDR_RESP_INV) 3601 return -EINVAL; 3602 3603 } else if (resp->flags & QEDR_RESP_INV) { 3604 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 3605 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3606 3607 if (resp->flags & QEDR_RESP_RDMA) 3608 return -EINVAL; 3609 3610 } else if (resp->flags & QEDR_RESP_RDMA) { 3611 return -EINVAL; 3612 } 3613 3614 return 0; 3615 } 3616 3617 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 3618 struct qedr_cq *cq, struct ib_wc *wc, 3619 struct rdma_cqe_responder *resp, u64 wr_id) 3620 { 3621 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */ 3622 wc->opcode = IB_WC_RECV; 3623 wc->wc_flags = 0; 3624 3625 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) { 3626 if (qedr_set_ok_cqe_resp_wc(resp, wc)) 3627 DP_ERR(dev, 3628 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n", 3629 cq, cq->icid, resp->flags); 3630 3631 } else { 3632 wc->status = qedr_cqe_resp_status_to_ib(resp->status); 3633 if (wc->status == IB_WC_GENERAL_ERR) 3634 DP_ERR(dev, 3635 "CQ %p (icid=%d) contains an invalid CQE status %d\n", 3636 cq, cq->icid, resp->status); 3637 } 3638 3639 /* Fill the rest of the WC */ 3640 wc->vendor_err = 0; 3641 wc->src_qp = qp->id; 3642 wc->qp = &qp->ibqp; 3643 wc->wr_id = wr_id; 3644 } 3645 3646 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 3647 struct qedr_cq *cq, struct ib_wc *wc, 3648 struct rdma_cqe_responder *resp) 3649 { 3650 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 3651 3652 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 3653 3654 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 3655 qed_chain_consume(&qp->rq.pbl); 3656 qedr_inc_sw_cons(&qp->rq); 3657 3658 return 1; 3659 } 3660 3661 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, 3662 int num_entries, struct ib_wc *wc, u16 hw_cons) 3663 { 3664 u16 cnt = 0; 3665 3666 while (num_entries && qp->rq.wqe_cons != hw_cons) { 3667 /* fill WC */ 3668 wc->status = IB_WC_WR_FLUSH_ERR; 3669 wc->vendor_err = 0; 3670 wc->wc_flags = 0; 3671 wc->src_qp = qp->id; 3672 wc->byte_len = 0; 3673 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 3674 wc->qp = &qp->ibqp; 3675 num_entries--; 3676 wc++; 3677 cnt++; 3678 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 3679 qed_chain_consume(&qp->rq.pbl); 3680 qedr_inc_sw_cons(&qp->rq); 3681 } 3682 3683 return cnt; 3684 } 3685 3686 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 3687 struct rdma_cqe_responder *resp, int *update) 3688 { 3689 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) { 3690 consume_cqe(cq); 3691 *update |= 1; 3692 } 3693 } 3694 3695 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, 3696 struct qedr_cq *cq, int num_entries, 3697 struct ib_wc *wc, struct rdma_cqe_responder *resp, 3698 int *update) 3699 { 3700 int cnt; 3701 3702 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 3703 cnt = process_resp_flush(qp, cq, num_entries, wc, 3704 resp->rq_cons); 3705 try_consume_resp_cqe(cq, qp, resp, update); 3706 } else { 3707 cnt = process_resp_one(dev, qp, cq, wc, resp); 3708 consume_cqe(cq); 3709 *update |= 1; 3710 } 3711 3712 return cnt; 3713 } 3714 3715 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 3716 struct rdma_cqe_requester *req, int *update) 3717 { 3718 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 3719 consume_cqe(cq); 3720 *update |= 1; 3721 } 3722 } 3723 3724 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 3725 { 3726 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 3727 struct qedr_cq *cq = get_qedr_cq(ibcq); 3728 union rdma_cqe *cqe = cq->latest_cqe; 3729 u32 old_cons, new_cons; 3730 unsigned long flags; 3731 int update = 0; 3732 int done = 0; 3733 3734 if (cq->destroyed) { 3735 DP_ERR(dev, 3736 "warning: poll was invoked after destroy for cq %p (icid=%d)\n", 3737 cq, cq->icid); 3738 return 0; 3739 } 3740 3741 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 3742 return qedr_gsi_poll_cq(ibcq, num_entries, wc); 3743 3744 spin_lock_irqsave(&cq->cq_lock, flags); 3745 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 3746 while (num_entries && is_valid_cqe(cq, cqe)) { 3747 struct qedr_qp *qp; 3748 int cnt = 0; 3749 3750 /* prevent speculative reads of any field of CQE */ 3751 rmb(); 3752 3753 qp = cqe_get_qp(cqe); 3754 if (!qp) { 3755 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe); 3756 break; 3757 } 3758 3759 wc->qp = &qp->ibqp; 3760 3761 switch (cqe_get_type(cqe)) { 3762 case RDMA_CQE_TYPE_REQUESTER: 3763 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc, 3764 &cqe->req); 3765 try_consume_req_cqe(cq, qp, &cqe->req, &update); 3766 break; 3767 case RDMA_CQE_TYPE_RESPONDER_RQ: 3768 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc, 3769 &cqe->resp, &update); 3770 break; 3771 case RDMA_CQE_TYPE_INVALID: 3772 default: 3773 DP_ERR(dev, "Error: invalid CQE type = %d\n", 3774 cqe_get_type(cqe)); 3775 } 3776 num_entries -= cnt; 3777 wc += cnt; 3778 done += cnt; 3779 3780 cqe = get_cqe(cq); 3781 } 3782 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 3783 3784 cq->cq_cons += new_cons - old_cons; 3785 3786 if (update) 3787 /* doorbell notifies abount latest VALID entry, 3788 * but chain already point to the next INVALID one 3789 */ 3790 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 3791 3792 spin_unlock_irqrestore(&cq->cq_lock, flags); 3793 return done; 3794 } 3795 3796 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, 3797 u8 port_num, 3798 const struct ib_wc *in_wc, 3799 const struct ib_grh *in_grh, 3800 const struct ib_mad_hdr *mad_hdr, 3801 size_t in_mad_size, struct ib_mad_hdr *out_mad, 3802 size_t *out_mad_size, u16 *out_mad_pkey_index) 3803 { 3804 struct qedr_dev *dev = get_qedr_dev(ibdev); 3805 3806 DP_DEBUG(dev, QEDR_MSG_GSI, 3807 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n", 3808 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod, 3809 mad_hdr->class_specific, mad_hdr->class_version, 3810 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status); 3811 return IB_MAD_RESULT_SUCCESS; 3812 } 3813