1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/dma-mapping.h> 33 #include <linux/crc32.h> 34 #include <net/ip.h> 35 #include <net/ipv6.h> 36 #include <net/udp.h> 37 #include <linux/iommu.h> 38 39 #include <rdma/ib_verbs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/iw_cm.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/ib_addr.h> 44 #include <rdma/ib_cache.h> 45 46 #include <linux/qed/common_hsi.h> 47 #include "qedr_hsi_rdma.h" 48 #include <linux/qed/qed_if.h> 49 #include "qedr.h" 50 #include "verbs.h" 51 #include <rdma/qedr-abi.h> 52 #include "qedr_roce_cm.h" 53 54 #define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm) 55 #define RDMA_MAX_SGE_PER_SRQ (4) 56 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) 57 58 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 59 60 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, 61 size_t len) 62 { 63 size_t min_len = min_t(size_t, len, udata->outlen); 64 65 return ib_copy_to_udata(udata, src, min_len); 66 } 67 68 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 69 { 70 if (index > QEDR_ROCE_PKEY_TABLE_LEN) 71 return -EINVAL; 72 73 *pkey = QEDR_ROCE_PKEY_DEFAULT; 74 return 0; 75 } 76 77 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port, 78 int index, union ib_gid *sgid) 79 { 80 struct qedr_dev *dev = get_qedr_dev(ibdev); 81 82 memset(sgid->raw, 0, sizeof(sgid->raw)); 83 ether_addr_copy(sgid->raw, dev->ndev->dev_addr); 84 85 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index, 86 sgid->global.interface_id, sgid->global.subnet_prefix); 87 88 return 0; 89 } 90 91 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 92 { 93 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 94 struct qedr_device_attr *qattr = &dev->attr; 95 struct qedr_srq *srq = get_qedr_srq(ibsrq); 96 97 srq_attr->srq_limit = srq->srq_limit; 98 srq_attr->max_wr = qattr->max_srq_wr; 99 srq_attr->max_sge = qattr->max_sge; 100 101 return 0; 102 } 103 104 int qedr_query_device(struct ib_device *ibdev, 105 struct ib_device_attr *attr, struct ib_udata *udata) 106 { 107 struct qedr_dev *dev = get_qedr_dev(ibdev); 108 struct qedr_device_attr *qattr = &dev->attr; 109 110 if (!dev->rdma_ctx) { 111 DP_ERR(dev, 112 "qedr_query_device called with invalid params rdma_ctx=%p\n", 113 dev->rdma_ctx); 114 return -EINVAL; 115 } 116 117 memset(attr, 0, sizeof(*attr)); 118 119 attr->fw_ver = qattr->fw_ver; 120 attr->sys_image_guid = qattr->sys_image_guid; 121 attr->max_mr_size = qattr->max_mr_size; 122 attr->page_size_cap = qattr->page_size_caps; 123 attr->vendor_id = qattr->vendor_id; 124 attr->vendor_part_id = qattr->vendor_part_id; 125 attr->hw_ver = qattr->hw_ver; 126 attr->max_qp = qattr->max_qp; 127 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe); 128 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 129 IB_DEVICE_RC_RNR_NAK_GEN | 130 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS; 131 132 attr->max_send_sge = qattr->max_sge; 133 attr->max_recv_sge = qattr->max_sge; 134 attr->max_sge_rd = qattr->max_sge; 135 attr->max_cq = qattr->max_cq; 136 attr->max_cqe = qattr->max_cqe; 137 attr->max_mr = qattr->max_mr; 138 attr->max_mw = qattr->max_mw; 139 attr->max_pd = qattr->max_pd; 140 attr->atomic_cap = dev->atomic_cap; 141 attr->max_fmr = qattr->max_fmr; 142 attr->max_map_per_fmr = 16; 143 attr->max_qp_init_rd_atom = 144 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 145 attr->max_qp_rd_atom = 146 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 147 attr->max_qp_init_rd_atom); 148 149 attr->max_srq = qattr->max_srq; 150 attr->max_srq_sge = qattr->max_srq_sge; 151 attr->max_srq_wr = qattr->max_srq_wr; 152 153 attr->local_ca_ack_delay = qattr->dev_ack_delay; 154 attr->max_fast_reg_page_list_len = qattr->max_mr / 8; 155 attr->max_pkeys = QEDR_ROCE_PKEY_MAX; 156 attr->max_ah = qattr->max_ah; 157 158 return 0; 159 } 160 161 #define QEDR_SPEED_SDR (1) 162 #define QEDR_SPEED_DDR (2) 163 #define QEDR_SPEED_QDR (4) 164 #define QEDR_SPEED_FDR10 (8) 165 #define QEDR_SPEED_FDR (16) 166 #define QEDR_SPEED_EDR (32) 167 168 static inline void get_link_speed_and_width(int speed, u8 *ib_speed, 169 u8 *ib_width) 170 { 171 switch (speed) { 172 case 1000: 173 *ib_speed = QEDR_SPEED_SDR; 174 *ib_width = IB_WIDTH_1X; 175 break; 176 case 10000: 177 *ib_speed = QEDR_SPEED_QDR; 178 *ib_width = IB_WIDTH_1X; 179 break; 180 181 case 20000: 182 *ib_speed = QEDR_SPEED_DDR; 183 *ib_width = IB_WIDTH_4X; 184 break; 185 186 case 25000: 187 *ib_speed = QEDR_SPEED_EDR; 188 *ib_width = IB_WIDTH_1X; 189 break; 190 191 case 40000: 192 *ib_speed = QEDR_SPEED_QDR; 193 *ib_width = IB_WIDTH_4X; 194 break; 195 196 case 50000: 197 *ib_speed = QEDR_SPEED_QDR; 198 *ib_width = IB_WIDTH_4X; 199 break; 200 201 case 100000: 202 *ib_speed = QEDR_SPEED_EDR; 203 *ib_width = IB_WIDTH_4X; 204 break; 205 206 default: 207 /* Unsupported */ 208 *ib_speed = QEDR_SPEED_SDR; 209 *ib_width = IB_WIDTH_1X; 210 } 211 } 212 213 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr) 214 { 215 struct qedr_dev *dev; 216 struct qed_rdma_port *rdma_port; 217 218 dev = get_qedr_dev(ibdev); 219 if (port > 1) { 220 DP_ERR(dev, "invalid_port=0x%x\n", port); 221 return -EINVAL; 222 } 223 224 if (!dev->rdma_ctx) { 225 DP_ERR(dev, "rdma_ctx is NULL\n"); 226 return -EINVAL; 227 } 228 229 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx); 230 231 /* *attr being zeroed by the caller, avoid zeroing it here */ 232 if (rdma_port->port_state == QED_RDMA_PORT_UP) { 233 attr->state = IB_PORT_ACTIVE; 234 attr->phys_state = 5; 235 } else { 236 attr->state = IB_PORT_DOWN; 237 attr->phys_state = 3; 238 } 239 attr->max_mtu = IB_MTU_4096; 240 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); 241 attr->lid = 0; 242 attr->lmc = 0; 243 attr->sm_lid = 0; 244 attr->sm_sl = 0; 245 attr->ip_gids = true; 246 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 247 attr->gid_tbl_len = 1; 248 attr->pkey_tbl_len = 1; 249 } else { 250 attr->gid_tbl_len = QEDR_MAX_SGID; 251 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; 252 } 253 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 254 attr->qkey_viol_cntr = 0; 255 get_link_speed_and_width(rdma_port->link_speed, 256 &attr->active_speed, &attr->active_width); 257 attr->max_msg_sz = rdma_port->max_msg_size; 258 attr->max_vl_num = 4; 259 260 return 0; 261 } 262 263 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask, 264 struct ib_port_modify *props) 265 { 266 struct qedr_dev *dev; 267 268 dev = get_qedr_dev(ibdev); 269 if (port > 1) { 270 DP_ERR(dev, "invalid_port=0x%x\n", port); 271 return -EINVAL; 272 } 273 274 return 0; 275 } 276 277 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr, 278 unsigned long len) 279 { 280 struct qedr_mm *mm; 281 282 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 283 if (!mm) 284 return -ENOMEM; 285 286 mm->key.phy_addr = phy_addr; 287 /* This function might be called with a length which is not a multiple 288 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel 289 * forces this granularity by increasing the requested size if needed. 290 * When qedr_mmap is called, it will search the list with the updated 291 * length as a key. To prevent search failures, the length is rounded up 292 * in advance to PAGE_SIZE. 293 */ 294 mm->key.len = roundup(len, PAGE_SIZE); 295 INIT_LIST_HEAD(&mm->entry); 296 297 mutex_lock(&uctx->mm_list_lock); 298 list_add(&mm->entry, &uctx->mm_head); 299 mutex_unlock(&uctx->mm_list_lock); 300 301 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 302 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n", 303 (unsigned long long)mm->key.phy_addr, 304 (unsigned long)mm->key.len, uctx); 305 306 return 0; 307 } 308 309 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr, 310 unsigned long len) 311 { 312 bool found = false; 313 struct qedr_mm *mm; 314 315 mutex_lock(&uctx->mm_list_lock); 316 list_for_each_entry(mm, &uctx->mm_head, entry) { 317 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 318 continue; 319 320 found = true; 321 break; 322 } 323 mutex_unlock(&uctx->mm_list_lock); 324 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 325 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n", 326 mm->key.phy_addr, mm->key.len, uctx, found); 327 328 return found; 329 } 330 331 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev, 332 struct ib_udata *udata) 333 { 334 int rc; 335 struct qedr_ucontext *ctx; 336 struct qedr_alloc_ucontext_resp uresp; 337 struct qedr_dev *dev = get_qedr_dev(ibdev); 338 struct qed_rdma_add_user_out_params oparams; 339 340 if (!udata) 341 return ERR_PTR(-EFAULT); 342 343 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 344 if (!ctx) 345 return ERR_PTR(-ENOMEM); 346 347 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams); 348 if (rc) { 349 DP_ERR(dev, 350 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n", 351 rc); 352 goto err; 353 } 354 355 ctx->dpi = oparams.dpi; 356 ctx->dpi_addr = oparams.dpi_addr; 357 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 358 ctx->dpi_size = oparams.dpi_size; 359 INIT_LIST_HEAD(&ctx->mm_head); 360 mutex_init(&ctx->mm_list_lock); 361 362 memset(&uresp, 0, sizeof(uresp)); 363 364 uresp.dpm_enabled = dev->user_dpm_enabled; 365 uresp.wids_enabled = 1; 366 uresp.wid_count = oparams.wid_count; 367 uresp.db_pa = ctx->dpi_phys_addr; 368 uresp.db_size = ctx->dpi_size; 369 uresp.max_send_wr = dev->attr.max_sqe; 370 uresp.max_recv_wr = dev->attr.max_rqe; 371 uresp.max_srq_wr = dev->attr.max_srq_wr; 372 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE; 373 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE; 374 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 375 uresp.max_cqes = QEDR_MAX_CQES; 376 377 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 378 if (rc) 379 goto err; 380 381 ctx->dev = dev; 382 383 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size); 384 if (rc) 385 goto err; 386 387 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n", 388 &ctx->ibucontext); 389 return &ctx->ibucontext; 390 391 err: 392 kfree(ctx); 393 return ERR_PTR(rc); 394 } 395 396 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx) 397 { 398 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx); 399 struct qedr_mm *mm, *tmp; 400 int status = 0; 401 402 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n", 403 uctx); 404 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi); 405 406 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 407 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 408 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n", 409 mm->key.phy_addr, mm->key.len, uctx); 410 list_del(&mm->entry); 411 kfree(mm); 412 } 413 414 kfree(uctx); 415 return status; 416 } 417 418 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 419 { 420 struct qedr_ucontext *ucontext = get_qedr_ucontext(context); 421 struct qedr_dev *dev = get_qedr_dev(context->device); 422 unsigned long phys_addr = vma->vm_pgoff << PAGE_SHIFT; 423 unsigned long len = (vma->vm_end - vma->vm_start); 424 unsigned long dpi_start; 425 426 dpi_start = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size); 427 428 DP_DEBUG(dev, QEDR_MSG_INIT, 429 "mmap invoked with vm_start=0x%pK, vm_end=0x%pK,vm_pgoff=0x%pK; dpi_start=0x%pK dpi_size=0x%x\n", 430 (void *)vma->vm_start, (void *)vma->vm_end, 431 (void *)vma->vm_pgoff, (void *)dpi_start, ucontext->dpi_size); 432 433 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) { 434 DP_ERR(dev, 435 "failed mmap, addresses must be page aligned: start=0x%pK, end=0x%pK\n", 436 (void *)vma->vm_start, (void *)vma->vm_end); 437 return -EINVAL; 438 } 439 440 if (!qedr_search_mmap(ucontext, phys_addr, len)) { 441 DP_ERR(dev, "failed mmap, vm_pgoff=0x%lx is not authorized\n", 442 vma->vm_pgoff); 443 return -EINVAL; 444 } 445 446 if (phys_addr < dpi_start || 447 ((phys_addr + len) > (dpi_start + ucontext->dpi_size))) { 448 DP_ERR(dev, 449 "failed mmap, pages are outside of dpi; page address=0x%pK, dpi_start=0x%pK, dpi_size=0x%x\n", 450 (void *)phys_addr, (void *)dpi_start, 451 ucontext->dpi_size); 452 return -EINVAL; 453 } 454 455 if (vma->vm_flags & VM_READ) { 456 DP_ERR(dev, "failed mmap, cannot map doorbell bar for read\n"); 457 return -EINVAL; 458 } 459 460 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 461 return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len, 462 vma->vm_page_prot); 463 } 464 465 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev, 466 struct ib_ucontext *context, struct ib_udata *udata) 467 { 468 struct qedr_dev *dev = get_qedr_dev(ibdev); 469 struct qedr_pd *pd; 470 u16 pd_id; 471 int rc; 472 473 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n", 474 (udata && context) ? "User Lib" : "Kernel"); 475 476 if (!dev->rdma_ctx) { 477 DP_ERR(dev, "invalid RDMA context\n"); 478 return ERR_PTR(-EINVAL); 479 } 480 481 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 482 if (!pd) 483 return ERR_PTR(-ENOMEM); 484 485 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id); 486 if (rc) 487 goto err; 488 489 pd->pd_id = pd_id; 490 491 if (udata && context) { 492 struct qedr_alloc_pd_uresp uresp = { 493 .pd_id = pd_id, 494 }; 495 496 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 497 if (rc) { 498 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); 499 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); 500 goto err; 501 } 502 503 pd->uctx = get_qedr_ucontext(context); 504 pd->uctx->pd = pd; 505 } 506 507 return &pd->ibpd; 508 509 err: 510 kfree(pd); 511 return ERR_PTR(rc); 512 } 513 514 int qedr_dealloc_pd(struct ib_pd *ibpd) 515 { 516 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 517 struct qedr_pd *pd = get_qedr_pd(ibpd); 518 519 if (!pd) { 520 pr_err("Invalid PD received in dealloc_pd\n"); 521 return -EINVAL; 522 } 523 524 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id); 525 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id); 526 527 kfree(pd); 528 529 return 0; 530 } 531 532 static void qedr_free_pbl(struct qedr_dev *dev, 533 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) 534 { 535 struct pci_dev *pdev = dev->pdev; 536 int i; 537 538 for (i = 0; i < pbl_info->num_pbls; i++) { 539 if (!pbl[i].va) 540 continue; 541 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 542 pbl[i].va, pbl[i].pa); 543 } 544 545 kfree(pbl); 546 } 547 548 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024) 549 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024) 550 551 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 552 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 553 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE) 554 555 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev, 556 struct qedr_pbl_info *pbl_info, 557 gfp_t flags) 558 { 559 struct pci_dev *pdev = dev->pdev; 560 struct qedr_pbl *pbl_table; 561 dma_addr_t *pbl_main_tbl; 562 dma_addr_t pa; 563 void *va; 564 int i; 565 566 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags); 567 if (!pbl_table) 568 return ERR_PTR(-ENOMEM); 569 570 for (i = 0; i < pbl_info->num_pbls; i++) { 571 va = dma_zalloc_coherent(&pdev->dev, pbl_info->pbl_size, 572 &pa, flags); 573 if (!va) 574 goto err; 575 576 pbl_table[i].va = va; 577 pbl_table[i].pa = pa; 578 } 579 580 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 581 * the first one with physical pointers to all of the rest 582 */ 583 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 584 for (i = 0; i < pbl_info->num_pbls - 1; i++) 585 pbl_main_tbl[i] = pbl_table[i + 1].pa; 586 587 return pbl_table; 588 589 err: 590 for (i--; i >= 0; i--) 591 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 592 pbl_table[i].va, pbl_table[i].pa); 593 594 qedr_free_pbl(dev, pbl_info, pbl_table); 595 596 return ERR_PTR(-ENOMEM); 597 } 598 599 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev, 600 struct qedr_pbl_info *pbl_info, 601 u32 num_pbes, int two_layer_capable) 602 { 603 u32 pbl_capacity; 604 u32 pbl_size; 605 u32 num_pbls; 606 607 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 608 if (num_pbes > MAX_PBES_TWO_LAYER) { 609 DP_ERR(dev, "prepare pbl table: too many pages %d\n", 610 num_pbes); 611 return -EINVAL; 612 } 613 614 /* calculate required pbl page size */ 615 pbl_size = MIN_FW_PBL_PAGE_SIZE; 616 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 617 NUM_PBES_ON_PAGE(pbl_size); 618 619 while (pbl_capacity < num_pbes) { 620 pbl_size *= 2; 621 pbl_capacity = pbl_size / sizeof(u64); 622 pbl_capacity = pbl_capacity * pbl_capacity; 623 } 624 625 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 626 num_pbls++; /* One for the layer0 ( points to the pbls) */ 627 pbl_info->two_layered = true; 628 } else { 629 /* One layered PBL */ 630 num_pbls = 1; 631 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, 632 roundup_pow_of_two((num_pbes * sizeof(u64)))); 633 pbl_info->two_layered = false; 634 } 635 636 pbl_info->num_pbls = num_pbls; 637 pbl_info->pbl_size = pbl_size; 638 pbl_info->num_pbes = num_pbes; 639 640 DP_DEBUG(dev, QEDR_MSG_MR, 641 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n", 642 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 643 644 return 0; 645 } 646 647 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, 648 struct qedr_pbl *pbl, 649 struct qedr_pbl_info *pbl_info, u32 pg_shift) 650 { 651 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 652 u32 fw_pg_cnt, fw_pg_per_umem_pg; 653 struct qedr_pbl *pbl_tbl; 654 struct scatterlist *sg; 655 struct regpair *pbe; 656 u64 pg_addr; 657 int entry; 658 659 if (!pbl_info->num_pbes) 660 return; 661 662 /* If we have a two layered pbl, the first pbl points to the rest 663 * of the pbls and the first entry lays on the second pbl in the table 664 */ 665 if (pbl_info->two_layered) 666 pbl_tbl = &pbl[1]; 667 else 668 pbl_tbl = pbl; 669 670 pbe = (struct regpair *)pbl_tbl->va; 671 if (!pbe) { 672 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n"); 673 return; 674 } 675 676 pbe_cnt = 0; 677 678 shift = umem->page_shift; 679 680 fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift); 681 682 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 683 pages = sg_dma_len(sg) >> shift; 684 pg_addr = sg_dma_address(sg); 685 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 686 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) { 687 pbe->lo = cpu_to_le32(pg_addr); 688 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); 689 690 pg_addr += BIT(pg_shift); 691 pbe_cnt++; 692 total_num_pbes++; 693 pbe++; 694 695 if (total_num_pbes == pbl_info->num_pbes) 696 return; 697 698 /* If the given pbl is full storing the pbes, 699 * move to next pbl. 700 */ 701 if (pbe_cnt == 702 (pbl_info->pbl_size / sizeof(u64))) { 703 pbl_tbl++; 704 pbe = (struct regpair *)pbl_tbl->va; 705 pbe_cnt = 0; 706 } 707 708 fw_pg_cnt++; 709 } 710 } 711 } 712 } 713 714 static int qedr_copy_cq_uresp(struct qedr_dev *dev, 715 struct qedr_cq *cq, struct ib_udata *udata) 716 { 717 struct qedr_create_cq_uresp uresp; 718 int rc; 719 720 memset(&uresp, 0, sizeof(uresp)); 721 722 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 723 uresp.icid = cq->icid; 724 725 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 726 if (rc) 727 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); 728 729 return rc; 730 } 731 732 static void consume_cqe(struct qedr_cq *cq) 733 { 734 if (cq->latest_cqe == cq->toggle_cqe) 735 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 736 737 cq->latest_cqe = qed_chain_consume(&cq->pbl); 738 } 739 740 static inline int qedr_align_cq_entries(int entries) 741 { 742 u64 size, aligned_size; 743 744 /* We allocate an extra entry that we don't report to the FW. */ 745 size = (entries + 1) * QEDR_CQE_SIZE; 746 aligned_size = ALIGN(size, PAGE_SIZE); 747 748 return aligned_size / QEDR_CQE_SIZE; 749 } 750 751 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx, 752 struct qedr_dev *dev, 753 struct qedr_userq *q, 754 u64 buf_addr, size_t buf_len, 755 int access, int dmasync, 756 int alloc_and_init) 757 { 758 u32 fw_pages; 759 int rc; 760 761 q->buf_addr = buf_addr; 762 q->buf_len = buf_len; 763 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync); 764 if (IS_ERR(q->umem)) { 765 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", 766 PTR_ERR(q->umem)); 767 return PTR_ERR(q->umem); 768 } 769 770 fw_pages = ib_umem_page_count(q->umem) << 771 (q->umem->page_shift - FW_PAGE_SHIFT); 772 773 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); 774 if (rc) 775 goto err0; 776 777 if (alloc_and_init) { 778 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 779 if (IS_ERR(q->pbl_tbl)) { 780 rc = PTR_ERR(q->pbl_tbl); 781 goto err0; 782 } 783 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info, 784 FW_PAGE_SHIFT); 785 } else { 786 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); 787 if (!q->pbl_tbl) { 788 rc = -ENOMEM; 789 goto err0; 790 } 791 } 792 793 return 0; 794 795 err0: 796 ib_umem_release(q->umem); 797 q->umem = NULL; 798 799 return rc; 800 } 801 802 static inline void qedr_init_cq_params(struct qedr_cq *cq, 803 struct qedr_ucontext *ctx, 804 struct qedr_dev *dev, int vector, 805 int chain_entries, int page_cnt, 806 u64 pbl_ptr, 807 struct qed_rdma_create_cq_in_params 808 *params) 809 { 810 memset(params, 0, sizeof(*params)); 811 params->cq_handle_hi = upper_32_bits((uintptr_t)cq); 812 params->cq_handle_lo = lower_32_bits((uintptr_t)cq); 813 params->cnq_id = vector; 814 params->cq_size = chain_entries - 1; 815 params->dpi = (ctx) ? ctx->dpi : dev->dpi; 816 params->pbl_num_pages = page_cnt; 817 params->pbl_ptr = pbl_ptr; 818 params->pbl_two_level = 0; 819 } 820 821 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags) 822 { 823 cq->db.data.agg_flags = flags; 824 cq->db.data.value = cpu_to_le32(cons); 825 writeq(cq->db.raw, cq->db_addr); 826 827 /* Make sure write would stick */ 828 mmiowb(); 829 } 830 831 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 832 { 833 struct qedr_cq *cq = get_qedr_cq(ibcq); 834 unsigned long sflags; 835 struct qedr_dev *dev; 836 837 dev = get_qedr_dev(ibcq->device); 838 839 if (cq->destroyed) { 840 DP_ERR(dev, 841 "warning: arm was invoked after destroy for cq %p (icid=%d)\n", 842 cq, cq->icid); 843 return -EINVAL; 844 } 845 846 847 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 848 return 0; 849 850 spin_lock_irqsave(&cq->cq_lock, sflags); 851 852 cq->arm_flags = 0; 853 854 if (flags & IB_CQ_SOLICITED) 855 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 856 857 if (flags & IB_CQ_NEXT_COMP) 858 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 859 860 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 861 862 spin_unlock_irqrestore(&cq->cq_lock, sflags); 863 864 return 0; 865 } 866 867 struct ib_cq *qedr_create_cq(struct ib_device *ibdev, 868 const struct ib_cq_init_attr *attr, 869 struct ib_ucontext *ib_ctx, struct ib_udata *udata) 870 { 871 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx); 872 struct qed_rdma_destroy_cq_out_params destroy_oparams; 873 struct qed_rdma_destroy_cq_in_params destroy_iparams; 874 struct qedr_dev *dev = get_qedr_dev(ibdev); 875 struct qed_rdma_create_cq_in_params params; 876 struct qedr_create_cq_ureq ureq; 877 int vector = attr->comp_vector; 878 int entries = attr->cqe; 879 struct qedr_cq *cq; 880 int chain_entries; 881 int page_cnt; 882 u64 pbl_ptr; 883 u16 icid; 884 int rc; 885 886 DP_DEBUG(dev, QEDR_MSG_INIT, 887 "create_cq: called from %s. entries=%d, vector=%d\n", 888 udata ? "User Lib" : "Kernel", entries, vector); 889 890 if (entries > QEDR_MAX_CQES) { 891 DP_ERR(dev, 892 "create cq: the number of entries %d is too high. Must be equal or below %d.\n", 893 entries, QEDR_MAX_CQES); 894 return ERR_PTR(-EINVAL); 895 } 896 897 chain_entries = qedr_align_cq_entries(entries); 898 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); 899 900 cq = kzalloc(sizeof(*cq), GFP_KERNEL); 901 if (!cq) 902 return ERR_PTR(-ENOMEM); 903 904 if (udata) { 905 memset(&ureq, 0, sizeof(ureq)); 906 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { 907 DP_ERR(dev, 908 "create cq: problem copying data from user space\n"); 909 goto err0; 910 } 911 912 if (!ureq.len) { 913 DP_ERR(dev, 914 "create cq: cannot create a cq with 0 entries\n"); 915 goto err0; 916 } 917 918 cq->cq_type = QEDR_CQ_TYPE_USER; 919 920 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, 921 ureq.len, IB_ACCESS_LOCAL_WRITE, 922 1, 1); 923 if (rc) 924 goto err0; 925 926 pbl_ptr = cq->q.pbl_tbl->pa; 927 page_cnt = cq->q.pbl_info.num_pbes; 928 929 cq->ibcq.cqe = chain_entries; 930 } else { 931 cq->cq_type = QEDR_CQ_TYPE_KERNEL; 932 933 rc = dev->ops->common->chain_alloc(dev->cdev, 934 QED_CHAIN_USE_TO_CONSUME, 935 QED_CHAIN_MODE_PBL, 936 QED_CHAIN_CNT_TYPE_U32, 937 chain_entries, 938 sizeof(union rdma_cqe), 939 &cq->pbl, NULL); 940 if (rc) 941 goto err1; 942 943 page_cnt = qed_chain_get_page_cnt(&cq->pbl); 944 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); 945 cq->ibcq.cqe = cq->pbl.capacity; 946 } 947 948 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, 949 pbl_ptr, ¶ms); 950 951 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 952 if (rc) 953 goto err2; 954 955 cq->icid = icid; 956 cq->sig = QEDR_CQ_MAGIC_NUMBER; 957 spin_lock_init(&cq->cq_lock); 958 959 if (ib_ctx) { 960 rc = qedr_copy_cq_uresp(dev, cq, udata); 961 if (rc) 962 goto err3; 963 } else { 964 /* Generate doorbell address. */ 965 cq->db_addr = dev->db_addr + 966 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 967 cq->db.data.icid = cq->icid; 968 cq->db.data.params = DB_AGG_CMD_SET << 969 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 970 971 /* point to the very last element, passing it we will toggle */ 972 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); 973 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 974 cq->latest_cqe = NULL; 975 consume_cqe(cq); 976 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 977 } 978 979 DP_DEBUG(dev, QEDR_MSG_CQ, 980 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", 981 cq->icid, cq, params.cq_size); 982 983 return &cq->ibcq; 984 985 err3: 986 destroy_iparams.icid = cq->icid; 987 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, 988 &destroy_oparams); 989 err2: 990 if (udata) 991 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 992 else 993 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 994 err1: 995 if (udata) 996 ib_umem_release(cq->q.umem); 997 err0: 998 kfree(cq); 999 return ERR_PTR(-EINVAL); 1000 } 1001 1002 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 1003 { 1004 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1005 struct qedr_cq *cq = get_qedr_cq(ibcq); 1006 1007 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq); 1008 1009 return 0; 1010 } 1011 1012 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) 1013 #define QEDR_DESTROY_CQ_ITER_DURATION (10) 1014 1015 int qedr_destroy_cq(struct ib_cq *ibcq) 1016 { 1017 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1018 struct qed_rdma_destroy_cq_out_params oparams; 1019 struct qed_rdma_destroy_cq_in_params iparams; 1020 struct qedr_cq *cq = get_qedr_cq(ibcq); 1021 int iter; 1022 int rc; 1023 1024 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); 1025 1026 cq->destroyed = 1; 1027 1028 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 1029 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 1030 goto done; 1031 1032 iparams.icid = cq->icid; 1033 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 1034 if (rc) 1035 return rc; 1036 1037 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1038 1039 if (ibcq->uobject && ibcq->uobject->context) { 1040 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1041 ib_umem_release(cq->q.umem); 1042 } 1043 1044 /* We don't want the IRQ handler to handle a non-existing CQ so we 1045 * wait until all CNQ interrupts, if any, are received. This will always 1046 * happen and will always happen very fast. If not, then a serious error 1047 * has occured. That is why we can use a long delay. 1048 * We spin for a short time so we don’t lose time on context switching 1049 * in case all the completions are handled in that span. Otherwise 1050 * we sleep for a while and check again. Since the CNQ may be 1051 * associated with (only) the current CPU we use msleep to allow the 1052 * current CPU to be freed. 1053 * The CNQ notification is increased in qedr_irq_handler(). 1054 */ 1055 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1056 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1057 udelay(QEDR_DESTROY_CQ_ITER_DURATION); 1058 iter--; 1059 } 1060 1061 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1062 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1063 msleep(QEDR_DESTROY_CQ_ITER_DURATION); 1064 iter--; 1065 } 1066 1067 if (oparams.num_cq_notif != cq->cnq_notif) 1068 goto err; 1069 1070 /* Note that we don't need to have explicit code to wait for the 1071 * completion of the event handler because it is invoked from the EQ. 1072 * Since the destroy CQ ramrod has also been received on the EQ we can 1073 * be certain that there's no event handler in process. 1074 */ 1075 done: 1076 cq->sig = ~cq->sig; 1077 1078 kfree(cq); 1079 1080 return 0; 1081 1082 err: 1083 DP_ERR(dev, 1084 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n", 1085 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif); 1086 1087 return -EINVAL; 1088 } 1089 1090 static inline int get_gid_info_from_table(struct ib_qp *ibqp, 1091 struct ib_qp_attr *attr, 1092 int attr_mask, 1093 struct qed_rdma_modify_qp_in_params 1094 *qp_params) 1095 { 1096 const struct ib_gid_attr *gid_attr; 1097 enum rdma_network_type nw_type; 1098 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1099 u32 ipv4_addr; 1100 int i; 1101 1102 gid_attr = grh->sgid_attr; 1103 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr->ndev); 1104 1105 nw_type = rdma_gid_attr_network_type(gid_attr); 1106 switch (nw_type) { 1107 case RDMA_NETWORK_IPV6: 1108 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1109 sizeof(qp_params->sgid)); 1110 memcpy(&qp_params->dgid.bytes[0], 1111 &grh->dgid, 1112 sizeof(qp_params->dgid)); 1113 qp_params->roce_mode = ROCE_V2_IPV6; 1114 SET_FIELD(qp_params->modify_flags, 1115 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1116 break; 1117 case RDMA_NETWORK_IB: 1118 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1119 sizeof(qp_params->sgid)); 1120 memcpy(&qp_params->dgid.bytes[0], 1121 &grh->dgid, 1122 sizeof(qp_params->dgid)); 1123 qp_params->roce_mode = ROCE_V1; 1124 break; 1125 case RDMA_NETWORK_IPV4: 1126 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid)); 1127 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid)); 1128 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw); 1129 qp_params->sgid.ipv4_addr = ipv4_addr; 1130 ipv4_addr = 1131 qedr_get_ipv4_from_gid(grh->dgid.raw); 1132 qp_params->dgid.ipv4_addr = ipv4_addr; 1133 SET_FIELD(qp_params->modify_flags, 1134 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1135 qp_params->roce_mode = ROCE_V2_IPV4; 1136 break; 1137 } 1138 1139 for (i = 0; i < 4; i++) { 1140 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 1141 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 1142 } 1143 1144 if (qp_params->vlan_id >= VLAN_CFI_MASK) 1145 qp_params->vlan_id = 0; 1146 1147 return 0; 1148 } 1149 1150 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, 1151 struct ib_qp_init_attr *attrs) 1152 { 1153 struct qedr_device_attr *qattr = &dev->attr; 1154 1155 /* QP0... attrs->qp_type == IB_QPT_GSI */ 1156 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { 1157 DP_DEBUG(dev, QEDR_MSG_QP, 1158 "create qp: unsupported qp type=0x%x requested\n", 1159 attrs->qp_type); 1160 return -EINVAL; 1161 } 1162 1163 if (attrs->cap.max_send_wr > qattr->max_sqe) { 1164 DP_ERR(dev, 1165 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n", 1166 attrs->cap.max_send_wr, qattr->max_sqe); 1167 return -EINVAL; 1168 } 1169 1170 if (attrs->cap.max_inline_data > qattr->max_inline) { 1171 DP_ERR(dev, 1172 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n", 1173 attrs->cap.max_inline_data, qattr->max_inline); 1174 return -EINVAL; 1175 } 1176 1177 if (attrs->cap.max_send_sge > qattr->max_sge) { 1178 DP_ERR(dev, 1179 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n", 1180 attrs->cap.max_send_sge, qattr->max_sge); 1181 return -EINVAL; 1182 } 1183 1184 if (attrs->cap.max_recv_sge > qattr->max_sge) { 1185 DP_ERR(dev, 1186 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n", 1187 attrs->cap.max_recv_sge, qattr->max_sge); 1188 return -EINVAL; 1189 } 1190 1191 /* Unprivileged user space cannot create special QP */ 1192 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) { 1193 DP_ERR(dev, 1194 "create qp: userspace can't create special QPs of type=0x%x\n", 1195 attrs->qp_type); 1196 return -EINVAL; 1197 } 1198 1199 return 0; 1200 } 1201 1202 static int qedr_copy_srq_uresp(struct qedr_dev *dev, 1203 struct qedr_srq *srq, struct ib_udata *udata) 1204 { 1205 struct qedr_create_srq_uresp uresp = {}; 1206 int rc; 1207 1208 uresp.srq_id = srq->srq_id; 1209 1210 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1211 if (rc) 1212 DP_ERR(dev, "create srq: problem copying data to user space\n"); 1213 1214 return rc; 1215 } 1216 1217 static void qedr_copy_rq_uresp(struct qedr_dev *dev, 1218 struct qedr_create_qp_uresp *uresp, 1219 struct qedr_qp *qp) 1220 { 1221 /* iWARP requires two doorbells per RQ. */ 1222 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1223 uresp->rq_db_offset = 1224 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1225 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1226 } else { 1227 uresp->rq_db_offset = 1228 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1229 } 1230 1231 uresp->rq_icid = qp->icid; 1232 } 1233 1234 static void qedr_copy_sq_uresp(struct qedr_dev *dev, 1235 struct qedr_create_qp_uresp *uresp, 1236 struct qedr_qp *qp) 1237 { 1238 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1239 1240 /* iWARP uses the same cid for rq and sq */ 1241 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1242 uresp->sq_icid = qp->icid; 1243 else 1244 uresp->sq_icid = qp->icid + 1; 1245 } 1246 1247 static int qedr_copy_qp_uresp(struct qedr_dev *dev, 1248 struct qedr_qp *qp, struct ib_udata *udata) 1249 { 1250 struct qedr_create_qp_uresp uresp; 1251 int rc; 1252 1253 memset(&uresp, 0, sizeof(uresp)); 1254 qedr_copy_sq_uresp(dev, &uresp, qp); 1255 qedr_copy_rq_uresp(dev, &uresp, qp); 1256 1257 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 1258 uresp.qp_id = qp->qp_id; 1259 1260 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1261 if (rc) 1262 DP_ERR(dev, 1263 "create qp: failed a copy to user space with qp icid=0x%x.\n", 1264 qp->icid); 1265 1266 return rc; 1267 } 1268 1269 static void qedr_set_common_qp_params(struct qedr_dev *dev, 1270 struct qedr_qp *qp, 1271 struct qedr_pd *pd, 1272 struct ib_qp_init_attr *attrs) 1273 { 1274 spin_lock_init(&qp->q_lock); 1275 atomic_set(&qp->refcnt, 1); 1276 qp->pd = pd; 1277 qp->qp_type = attrs->qp_type; 1278 qp->max_inline_data = attrs->cap.max_inline_data; 1279 qp->sq.max_sges = attrs->cap.max_send_sge; 1280 qp->state = QED_ROCE_QP_STATE_RESET; 1281 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1282 qp->sq_cq = get_qedr_cq(attrs->send_cq); 1283 qp->dev = dev; 1284 1285 if (attrs->srq) { 1286 qp->srq = get_qedr_srq(attrs->srq); 1287 } else { 1288 qp->rq_cq = get_qedr_cq(attrs->recv_cq); 1289 qp->rq.max_sges = attrs->cap.max_recv_sge; 1290 DP_DEBUG(dev, QEDR_MSG_QP, 1291 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 1292 qp->rq.max_sges, qp->rq_cq->icid); 1293 } 1294 1295 DP_DEBUG(dev, QEDR_MSG_QP, 1296 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n", 1297 pd->pd_id, qp->qp_type, qp->max_inline_data, 1298 qp->state, qp->signaled, (attrs->srq) ? 1 : 0); 1299 DP_DEBUG(dev, QEDR_MSG_QP, 1300 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1301 qp->sq.max_sges, qp->sq_cq->icid); 1302 } 1303 1304 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1305 { 1306 qp->sq.db = dev->db_addr + 1307 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1308 qp->sq.db_data.data.icid = qp->icid + 1; 1309 if (!qp->srq) { 1310 qp->rq.db = dev->db_addr + 1311 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1312 qp->rq.db_data.data.icid = qp->icid; 1313 } 1314 } 1315 1316 static int qedr_check_srq_params(struct ib_pd *ibpd, struct qedr_dev *dev, 1317 struct ib_srq_init_attr *attrs, 1318 struct ib_udata *udata) 1319 { 1320 struct qedr_device_attr *qattr = &dev->attr; 1321 1322 if (attrs->attr.max_wr > qattr->max_srq_wr) { 1323 DP_ERR(dev, 1324 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n", 1325 attrs->attr.max_wr, qattr->max_srq_wr); 1326 return -EINVAL; 1327 } 1328 1329 if (attrs->attr.max_sge > qattr->max_sge) { 1330 DP_ERR(dev, 1331 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", 1332 attrs->attr.max_sge, qattr->max_sge); 1333 return -EINVAL; 1334 } 1335 1336 return 0; 1337 } 1338 1339 static void qedr_free_srq_user_params(struct qedr_srq *srq) 1340 { 1341 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1342 ib_umem_release(srq->usrq.umem); 1343 ib_umem_release(srq->prod_umem); 1344 } 1345 1346 static void qedr_free_srq_kernel_params(struct qedr_srq *srq) 1347 { 1348 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1349 struct qedr_dev *dev = srq->dev; 1350 1351 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl); 1352 1353 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1354 hw_srq->virt_prod_pair_addr, 1355 hw_srq->phy_prod_pair_addr); 1356 } 1357 1358 static int qedr_init_srq_user_params(struct ib_ucontext *ib_ctx, 1359 struct qedr_srq *srq, 1360 struct qedr_create_srq_ureq *ureq, 1361 int access, int dmasync) 1362 { 1363 struct scatterlist *sg; 1364 int rc; 1365 1366 rc = qedr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr, 1367 ureq->srq_len, access, dmasync, 1); 1368 if (rc) 1369 return rc; 1370 1371 srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr, 1372 sizeof(struct rdma_srq_producers), 1373 access, dmasync); 1374 if (IS_ERR(srq->prod_umem)) { 1375 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1376 ib_umem_release(srq->usrq.umem); 1377 DP_ERR(srq->dev, 1378 "create srq: failed ib_umem_get for producer, got %ld\n", 1379 PTR_ERR(srq->prod_umem)); 1380 return PTR_ERR(srq->prod_umem); 1381 } 1382 1383 sg = srq->prod_umem->sg_head.sgl; 1384 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); 1385 1386 return 0; 1387 } 1388 1389 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq, 1390 struct qedr_dev *dev, 1391 struct ib_srq_init_attr *init_attr) 1392 { 1393 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1394 dma_addr_t phy_prod_pair_addr; 1395 u32 num_elems; 1396 void *va; 1397 int rc; 1398 1399 va = dma_alloc_coherent(&dev->pdev->dev, 1400 sizeof(struct rdma_srq_producers), 1401 &phy_prod_pair_addr, GFP_KERNEL); 1402 if (!va) { 1403 DP_ERR(dev, 1404 "create srq: failed to allocate dma memory for producer\n"); 1405 return -ENOMEM; 1406 } 1407 1408 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; 1409 hw_srq->virt_prod_pair_addr = va; 1410 1411 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE; 1412 rc = dev->ops->common->chain_alloc(dev->cdev, 1413 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1414 QED_CHAIN_MODE_PBL, 1415 QED_CHAIN_CNT_TYPE_U32, 1416 num_elems, 1417 QEDR_SRQ_WQE_ELEM_SIZE, 1418 &hw_srq->pbl, NULL); 1419 if (rc) 1420 goto err0; 1421 1422 hw_srq->num_elems = num_elems; 1423 1424 return 0; 1425 1426 err0: 1427 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1428 va, phy_prod_pair_addr); 1429 return rc; 1430 } 1431 1432 static int qedr_idr_add(struct qedr_dev *dev, struct qedr_idr *qidr, 1433 void *ptr, u32 id); 1434 static void qedr_idr_remove(struct qedr_dev *dev, 1435 struct qedr_idr *qidr, u32 id); 1436 1437 struct ib_srq *qedr_create_srq(struct ib_pd *ibpd, 1438 struct ib_srq_init_attr *init_attr, 1439 struct ib_udata *udata) 1440 { 1441 struct qed_rdma_destroy_srq_in_params destroy_in_params; 1442 struct qed_rdma_create_srq_in_params in_params = {}; 1443 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 1444 struct qed_rdma_create_srq_out_params out_params; 1445 struct qedr_pd *pd = get_qedr_pd(ibpd); 1446 struct qedr_create_srq_ureq ureq = {}; 1447 u64 pbl_base_addr, phy_prod_pair_addr; 1448 struct ib_ucontext *ib_ctx = NULL; 1449 struct qedr_srq_hwq_info *hw_srq; 1450 u32 page_cnt, page_size; 1451 struct qedr_srq *srq; 1452 int rc = 0; 1453 1454 DP_DEBUG(dev, QEDR_MSG_QP, 1455 "create SRQ called from %s (pd %p)\n", 1456 (udata) ? "User lib" : "kernel", pd); 1457 1458 rc = qedr_check_srq_params(ibpd, dev, init_attr, udata); 1459 if (rc) 1460 return ERR_PTR(-EINVAL); 1461 1462 srq = kzalloc(sizeof(*srq), GFP_KERNEL); 1463 if (!srq) 1464 return ERR_PTR(-ENOMEM); 1465 1466 srq->dev = dev; 1467 hw_srq = &srq->hw_srq; 1468 spin_lock_init(&srq->lock); 1469 1470 hw_srq->max_wr = init_attr->attr.max_wr; 1471 hw_srq->max_sges = init_attr->attr.max_sge; 1472 1473 if (udata && ibpd->uobject && ibpd->uobject->context) { 1474 ib_ctx = ibpd->uobject->context; 1475 1476 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { 1477 DP_ERR(dev, 1478 "create srq: problem copying data from user space\n"); 1479 goto err0; 1480 } 1481 1482 rc = qedr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0); 1483 if (rc) 1484 goto err0; 1485 1486 page_cnt = srq->usrq.pbl_info.num_pbes; 1487 pbl_base_addr = srq->usrq.pbl_tbl->pa; 1488 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1489 page_size = BIT(srq->usrq.umem->page_shift); 1490 } else { 1491 struct qed_chain *pbl; 1492 1493 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr); 1494 if (rc) 1495 goto err0; 1496 1497 pbl = &hw_srq->pbl; 1498 page_cnt = qed_chain_get_page_cnt(pbl); 1499 pbl_base_addr = qed_chain_get_pbl_phys(pbl); 1500 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1501 page_size = QED_CHAIN_PAGE_SIZE; 1502 } 1503 1504 in_params.pd_id = pd->pd_id; 1505 in_params.pbl_base_addr = pbl_base_addr; 1506 in_params.prod_pair_addr = phy_prod_pair_addr; 1507 in_params.num_pages = page_cnt; 1508 in_params.page_size = page_size; 1509 1510 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); 1511 if (rc) 1512 goto err1; 1513 1514 srq->srq_id = out_params.srq_id; 1515 1516 if (udata) { 1517 rc = qedr_copy_srq_uresp(dev, srq, udata); 1518 if (rc) 1519 goto err2; 1520 } 1521 1522 rc = qedr_idr_add(dev, &dev->srqidr, srq, srq->srq_id); 1523 if (rc) 1524 goto err2; 1525 1526 DP_DEBUG(dev, QEDR_MSG_SRQ, 1527 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id); 1528 return &srq->ibsrq; 1529 1530 err2: 1531 destroy_in_params.srq_id = srq->srq_id; 1532 1533 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); 1534 err1: 1535 if (udata) 1536 qedr_free_srq_user_params(srq); 1537 else 1538 qedr_free_srq_kernel_params(srq); 1539 err0: 1540 kfree(srq); 1541 1542 return ERR_PTR(-EFAULT); 1543 } 1544 1545 int qedr_destroy_srq(struct ib_srq *ibsrq) 1546 { 1547 struct qed_rdma_destroy_srq_in_params in_params = {}; 1548 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1549 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1550 1551 qedr_idr_remove(dev, &dev->srqidr, srq->srq_id); 1552 in_params.srq_id = srq->srq_id; 1553 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params); 1554 1555 if (ibsrq->pd->uobject) 1556 qedr_free_srq_user_params(srq); 1557 else 1558 qedr_free_srq_kernel_params(srq); 1559 1560 DP_DEBUG(dev, QEDR_MSG_SRQ, 1561 "destroy srq: destroyed srq with srq_id=0x%0x\n", 1562 srq->srq_id); 1563 kfree(srq); 1564 1565 return 0; 1566 } 1567 1568 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 1569 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 1570 { 1571 struct qed_rdma_modify_srq_in_params in_params = {}; 1572 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1573 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1574 int rc; 1575 1576 if (attr_mask & IB_SRQ_MAX_WR) { 1577 DP_ERR(dev, 1578 "modify srq: invalid attribute mask=0x%x specified for %p\n", 1579 attr_mask, srq); 1580 return -EINVAL; 1581 } 1582 1583 if (attr_mask & IB_SRQ_LIMIT) { 1584 if (attr->srq_limit >= srq->hw_srq.max_wr) { 1585 DP_ERR(dev, 1586 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n", 1587 attr->srq_limit, srq->hw_srq.max_wr); 1588 return -EINVAL; 1589 } 1590 1591 in_params.srq_id = srq->srq_id; 1592 in_params.wqe_limit = attr->srq_limit; 1593 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params); 1594 if (rc) 1595 return rc; 1596 } 1597 1598 srq->srq_limit = attr->srq_limit; 1599 1600 DP_DEBUG(dev, QEDR_MSG_SRQ, 1601 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id); 1602 1603 return 0; 1604 } 1605 1606 static inline void 1607 qedr_init_common_qp_in_params(struct qedr_dev *dev, 1608 struct qedr_pd *pd, 1609 struct qedr_qp *qp, 1610 struct ib_qp_init_attr *attrs, 1611 bool fmr_and_reserved_lkey, 1612 struct qed_rdma_create_qp_in_params *params) 1613 { 1614 /* QP handle to be written in an async event */ 1615 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp); 1616 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp); 1617 1618 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 1619 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 1620 params->pd = pd->pd_id; 1621 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 1622 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; 1623 params->stats_queue = 0; 1624 params->srq_id = 0; 1625 params->use_srq = false; 1626 1627 if (!qp->srq) { 1628 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1629 1630 } else { 1631 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1632 params->srq_id = qp->srq->srq_id; 1633 params->use_srq = true; 1634 } 1635 } 1636 1637 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) 1638 { 1639 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. " 1640 "qp=%p. " 1641 "sq_addr=0x%llx, " 1642 "sq_len=%zd, " 1643 "rq_addr=0x%llx, " 1644 "rq_len=%zd" 1645 "\n", 1646 qp, 1647 qp->usq.buf_addr, 1648 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len); 1649 } 1650 1651 static int qedr_idr_add(struct qedr_dev *dev, struct qedr_idr *qidr, 1652 void *ptr, u32 id) 1653 { 1654 int rc; 1655 1656 idr_preload(GFP_KERNEL); 1657 spin_lock_irq(&qidr->idr_lock); 1658 1659 rc = idr_alloc(&qidr->idr, ptr, id, id + 1, GFP_ATOMIC); 1660 1661 spin_unlock_irq(&qidr->idr_lock); 1662 idr_preload_end(); 1663 1664 return rc < 0 ? rc : 0; 1665 } 1666 1667 static void qedr_idr_remove(struct qedr_dev *dev, struct qedr_idr *qidr, u32 id) 1668 { 1669 spin_lock_irq(&qidr->idr_lock); 1670 idr_remove(&qidr->idr, id); 1671 spin_unlock_irq(&qidr->idr_lock); 1672 } 1673 1674 static inline void 1675 qedr_iwarp_populate_user_qp(struct qedr_dev *dev, 1676 struct qedr_qp *qp, 1677 struct qed_rdma_create_qp_out_params *out_params) 1678 { 1679 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 1680 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 1681 1682 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 1683 &qp->usq.pbl_info, FW_PAGE_SHIFT); 1684 if (!qp->srq) { 1685 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 1686 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 1687 } 1688 1689 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 1690 &qp->urq.pbl_info, FW_PAGE_SHIFT); 1691 } 1692 1693 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp) 1694 { 1695 if (qp->usq.umem) 1696 ib_umem_release(qp->usq.umem); 1697 qp->usq.umem = NULL; 1698 1699 if (qp->urq.umem) 1700 ib_umem_release(qp->urq.umem); 1701 qp->urq.umem = NULL; 1702 } 1703 1704 static int qedr_create_user_qp(struct qedr_dev *dev, 1705 struct qedr_qp *qp, 1706 struct ib_pd *ibpd, 1707 struct ib_udata *udata, 1708 struct ib_qp_init_attr *attrs) 1709 { 1710 struct qed_rdma_create_qp_in_params in_params; 1711 struct qed_rdma_create_qp_out_params out_params; 1712 struct qedr_pd *pd = get_qedr_pd(ibpd); 1713 struct ib_ucontext *ib_ctx = NULL; 1714 struct qedr_create_qp_ureq ureq; 1715 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1); 1716 int rc = -EINVAL; 1717 1718 ib_ctx = ibpd->uobject->context; 1719 1720 memset(&ureq, 0, sizeof(ureq)); 1721 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq)); 1722 if (rc) { 1723 DP_ERR(dev, "Problem copying data from user space\n"); 1724 return rc; 1725 } 1726 1727 /* SQ - read access only (0), dma sync not required (0) */ 1728 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr, 1729 ureq.sq_len, 0, 0, alloc_and_init); 1730 if (rc) 1731 return rc; 1732 1733 if (!qp->srq) { 1734 /* RQ - read access only (0), dma sync not required (0) */ 1735 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr, 1736 ureq.rq_len, 0, 0, alloc_and_init); 1737 if (rc) 1738 return rc; 1739 } 1740 1741 memset(&in_params, 0, sizeof(in_params)); 1742 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 1743 in_params.qp_handle_lo = ureq.qp_handle_lo; 1744 in_params.qp_handle_hi = ureq.qp_handle_hi; 1745 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 1746 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 1747 if (!qp->srq) { 1748 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 1749 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 1750 } 1751 1752 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1753 &in_params, &out_params); 1754 1755 if (!qp->qed_qp) { 1756 rc = -ENOMEM; 1757 goto err1; 1758 } 1759 1760 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1761 qedr_iwarp_populate_user_qp(dev, qp, &out_params); 1762 1763 qp->qp_id = out_params.qp_id; 1764 qp->icid = out_params.icid; 1765 1766 rc = qedr_copy_qp_uresp(dev, qp, udata); 1767 if (rc) 1768 goto err; 1769 1770 qedr_qp_user_print(dev, qp); 1771 1772 return 0; 1773 err: 1774 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1775 if (rc) 1776 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc); 1777 1778 err1: 1779 qedr_cleanup_user(dev, qp); 1780 return rc; 1781 } 1782 1783 static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1784 { 1785 qp->sq.db = dev->db_addr + 1786 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1787 qp->sq.db_data.data.icid = qp->icid; 1788 1789 qp->rq.db = dev->db_addr + 1790 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1791 qp->rq.db_data.data.icid = qp->icid; 1792 qp->rq.iwarp_db2 = dev->db_addr + 1793 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1794 qp->rq.iwarp_db2_data.data.icid = qp->icid; 1795 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 1796 } 1797 1798 static int 1799 qedr_roce_create_kernel_qp(struct qedr_dev *dev, 1800 struct qedr_qp *qp, 1801 struct qed_rdma_create_qp_in_params *in_params, 1802 u32 n_sq_elems, u32 n_rq_elems) 1803 { 1804 struct qed_rdma_create_qp_out_params out_params; 1805 int rc; 1806 1807 rc = dev->ops->common->chain_alloc(dev->cdev, 1808 QED_CHAIN_USE_TO_PRODUCE, 1809 QED_CHAIN_MODE_PBL, 1810 QED_CHAIN_CNT_TYPE_U32, 1811 n_sq_elems, 1812 QEDR_SQE_ELEMENT_SIZE, 1813 &qp->sq.pbl, NULL); 1814 1815 if (rc) 1816 return rc; 1817 1818 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl); 1819 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl); 1820 1821 rc = dev->ops->common->chain_alloc(dev->cdev, 1822 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1823 QED_CHAIN_MODE_PBL, 1824 QED_CHAIN_CNT_TYPE_U32, 1825 n_rq_elems, 1826 QEDR_RQE_ELEMENT_SIZE, 1827 &qp->rq.pbl, NULL); 1828 if (rc) 1829 return rc; 1830 1831 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl); 1832 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl); 1833 1834 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1835 in_params, &out_params); 1836 1837 if (!qp->qed_qp) 1838 return -EINVAL; 1839 1840 qp->qp_id = out_params.qp_id; 1841 qp->icid = out_params.icid; 1842 1843 qedr_set_roce_db_info(dev, qp); 1844 return rc; 1845 } 1846 1847 static int 1848 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev, 1849 struct qedr_qp *qp, 1850 struct qed_rdma_create_qp_in_params *in_params, 1851 u32 n_sq_elems, u32 n_rq_elems) 1852 { 1853 struct qed_rdma_create_qp_out_params out_params; 1854 struct qed_chain_ext_pbl ext_pbl; 1855 int rc; 1856 1857 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems, 1858 QEDR_SQE_ELEMENT_SIZE, 1859 QED_CHAIN_MODE_PBL); 1860 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems, 1861 QEDR_RQE_ELEMENT_SIZE, 1862 QED_CHAIN_MODE_PBL); 1863 1864 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1865 in_params, &out_params); 1866 1867 if (!qp->qed_qp) 1868 return -EINVAL; 1869 1870 /* Now we allocate the chain */ 1871 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt; 1872 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys; 1873 1874 rc = dev->ops->common->chain_alloc(dev->cdev, 1875 QED_CHAIN_USE_TO_PRODUCE, 1876 QED_CHAIN_MODE_PBL, 1877 QED_CHAIN_CNT_TYPE_U32, 1878 n_sq_elems, 1879 QEDR_SQE_ELEMENT_SIZE, 1880 &qp->sq.pbl, &ext_pbl); 1881 1882 if (rc) 1883 goto err; 1884 1885 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt; 1886 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys; 1887 1888 rc = dev->ops->common->chain_alloc(dev->cdev, 1889 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1890 QED_CHAIN_MODE_PBL, 1891 QED_CHAIN_CNT_TYPE_U32, 1892 n_rq_elems, 1893 QEDR_RQE_ELEMENT_SIZE, 1894 &qp->rq.pbl, &ext_pbl); 1895 1896 if (rc) 1897 goto err; 1898 1899 qp->qp_id = out_params.qp_id; 1900 qp->icid = out_params.icid; 1901 1902 qedr_set_iwarp_db_info(dev, qp); 1903 return rc; 1904 1905 err: 1906 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1907 1908 return rc; 1909 } 1910 1911 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) 1912 { 1913 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl); 1914 kfree(qp->wqe_wr_id); 1915 1916 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl); 1917 kfree(qp->rqe_wr_id); 1918 } 1919 1920 static int qedr_create_kernel_qp(struct qedr_dev *dev, 1921 struct qedr_qp *qp, 1922 struct ib_pd *ibpd, 1923 struct ib_qp_init_attr *attrs) 1924 { 1925 struct qed_rdma_create_qp_in_params in_params; 1926 struct qedr_pd *pd = get_qedr_pd(ibpd); 1927 int rc = -EINVAL; 1928 u32 n_rq_elems; 1929 u32 n_sq_elems; 1930 u32 n_sq_entries; 1931 1932 memset(&in_params, 0, sizeof(in_params)); 1933 1934 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in 1935 * the ring. The ring should allow at least a single WR, even if the 1936 * user requested none, due to allocation issues. 1937 * We should add an extra WR since the prod and cons indices of 1938 * wqe_wr_id are managed in such a way that the WQ is considered full 1939 * when (prod+1)%max_wr==cons. We currently don't do that because we 1940 * double the number of entries due an iSER issue that pushes far more 1941 * WRs than indicated. If we decline its ib_post_send() then we get 1942 * error prints in the dmesg we'd like to avoid. 1943 */ 1944 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 1945 dev->attr.max_sqe); 1946 1947 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id), 1948 GFP_KERNEL); 1949 if (!qp->wqe_wr_id) { 1950 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n"); 1951 return -ENOMEM; 1952 } 1953 1954 /* QP handle to be written in CQE */ 1955 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp); 1956 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp); 1957 1958 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in 1959 * the ring. There ring should allow at least a single WR, even if the 1960 * user requested none, due to allocation issues. 1961 */ 1962 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1); 1963 1964 /* Allocate driver internal RQ array */ 1965 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id), 1966 GFP_KERNEL); 1967 if (!qp->rqe_wr_id) { 1968 DP_ERR(dev, 1969 "create qp: failed RQ shadow memory allocation\n"); 1970 kfree(qp->wqe_wr_id); 1971 return -ENOMEM; 1972 } 1973 1974 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 1975 1976 n_sq_entries = attrs->cap.max_send_wr; 1977 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe); 1978 n_sq_entries = max_t(u32, n_sq_entries, 1); 1979 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE; 1980 1981 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE; 1982 1983 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1984 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params, 1985 n_sq_elems, n_rq_elems); 1986 else 1987 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params, 1988 n_sq_elems, n_rq_elems); 1989 if (rc) 1990 qedr_cleanup_kernel(dev, qp); 1991 1992 return rc; 1993 } 1994 1995 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, 1996 struct ib_qp_init_attr *attrs, 1997 struct ib_udata *udata) 1998 { 1999 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2000 struct qedr_pd *pd = get_qedr_pd(ibpd); 2001 struct qedr_qp *qp; 2002 struct ib_qp *ibqp; 2003 int rc = 0; 2004 2005 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", 2006 udata ? "user library" : "kernel", pd); 2007 2008 rc = qedr_check_qp_attrs(ibpd, dev, attrs); 2009 if (rc) 2010 return ERR_PTR(rc); 2011 2012 DP_DEBUG(dev, QEDR_MSG_QP, 2013 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 2014 udata ? "user library" : "kernel", attrs->event_handler, pd, 2015 get_qedr_cq(attrs->send_cq), 2016 get_qedr_cq(attrs->send_cq)->icid, 2017 get_qedr_cq(attrs->recv_cq), 2018 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0); 2019 2020 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 2021 if (!qp) { 2022 DP_ERR(dev, "create qp: failed allocating memory\n"); 2023 return ERR_PTR(-ENOMEM); 2024 } 2025 2026 qedr_set_common_qp_params(dev, qp, pd, attrs); 2027 2028 if (attrs->qp_type == IB_QPT_GSI) { 2029 ibqp = qedr_create_gsi_qp(dev, attrs, qp); 2030 if (IS_ERR(ibqp)) 2031 kfree(qp); 2032 return ibqp; 2033 } 2034 2035 if (udata) 2036 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); 2037 else 2038 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); 2039 2040 if (rc) 2041 goto err; 2042 2043 qp->ibqp.qp_num = qp->qp_id; 2044 2045 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 2046 rc = qedr_idr_add(dev, &dev->qpidr, qp, qp->qp_id); 2047 if (rc) 2048 goto err; 2049 } 2050 2051 return &qp->ibqp; 2052 2053 err: 2054 kfree(qp); 2055 2056 return ERR_PTR(-EFAULT); 2057 } 2058 2059 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) 2060 { 2061 switch (qp_state) { 2062 case QED_ROCE_QP_STATE_RESET: 2063 return IB_QPS_RESET; 2064 case QED_ROCE_QP_STATE_INIT: 2065 return IB_QPS_INIT; 2066 case QED_ROCE_QP_STATE_RTR: 2067 return IB_QPS_RTR; 2068 case QED_ROCE_QP_STATE_RTS: 2069 return IB_QPS_RTS; 2070 case QED_ROCE_QP_STATE_SQD: 2071 return IB_QPS_SQD; 2072 case QED_ROCE_QP_STATE_ERR: 2073 return IB_QPS_ERR; 2074 case QED_ROCE_QP_STATE_SQE: 2075 return IB_QPS_SQE; 2076 } 2077 return IB_QPS_ERR; 2078 } 2079 2080 static enum qed_roce_qp_state qedr_get_state_from_ibqp( 2081 enum ib_qp_state qp_state) 2082 { 2083 switch (qp_state) { 2084 case IB_QPS_RESET: 2085 return QED_ROCE_QP_STATE_RESET; 2086 case IB_QPS_INIT: 2087 return QED_ROCE_QP_STATE_INIT; 2088 case IB_QPS_RTR: 2089 return QED_ROCE_QP_STATE_RTR; 2090 case IB_QPS_RTS: 2091 return QED_ROCE_QP_STATE_RTS; 2092 case IB_QPS_SQD: 2093 return QED_ROCE_QP_STATE_SQD; 2094 case IB_QPS_ERR: 2095 return QED_ROCE_QP_STATE_ERR; 2096 default: 2097 return QED_ROCE_QP_STATE_ERR; 2098 } 2099 } 2100 2101 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) 2102 { 2103 qed_chain_reset(&qph->pbl); 2104 qph->prod = 0; 2105 qph->cons = 0; 2106 qph->wqe_cons = 0; 2107 qph->db_data.data.value = cpu_to_le16(0); 2108 } 2109 2110 static int qedr_update_qp_state(struct qedr_dev *dev, 2111 struct qedr_qp *qp, 2112 enum qed_roce_qp_state cur_state, 2113 enum qed_roce_qp_state new_state) 2114 { 2115 int status = 0; 2116 2117 if (new_state == cur_state) 2118 return 0; 2119 2120 switch (cur_state) { 2121 case QED_ROCE_QP_STATE_RESET: 2122 switch (new_state) { 2123 case QED_ROCE_QP_STATE_INIT: 2124 qp->prev_wqe_size = 0; 2125 qedr_reset_qp_hwq_info(&qp->sq); 2126 qedr_reset_qp_hwq_info(&qp->rq); 2127 break; 2128 default: 2129 status = -EINVAL; 2130 break; 2131 }; 2132 break; 2133 case QED_ROCE_QP_STATE_INIT: 2134 switch (new_state) { 2135 case QED_ROCE_QP_STATE_RTR: 2136 /* Update doorbell (in case post_recv was 2137 * done before move to RTR) 2138 */ 2139 2140 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2141 writel(qp->rq.db_data.raw, qp->rq.db); 2142 /* Make sure write takes effect */ 2143 mmiowb(); 2144 } 2145 break; 2146 case QED_ROCE_QP_STATE_ERR: 2147 break; 2148 default: 2149 /* Invalid state change. */ 2150 status = -EINVAL; 2151 break; 2152 }; 2153 break; 2154 case QED_ROCE_QP_STATE_RTR: 2155 /* RTR->XXX */ 2156 switch (new_state) { 2157 case QED_ROCE_QP_STATE_RTS: 2158 break; 2159 case QED_ROCE_QP_STATE_ERR: 2160 break; 2161 default: 2162 /* Invalid state change. */ 2163 status = -EINVAL; 2164 break; 2165 }; 2166 break; 2167 case QED_ROCE_QP_STATE_RTS: 2168 /* RTS->XXX */ 2169 switch (new_state) { 2170 case QED_ROCE_QP_STATE_SQD: 2171 break; 2172 case QED_ROCE_QP_STATE_ERR: 2173 break; 2174 default: 2175 /* Invalid state change. */ 2176 status = -EINVAL; 2177 break; 2178 }; 2179 break; 2180 case QED_ROCE_QP_STATE_SQD: 2181 /* SQD->XXX */ 2182 switch (new_state) { 2183 case QED_ROCE_QP_STATE_RTS: 2184 case QED_ROCE_QP_STATE_ERR: 2185 break; 2186 default: 2187 /* Invalid state change. */ 2188 status = -EINVAL; 2189 break; 2190 }; 2191 break; 2192 case QED_ROCE_QP_STATE_ERR: 2193 /* ERR->XXX */ 2194 switch (new_state) { 2195 case QED_ROCE_QP_STATE_RESET: 2196 if ((qp->rq.prod != qp->rq.cons) || 2197 (qp->sq.prod != qp->sq.cons)) { 2198 DP_NOTICE(dev, 2199 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n", 2200 qp->rq.prod, qp->rq.cons, qp->sq.prod, 2201 qp->sq.cons); 2202 status = -EINVAL; 2203 } 2204 break; 2205 default: 2206 status = -EINVAL; 2207 break; 2208 }; 2209 break; 2210 default: 2211 status = -EINVAL; 2212 break; 2213 }; 2214 2215 return status; 2216 } 2217 2218 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2219 int attr_mask, struct ib_udata *udata) 2220 { 2221 struct qedr_qp *qp = get_qedr_qp(ibqp); 2222 struct qed_rdma_modify_qp_in_params qp_params = { 0 }; 2223 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); 2224 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 2225 enum ib_qp_state old_qp_state, new_qp_state; 2226 enum qed_roce_qp_state cur_state; 2227 int rc = 0; 2228 2229 DP_DEBUG(dev, QEDR_MSG_QP, 2230 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask, 2231 attr->qp_state); 2232 2233 old_qp_state = qedr_get_ibqp_state(qp->state); 2234 if (attr_mask & IB_QP_STATE) 2235 new_qp_state = attr->qp_state; 2236 else 2237 new_qp_state = old_qp_state; 2238 2239 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2240 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state, 2241 ibqp->qp_type, attr_mask)) { 2242 DP_ERR(dev, 2243 "modify qp: invalid attribute mask=0x%x specified for\n" 2244 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n", 2245 attr_mask, qp->qp_id, ibqp->qp_type, 2246 old_qp_state, new_qp_state); 2247 rc = -EINVAL; 2248 goto err; 2249 } 2250 } 2251 2252 /* Translate the masks... */ 2253 if (attr_mask & IB_QP_STATE) { 2254 SET_FIELD(qp_params.modify_flags, 2255 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 2256 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state); 2257 } 2258 2259 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 2260 qp_params.sqd_async = true; 2261 2262 if (attr_mask & IB_QP_PKEY_INDEX) { 2263 SET_FIELD(qp_params.modify_flags, 2264 QED_ROCE_MODIFY_QP_VALID_PKEY, 1); 2265 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) { 2266 rc = -EINVAL; 2267 goto err; 2268 } 2269 2270 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT; 2271 } 2272 2273 if (attr_mask & IB_QP_QKEY) 2274 qp->qkey = attr->qkey; 2275 2276 if (attr_mask & IB_QP_ACCESS_FLAGS) { 2277 SET_FIELD(qp_params.modify_flags, 2278 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 2279 qp_params.incoming_rdma_read_en = attr->qp_access_flags & 2280 IB_ACCESS_REMOTE_READ; 2281 qp_params.incoming_rdma_write_en = attr->qp_access_flags & 2282 IB_ACCESS_REMOTE_WRITE; 2283 qp_params.incoming_atomic_en = attr->qp_access_flags & 2284 IB_ACCESS_REMOTE_ATOMIC; 2285 } 2286 2287 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 2288 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2289 return -EINVAL; 2290 2291 if (attr_mask & IB_QP_PATH_MTU) { 2292 if (attr->path_mtu < IB_MTU_256 || 2293 attr->path_mtu > IB_MTU_4096) { 2294 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n"); 2295 rc = -EINVAL; 2296 goto err; 2297 } 2298 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 2299 ib_mtu_enum_to_int(iboe_get_mtu 2300 (dev->ndev->mtu))); 2301 } 2302 2303 if (!qp->mtu) { 2304 qp->mtu = 2305 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2306 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu); 2307 } 2308 2309 SET_FIELD(qp_params.modify_flags, 2310 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1); 2311 2312 qp_params.traffic_class_tos = grh->traffic_class; 2313 qp_params.flow_label = grh->flow_label; 2314 qp_params.hop_limit_ttl = grh->hop_limit; 2315 2316 qp->sgid_idx = grh->sgid_index; 2317 2318 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params); 2319 if (rc) { 2320 DP_ERR(dev, 2321 "modify qp: problems with GID index %d (rc=%d)\n", 2322 grh->sgid_index, rc); 2323 return rc; 2324 } 2325 2326 rc = qedr_get_dmac(dev, &attr->ah_attr, 2327 qp_params.remote_mac_addr); 2328 if (rc) 2329 return rc; 2330 2331 qp_params.use_local_mac = true; 2332 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr); 2333 2334 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n", 2335 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 2336 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 2337 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n", 2338 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 2339 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 2340 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n", 2341 qp_params.remote_mac_addr); 2342 2343 qp_params.mtu = qp->mtu; 2344 qp_params.lb_indication = false; 2345 } 2346 2347 if (!qp_params.mtu) { 2348 /* Stay with current MTU */ 2349 if (qp->mtu) 2350 qp_params.mtu = qp->mtu; 2351 else 2352 qp_params.mtu = 2353 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2354 } 2355 2356 if (attr_mask & IB_QP_TIMEOUT) { 2357 SET_FIELD(qp_params.modify_flags, 2358 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 2359 2360 /* The received timeout value is an exponent used like this: 2361 * "12.7.34 LOCAL ACK TIMEOUT 2362 * Value representing the transport (ACK) timeout for use by 2363 * the remote, expressed as: 4.096 * 2^timeout [usec]" 2364 * The FW expects timeout in msec so we need to divide the usec 2365 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2, 2366 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8). 2367 * The value of zero means infinite so we use a 'max_t' to make 2368 * sure that sub 1 msec values will be configured as 1 msec. 2369 */ 2370 if (attr->timeout) 2371 qp_params.ack_timeout = 2372 1 << max_t(int, attr->timeout - 8, 0); 2373 else 2374 qp_params.ack_timeout = 0; 2375 } 2376 2377 if (attr_mask & IB_QP_RETRY_CNT) { 2378 SET_FIELD(qp_params.modify_flags, 2379 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 2380 qp_params.retry_cnt = attr->retry_cnt; 2381 } 2382 2383 if (attr_mask & IB_QP_RNR_RETRY) { 2384 SET_FIELD(qp_params.modify_flags, 2385 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1); 2386 qp_params.rnr_retry_cnt = attr->rnr_retry; 2387 } 2388 2389 if (attr_mask & IB_QP_RQ_PSN) { 2390 SET_FIELD(qp_params.modify_flags, 2391 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1); 2392 qp_params.rq_psn = attr->rq_psn; 2393 qp->rq_psn = attr->rq_psn; 2394 } 2395 2396 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 2397 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) { 2398 rc = -EINVAL; 2399 DP_ERR(dev, 2400 "unsupported max_rd_atomic=%d, supported=%d\n", 2401 attr->max_rd_atomic, 2402 dev->attr.max_qp_req_rd_atomic_resc); 2403 goto err; 2404 } 2405 2406 SET_FIELD(qp_params.modify_flags, 2407 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1); 2408 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 2409 } 2410 2411 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 2412 SET_FIELD(qp_params.modify_flags, 2413 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1); 2414 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 2415 } 2416 2417 if (attr_mask & IB_QP_SQ_PSN) { 2418 SET_FIELD(qp_params.modify_flags, 2419 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1); 2420 qp_params.sq_psn = attr->sq_psn; 2421 qp->sq_psn = attr->sq_psn; 2422 } 2423 2424 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 2425 if (attr->max_dest_rd_atomic > 2426 dev->attr.max_qp_resp_rd_atomic_resc) { 2427 DP_ERR(dev, 2428 "unsupported max_dest_rd_atomic=%d, supported=%d\n", 2429 attr->max_dest_rd_atomic, 2430 dev->attr.max_qp_resp_rd_atomic_resc); 2431 2432 rc = -EINVAL; 2433 goto err; 2434 } 2435 2436 SET_FIELD(qp_params.modify_flags, 2437 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1); 2438 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 2439 } 2440 2441 if (attr_mask & IB_QP_DEST_QPN) { 2442 SET_FIELD(qp_params.modify_flags, 2443 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1); 2444 2445 qp_params.dest_qp = attr->dest_qp_num; 2446 qp->dest_qp_num = attr->dest_qp_num; 2447 } 2448 2449 cur_state = qp->state; 2450 2451 /* Update the QP state before the actual ramrod to prevent a race with 2452 * fast path. Modifying the QP state to error will cause the device to 2453 * flush the CQEs and while polling the flushed CQEs will considered as 2454 * a potential issue if the QP isn't in error state. 2455 */ 2456 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI && 2457 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR) 2458 qp->state = QED_ROCE_QP_STATE_ERR; 2459 2460 if (qp->qp_type != IB_QPT_GSI) 2461 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx, 2462 qp->qed_qp, &qp_params); 2463 2464 if (attr_mask & IB_QP_STATE) { 2465 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 2466 rc = qedr_update_qp_state(dev, qp, cur_state, 2467 qp_params.new_state); 2468 qp->state = qp_params.new_state; 2469 } 2470 2471 err: 2472 return rc; 2473 } 2474 2475 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params) 2476 { 2477 int ib_qp_acc_flags = 0; 2478 2479 if (params->incoming_rdma_write_en) 2480 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 2481 if (params->incoming_rdma_read_en) 2482 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 2483 if (params->incoming_atomic_en) 2484 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 2485 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 2486 return ib_qp_acc_flags; 2487 } 2488 2489 int qedr_query_qp(struct ib_qp *ibqp, 2490 struct ib_qp_attr *qp_attr, 2491 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 2492 { 2493 struct qed_rdma_query_qp_out_params params; 2494 struct qedr_qp *qp = get_qedr_qp(ibqp); 2495 struct qedr_dev *dev = qp->dev; 2496 int rc = 0; 2497 2498 memset(¶ms, 0, sizeof(params)); 2499 2500 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms); 2501 if (rc) 2502 goto err; 2503 2504 memset(qp_attr, 0, sizeof(*qp_attr)); 2505 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 2506 2507 qp_attr->qp_state = qedr_get_ibqp_state(params.state); 2508 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state); 2509 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu); 2510 qp_attr->path_mig_state = IB_MIG_MIGRATED; 2511 qp_attr->rq_psn = params.rq_psn; 2512 qp_attr->sq_psn = params.sq_psn; 2513 qp_attr->dest_qp_num = params.dest_qp; 2514 2515 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms); 2516 2517 qp_attr->cap.max_send_wr = qp->sq.max_wr; 2518 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 2519 qp_attr->cap.max_send_sge = qp->sq.max_sges; 2520 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 2521 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; 2522 qp_init_attr->cap = qp_attr->cap; 2523 2524 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2525 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 2526 params.flow_label, qp->sgid_idx, 2527 params.hop_limit_ttl, params.traffic_class_tos); 2528 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); 2529 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 2530 rdma_ah_set_sl(&qp_attr->ah_attr, 0); 2531 qp_attr->timeout = params.timeout; 2532 qp_attr->rnr_retry = params.rnr_retry; 2533 qp_attr->retry_cnt = params.retry_cnt; 2534 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 2535 qp_attr->pkey_index = params.pkey_index; 2536 qp_attr->port_num = 1; 2537 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 2538 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 2539 qp_attr->alt_pkey_index = 0; 2540 qp_attr->alt_port_num = 0; 2541 qp_attr->alt_timeout = 0; 2542 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 2543 2544 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0; 2545 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 2546 qp_attr->max_rd_atomic = params.max_rd_atomic; 2547 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0; 2548 2549 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n", 2550 qp_attr->cap.max_inline_data); 2551 2552 err: 2553 return rc; 2554 } 2555 2556 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp) 2557 { 2558 int rc = 0; 2559 2560 if (qp->qp_type != IB_QPT_GSI) { 2561 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2562 if (rc) 2563 return rc; 2564 } 2565 2566 if (qp->ibqp.uobject && qp->ibqp.uobject->context) 2567 qedr_cleanup_user(dev, qp); 2568 else 2569 qedr_cleanup_kernel(dev, qp); 2570 2571 return 0; 2572 } 2573 2574 int qedr_destroy_qp(struct ib_qp *ibqp) 2575 { 2576 struct qedr_qp *qp = get_qedr_qp(ibqp); 2577 struct qedr_dev *dev = qp->dev; 2578 struct ib_qp_attr attr; 2579 int attr_mask = 0; 2580 int rc = 0; 2581 2582 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n", 2583 qp, qp->qp_type); 2584 2585 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2586 if ((qp->state != QED_ROCE_QP_STATE_RESET) && 2587 (qp->state != QED_ROCE_QP_STATE_ERR) && 2588 (qp->state != QED_ROCE_QP_STATE_INIT)) { 2589 2590 attr.qp_state = IB_QPS_ERR; 2591 attr_mask |= IB_QP_STATE; 2592 2593 /* Change the QP state to ERROR */ 2594 qedr_modify_qp(ibqp, &attr, attr_mask, NULL); 2595 } 2596 } else { 2597 /* Wait for the connect/accept to complete */ 2598 if (qp->ep) { 2599 int wait_count = 1; 2600 2601 while (qp->ep->during_connect) { 2602 DP_DEBUG(dev, QEDR_MSG_QP, 2603 "Still in during connect/accept\n"); 2604 2605 msleep(100); 2606 if (wait_count++ > 200) { 2607 DP_NOTICE(dev, 2608 "during connect timeout\n"); 2609 break; 2610 } 2611 } 2612 } 2613 } 2614 2615 if (qp->qp_type == IB_QPT_GSI) 2616 qedr_destroy_gsi_qp(dev); 2617 2618 qedr_free_qp_resources(dev, qp); 2619 2620 if (atomic_dec_and_test(&qp->refcnt) && 2621 rdma_protocol_iwarp(&dev->ibdev, 1)) { 2622 qedr_idr_remove(dev, &dev->qpidr, qp->qp_id); 2623 kfree(qp); 2624 } 2625 return rc; 2626 } 2627 2628 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr, 2629 struct ib_udata *udata) 2630 { 2631 struct qedr_ah *ah; 2632 2633 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 2634 if (!ah) 2635 return ERR_PTR(-ENOMEM); 2636 2637 rdma_copy_ah_attr(&ah->attr, attr); 2638 2639 return &ah->ibah; 2640 } 2641 2642 int qedr_destroy_ah(struct ib_ah *ibah) 2643 { 2644 struct qedr_ah *ah = get_qedr_ah(ibah); 2645 2646 rdma_destroy_ah_attr(&ah->attr); 2647 kfree(ah); 2648 return 0; 2649 } 2650 2651 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) 2652 { 2653 struct qedr_pbl *pbl, *tmp; 2654 2655 if (info->pbl_table) 2656 list_add_tail(&info->pbl_table->list_entry, 2657 &info->free_pbl_list); 2658 2659 if (!list_empty(&info->inuse_pbl_list)) 2660 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 2661 2662 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 2663 list_del(&pbl->list_entry); 2664 qedr_free_pbl(dev, &info->pbl_info, pbl); 2665 } 2666 } 2667 2668 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info, 2669 size_t page_list_len, bool two_layered) 2670 { 2671 struct qedr_pbl *tmp; 2672 int rc; 2673 2674 INIT_LIST_HEAD(&info->free_pbl_list); 2675 INIT_LIST_HEAD(&info->inuse_pbl_list); 2676 2677 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info, 2678 page_list_len, two_layered); 2679 if (rc) 2680 goto done; 2681 2682 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2683 if (IS_ERR(info->pbl_table)) { 2684 rc = PTR_ERR(info->pbl_table); 2685 goto done; 2686 } 2687 2688 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n", 2689 &info->pbl_table->pa); 2690 2691 /* in usual case we use 2 PBLs, so we add one to free 2692 * list and allocating another one 2693 */ 2694 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2695 if (IS_ERR(tmp)) { 2696 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n"); 2697 goto done; 2698 } 2699 2700 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 2701 2702 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa); 2703 2704 done: 2705 if (rc) 2706 free_mr_info(dev, info); 2707 2708 return rc; 2709 } 2710 2711 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 2712 u64 usr_addr, int acc, struct ib_udata *udata) 2713 { 2714 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2715 struct qedr_mr *mr; 2716 struct qedr_pd *pd; 2717 int rc = -ENOMEM; 2718 2719 pd = get_qedr_pd(ibpd); 2720 DP_DEBUG(dev, QEDR_MSG_MR, 2721 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 2722 pd->pd_id, start, len, usr_addr, acc); 2723 2724 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 2725 return ERR_PTR(-EINVAL); 2726 2727 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2728 if (!mr) 2729 return ERR_PTR(rc); 2730 2731 mr->type = QEDR_MR_USER; 2732 2733 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0); 2734 if (IS_ERR(mr->umem)) { 2735 rc = -EFAULT; 2736 goto err0; 2737 } 2738 2739 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); 2740 if (rc) 2741 goto err1; 2742 2743 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 2744 &mr->info.pbl_info, mr->umem->page_shift); 2745 2746 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2747 if (rc) { 2748 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2749 goto err1; 2750 } 2751 2752 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2753 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2754 mr->hw_mr.key = 0; 2755 mr->hw_mr.pd = pd->pd_id; 2756 mr->hw_mr.local_read = 1; 2757 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2758 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2759 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2760 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2761 mr->hw_mr.mw_bind = false; 2762 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 2763 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2764 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2765 mr->hw_mr.page_size_log = mr->umem->page_shift; 2766 mr->hw_mr.fbo = ib_umem_offset(mr->umem); 2767 mr->hw_mr.length = len; 2768 mr->hw_mr.vaddr = usr_addr; 2769 mr->hw_mr.zbva = false; 2770 mr->hw_mr.phy_mr = false; 2771 mr->hw_mr.dma_mr = false; 2772 2773 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2774 if (rc) { 2775 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2776 goto err2; 2777 } 2778 2779 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2780 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2781 mr->hw_mr.remote_atomic) 2782 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2783 2784 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n", 2785 mr->ibmr.lkey); 2786 return &mr->ibmr; 2787 2788 err2: 2789 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2790 err1: 2791 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2792 err0: 2793 kfree(mr); 2794 return ERR_PTR(rc); 2795 } 2796 2797 int qedr_dereg_mr(struct ib_mr *ib_mr) 2798 { 2799 struct qedr_mr *mr = get_qedr_mr(ib_mr); 2800 struct qedr_dev *dev = get_qedr_dev(ib_mr->device); 2801 int rc = 0; 2802 2803 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid); 2804 if (rc) 2805 return rc; 2806 2807 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2808 2809 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR)) 2810 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2811 2812 /* it could be user registered memory. */ 2813 if (mr->umem) 2814 ib_umem_release(mr->umem); 2815 2816 kfree(mr); 2817 2818 return rc; 2819 } 2820 2821 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, 2822 int max_page_list_len) 2823 { 2824 struct qedr_pd *pd = get_qedr_pd(ibpd); 2825 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2826 struct qedr_mr *mr; 2827 int rc = -ENOMEM; 2828 2829 DP_DEBUG(dev, QEDR_MSG_MR, 2830 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id, 2831 max_page_list_len); 2832 2833 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2834 if (!mr) 2835 return ERR_PTR(rc); 2836 2837 mr->dev = dev; 2838 mr->type = QEDR_MR_FRMR; 2839 2840 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1); 2841 if (rc) 2842 goto err0; 2843 2844 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2845 if (rc) { 2846 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2847 goto err0; 2848 } 2849 2850 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2851 mr->hw_mr.tid_type = QED_RDMA_TID_FMR; 2852 mr->hw_mr.key = 0; 2853 mr->hw_mr.pd = pd->pd_id; 2854 mr->hw_mr.local_read = 1; 2855 mr->hw_mr.local_write = 0; 2856 mr->hw_mr.remote_read = 0; 2857 mr->hw_mr.remote_write = 0; 2858 mr->hw_mr.remote_atomic = 0; 2859 mr->hw_mr.mw_bind = false; 2860 mr->hw_mr.pbl_ptr = 0; 2861 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2862 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2863 mr->hw_mr.fbo = 0; 2864 mr->hw_mr.length = 0; 2865 mr->hw_mr.vaddr = 0; 2866 mr->hw_mr.zbva = false; 2867 mr->hw_mr.phy_mr = true; 2868 mr->hw_mr.dma_mr = false; 2869 2870 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2871 if (rc) { 2872 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2873 goto err1; 2874 } 2875 2876 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2877 mr->ibmr.rkey = mr->ibmr.lkey; 2878 2879 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey); 2880 return mr; 2881 2882 err1: 2883 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2884 err0: 2885 kfree(mr); 2886 return ERR_PTR(rc); 2887 } 2888 2889 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, 2890 enum ib_mr_type mr_type, u32 max_num_sg) 2891 { 2892 struct qedr_mr *mr; 2893 2894 if (mr_type != IB_MR_TYPE_MEM_REG) 2895 return ERR_PTR(-EINVAL); 2896 2897 mr = __qedr_alloc_mr(ibpd, max_num_sg); 2898 2899 if (IS_ERR(mr)) 2900 return ERR_PTR(-EINVAL); 2901 2902 return &mr->ibmr; 2903 } 2904 2905 static int qedr_set_page(struct ib_mr *ibmr, u64 addr) 2906 { 2907 struct qedr_mr *mr = get_qedr_mr(ibmr); 2908 struct qedr_pbl *pbl_table; 2909 struct regpair *pbe; 2910 u32 pbes_in_page; 2911 2912 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 2913 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages); 2914 return -ENOMEM; 2915 } 2916 2917 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n", 2918 mr->npages, addr); 2919 2920 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 2921 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 2922 pbe = (struct regpair *)pbl_table->va; 2923 pbe += mr->npages % pbes_in_page; 2924 pbe->lo = cpu_to_le32((u32)addr); 2925 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 2926 2927 mr->npages++; 2928 2929 return 0; 2930 } 2931 2932 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info) 2933 { 2934 int work = info->completed - info->completed_handled - 1; 2935 2936 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work); 2937 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 2938 struct qedr_pbl *pbl; 2939 2940 /* Free all the page list that are possible to be freed 2941 * (all the ones that were invalidated), under the assumption 2942 * that if an FMR was completed successfully that means that 2943 * if there was an invalidate operation before it also ended 2944 */ 2945 pbl = list_first_entry(&info->inuse_pbl_list, 2946 struct qedr_pbl, list_entry); 2947 list_move_tail(&pbl->list_entry, &info->free_pbl_list); 2948 info->completed_handled++; 2949 } 2950 } 2951 2952 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 2953 int sg_nents, unsigned int *sg_offset) 2954 { 2955 struct qedr_mr *mr = get_qedr_mr(ibmr); 2956 2957 mr->npages = 0; 2958 2959 handle_completed_mrs(mr->dev, &mr->info); 2960 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page); 2961 } 2962 2963 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc) 2964 { 2965 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2966 struct qedr_pd *pd = get_qedr_pd(ibpd); 2967 struct qedr_mr *mr; 2968 int rc; 2969 2970 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2971 if (!mr) 2972 return ERR_PTR(-ENOMEM); 2973 2974 mr->type = QEDR_MR_DMA; 2975 2976 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2977 if (rc) { 2978 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2979 goto err1; 2980 } 2981 2982 /* index only, 18 bit long, lkey = itid << 8 | key */ 2983 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2984 mr->hw_mr.pd = pd->pd_id; 2985 mr->hw_mr.local_read = 1; 2986 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2987 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2988 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2989 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2990 mr->hw_mr.dma_mr = true; 2991 2992 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2993 if (rc) { 2994 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2995 goto err2; 2996 } 2997 2998 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2999 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 3000 mr->hw_mr.remote_atomic) 3001 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3002 3003 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey); 3004 return &mr->ibmr; 3005 3006 err2: 3007 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3008 err1: 3009 kfree(mr); 3010 return ERR_PTR(rc); 3011 } 3012 3013 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq) 3014 { 3015 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 3016 } 3017 3018 static int sge_data_len(struct ib_sge *sg_list, int num_sge) 3019 { 3020 int i, len = 0; 3021 3022 for (i = 0; i < num_sge; i++) 3023 len += sg_list[i].length; 3024 3025 return len; 3026 } 3027 3028 static void swap_wqe_data64(u64 *p) 3029 { 3030 int i; 3031 3032 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 3033 *p = cpu_to_be64(cpu_to_le64(*p)); 3034 } 3035 3036 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev, 3037 struct qedr_qp *qp, u8 *wqe_size, 3038 const struct ib_send_wr *wr, 3039 const struct ib_send_wr **bad_wr, 3040 u8 *bits, u8 bit) 3041 { 3042 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 3043 char *seg_prt, *wqe; 3044 int i, seg_siz; 3045 3046 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 3047 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size); 3048 *bad_wr = wr; 3049 return 0; 3050 } 3051 3052 if (!data_size) 3053 return data_size; 3054 3055 *bits |= bit; 3056 3057 seg_prt = NULL; 3058 wqe = NULL; 3059 seg_siz = 0; 3060 3061 /* Copy data inline */ 3062 for (i = 0; i < wr->num_sge; i++) { 3063 u32 len = wr->sg_list[i].length; 3064 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 3065 3066 while (len > 0) { 3067 u32 cur; 3068 3069 /* New segment required */ 3070 if (!seg_siz) { 3071 wqe = (char *)qed_chain_produce(&qp->sq.pbl); 3072 seg_prt = wqe; 3073 seg_siz = sizeof(struct rdma_sq_common_wqe); 3074 (*wqe_size)++; 3075 } 3076 3077 /* Calculate currently allowed length */ 3078 cur = min_t(u32, len, seg_siz); 3079 memcpy(seg_prt, src, cur); 3080 3081 /* Update segment variables */ 3082 seg_prt += cur; 3083 seg_siz -= cur; 3084 3085 /* Update sge variables */ 3086 src += cur; 3087 len -= cur; 3088 3089 /* Swap fully-completed segments */ 3090 if (!seg_siz) 3091 swap_wqe_data64((u64 *)wqe); 3092 } 3093 } 3094 3095 /* swap last not completed segment */ 3096 if (seg_siz) 3097 swap_wqe_data64((u64 *)wqe); 3098 3099 return data_size; 3100 } 3101 3102 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 3103 do { \ 3104 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3105 (sge)->length = cpu_to_le32(vlength); \ 3106 (sge)->flags = cpu_to_le32(vflags); \ 3107 } while (0) 3108 3109 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 3110 do { \ 3111 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \ 3112 (hdr)->num_sges = num_sge; \ 3113 } while (0) 3114 3115 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 3116 do { \ 3117 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3118 (sge)->length = cpu_to_le32(vlength); \ 3119 (sge)->l_key = cpu_to_le32(vlkey); \ 3120 } while (0) 3121 3122 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, 3123 const struct ib_send_wr *wr) 3124 { 3125 u32 data_size = 0; 3126 int i; 3127 3128 for (i = 0; i < wr->num_sge; i++) { 3129 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl); 3130 3131 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr); 3132 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 3133 sge->length = cpu_to_le32(wr->sg_list[i].length); 3134 data_size += wr->sg_list[i].length; 3135 } 3136 3137 if (wqe_size) 3138 *wqe_size += wr->num_sge; 3139 3140 return data_size; 3141 } 3142 3143 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev, 3144 struct qedr_qp *qp, 3145 struct rdma_sq_rdma_wqe_1st *rwqe, 3146 struct rdma_sq_rdma_wqe_2nd *rwqe2, 3147 const struct ib_send_wr *wr, 3148 const struct ib_send_wr **bad_wr) 3149 { 3150 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 3151 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr); 3152 3153 if (wr->send_flags & IB_SEND_INLINE && 3154 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 3155 wr->opcode == IB_WR_RDMA_WRITE)) { 3156 u8 flags = 0; 3157 3158 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 3159 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, 3160 bad_wr, &rwqe->flags, flags); 3161 } 3162 3163 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr); 3164 } 3165 3166 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev, 3167 struct qedr_qp *qp, 3168 struct rdma_sq_send_wqe_1st *swqe, 3169 struct rdma_sq_send_wqe_2st *swqe2, 3170 const struct ib_send_wr *wr, 3171 const struct ib_send_wr **bad_wr) 3172 { 3173 memset(swqe2, 0, sizeof(*swqe2)); 3174 if (wr->send_flags & IB_SEND_INLINE) { 3175 u8 flags = 0; 3176 3177 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 3178 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, 3179 bad_wr, &swqe->flags, flags); 3180 } 3181 3182 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr); 3183 } 3184 3185 static int qedr_prepare_reg(struct qedr_qp *qp, 3186 struct rdma_sq_fmr_wqe_1st *fwqe1, 3187 const struct ib_reg_wr *wr) 3188 { 3189 struct qedr_mr *mr = get_qedr_mr(wr->mr); 3190 struct rdma_sq_fmr_wqe_2nd *fwqe2; 3191 3192 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl); 3193 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 3194 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 3195 fwqe1->l_key = wr->key; 3196 3197 fwqe2->access_ctrl = 0; 3198 3199 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 3200 !!(wr->access & IB_ACCESS_REMOTE_READ)); 3201 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 3202 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 3203 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 3204 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 3205 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 3206 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 3207 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 3208 fwqe2->fmr_ctrl = 0; 3209 3210 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 3211 ilog2(mr->ibmr.page_size) - 12); 3212 3213 fwqe2->length_hi = 0; 3214 fwqe2->length_lo = mr->ibmr.length; 3215 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 3216 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 3217 3218 qp->wqe_wr_id[qp->sq.prod].mr = mr; 3219 3220 return 0; 3221 } 3222 3223 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 3224 { 3225 switch (opcode) { 3226 case IB_WR_RDMA_WRITE: 3227 case IB_WR_RDMA_WRITE_WITH_IMM: 3228 return IB_WC_RDMA_WRITE; 3229 case IB_WR_SEND_WITH_IMM: 3230 case IB_WR_SEND: 3231 case IB_WR_SEND_WITH_INV: 3232 return IB_WC_SEND; 3233 case IB_WR_RDMA_READ: 3234 case IB_WR_RDMA_READ_WITH_INV: 3235 return IB_WC_RDMA_READ; 3236 case IB_WR_ATOMIC_CMP_AND_SWP: 3237 return IB_WC_COMP_SWAP; 3238 case IB_WR_ATOMIC_FETCH_AND_ADD: 3239 return IB_WC_FETCH_ADD; 3240 case IB_WR_REG_MR: 3241 return IB_WC_REG_MR; 3242 case IB_WR_LOCAL_INV: 3243 return IB_WC_LOCAL_INV; 3244 default: 3245 return IB_WC_SEND; 3246 } 3247 } 3248 3249 static inline bool qedr_can_post_send(struct qedr_qp *qp, 3250 const struct ib_send_wr *wr) 3251 { 3252 int wq_is_full, err_wr, pbl_is_full; 3253 struct qedr_dev *dev = qp->dev; 3254 3255 /* prevent SQ overflow and/or processing of a bad WR */ 3256 err_wr = wr->num_sge > qp->sq.max_sges; 3257 wq_is_full = qedr_wq_is_full(&qp->sq); 3258 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) < 3259 QEDR_MAX_SQE_ELEMENTS_PER_SQE; 3260 if (wq_is_full || err_wr || pbl_is_full) { 3261 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) { 3262 DP_ERR(dev, 3263 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n", 3264 qp); 3265 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL; 3266 } 3267 3268 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) { 3269 DP_ERR(dev, 3270 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n", 3271 qp); 3272 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR; 3273 } 3274 3275 if (pbl_is_full && 3276 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) { 3277 DP_ERR(dev, 3278 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n", 3279 qp); 3280 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL; 3281 } 3282 return false; 3283 } 3284 return true; 3285 } 3286 3287 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3288 const struct ib_send_wr **bad_wr) 3289 { 3290 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3291 struct qedr_qp *qp = get_qedr_qp(ibqp); 3292 struct rdma_sq_atomic_wqe_1st *awqe1; 3293 struct rdma_sq_atomic_wqe_2nd *awqe2; 3294 struct rdma_sq_atomic_wqe_3rd *awqe3; 3295 struct rdma_sq_send_wqe_2st *swqe2; 3296 struct rdma_sq_local_inv_wqe *iwqe; 3297 struct rdma_sq_rdma_wqe_2nd *rwqe2; 3298 struct rdma_sq_send_wqe_1st *swqe; 3299 struct rdma_sq_rdma_wqe_1st *rwqe; 3300 struct rdma_sq_fmr_wqe_1st *fwqe1; 3301 struct rdma_sq_common_wqe *wqe; 3302 u32 length; 3303 int rc = 0; 3304 bool comp; 3305 3306 if (!qedr_can_post_send(qp, wr)) { 3307 *bad_wr = wr; 3308 return -ENOMEM; 3309 } 3310 3311 wqe = qed_chain_produce(&qp->sq.pbl); 3312 qp->wqe_wr_id[qp->sq.prod].signaled = 3313 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 3314 3315 wqe->flags = 0; 3316 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, 3317 !!(wr->send_flags & IB_SEND_SOLICITED)); 3318 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled; 3319 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 3320 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, 3321 !!(wr->send_flags & IB_SEND_FENCE)); 3322 wqe->prev_wqe_size = qp->prev_wqe_size; 3323 3324 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode); 3325 3326 switch (wr->opcode) { 3327 case IB_WR_SEND_WITH_IMM: 3328 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3329 rc = -EINVAL; 3330 *bad_wr = wr; 3331 break; 3332 } 3333 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 3334 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3335 swqe->wqe_size = 2; 3336 swqe2 = qed_chain_produce(&qp->sq.pbl); 3337 3338 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 3339 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3340 wr, bad_wr); 3341 swqe->length = cpu_to_le32(length); 3342 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3343 qp->prev_wqe_size = swqe->wqe_size; 3344 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3345 break; 3346 case IB_WR_SEND: 3347 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 3348 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3349 3350 swqe->wqe_size = 2; 3351 swqe2 = qed_chain_produce(&qp->sq.pbl); 3352 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3353 wr, bad_wr); 3354 swqe->length = cpu_to_le32(length); 3355 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3356 qp->prev_wqe_size = swqe->wqe_size; 3357 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3358 break; 3359 case IB_WR_SEND_WITH_INV: 3360 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 3361 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3362 swqe2 = qed_chain_produce(&qp->sq.pbl); 3363 swqe->wqe_size = 2; 3364 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey); 3365 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3366 wr, bad_wr); 3367 swqe->length = cpu_to_le32(length); 3368 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3369 qp->prev_wqe_size = swqe->wqe_size; 3370 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3371 break; 3372 3373 case IB_WR_RDMA_WRITE_WITH_IMM: 3374 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3375 rc = -EINVAL; 3376 *bad_wr = wr; 3377 break; 3378 } 3379 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 3380 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3381 3382 rwqe->wqe_size = 2; 3383 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 3384 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3385 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3386 wr, bad_wr); 3387 rwqe->length = cpu_to_le32(length); 3388 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3389 qp->prev_wqe_size = rwqe->wqe_size; 3390 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3391 break; 3392 case IB_WR_RDMA_WRITE: 3393 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 3394 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3395 3396 rwqe->wqe_size = 2; 3397 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3398 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3399 wr, bad_wr); 3400 rwqe->length = cpu_to_le32(length); 3401 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3402 qp->prev_wqe_size = rwqe->wqe_size; 3403 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3404 break; 3405 case IB_WR_RDMA_READ_WITH_INV: 3406 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1); 3407 /* fallthrough -- same is identical to RDMA READ */ 3408 3409 case IB_WR_RDMA_READ: 3410 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 3411 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3412 3413 rwqe->wqe_size = 2; 3414 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3415 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3416 wr, bad_wr); 3417 rwqe->length = cpu_to_le32(length); 3418 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3419 qp->prev_wqe_size = rwqe->wqe_size; 3420 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3421 break; 3422 3423 case IB_WR_ATOMIC_CMP_AND_SWP: 3424 case IB_WR_ATOMIC_FETCH_AND_ADD: 3425 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe; 3426 awqe1->wqe_size = 4; 3427 3428 awqe2 = qed_chain_produce(&qp->sq.pbl); 3429 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr); 3430 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 3431 3432 awqe3 = qed_chain_produce(&qp->sq.pbl); 3433 3434 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 3435 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 3436 DMA_REGPAIR_LE(awqe3->swap_data, 3437 atomic_wr(wr)->compare_add); 3438 } else { 3439 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 3440 DMA_REGPAIR_LE(awqe3->swap_data, 3441 atomic_wr(wr)->swap); 3442 DMA_REGPAIR_LE(awqe3->cmp_data, 3443 atomic_wr(wr)->compare_add); 3444 } 3445 3446 qedr_prepare_sq_sges(qp, NULL, wr); 3447 3448 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size; 3449 qp->prev_wqe_size = awqe1->wqe_size; 3450 break; 3451 3452 case IB_WR_LOCAL_INV: 3453 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 3454 iwqe->wqe_size = 1; 3455 3456 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 3457 iwqe->inv_l_key = wr->ex.invalidate_rkey; 3458 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size; 3459 qp->prev_wqe_size = iwqe->wqe_size; 3460 break; 3461 case IB_WR_REG_MR: 3462 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n"); 3463 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 3464 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 3465 fwqe1->wqe_size = 2; 3466 3467 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr)); 3468 if (rc) { 3469 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc); 3470 *bad_wr = wr; 3471 break; 3472 } 3473 3474 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 3475 qp->prev_wqe_size = fwqe1->wqe_size; 3476 break; 3477 default: 3478 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode); 3479 rc = -EINVAL; 3480 *bad_wr = wr; 3481 break; 3482 } 3483 3484 if (*bad_wr) { 3485 u16 value; 3486 3487 /* Restore prod to its position before 3488 * this WR was processed 3489 */ 3490 value = le16_to_cpu(qp->sq.db_data.data.value); 3491 qed_chain_set_prod(&qp->sq.pbl, value, wqe); 3492 3493 /* Restore prev_wqe_size */ 3494 qp->prev_wqe_size = wqe->prev_wqe_size; 3495 rc = -EINVAL; 3496 DP_ERR(dev, "POST SEND FAILED\n"); 3497 } 3498 3499 return rc; 3500 } 3501 3502 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3503 const struct ib_send_wr **bad_wr) 3504 { 3505 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3506 struct qedr_qp *qp = get_qedr_qp(ibqp); 3507 unsigned long flags; 3508 int rc = 0; 3509 3510 *bad_wr = NULL; 3511 3512 if (qp->qp_type == IB_QPT_GSI) 3513 return qedr_gsi_post_send(ibqp, wr, bad_wr); 3514 3515 spin_lock_irqsave(&qp->q_lock, flags); 3516 3517 if (rdma_protocol_roce(&dev->ibdev, 1)) { 3518 if ((qp->state != QED_ROCE_QP_STATE_RTS) && 3519 (qp->state != QED_ROCE_QP_STATE_ERR) && 3520 (qp->state != QED_ROCE_QP_STATE_SQD)) { 3521 spin_unlock_irqrestore(&qp->q_lock, flags); 3522 *bad_wr = wr; 3523 DP_DEBUG(dev, QEDR_MSG_CQ, 3524 "QP in wrong state! QP icid=0x%x state %d\n", 3525 qp->icid, qp->state); 3526 return -EINVAL; 3527 } 3528 } 3529 3530 while (wr) { 3531 rc = __qedr_post_send(ibqp, wr, bad_wr); 3532 if (rc) 3533 break; 3534 3535 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 3536 3537 qedr_inc_sw_prod(&qp->sq); 3538 3539 qp->sq.db_data.data.value++; 3540 3541 wr = wr->next; 3542 } 3543 3544 /* Trigger doorbell 3545 * If there was a failure in the first WR then it will be triggered in 3546 * vane. However this is not harmful (as long as the producer value is 3547 * unchanged). For performance reasons we avoid checking for this 3548 * redundant doorbell. 3549 * 3550 * qp->wqe_wr_id is accessed during qedr_poll_cq, as 3551 * soon as we give the doorbell, we could get a completion 3552 * for this wr, therefore we need to make sure that the 3553 * memory is updated before giving the doorbell. 3554 * During qedr_poll_cq, rmb is called before accessing the 3555 * cqe. This covers for the smp_rmb as well. 3556 */ 3557 smp_wmb(); 3558 writel(qp->sq.db_data.raw, qp->sq.db); 3559 3560 /* Make sure write sticks */ 3561 mmiowb(); 3562 3563 spin_unlock_irqrestore(&qp->q_lock, flags); 3564 3565 return rc; 3566 } 3567 3568 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq) 3569 { 3570 u32 used; 3571 3572 /* Calculate number of elements used based on producer 3573 * count and consumer count and subtract it from max 3574 * work request supported so that we get elements left. 3575 */ 3576 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt; 3577 3578 return hw_srq->max_wr - used; 3579 } 3580 3581 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 3582 const struct ib_recv_wr **bad_wr) 3583 { 3584 struct qedr_srq *srq = get_qedr_srq(ibsrq); 3585 struct qedr_srq_hwq_info *hw_srq; 3586 struct qedr_dev *dev = srq->dev; 3587 struct qed_chain *pbl; 3588 unsigned long flags; 3589 int status = 0; 3590 u32 num_sge; 3591 u32 offset; 3592 3593 spin_lock_irqsave(&srq->lock, flags); 3594 3595 hw_srq = &srq->hw_srq; 3596 pbl = &srq->hw_srq.pbl; 3597 while (wr) { 3598 struct rdma_srq_wqe_header *hdr; 3599 int i; 3600 3601 if (!qedr_srq_elem_left(hw_srq) || 3602 wr->num_sge > srq->hw_srq.max_sges) { 3603 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n", 3604 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt, 3605 wr->num_sge, srq->hw_srq.max_sges); 3606 status = -ENOMEM; 3607 *bad_wr = wr; 3608 break; 3609 } 3610 3611 hdr = qed_chain_produce(pbl); 3612 num_sge = wr->num_sge; 3613 /* Set number of sge and work request id in header */ 3614 SRQ_HDR_SET(hdr, wr->wr_id, num_sge); 3615 3616 srq->hw_srq.wr_prod_cnt++; 3617 hw_srq->wqe_prod++; 3618 hw_srq->sge_prod++; 3619 3620 DP_DEBUG(dev, QEDR_MSG_SRQ, 3621 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n", 3622 wr->num_sge, hw_srq->wqe_prod, wr->wr_id); 3623 3624 for (i = 0; i < wr->num_sge; i++) { 3625 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl); 3626 3627 /* Set SGE length, lkey and address */ 3628 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, 3629 wr->sg_list[i].length, wr->sg_list[i].lkey); 3630 3631 DP_DEBUG(dev, QEDR_MSG_SRQ, 3632 "[%d]: len %d key %x addr %x:%x\n", 3633 i, srq_sge->length, srq_sge->l_key, 3634 srq_sge->addr.hi, srq_sge->addr.lo); 3635 hw_srq->sge_prod++; 3636 } 3637 3638 /* Flush WQE and SGE information before 3639 * updating producer. 3640 */ 3641 wmb(); 3642 3643 /* SRQ producer is 8 bytes. Need to update SGE producer index 3644 * in first 4 bytes and need to update WQE producer in 3645 * next 4 bytes. 3646 */ 3647 *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod; 3648 offset = offsetof(struct rdma_srq_producers, wqe_prod); 3649 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) = 3650 hw_srq->wqe_prod; 3651 3652 /* Flush producer after updating it. */ 3653 wmb(); 3654 wr = wr->next; 3655 } 3656 3657 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n", 3658 qed_chain_get_elem_left(pbl)); 3659 spin_unlock_irqrestore(&srq->lock, flags); 3660 3661 return status; 3662 } 3663 3664 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 3665 const struct ib_recv_wr **bad_wr) 3666 { 3667 struct qedr_qp *qp = get_qedr_qp(ibqp); 3668 struct qedr_dev *dev = qp->dev; 3669 unsigned long flags; 3670 int status = 0; 3671 3672 if (qp->qp_type == IB_QPT_GSI) 3673 return qedr_gsi_post_recv(ibqp, wr, bad_wr); 3674 3675 spin_lock_irqsave(&qp->q_lock, flags); 3676 3677 if (qp->state == QED_ROCE_QP_STATE_RESET) { 3678 spin_unlock_irqrestore(&qp->q_lock, flags); 3679 *bad_wr = wr; 3680 return -EINVAL; 3681 } 3682 3683 while (wr) { 3684 int i; 3685 3686 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) < 3687 QEDR_MAX_RQE_ELEMENTS_PER_RQE || 3688 wr->num_sge > qp->rq.max_sges) { 3689 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n", 3690 qed_chain_get_elem_left_u32(&qp->rq.pbl), 3691 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge, 3692 qp->rq.max_sges); 3693 status = -ENOMEM; 3694 *bad_wr = wr; 3695 break; 3696 } 3697 for (i = 0; i < wr->num_sge; i++) { 3698 u32 flags = 0; 3699 struct rdma_rq_sge *rqe = 3700 qed_chain_produce(&qp->rq.pbl); 3701 3702 /* First one must include the number 3703 * of SGE in the list 3704 */ 3705 if (!i) 3706 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 3707 wr->num_sge); 3708 3709 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 3710 wr->sg_list[i].lkey); 3711 3712 RQ_SGE_SET(rqe, wr->sg_list[i].addr, 3713 wr->sg_list[i].length, flags); 3714 } 3715 3716 /* Special case of no sges. FW requires between 1-4 sges... 3717 * in this case we need to post 1 sge with length zero. this is 3718 * because rdma write with immediate consumes an RQ. 3719 */ 3720 if (!wr->num_sge) { 3721 u32 flags = 0; 3722 struct rdma_rq_sge *rqe = 3723 qed_chain_produce(&qp->rq.pbl); 3724 3725 /* First one must include the number 3726 * of SGE in the list 3727 */ 3728 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0); 3729 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 3730 3731 RQ_SGE_SET(rqe, 0, 0, flags); 3732 i = 1; 3733 } 3734 3735 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 3736 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 3737 3738 qedr_inc_sw_prod(&qp->rq); 3739 3740 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as 3741 * soon as we give the doorbell, we could get a completion 3742 * for this wr, therefore we need to make sure that the 3743 * memory is update before giving the doorbell. 3744 * During qedr_poll_cq, rmb is called before accessing the 3745 * cqe. This covers for the smp_rmb as well. 3746 */ 3747 smp_wmb(); 3748 3749 qp->rq.db_data.data.value++; 3750 3751 writel(qp->rq.db_data.raw, qp->rq.db); 3752 3753 /* Make sure write sticks */ 3754 mmiowb(); 3755 3756 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 3757 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 3758 mmiowb(); /* for second doorbell */ 3759 } 3760 3761 wr = wr->next; 3762 } 3763 3764 spin_unlock_irqrestore(&qp->q_lock, flags); 3765 3766 return status; 3767 } 3768 3769 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe) 3770 { 3771 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3772 3773 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) == 3774 cq->pbl_toggle; 3775 } 3776 3777 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe) 3778 { 3779 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3780 struct qedr_qp *qp; 3781 3782 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi, 3783 resp_cqe->qp_handle.lo, 3784 u64); 3785 return qp; 3786 } 3787 3788 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe) 3789 { 3790 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3791 3792 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 3793 } 3794 3795 /* Return latest CQE (needs processing) */ 3796 static union rdma_cqe *get_cqe(struct qedr_cq *cq) 3797 { 3798 return cq->latest_cqe; 3799 } 3800 3801 /* In fmr we need to increase the number of fmr completed counter for the fmr 3802 * algorithm determining whether we can free a pbl or not. 3803 * we need to perform this whether the work request was signaled or not. for 3804 * this purpose we call this function from the condition that checks if a wr 3805 * should be skipped, to make sure we don't miss it ( possibly this fmr 3806 * operation was not signalted) 3807 */ 3808 static inline void qedr_chk_if_fmr(struct qedr_qp *qp) 3809 { 3810 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 3811 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3812 } 3813 3814 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, 3815 struct qedr_cq *cq, int num_entries, 3816 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, 3817 int force) 3818 { 3819 u16 cnt = 0; 3820 3821 while (num_entries && qp->sq.wqe_cons != hw_cons) { 3822 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 3823 qedr_chk_if_fmr(qp); 3824 /* skip WC */ 3825 goto next_cqe; 3826 } 3827 3828 /* fill WC */ 3829 wc->status = status; 3830 wc->vendor_err = 0; 3831 wc->wc_flags = 0; 3832 wc->src_qp = qp->id; 3833 wc->qp = &qp->ibqp; 3834 3835 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 3836 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 3837 3838 switch (wc->opcode) { 3839 case IB_WC_RDMA_WRITE: 3840 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3841 break; 3842 case IB_WC_COMP_SWAP: 3843 case IB_WC_FETCH_ADD: 3844 wc->byte_len = 8; 3845 break; 3846 case IB_WC_REG_MR: 3847 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3848 break; 3849 case IB_WC_RDMA_READ: 3850 case IB_WC_SEND: 3851 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3852 break; 3853 default: 3854 break; 3855 } 3856 3857 num_entries--; 3858 wc++; 3859 cnt++; 3860 next_cqe: 3861 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 3862 qed_chain_consume(&qp->sq.pbl); 3863 qedr_inc_sw_cons(&qp->sq); 3864 } 3865 3866 return cnt; 3867 } 3868 3869 static int qedr_poll_cq_req(struct qedr_dev *dev, 3870 struct qedr_qp *qp, struct qedr_cq *cq, 3871 int num_entries, struct ib_wc *wc, 3872 struct rdma_cqe_requester *req) 3873 { 3874 int cnt = 0; 3875 3876 switch (req->status) { 3877 case RDMA_CQE_REQ_STS_OK: 3878 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3879 IB_WC_SUCCESS, 0); 3880 break; 3881 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 3882 if (qp->state != QED_ROCE_QP_STATE_ERR) 3883 DP_DEBUG(dev, QEDR_MSG_CQ, 3884 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3885 cq->icid, qp->icid); 3886 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3887 IB_WC_WR_FLUSH_ERR, 1); 3888 break; 3889 default: 3890 /* process all WQE before the cosumer */ 3891 qp->state = QED_ROCE_QP_STATE_ERR; 3892 cnt = process_req(dev, qp, cq, num_entries, wc, 3893 req->sq_cons - 1, IB_WC_SUCCESS, 0); 3894 wc += cnt; 3895 /* if we have extra WC fill it with actual error info */ 3896 if (cnt < num_entries) { 3897 enum ib_wc_status wc_status; 3898 3899 switch (req->status) { 3900 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 3901 DP_ERR(dev, 3902 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3903 cq->icid, qp->icid); 3904 wc_status = IB_WC_BAD_RESP_ERR; 3905 break; 3906 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 3907 DP_ERR(dev, 3908 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3909 cq->icid, qp->icid); 3910 wc_status = IB_WC_LOC_LEN_ERR; 3911 break; 3912 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 3913 DP_ERR(dev, 3914 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3915 cq->icid, qp->icid); 3916 wc_status = IB_WC_LOC_QP_OP_ERR; 3917 break; 3918 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 3919 DP_ERR(dev, 3920 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3921 cq->icid, qp->icid); 3922 wc_status = IB_WC_LOC_PROT_ERR; 3923 break; 3924 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 3925 DP_ERR(dev, 3926 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3927 cq->icid, qp->icid); 3928 wc_status = IB_WC_MW_BIND_ERR; 3929 break; 3930 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 3931 DP_ERR(dev, 3932 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3933 cq->icid, qp->icid); 3934 wc_status = IB_WC_REM_INV_REQ_ERR; 3935 break; 3936 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 3937 DP_ERR(dev, 3938 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3939 cq->icid, qp->icid); 3940 wc_status = IB_WC_REM_ACCESS_ERR; 3941 break; 3942 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 3943 DP_ERR(dev, 3944 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3945 cq->icid, qp->icid); 3946 wc_status = IB_WC_REM_OP_ERR; 3947 break; 3948 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 3949 DP_ERR(dev, 3950 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3951 cq->icid, qp->icid); 3952 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 3953 break; 3954 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 3955 DP_ERR(dev, 3956 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3957 cq->icid, qp->icid); 3958 wc_status = IB_WC_RETRY_EXC_ERR; 3959 break; 3960 default: 3961 DP_ERR(dev, 3962 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3963 cq->icid, qp->icid); 3964 wc_status = IB_WC_GENERAL_ERR; 3965 } 3966 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 3967 wc_status, 1); 3968 } 3969 } 3970 3971 return cnt; 3972 } 3973 3974 static inline int qedr_cqe_resp_status_to_ib(u8 status) 3975 { 3976 switch (status) { 3977 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 3978 return IB_WC_LOC_ACCESS_ERR; 3979 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 3980 return IB_WC_LOC_LEN_ERR; 3981 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 3982 return IB_WC_LOC_QP_OP_ERR; 3983 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 3984 return IB_WC_LOC_PROT_ERR; 3985 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 3986 return IB_WC_MW_BIND_ERR; 3987 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 3988 return IB_WC_REM_INV_RD_REQ_ERR; 3989 case RDMA_CQE_RESP_STS_OK: 3990 return IB_WC_SUCCESS; 3991 default: 3992 return IB_WC_GENERAL_ERR; 3993 } 3994 } 3995 3996 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp, 3997 struct ib_wc *wc) 3998 { 3999 wc->status = IB_WC_SUCCESS; 4000 wc->byte_len = le32_to_cpu(resp->length); 4001 4002 if (resp->flags & QEDR_RESP_IMM) { 4003 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key)); 4004 wc->wc_flags |= IB_WC_WITH_IMM; 4005 4006 if (resp->flags & QEDR_RESP_RDMA) 4007 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 4008 4009 if (resp->flags & QEDR_RESP_INV) 4010 return -EINVAL; 4011 4012 } else if (resp->flags & QEDR_RESP_INV) { 4013 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 4014 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4015 4016 if (resp->flags & QEDR_RESP_RDMA) 4017 return -EINVAL; 4018 4019 } else if (resp->flags & QEDR_RESP_RDMA) { 4020 return -EINVAL; 4021 } 4022 4023 return 0; 4024 } 4025 4026 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4027 struct qedr_cq *cq, struct ib_wc *wc, 4028 struct rdma_cqe_responder *resp, u64 wr_id) 4029 { 4030 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */ 4031 wc->opcode = IB_WC_RECV; 4032 wc->wc_flags = 0; 4033 4034 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) { 4035 if (qedr_set_ok_cqe_resp_wc(resp, wc)) 4036 DP_ERR(dev, 4037 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n", 4038 cq, cq->icid, resp->flags); 4039 4040 } else { 4041 wc->status = qedr_cqe_resp_status_to_ib(resp->status); 4042 if (wc->status == IB_WC_GENERAL_ERR) 4043 DP_ERR(dev, 4044 "CQ %p (icid=%d) contains an invalid CQE status %d\n", 4045 cq, cq->icid, resp->status); 4046 } 4047 4048 /* Fill the rest of the WC */ 4049 wc->vendor_err = 0; 4050 wc->src_qp = qp->id; 4051 wc->qp = &qp->ibqp; 4052 wc->wr_id = wr_id; 4053 } 4054 4055 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4056 struct qedr_cq *cq, struct ib_wc *wc, 4057 struct rdma_cqe_responder *resp) 4058 { 4059 struct qedr_srq *srq = qp->srq; 4060 u64 wr_id; 4061 4062 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi), 4063 le32_to_cpu(resp->srq_wr_id.lo), u64); 4064 4065 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4066 wc->status = IB_WC_WR_FLUSH_ERR; 4067 wc->vendor_err = 0; 4068 wc->wr_id = wr_id; 4069 wc->byte_len = 0; 4070 wc->src_qp = qp->id; 4071 wc->qp = &qp->ibqp; 4072 wc->wr_id = wr_id; 4073 } else { 4074 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4075 } 4076 srq->hw_srq.wr_cons_cnt++; 4077 4078 return 1; 4079 } 4080 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4081 struct qedr_cq *cq, struct ib_wc *wc, 4082 struct rdma_cqe_responder *resp) 4083 { 4084 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4085 4086 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4087 4088 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4089 qed_chain_consume(&qp->rq.pbl); 4090 qedr_inc_sw_cons(&qp->rq); 4091 4092 return 1; 4093 } 4094 4095 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, 4096 int num_entries, struct ib_wc *wc, u16 hw_cons) 4097 { 4098 u16 cnt = 0; 4099 4100 while (num_entries && qp->rq.wqe_cons != hw_cons) { 4101 /* fill WC */ 4102 wc->status = IB_WC_WR_FLUSH_ERR; 4103 wc->vendor_err = 0; 4104 wc->wc_flags = 0; 4105 wc->src_qp = qp->id; 4106 wc->byte_len = 0; 4107 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4108 wc->qp = &qp->ibqp; 4109 num_entries--; 4110 wc++; 4111 cnt++; 4112 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4113 qed_chain_consume(&qp->rq.pbl); 4114 qedr_inc_sw_cons(&qp->rq); 4115 } 4116 4117 return cnt; 4118 } 4119 4120 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4121 struct rdma_cqe_responder *resp, int *update) 4122 { 4123 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) { 4124 consume_cqe(cq); 4125 *update |= 1; 4126 } 4127 } 4128 4129 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4130 struct qedr_cq *cq, int num_entries, 4131 struct ib_wc *wc, 4132 struct rdma_cqe_responder *resp) 4133 { 4134 int cnt; 4135 4136 cnt = process_resp_one_srq(dev, qp, cq, wc, resp); 4137 consume_cqe(cq); 4138 4139 return cnt; 4140 } 4141 4142 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, 4143 struct qedr_cq *cq, int num_entries, 4144 struct ib_wc *wc, struct rdma_cqe_responder *resp, 4145 int *update) 4146 { 4147 int cnt; 4148 4149 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4150 cnt = process_resp_flush(qp, cq, num_entries, wc, 4151 resp->rq_cons_or_srq_id); 4152 try_consume_resp_cqe(cq, qp, resp, update); 4153 } else { 4154 cnt = process_resp_one(dev, qp, cq, wc, resp); 4155 consume_cqe(cq); 4156 *update |= 1; 4157 } 4158 4159 return cnt; 4160 } 4161 4162 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4163 struct rdma_cqe_requester *req, int *update) 4164 { 4165 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 4166 consume_cqe(cq); 4167 *update |= 1; 4168 } 4169 } 4170 4171 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 4172 { 4173 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 4174 struct qedr_cq *cq = get_qedr_cq(ibcq); 4175 union rdma_cqe *cqe; 4176 u32 old_cons, new_cons; 4177 unsigned long flags; 4178 int update = 0; 4179 int done = 0; 4180 4181 if (cq->destroyed) { 4182 DP_ERR(dev, 4183 "warning: poll was invoked after destroy for cq %p (icid=%d)\n", 4184 cq, cq->icid); 4185 return 0; 4186 } 4187 4188 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 4189 return qedr_gsi_poll_cq(ibcq, num_entries, wc); 4190 4191 spin_lock_irqsave(&cq->cq_lock, flags); 4192 cqe = cq->latest_cqe; 4193 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4194 while (num_entries && is_valid_cqe(cq, cqe)) { 4195 struct qedr_qp *qp; 4196 int cnt = 0; 4197 4198 /* prevent speculative reads of any field of CQE */ 4199 rmb(); 4200 4201 qp = cqe_get_qp(cqe); 4202 if (!qp) { 4203 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe); 4204 break; 4205 } 4206 4207 wc->qp = &qp->ibqp; 4208 4209 switch (cqe_get_type(cqe)) { 4210 case RDMA_CQE_TYPE_REQUESTER: 4211 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc, 4212 &cqe->req); 4213 try_consume_req_cqe(cq, qp, &cqe->req, &update); 4214 break; 4215 case RDMA_CQE_TYPE_RESPONDER_RQ: 4216 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc, 4217 &cqe->resp, &update); 4218 break; 4219 case RDMA_CQE_TYPE_RESPONDER_SRQ: 4220 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries, 4221 wc, &cqe->resp); 4222 update = 1; 4223 break; 4224 case RDMA_CQE_TYPE_INVALID: 4225 default: 4226 DP_ERR(dev, "Error: invalid CQE type = %d\n", 4227 cqe_get_type(cqe)); 4228 } 4229 num_entries -= cnt; 4230 wc += cnt; 4231 done += cnt; 4232 4233 cqe = get_cqe(cq); 4234 } 4235 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4236 4237 cq->cq_cons += new_cons - old_cons; 4238 4239 if (update) 4240 /* doorbell notifies abount latest VALID entry, 4241 * but chain already point to the next INVALID one 4242 */ 4243 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 4244 4245 spin_unlock_irqrestore(&cq->cq_lock, flags); 4246 return done; 4247 } 4248 4249 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, 4250 u8 port_num, 4251 const struct ib_wc *in_wc, 4252 const struct ib_grh *in_grh, 4253 const struct ib_mad_hdr *mad_hdr, 4254 size_t in_mad_size, struct ib_mad_hdr *out_mad, 4255 size_t *out_mad_size, u16 *out_mad_pkey_index) 4256 { 4257 struct qedr_dev *dev = get_qedr_dev(ibdev); 4258 4259 DP_DEBUG(dev, QEDR_MSG_GSI, 4260 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n", 4261 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod, 4262 mad_hdr->class_specific, mad_hdr->class_version, 4263 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status); 4264 return IB_MAD_RESULT_SUCCESS; 4265 } 4266