1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/dma-mapping.h> 33 #include <linux/crc32.h> 34 #include <net/ip.h> 35 #include <net/ipv6.h> 36 #include <net/udp.h> 37 #include <linux/iommu.h> 38 39 #include <rdma/ib_verbs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/iw_cm.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/ib_addr.h> 44 #include <rdma/ib_cache.h> 45 46 #include <linux/qed/common_hsi.h> 47 #include "qedr_hsi_rdma.h" 48 #include <linux/qed/qed_if.h> 49 #include "qedr.h" 50 #include "verbs.h" 51 #include <rdma/qedr-abi.h> 52 #include "qedr_cm.h" 53 54 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 55 56 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, 57 size_t len) 58 { 59 size_t min_len = min_t(size_t, len, udata->outlen); 60 61 return ib_copy_to_udata(udata, src, min_len); 62 } 63 64 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 65 { 66 if (index > QEDR_ROCE_PKEY_TABLE_LEN) 67 return -EINVAL; 68 69 *pkey = QEDR_ROCE_PKEY_DEFAULT; 70 return 0; 71 } 72 73 int qedr_query_gid(struct ib_device *ibdev, u8 port, int index, 74 union ib_gid *sgid) 75 { 76 struct qedr_dev *dev = get_qedr_dev(ibdev); 77 int rc = 0; 78 79 if (!rdma_cap_roce_gid_table(ibdev, port)) 80 return -ENODEV; 81 82 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL); 83 if (rc == -EAGAIN) { 84 memcpy(sgid, &zgid, sizeof(*sgid)); 85 return 0; 86 } 87 88 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index, 89 sgid->global.interface_id, sgid->global.subnet_prefix); 90 91 return rc; 92 } 93 94 int qedr_add_gid(struct ib_device *device, u8 port_num, 95 unsigned int index, const union ib_gid *gid, 96 const struct ib_gid_attr *attr, void **context) 97 { 98 if (!rdma_cap_roce_gid_table(device, port_num)) 99 return -EINVAL; 100 101 if (port_num > QEDR_MAX_PORT) 102 return -EINVAL; 103 104 if (!context) 105 return -EINVAL; 106 107 return 0; 108 } 109 110 int qedr_del_gid(struct ib_device *device, u8 port_num, 111 unsigned int index, void **context) 112 { 113 if (!rdma_cap_roce_gid_table(device, port_num)) 114 return -EINVAL; 115 116 if (port_num > QEDR_MAX_PORT) 117 return -EINVAL; 118 119 if (!context) 120 return -EINVAL; 121 122 return 0; 123 } 124 125 int qedr_query_device(struct ib_device *ibdev, 126 struct ib_device_attr *attr, struct ib_udata *udata) 127 { 128 struct qedr_dev *dev = get_qedr_dev(ibdev); 129 struct qedr_device_attr *qattr = &dev->attr; 130 131 if (!dev->rdma_ctx) { 132 DP_ERR(dev, 133 "qedr_query_device called with invalid params rdma_ctx=%p\n", 134 dev->rdma_ctx); 135 return -EINVAL; 136 } 137 138 memset(attr, 0, sizeof(*attr)); 139 140 attr->fw_ver = qattr->fw_ver; 141 attr->sys_image_guid = qattr->sys_image_guid; 142 attr->max_mr_size = qattr->max_mr_size; 143 attr->page_size_cap = qattr->page_size_caps; 144 attr->vendor_id = qattr->vendor_id; 145 attr->vendor_part_id = qattr->vendor_part_id; 146 attr->hw_ver = qattr->hw_ver; 147 attr->max_qp = qattr->max_qp; 148 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe); 149 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 150 IB_DEVICE_RC_RNR_NAK_GEN | 151 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS; 152 153 attr->max_sge = qattr->max_sge; 154 attr->max_sge_rd = qattr->max_sge; 155 attr->max_cq = qattr->max_cq; 156 attr->max_cqe = qattr->max_cqe; 157 attr->max_mr = qattr->max_mr; 158 attr->max_mw = qattr->max_mw; 159 attr->max_pd = qattr->max_pd; 160 attr->atomic_cap = dev->atomic_cap; 161 attr->max_fmr = qattr->max_fmr; 162 attr->max_map_per_fmr = 16; 163 attr->max_qp_init_rd_atom = 164 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 165 attr->max_qp_rd_atom = 166 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 167 attr->max_qp_init_rd_atom); 168 169 attr->max_srq = qattr->max_srq; 170 attr->max_srq_sge = qattr->max_srq_sge; 171 attr->max_srq_wr = qattr->max_srq_wr; 172 173 attr->local_ca_ack_delay = qattr->dev_ack_delay; 174 attr->max_fast_reg_page_list_len = qattr->max_mr / 8; 175 attr->max_pkeys = QEDR_ROCE_PKEY_MAX; 176 attr->max_ah = qattr->max_ah; 177 178 return 0; 179 } 180 181 #define QEDR_SPEED_SDR (1) 182 #define QEDR_SPEED_DDR (2) 183 #define QEDR_SPEED_QDR (4) 184 #define QEDR_SPEED_FDR10 (8) 185 #define QEDR_SPEED_FDR (16) 186 #define QEDR_SPEED_EDR (32) 187 188 static inline void get_link_speed_and_width(int speed, u8 *ib_speed, 189 u8 *ib_width) 190 { 191 switch (speed) { 192 case 1000: 193 *ib_speed = QEDR_SPEED_SDR; 194 *ib_width = IB_WIDTH_1X; 195 break; 196 case 10000: 197 *ib_speed = QEDR_SPEED_QDR; 198 *ib_width = IB_WIDTH_1X; 199 break; 200 201 case 20000: 202 *ib_speed = QEDR_SPEED_DDR; 203 *ib_width = IB_WIDTH_4X; 204 break; 205 206 case 25000: 207 *ib_speed = QEDR_SPEED_EDR; 208 *ib_width = IB_WIDTH_1X; 209 break; 210 211 case 40000: 212 *ib_speed = QEDR_SPEED_QDR; 213 *ib_width = IB_WIDTH_4X; 214 break; 215 216 case 50000: 217 *ib_speed = QEDR_SPEED_QDR; 218 *ib_width = IB_WIDTH_4X; 219 break; 220 221 case 100000: 222 *ib_speed = QEDR_SPEED_EDR; 223 *ib_width = IB_WIDTH_4X; 224 break; 225 226 default: 227 /* Unsupported */ 228 *ib_speed = QEDR_SPEED_SDR; 229 *ib_width = IB_WIDTH_1X; 230 } 231 } 232 233 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr) 234 { 235 struct qedr_dev *dev; 236 struct qed_rdma_port *rdma_port; 237 238 dev = get_qedr_dev(ibdev); 239 if (port > 1) { 240 DP_ERR(dev, "invalid_port=0x%x\n", port); 241 return -EINVAL; 242 } 243 244 if (!dev->rdma_ctx) { 245 DP_ERR(dev, "rdma_ctx is NULL\n"); 246 return -EINVAL; 247 } 248 249 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx); 250 251 /* *attr being zeroed by the caller, avoid zeroing it here */ 252 if (rdma_port->port_state == QED_RDMA_PORT_UP) { 253 attr->state = IB_PORT_ACTIVE; 254 attr->phys_state = 5; 255 } else { 256 attr->state = IB_PORT_DOWN; 257 attr->phys_state = 3; 258 } 259 attr->max_mtu = IB_MTU_4096; 260 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); 261 attr->lid = 0; 262 attr->lmc = 0; 263 attr->sm_lid = 0; 264 attr->sm_sl = 0; 265 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS; 266 attr->gid_tbl_len = QEDR_MAX_SGID; 267 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; 268 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 269 attr->qkey_viol_cntr = 0; 270 get_link_speed_and_width(rdma_port->link_speed, 271 &attr->active_speed, &attr->active_width); 272 attr->max_msg_sz = rdma_port->max_msg_size; 273 attr->max_vl_num = 4; 274 275 return 0; 276 } 277 278 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask, 279 struct ib_port_modify *props) 280 { 281 struct qedr_dev *dev; 282 283 dev = get_qedr_dev(ibdev); 284 if (port > 1) { 285 DP_ERR(dev, "invalid_port=0x%x\n", port); 286 return -EINVAL; 287 } 288 289 return 0; 290 } 291 292 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr, 293 unsigned long len) 294 { 295 struct qedr_mm *mm; 296 297 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 298 if (!mm) 299 return -ENOMEM; 300 301 mm->key.phy_addr = phy_addr; 302 /* This function might be called with a length which is not a multiple 303 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel 304 * forces this granularity by increasing the requested size if needed. 305 * When qedr_mmap is called, it will search the list with the updated 306 * length as a key. To prevent search failures, the length is rounded up 307 * in advance to PAGE_SIZE. 308 */ 309 mm->key.len = roundup(len, PAGE_SIZE); 310 INIT_LIST_HEAD(&mm->entry); 311 312 mutex_lock(&uctx->mm_list_lock); 313 list_add(&mm->entry, &uctx->mm_head); 314 mutex_unlock(&uctx->mm_list_lock); 315 316 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 317 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n", 318 (unsigned long long)mm->key.phy_addr, 319 (unsigned long)mm->key.len, uctx); 320 321 return 0; 322 } 323 324 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr, 325 unsigned long len) 326 { 327 bool found = false; 328 struct qedr_mm *mm; 329 330 mutex_lock(&uctx->mm_list_lock); 331 list_for_each_entry(mm, &uctx->mm_head, entry) { 332 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 333 continue; 334 335 found = true; 336 break; 337 } 338 mutex_unlock(&uctx->mm_list_lock); 339 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 340 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n", 341 mm->key.phy_addr, mm->key.len, uctx, found); 342 343 return found; 344 } 345 346 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev, 347 struct ib_udata *udata) 348 { 349 int rc; 350 struct qedr_ucontext *ctx; 351 struct qedr_alloc_ucontext_resp uresp; 352 struct qedr_dev *dev = get_qedr_dev(ibdev); 353 struct qed_rdma_add_user_out_params oparams; 354 355 if (!udata) 356 return ERR_PTR(-EFAULT); 357 358 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 359 if (!ctx) 360 return ERR_PTR(-ENOMEM); 361 362 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams); 363 if (rc) { 364 DP_ERR(dev, 365 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n", 366 rc); 367 goto err; 368 } 369 370 ctx->dpi = oparams.dpi; 371 ctx->dpi_addr = oparams.dpi_addr; 372 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 373 ctx->dpi_size = oparams.dpi_size; 374 INIT_LIST_HEAD(&ctx->mm_head); 375 mutex_init(&ctx->mm_list_lock); 376 377 memset(&uresp, 0, sizeof(uresp)); 378 379 uresp.dpm_enabled = dev->user_dpm_enabled; 380 uresp.wids_enabled = 1; 381 uresp.wid_count = oparams.wid_count; 382 uresp.db_pa = ctx->dpi_phys_addr; 383 uresp.db_size = ctx->dpi_size; 384 uresp.max_send_wr = dev->attr.max_sqe; 385 uresp.max_recv_wr = dev->attr.max_rqe; 386 uresp.max_srq_wr = dev->attr.max_srq_wr; 387 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE; 388 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE; 389 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 390 uresp.max_cqes = QEDR_MAX_CQES; 391 392 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 393 if (rc) 394 goto err; 395 396 ctx->dev = dev; 397 398 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size); 399 if (rc) 400 goto err; 401 402 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n", 403 &ctx->ibucontext); 404 return &ctx->ibucontext; 405 406 err: 407 kfree(ctx); 408 return ERR_PTR(rc); 409 } 410 411 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx) 412 { 413 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx); 414 struct qedr_mm *mm, *tmp; 415 int status = 0; 416 417 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n", 418 uctx); 419 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi); 420 421 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 422 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 423 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n", 424 mm->key.phy_addr, mm->key.len, uctx); 425 list_del(&mm->entry); 426 kfree(mm); 427 } 428 429 kfree(uctx); 430 return status; 431 } 432 433 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 434 { 435 struct qedr_ucontext *ucontext = get_qedr_ucontext(context); 436 struct qedr_dev *dev = get_qedr_dev(context->device); 437 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 438 u64 unmapped_db = dev->db_phys_addr; 439 unsigned long len = (vma->vm_end - vma->vm_start); 440 int rc = 0; 441 bool found; 442 443 DP_DEBUG(dev, QEDR_MSG_INIT, 444 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n", 445 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len); 446 if (vma->vm_start & (PAGE_SIZE - 1)) { 447 DP_ERR(dev, "Vma_start not page aligned = %ld\n", 448 vma->vm_start); 449 return -EINVAL; 450 } 451 452 found = qedr_search_mmap(ucontext, vm_page, len); 453 if (!found) { 454 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n", 455 vma->vm_pgoff); 456 return -EINVAL; 457 } 458 459 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n"); 460 461 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + 462 dev->db_size))) { 463 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n"); 464 if (vma->vm_flags & VM_READ) { 465 DP_ERR(dev, "Trying to map doorbell bar for read\n"); 466 return -EPERM; 467 } 468 469 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 470 471 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 472 PAGE_SIZE, vma->vm_page_prot); 473 } else { 474 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n"); 475 rc = remap_pfn_range(vma, vma->vm_start, 476 vma->vm_pgoff, len, vma->vm_page_prot); 477 } 478 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc); 479 return rc; 480 } 481 482 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev, 483 struct ib_ucontext *context, struct ib_udata *udata) 484 { 485 struct qedr_dev *dev = get_qedr_dev(ibdev); 486 struct qedr_pd *pd; 487 u16 pd_id; 488 int rc; 489 490 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n", 491 (udata && context) ? "User Lib" : "Kernel"); 492 493 if (!dev->rdma_ctx) { 494 DP_ERR(dev, "invalid RDMA context\n"); 495 return ERR_PTR(-EINVAL); 496 } 497 498 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 499 if (!pd) 500 return ERR_PTR(-ENOMEM); 501 502 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id); 503 if (rc) 504 goto err; 505 506 pd->pd_id = pd_id; 507 508 if (udata && context) { 509 struct qedr_alloc_pd_uresp uresp; 510 511 uresp.pd_id = pd_id; 512 513 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 514 if (rc) { 515 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); 516 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); 517 goto err; 518 } 519 520 pd->uctx = get_qedr_ucontext(context); 521 pd->uctx->pd = pd; 522 } 523 524 return &pd->ibpd; 525 526 err: 527 kfree(pd); 528 return ERR_PTR(rc); 529 } 530 531 int qedr_dealloc_pd(struct ib_pd *ibpd) 532 { 533 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 534 struct qedr_pd *pd = get_qedr_pd(ibpd); 535 536 if (!pd) { 537 pr_err("Invalid PD received in dealloc_pd\n"); 538 return -EINVAL; 539 } 540 541 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id); 542 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id); 543 544 kfree(pd); 545 546 return 0; 547 } 548 549 static void qedr_free_pbl(struct qedr_dev *dev, 550 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) 551 { 552 struct pci_dev *pdev = dev->pdev; 553 int i; 554 555 for (i = 0; i < pbl_info->num_pbls; i++) { 556 if (!pbl[i].va) 557 continue; 558 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 559 pbl[i].va, pbl[i].pa); 560 } 561 562 kfree(pbl); 563 } 564 565 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024) 566 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024) 567 568 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 569 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 570 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE) 571 572 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev, 573 struct qedr_pbl_info *pbl_info, 574 gfp_t flags) 575 { 576 struct pci_dev *pdev = dev->pdev; 577 struct qedr_pbl *pbl_table; 578 dma_addr_t *pbl_main_tbl; 579 dma_addr_t pa; 580 void *va; 581 int i; 582 583 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags); 584 if (!pbl_table) 585 return ERR_PTR(-ENOMEM); 586 587 for (i = 0; i < pbl_info->num_pbls; i++) { 588 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, 589 &pa, flags); 590 if (!va) 591 goto err; 592 593 memset(va, 0, pbl_info->pbl_size); 594 pbl_table[i].va = va; 595 pbl_table[i].pa = pa; 596 } 597 598 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 599 * the first one with physical pointers to all of the rest 600 */ 601 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 602 for (i = 0; i < pbl_info->num_pbls - 1; i++) 603 pbl_main_tbl[i] = pbl_table[i + 1].pa; 604 605 return pbl_table; 606 607 err: 608 for (i--; i >= 0; i--) 609 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 610 pbl_table[i].va, pbl_table[i].pa); 611 612 qedr_free_pbl(dev, pbl_info, pbl_table); 613 614 return ERR_PTR(-ENOMEM); 615 } 616 617 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev, 618 struct qedr_pbl_info *pbl_info, 619 u32 num_pbes, int two_layer_capable) 620 { 621 u32 pbl_capacity; 622 u32 pbl_size; 623 u32 num_pbls; 624 625 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 626 if (num_pbes > MAX_PBES_TWO_LAYER) { 627 DP_ERR(dev, "prepare pbl table: too many pages %d\n", 628 num_pbes); 629 return -EINVAL; 630 } 631 632 /* calculate required pbl page size */ 633 pbl_size = MIN_FW_PBL_PAGE_SIZE; 634 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 635 NUM_PBES_ON_PAGE(pbl_size); 636 637 while (pbl_capacity < num_pbes) { 638 pbl_size *= 2; 639 pbl_capacity = pbl_size / sizeof(u64); 640 pbl_capacity = pbl_capacity * pbl_capacity; 641 } 642 643 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 644 num_pbls++; /* One for the layer0 ( points to the pbls) */ 645 pbl_info->two_layered = true; 646 } else { 647 /* One layered PBL */ 648 num_pbls = 1; 649 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, 650 roundup_pow_of_two((num_pbes * sizeof(u64)))); 651 pbl_info->two_layered = false; 652 } 653 654 pbl_info->num_pbls = num_pbls; 655 pbl_info->pbl_size = pbl_size; 656 pbl_info->num_pbes = num_pbes; 657 658 DP_DEBUG(dev, QEDR_MSG_MR, 659 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n", 660 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 661 662 return 0; 663 } 664 665 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, 666 struct qedr_pbl *pbl, 667 struct qedr_pbl_info *pbl_info, u32 pg_shift) 668 { 669 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 670 u32 fw_pg_cnt, fw_pg_per_umem_pg; 671 struct qedr_pbl *pbl_tbl; 672 struct scatterlist *sg; 673 struct regpair *pbe; 674 u64 pg_addr; 675 int entry; 676 677 if (!pbl_info->num_pbes) 678 return; 679 680 /* If we have a two layered pbl, the first pbl points to the rest 681 * of the pbls and the first entry lays on the second pbl in the table 682 */ 683 if (pbl_info->two_layered) 684 pbl_tbl = &pbl[1]; 685 else 686 pbl_tbl = pbl; 687 688 pbe = (struct regpair *)pbl_tbl->va; 689 if (!pbe) { 690 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n"); 691 return; 692 } 693 694 pbe_cnt = 0; 695 696 shift = umem->page_shift; 697 698 fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift); 699 700 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 701 pages = sg_dma_len(sg) >> shift; 702 pg_addr = sg_dma_address(sg); 703 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 704 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) { 705 pbe->lo = cpu_to_le32(pg_addr); 706 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); 707 708 pg_addr += BIT(pg_shift); 709 pbe_cnt++; 710 total_num_pbes++; 711 pbe++; 712 713 if (total_num_pbes == pbl_info->num_pbes) 714 return; 715 716 /* If the given pbl is full storing the pbes, 717 * move to next pbl. 718 */ 719 if (pbe_cnt == 720 (pbl_info->pbl_size / sizeof(u64))) { 721 pbl_tbl++; 722 pbe = (struct regpair *)pbl_tbl->va; 723 pbe_cnt = 0; 724 } 725 726 fw_pg_cnt++; 727 } 728 } 729 } 730 } 731 732 static int qedr_copy_cq_uresp(struct qedr_dev *dev, 733 struct qedr_cq *cq, struct ib_udata *udata) 734 { 735 struct qedr_create_cq_uresp uresp; 736 int rc; 737 738 memset(&uresp, 0, sizeof(uresp)); 739 740 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 741 uresp.icid = cq->icid; 742 743 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 744 if (rc) 745 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); 746 747 return rc; 748 } 749 750 static void consume_cqe(struct qedr_cq *cq) 751 { 752 if (cq->latest_cqe == cq->toggle_cqe) 753 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 754 755 cq->latest_cqe = qed_chain_consume(&cq->pbl); 756 } 757 758 static inline int qedr_align_cq_entries(int entries) 759 { 760 u64 size, aligned_size; 761 762 /* We allocate an extra entry that we don't report to the FW. */ 763 size = (entries + 1) * QEDR_CQE_SIZE; 764 aligned_size = ALIGN(size, PAGE_SIZE); 765 766 return aligned_size / QEDR_CQE_SIZE; 767 } 768 769 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx, 770 struct qedr_dev *dev, 771 struct qedr_userq *q, 772 u64 buf_addr, size_t buf_len, 773 int access, int dmasync) 774 { 775 u32 fw_pages; 776 int rc; 777 778 q->buf_addr = buf_addr; 779 q->buf_len = buf_len; 780 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync); 781 if (IS_ERR(q->umem)) { 782 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", 783 PTR_ERR(q->umem)); 784 return PTR_ERR(q->umem); 785 } 786 787 fw_pages = ib_umem_page_count(q->umem) << 788 (q->umem->page_shift - FW_PAGE_SHIFT); 789 790 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); 791 if (rc) 792 goto err0; 793 794 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 795 if (IS_ERR(q->pbl_tbl)) { 796 rc = PTR_ERR(q->pbl_tbl); 797 goto err0; 798 } 799 800 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info, 801 FW_PAGE_SHIFT); 802 803 return 0; 804 805 err0: 806 ib_umem_release(q->umem); 807 808 return rc; 809 } 810 811 static inline void qedr_init_cq_params(struct qedr_cq *cq, 812 struct qedr_ucontext *ctx, 813 struct qedr_dev *dev, int vector, 814 int chain_entries, int page_cnt, 815 u64 pbl_ptr, 816 struct qed_rdma_create_cq_in_params 817 *params) 818 { 819 memset(params, 0, sizeof(*params)); 820 params->cq_handle_hi = upper_32_bits((uintptr_t)cq); 821 params->cq_handle_lo = lower_32_bits((uintptr_t)cq); 822 params->cnq_id = vector; 823 params->cq_size = chain_entries - 1; 824 params->dpi = (ctx) ? ctx->dpi : dev->dpi; 825 params->pbl_num_pages = page_cnt; 826 params->pbl_ptr = pbl_ptr; 827 params->pbl_two_level = 0; 828 } 829 830 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags) 831 { 832 /* Flush data before signalling doorbell */ 833 wmb(); 834 cq->db.data.agg_flags = flags; 835 cq->db.data.value = cpu_to_le32(cons); 836 writeq(cq->db.raw, cq->db_addr); 837 838 /* Make sure write would stick */ 839 mmiowb(); 840 } 841 842 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 843 { 844 struct qedr_cq *cq = get_qedr_cq(ibcq); 845 unsigned long sflags; 846 struct qedr_dev *dev; 847 848 dev = get_qedr_dev(ibcq->device); 849 850 if (cq->destroyed) { 851 DP_ERR(dev, 852 "warning: arm was invoked after destroy for cq %p (icid=%d)\n", 853 cq, cq->icid); 854 return -EINVAL; 855 } 856 857 858 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 859 return 0; 860 861 spin_lock_irqsave(&cq->cq_lock, sflags); 862 863 cq->arm_flags = 0; 864 865 if (flags & IB_CQ_SOLICITED) 866 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 867 868 if (flags & IB_CQ_NEXT_COMP) 869 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 870 871 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 872 873 spin_unlock_irqrestore(&cq->cq_lock, sflags); 874 875 return 0; 876 } 877 878 struct ib_cq *qedr_create_cq(struct ib_device *ibdev, 879 const struct ib_cq_init_attr *attr, 880 struct ib_ucontext *ib_ctx, struct ib_udata *udata) 881 { 882 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx); 883 struct qed_rdma_destroy_cq_out_params destroy_oparams; 884 struct qed_rdma_destroy_cq_in_params destroy_iparams; 885 struct qedr_dev *dev = get_qedr_dev(ibdev); 886 struct qed_rdma_create_cq_in_params params; 887 struct qedr_create_cq_ureq ureq; 888 int vector = attr->comp_vector; 889 int entries = attr->cqe; 890 struct qedr_cq *cq; 891 int chain_entries; 892 int page_cnt; 893 u64 pbl_ptr; 894 u16 icid; 895 int rc; 896 897 DP_DEBUG(dev, QEDR_MSG_INIT, 898 "create_cq: called from %s. entries=%d, vector=%d\n", 899 udata ? "User Lib" : "Kernel", entries, vector); 900 901 if (entries > QEDR_MAX_CQES) { 902 DP_ERR(dev, 903 "create cq: the number of entries %d is too high. Must be equal or below %d.\n", 904 entries, QEDR_MAX_CQES); 905 return ERR_PTR(-EINVAL); 906 } 907 908 chain_entries = qedr_align_cq_entries(entries); 909 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); 910 911 cq = kzalloc(sizeof(*cq), GFP_KERNEL); 912 if (!cq) 913 return ERR_PTR(-ENOMEM); 914 915 if (udata) { 916 memset(&ureq, 0, sizeof(ureq)); 917 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { 918 DP_ERR(dev, 919 "create cq: problem copying data from user space\n"); 920 goto err0; 921 } 922 923 if (!ureq.len) { 924 DP_ERR(dev, 925 "create cq: cannot create a cq with 0 entries\n"); 926 goto err0; 927 } 928 929 cq->cq_type = QEDR_CQ_TYPE_USER; 930 931 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, 932 ureq.len, IB_ACCESS_LOCAL_WRITE, 1); 933 if (rc) 934 goto err0; 935 936 pbl_ptr = cq->q.pbl_tbl->pa; 937 page_cnt = cq->q.pbl_info.num_pbes; 938 939 cq->ibcq.cqe = chain_entries; 940 } else { 941 cq->cq_type = QEDR_CQ_TYPE_KERNEL; 942 943 rc = dev->ops->common->chain_alloc(dev->cdev, 944 QED_CHAIN_USE_TO_CONSUME, 945 QED_CHAIN_MODE_PBL, 946 QED_CHAIN_CNT_TYPE_U32, 947 chain_entries, 948 sizeof(union rdma_cqe), 949 &cq->pbl, NULL); 950 if (rc) 951 goto err1; 952 953 page_cnt = qed_chain_get_page_cnt(&cq->pbl); 954 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); 955 cq->ibcq.cqe = cq->pbl.capacity; 956 } 957 958 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, 959 pbl_ptr, ¶ms); 960 961 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 962 if (rc) 963 goto err2; 964 965 cq->icid = icid; 966 cq->sig = QEDR_CQ_MAGIC_NUMBER; 967 spin_lock_init(&cq->cq_lock); 968 969 if (ib_ctx) { 970 rc = qedr_copy_cq_uresp(dev, cq, udata); 971 if (rc) 972 goto err3; 973 } else { 974 /* Generate doorbell address. */ 975 cq->db_addr = dev->db_addr + 976 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 977 cq->db.data.icid = cq->icid; 978 cq->db.data.params = DB_AGG_CMD_SET << 979 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 980 981 /* point to the very last element, passing it we will toggle */ 982 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); 983 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 984 cq->latest_cqe = NULL; 985 consume_cqe(cq); 986 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 987 } 988 989 DP_DEBUG(dev, QEDR_MSG_CQ, 990 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", 991 cq->icid, cq, params.cq_size); 992 993 return &cq->ibcq; 994 995 err3: 996 destroy_iparams.icid = cq->icid; 997 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, 998 &destroy_oparams); 999 err2: 1000 if (udata) 1001 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1002 else 1003 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1004 err1: 1005 if (udata) 1006 ib_umem_release(cq->q.umem); 1007 err0: 1008 kfree(cq); 1009 return ERR_PTR(-EINVAL); 1010 } 1011 1012 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 1013 { 1014 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1015 struct qedr_cq *cq = get_qedr_cq(ibcq); 1016 1017 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq); 1018 1019 return 0; 1020 } 1021 1022 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) 1023 #define QEDR_DESTROY_CQ_ITER_DURATION (10) 1024 1025 int qedr_destroy_cq(struct ib_cq *ibcq) 1026 { 1027 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1028 struct qed_rdma_destroy_cq_out_params oparams; 1029 struct qed_rdma_destroy_cq_in_params iparams; 1030 struct qedr_cq *cq = get_qedr_cq(ibcq); 1031 int iter; 1032 int rc; 1033 1034 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); 1035 1036 cq->destroyed = 1; 1037 1038 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 1039 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 1040 goto done; 1041 1042 iparams.icid = cq->icid; 1043 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 1044 if (rc) 1045 return rc; 1046 1047 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1048 1049 if (ibcq->uobject && ibcq->uobject->context) { 1050 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1051 ib_umem_release(cq->q.umem); 1052 } 1053 1054 /* We don't want the IRQ handler to handle a non-existing CQ so we 1055 * wait until all CNQ interrupts, if any, are received. This will always 1056 * happen and will always happen very fast. If not, then a serious error 1057 * has occured. That is why we can use a long delay. 1058 * We spin for a short time so we don’t lose time on context switching 1059 * in case all the completions are handled in that span. Otherwise 1060 * we sleep for a while and check again. Since the CNQ may be 1061 * associated with (only) the current CPU we use msleep to allow the 1062 * current CPU to be freed. 1063 * The CNQ notification is increased in qedr_irq_handler(). 1064 */ 1065 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1066 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1067 udelay(QEDR_DESTROY_CQ_ITER_DURATION); 1068 iter--; 1069 } 1070 1071 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1072 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1073 msleep(QEDR_DESTROY_CQ_ITER_DURATION); 1074 iter--; 1075 } 1076 1077 if (oparams.num_cq_notif != cq->cnq_notif) 1078 goto err; 1079 1080 /* Note that we don't need to have explicit code to wait for the 1081 * completion of the event handler because it is invoked from the EQ. 1082 * Since the destroy CQ ramrod has also been received on the EQ we can 1083 * be certain that there's no event handler in process. 1084 */ 1085 done: 1086 cq->sig = ~cq->sig; 1087 1088 kfree(cq); 1089 1090 return 0; 1091 1092 err: 1093 DP_ERR(dev, 1094 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n", 1095 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif); 1096 1097 return -EINVAL; 1098 } 1099 1100 static inline int get_gid_info_from_table(struct ib_qp *ibqp, 1101 struct ib_qp_attr *attr, 1102 int attr_mask, 1103 struct qed_rdma_modify_qp_in_params 1104 *qp_params) 1105 { 1106 enum rdma_network_type nw_type; 1107 struct ib_gid_attr gid_attr; 1108 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1109 union ib_gid gid; 1110 u32 ipv4_addr; 1111 int rc = 0; 1112 int i; 1113 1114 rc = ib_get_cached_gid(ibqp->device, 1115 rdma_ah_get_port_num(&attr->ah_attr), 1116 grh->sgid_index, &gid, &gid_attr); 1117 if (rc) 1118 return rc; 1119 1120 if (!memcmp(&gid, &zgid, sizeof(gid))) 1121 return -ENOENT; 1122 1123 if (gid_attr.ndev) { 1124 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev); 1125 1126 dev_put(gid_attr.ndev); 1127 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid); 1128 switch (nw_type) { 1129 case RDMA_NETWORK_IPV6: 1130 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0], 1131 sizeof(qp_params->sgid)); 1132 memcpy(&qp_params->dgid.bytes[0], 1133 &grh->dgid, 1134 sizeof(qp_params->dgid)); 1135 qp_params->roce_mode = ROCE_V2_IPV6; 1136 SET_FIELD(qp_params->modify_flags, 1137 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1138 break; 1139 case RDMA_NETWORK_IB: 1140 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0], 1141 sizeof(qp_params->sgid)); 1142 memcpy(&qp_params->dgid.bytes[0], 1143 &grh->dgid, 1144 sizeof(qp_params->dgid)); 1145 qp_params->roce_mode = ROCE_V1; 1146 break; 1147 case RDMA_NETWORK_IPV4: 1148 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid)); 1149 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid)); 1150 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw); 1151 qp_params->sgid.ipv4_addr = ipv4_addr; 1152 ipv4_addr = 1153 qedr_get_ipv4_from_gid(grh->dgid.raw); 1154 qp_params->dgid.ipv4_addr = ipv4_addr; 1155 SET_FIELD(qp_params->modify_flags, 1156 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1157 qp_params->roce_mode = ROCE_V2_IPV4; 1158 break; 1159 } 1160 } 1161 1162 for (i = 0; i < 4; i++) { 1163 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 1164 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 1165 } 1166 1167 if (qp_params->vlan_id >= VLAN_CFI_MASK) 1168 qp_params->vlan_id = 0; 1169 1170 return 0; 1171 } 1172 1173 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, 1174 struct ib_qp_init_attr *attrs) 1175 { 1176 struct qedr_device_attr *qattr = &dev->attr; 1177 1178 /* QP0... attrs->qp_type == IB_QPT_GSI */ 1179 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { 1180 DP_DEBUG(dev, QEDR_MSG_QP, 1181 "create qp: unsupported qp type=0x%x requested\n", 1182 attrs->qp_type); 1183 return -EINVAL; 1184 } 1185 1186 if (attrs->cap.max_send_wr > qattr->max_sqe) { 1187 DP_ERR(dev, 1188 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n", 1189 attrs->cap.max_send_wr, qattr->max_sqe); 1190 return -EINVAL; 1191 } 1192 1193 if (attrs->cap.max_inline_data > qattr->max_inline) { 1194 DP_ERR(dev, 1195 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n", 1196 attrs->cap.max_inline_data, qattr->max_inline); 1197 return -EINVAL; 1198 } 1199 1200 if (attrs->cap.max_send_sge > qattr->max_sge) { 1201 DP_ERR(dev, 1202 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n", 1203 attrs->cap.max_send_sge, qattr->max_sge); 1204 return -EINVAL; 1205 } 1206 1207 if (attrs->cap.max_recv_sge > qattr->max_sge) { 1208 DP_ERR(dev, 1209 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n", 1210 attrs->cap.max_recv_sge, qattr->max_sge); 1211 return -EINVAL; 1212 } 1213 1214 /* Unprivileged user space cannot create special QP */ 1215 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) { 1216 DP_ERR(dev, 1217 "create qp: userspace can't create special QPs of type=0x%x\n", 1218 attrs->qp_type); 1219 return -EINVAL; 1220 } 1221 1222 return 0; 1223 } 1224 1225 static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp, 1226 struct qedr_qp *qp) 1227 { 1228 uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1229 uresp->rq_icid = qp->icid; 1230 } 1231 1232 static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp, 1233 struct qedr_qp *qp) 1234 { 1235 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1236 uresp->sq_icid = qp->icid + 1; 1237 } 1238 1239 static int qedr_copy_qp_uresp(struct qedr_dev *dev, 1240 struct qedr_qp *qp, struct ib_udata *udata) 1241 { 1242 struct qedr_create_qp_uresp uresp; 1243 int rc; 1244 1245 memset(&uresp, 0, sizeof(uresp)); 1246 qedr_copy_sq_uresp(&uresp, qp); 1247 qedr_copy_rq_uresp(&uresp, qp); 1248 1249 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 1250 uresp.qp_id = qp->qp_id; 1251 1252 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1253 if (rc) 1254 DP_ERR(dev, 1255 "create qp: failed a copy to user space with qp icid=0x%x.\n", 1256 qp->icid); 1257 1258 return rc; 1259 } 1260 1261 static void qedr_set_common_qp_params(struct qedr_dev *dev, 1262 struct qedr_qp *qp, 1263 struct qedr_pd *pd, 1264 struct ib_qp_init_attr *attrs) 1265 { 1266 spin_lock_init(&qp->q_lock); 1267 qp->pd = pd; 1268 qp->qp_type = attrs->qp_type; 1269 qp->max_inline_data = attrs->cap.max_inline_data; 1270 qp->sq.max_sges = attrs->cap.max_send_sge; 1271 qp->state = QED_ROCE_QP_STATE_RESET; 1272 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1273 qp->sq_cq = get_qedr_cq(attrs->send_cq); 1274 qp->rq_cq = get_qedr_cq(attrs->recv_cq); 1275 qp->dev = dev; 1276 qp->rq.max_sges = attrs->cap.max_recv_sge; 1277 1278 DP_DEBUG(dev, QEDR_MSG_QP, 1279 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 1280 qp->rq.max_sges, qp->rq_cq->icid); 1281 DP_DEBUG(dev, QEDR_MSG_QP, 1282 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n", 1283 pd->pd_id, qp->qp_type, qp->max_inline_data, 1284 qp->state, qp->signaled, (attrs->srq) ? 1 : 0); 1285 DP_DEBUG(dev, QEDR_MSG_QP, 1286 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1287 qp->sq.max_sges, qp->sq_cq->icid); 1288 } 1289 1290 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1291 { 1292 qp->sq.db = dev->db_addr + 1293 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1294 qp->sq.db_data.data.icid = qp->icid + 1; 1295 qp->rq.db = dev->db_addr + 1296 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1297 qp->rq.db_data.data.icid = qp->icid; 1298 } 1299 1300 static inline void 1301 qedr_init_common_qp_in_params(struct qedr_dev *dev, 1302 struct qedr_pd *pd, 1303 struct qedr_qp *qp, 1304 struct ib_qp_init_attr *attrs, 1305 bool fmr_and_reserved_lkey, 1306 struct qed_rdma_create_qp_in_params *params) 1307 { 1308 /* QP handle to be written in an async event */ 1309 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp); 1310 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp); 1311 1312 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 1313 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 1314 params->pd = pd->pd_id; 1315 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 1316 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; 1317 params->stats_queue = 0; 1318 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1319 params->srq_id = 0; 1320 params->use_srq = false; 1321 } 1322 1323 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) 1324 { 1325 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. " 1326 "qp=%p. " 1327 "sq_addr=0x%llx, " 1328 "sq_len=%zd, " 1329 "rq_addr=0x%llx, " 1330 "rq_len=%zd" 1331 "\n", 1332 qp, 1333 qp->usq.buf_addr, 1334 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len); 1335 } 1336 1337 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp) 1338 { 1339 if (qp->usq.umem) 1340 ib_umem_release(qp->usq.umem); 1341 qp->usq.umem = NULL; 1342 1343 if (qp->urq.umem) 1344 ib_umem_release(qp->urq.umem); 1345 qp->urq.umem = NULL; 1346 } 1347 1348 static int qedr_create_user_qp(struct qedr_dev *dev, 1349 struct qedr_qp *qp, 1350 struct ib_pd *ibpd, 1351 struct ib_udata *udata, 1352 struct ib_qp_init_attr *attrs) 1353 { 1354 struct qed_rdma_create_qp_in_params in_params; 1355 struct qed_rdma_create_qp_out_params out_params; 1356 struct qedr_pd *pd = get_qedr_pd(ibpd); 1357 struct ib_ucontext *ib_ctx = NULL; 1358 struct qedr_ucontext *ctx = NULL; 1359 struct qedr_create_qp_ureq ureq; 1360 int rc = -EINVAL; 1361 1362 ib_ctx = ibpd->uobject->context; 1363 ctx = get_qedr_ucontext(ib_ctx); 1364 1365 memset(&ureq, 0, sizeof(ureq)); 1366 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq)); 1367 if (rc) { 1368 DP_ERR(dev, "Problem copying data from user space\n"); 1369 return rc; 1370 } 1371 1372 /* SQ - read access only (0), dma sync not required (0) */ 1373 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr, 1374 ureq.sq_len, 0, 0); 1375 if (rc) 1376 return rc; 1377 1378 /* RQ - read access only (0), dma sync not required (0) */ 1379 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr, 1380 ureq.rq_len, 0, 0); 1381 1382 if (rc) 1383 return rc; 1384 1385 memset(&in_params, 0, sizeof(in_params)); 1386 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 1387 in_params.qp_handle_lo = ureq.qp_handle_lo; 1388 in_params.qp_handle_hi = ureq.qp_handle_hi; 1389 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 1390 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 1391 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 1392 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 1393 1394 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1395 &in_params, &out_params); 1396 1397 if (!qp->qed_qp) { 1398 rc = -ENOMEM; 1399 goto err1; 1400 } 1401 1402 qp->qp_id = out_params.qp_id; 1403 qp->icid = out_params.icid; 1404 1405 rc = qedr_copy_qp_uresp(dev, qp, udata); 1406 if (rc) 1407 goto err; 1408 1409 qedr_qp_user_print(dev, qp); 1410 1411 return 0; 1412 err: 1413 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1414 if (rc) 1415 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc); 1416 1417 err1: 1418 qedr_cleanup_user(dev, qp); 1419 return rc; 1420 } 1421 1422 static int 1423 qedr_roce_create_kernel_qp(struct qedr_dev *dev, 1424 struct qedr_qp *qp, 1425 struct qed_rdma_create_qp_in_params *in_params, 1426 u32 n_sq_elems, u32 n_rq_elems) 1427 { 1428 struct qed_rdma_create_qp_out_params out_params; 1429 int rc; 1430 1431 rc = dev->ops->common->chain_alloc(dev->cdev, 1432 QED_CHAIN_USE_TO_PRODUCE, 1433 QED_CHAIN_MODE_PBL, 1434 QED_CHAIN_CNT_TYPE_U32, 1435 n_sq_elems, 1436 QEDR_SQE_ELEMENT_SIZE, 1437 &qp->sq.pbl, NULL); 1438 1439 if (rc) 1440 return rc; 1441 1442 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl); 1443 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl); 1444 1445 rc = dev->ops->common->chain_alloc(dev->cdev, 1446 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1447 QED_CHAIN_MODE_PBL, 1448 QED_CHAIN_CNT_TYPE_U32, 1449 n_rq_elems, 1450 QEDR_RQE_ELEMENT_SIZE, 1451 &qp->rq.pbl, NULL); 1452 if (rc) 1453 return rc; 1454 1455 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl); 1456 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl); 1457 1458 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1459 in_params, &out_params); 1460 1461 if (!qp->qed_qp) 1462 return -EINVAL; 1463 1464 qp->qp_id = out_params.qp_id; 1465 qp->icid = out_params.icid; 1466 1467 qedr_set_roce_db_info(dev, qp); 1468 1469 return 0; 1470 } 1471 1472 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) 1473 { 1474 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl); 1475 kfree(qp->wqe_wr_id); 1476 1477 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl); 1478 kfree(qp->rqe_wr_id); 1479 } 1480 1481 static int qedr_create_kernel_qp(struct qedr_dev *dev, 1482 struct qedr_qp *qp, 1483 struct ib_pd *ibpd, 1484 struct ib_qp_init_attr *attrs) 1485 { 1486 struct qed_rdma_create_qp_in_params in_params; 1487 struct qedr_pd *pd = get_qedr_pd(ibpd); 1488 int rc = -EINVAL; 1489 u32 n_rq_elems; 1490 u32 n_sq_elems; 1491 u32 n_sq_entries; 1492 1493 memset(&in_params, 0, sizeof(in_params)); 1494 1495 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in 1496 * the ring. The ring should allow at least a single WR, even if the 1497 * user requested none, due to allocation issues. 1498 * We should add an extra WR since the prod and cons indices of 1499 * wqe_wr_id are managed in such a way that the WQ is considered full 1500 * when (prod+1)%max_wr==cons. We currently don't do that because we 1501 * double the number of entries due an iSER issue that pushes far more 1502 * WRs than indicated. If we decline its ib_post_send() then we get 1503 * error prints in the dmesg we'd like to avoid. 1504 */ 1505 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 1506 dev->attr.max_sqe); 1507 1508 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id), 1509 GFP_KERNEL); 1510 if (!qp->wqe_wr_id) { 1511 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n"); 1512 return -ENOMEM; 1513 } 1514 1515 /* QP handle to be written in CQE */ 1516 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp); 1517 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp); 1518 1519 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in 1520 * the ring. There ring should allow at least a single WR, even if the 1521 * user requested none, due to allocation issues. 1522 */ 1523 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1); 1524 1525 /* Allocate driver internal RQ array */ 1526 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id), 1527 GFP_KERNEL); 1528 if (!qp->rqe_wr_id) { 1529 DP_ERR(dev, 1530 "create qp: failed RQ shadow memory allocation\n"); 1531 kfree(qp->wqe_wr_id); 1532 return -ENOMEM; 1533 } 1534 1535 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 1536 1537 n_sq_entries = attrs->cap.max_send_wr; 1538 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe); 1539 n_sq_entries = max_t(u32, n_sq_entries, 1); 1540 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE; 1541 1542 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE; 1543 1544 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params, 1545 n_sq_elems, n_rq_elems); 1546 if (rc) 1547 qedr_cleanup_kernel(dev, qp); 1548 1549 return rc; 1550 } 1551 1552 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, 1553 struct ib_qp_init_attr *attrs, 1554 struct ib_udata *udata) 1555 { 1556 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 1557 struct qedr_pd *pd = get_qedr_pd(ibpd); 1558 struct qedr_qp *qp; 1559 struct ib_qp *ibqp; 1560 int rc = 0; 1561 1562 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", 1563 udata ? "user library" : "kernel", pd); 1564 1565 rc = qedr_check_qp_attrs(ibpd, dev, attrs); 1566 if (rc) 1567 return ERR_PTR(rc); 1568 1569 if (attrs->srq) 1570 return ERR_PTR(-EINVAL); 1571 1572 DP_DEBUG(dev, QEDR_MSG_QP, 1573 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 1574 udata ? "user library" : "kernel", attrs->event_handler, pd, 1575 get_qedr_cq(attrs->send_cq), 1576 get_qedr_cq(attrs->send_cq)->icid, 1577 get_qedr_cq(attrs->recv_cq), 1578 get_qedr_cq(attrs->recv_cq)->icid); 1579 1580 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1581 if (!qp) { 1582 DP_ERR(dev, "create qp: failed allocating memory\n"); 1583 return ERR_PTR(-ENOMEM); 1584 } 1585 1586 qedr_set_common_qp_params(dev, qp, pd, attrs); 1587 1588 if (attrs->qp_type == IB_QPT_GSI) { 1589 ibqp = qedr_create_gsi_qp(dev, attrs, qp); 1590 if (IS_ERR(ibqp)) 1591 kfree(qp); 1592 return ibqp; 1593 } 1594 1595 if (udata) 1596 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); 1597 else 1598 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); 1599 1600 if (rc) 1601 goto err; 1602 1603 qp->ibqp.qp_num = qp->qp_id; 1604 1605 return &qp->ibqp; 1606 1607 err: 1608 kfree(qp); 1609 1610 return ERR_PTR(-EFAULT); 1611 } 1612 1613 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) 1614 { 1615 switch (qp_state) { 1616 case QED_ROCE_QP_STATE_RESET: 1617 return IB_QPS_RESET; 1618 case QED_ROCE_QP_STATE_INIT: 1619 return IB_QPS_INIT; 1620 case QED_ROCE_QP_STATE_RTR: 1621 return IB_QPS_RTR; 1622 case QED_ROCE_QP_STATE_RTS: 1623 return IB_QPS_RTS; 1624 case QED_ROCE_QP_STATE_SQD: 1625 return IB_QPS_SQD; 1626 case QED_ROCE_QP_STATE_ERR: 1627 return IB_QPS_ERR; 1628 case QED_ROCE_QP_STATE_SQE: 1629 return IB_QPS_SQE; 1630 } 1631 return IB_QPS_ERR; 1632 } 1633 1634 static enum qed_roce_qp_state qedr_get_state_from_ibqp( 1635 enum ib_qp_state qp_state) 1636 { 1637 switch (qp_state) { 1638 case IB_QPS_RESET: 1639 return QED_ROCE_QP_STATE_RESET; 1640 case IB_QPS_INIT: 1641 return QED_ROCE_QP_STATE_INIT; 1642 case IB_QPS_RTR: 1643 return QED_ROCE_QP_STATE_RTR; 1644 case IB_QPS_RTS: 1645 return QED_ROCE_QP_STATE_RTS; 1646 case IB_QPS_SQD: 1647 return QED_ROCE_QP_STATE_SQD; 1648 case IB_QPS_ERR: 1649 return QED_ROCE_QP_STATE_ERR; 1650 default: 1651 return QED_ROCE_QP_STATE_ERR; 1652 } 1653 } 1654 1655 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) 1656 { 1657 qed_chain_reset(&qph->pbl); 1658 qph->prod = 0; 1659 qph->cons = 0; 1660 qph->wqe_cons = 0; 1661 qph->db_data.data.value = cpu_to_le16(0); 1662 } 1663 1664 static int qedr_update_qp_state(struct qedr_dev *dev, 1665 struct qedr_qp *qp, 1666 enum qed_roce_qp_state new_state) 1667 { 1668 int status = 0; 1669 1670 if (new_state == qp->state) 1671 return 0; 1672 1673 switch (qp->state) { 1674 case QED_ROCE_QP_STATE_RESET: 1675 switch (new_state) { 1676 case QED_ROCE_QP_STATE_INIT: 1677 qp->prev_wqe_size = 0; 1678 qedr_reset_qp_hwq_info(&qp->sq); 1679 qedr_reset_qp_hwq_info(&qp->rq); 1680 break; 1681 default: 1682 status = -EINVAL; 1683 break; 1684 }; 1685 break; 1686 case QED_ROCE_QP_STATE_INIT: 1687 switch (new_state) { 1688 case QED_ROCE_QP_STATE_RTR: 1689 /* Update doorbell (in case post_recv was 1690 * done before move to RTR) 1691 */ 1692 wmb(); 1693 writel(qp->rq.db_data.raw, qp->rq.db); 1694 /* Make sure write takes effect */ 1695 mmiowb(); 1696 break; 1697 case QED_ROCE_QP_STATE_ERR: 1698 break; 1699 default: 1700 /* Invalid state change. */ 1701 status = -EINVAL; 1702 break; 1703 }; 1704 break; 1705 case QED_ROCE_QP_STATE_RTR: 1706 /* RTR->XXX */ 1707 switch (new_state) { 1708 case QED_ROCE_QP_STATE_RTS: 1709 break; 1710 case QED_ROCE_QP_STATE_ERR: 1711 break; 1712 default: 1713 /* Invalid state change. */ 1714 status = -EINVAL; 1715 break; 1716 }; 1717 break; 1718 case QED_ROCE_QP_STATE_RTS: 1719 /* RTS->XXX */ 1720 switch (new_state) { 1721 case QED_ROCE_QP_STATE_SQD: 1722 break; 1723 case QED_ROCE_QP_STATE_ERR: 1724 break; 1725 default: 1726 /* Invalid state change. */ 1727 status = -EINVAL; 1728 break; 1729 }; 1730 break; 1731 case QED_ROCE_QP_STATE_SQD: 1732 /* SQD->XXX */ 1733 switch (new_state) { 1734 case QED_ROCE_QP_STATE_RTS: 1735 case QED_ROCE_QP_STATE_ERR: 1736 break; 1737 default: 1738 /* Invalid state change. */ 1739 status = -EINVAL; 1740 break; 1741 }; 1742 break; 1743 case QED_ROCE_QP_STATE_ERR: 1744 /* ERR->XXX */ 1745 switch (new_state) { 1746 case QED_ROCE_QP_STATE_RESET: 1747 if ((qp->rq.prod != qp->rq.cons) || 1748 (qp->sq.prod != qp->sq.cons)) { 1749 DP_NOTICE(dev, 1750 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n", 1751 qp->rq.prod, qp->rq.cons, qp->sq.prod, 1752 qp->sq.cons); 1753 status = -EINVAL; 1754 } 1755 break; 1756 default: 1757 status = -EINVAL; 1758 break; 1759 }; 1760 break; 1761 default: 1762 status = -EINVAL; 1763 break; 1764 }; 1765 1766 return status; 1767 } 1768 1769 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1770 int attr_mask, struct ib_udata *udata) 1771 { 1772 struct qedr_qp *qp = get_qedr_qp(ibqp); 1773 struct qed_rdma_modify_qp_in_params qp_params = { 0 }; 1774 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); 1775 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1776 enum ib_qp_state old_qp_state, new_qp_state; 1777 int rc = 0; 1778 1779 DP_DEBUG(dev, QEDR_MSG_QP, 1780 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask, 1781 attr->qp_state); 1782 1783 old_qp_state = qedr_get_ibqp_state(qp->state); 1784 if (attr_mask & IB_QP_STATE) 1785 new_qp_state = attr->qp_state; 1786 else 1787 new_qp_state = old_qp_state; 1788 1789 if (!ib_modify_qp_is_ok 1790 (old_qp_state, new_qp_state, ibqp->qp_type, attr_mask, 1791 IB_LINK_LAYER_ETHERNET)) { 1792 DP_ERR(dev, 1793 "modify qp: invalid attribute mask=0x%x specified for\n" 1794 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n", 1795 attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state, 1796 new_qp_state); 1797 rc = -EINVAL; 1798 goto err; 1799 } 1800 1801 /* Translate the masks... */ 1802 if (attr_mask & IB_QP_STATE) { 1803 SET_FIELD(qp_params.modify_flags, 1804 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 1805 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state); 1806 } 1807 1808 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 1809 qp_params.sqd_async = true; 1810 1811 if (attr_mask & IB_QP_PKEY_INDEX) { 1812 SET_FIELD(qp_params.modify_flags, 1813 QED_ROCE_MODIFY_QP_VALID_PKEY, 1); 1814 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) { 1815 rc = -EINVAL; 1816 goto err; 1817 } 1818 1819 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT; 1820 } 1821 1822 if (attr_mask & IB_QP_QKEY) 1823 qp->qkey = attr->qkey; 1824 1825 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1826 SET_FIELD(qp_params.modify_flags, 1827 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 1828 qp_params.incoming_rdma_read_en = attr->qp_access_flags & 1829 IB_ACCESS_REMOTE_READ; 1830 qp_params.incoming_rdma_write_en = attr->qp_access_flags & 1831 IB_ACCESS_REMOTE_WRITE; 1832 qp_params.incoming_atomic_en = attr->qp_access_flags & 1833 IB_ACCESS_REMOTE_ATOMIC; 1834 } 1835 1836 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 1837 if (attr_mask & IB_QP_PATH_MTU) { 1838 if (attr->path_mtu < IB_MTU_256 || 1839 attr->path_mtu > IB_MTU_4096) { 1840 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n"); 1841 rc = -EINVAL; 1842 goto err; 1843 } 1844 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 1845 ib_mtu_enum_to_int(iboe_get_mtu 1846 (dev->ndev->mtu))); 1847 } 1848 1849 if (!qp->mtu) { 1850 qp->mtu = 1851 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 1852 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu); 1853 } 1854 1855 SET_FIELD(qp_params.modify_flags, 1856 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1); 1857 1858 qp_params.traffic_class_tos = grh->traffic_class; 1859 qp_params.flow_label = grh->flow_label; 1860 qp_params.hop_limit_ttl = grh->hop_limit; 1861 1862 qp->sgid_idx = grh->sgid_index; 1863 1864 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params); 1865 if (rc) { 1866 DP_ERR(dev, 1867 "modify qp: problems with GID index %d (rc=%d)\n", 1868 grh->sgid_index, rc); 1869 return rc; 1870 } 1871 1872 rc = qedr_get_dmac(dev, &attr->ah_attr, 1873 qp_params.remote_mac_addr); 1874 if (rc) 1875 return rc; 1876 1877 qp_params.use_local_mac = true; 1878 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr); 1879 1880 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n", 1881 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 1882 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 1883 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n", 1884 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 1885 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 1886 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n", 1887 qp_params.remote_mac_addr); 1888 1889 qp_params.mtu = qp->mtu; 1890 qp_params.lb_indication = false; 1891 } 1892 1893 if (!qp_params.mtu) { 1894 /* Stay with current MTU */ 1895 if (qp->mtu) 1896 qp_params.mtu = qp->mtu; 1897 else 1898 qp_params.mtu = 1899 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 1900 } 1901 1902 if (attr_mask & IB_QP_TIMEOUT) { 1903 SET_FIELD(qp_params.modify_flags, 1904 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 1905 1906 qp_params.ack_timeout = attr->timeout; 1907 if (attr->timeout) { 1908 u32 temp; 1909 1910 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000; 1911 /* FW requires [msec] */ 1912 qp_params.ack_timeout = temp; 1913 } else { 1914 /* Infinite */ 1915 qp_params.ack_timeout = 0; 1916 } 1917 } 1918 if (attr_mask & IB_QP_RETRY_CNT) { 1919 SET_FIELD(qp_params.modify_flags, 1920 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 1921 qp_params.retry_cnt = attr->retry_cnt; 1922 } 1923 1924 if (attr_mask & IB_QP_RNR_RETRY) { 1925 SET_FIELD(qp_params.modify_flags, 1926 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1); 1927 qp_params.rnr_retry_cnt = attr->rnr_retry; 1928 } 1929 1930 if (attr_mask & IB_QP_RQ_PSN) { 1931 SET_FIELD(qp_params.modify_flags, 1932 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1); 1933 qp_params.rq_psn = attr->rq_psn; 1934 qp->rq_psn = attr->rq_psn; 1935 } 1936 1937 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1938 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) { 1939 rc = -EINVAL; 1940 DP_ERR(dev, 1941 "unsupported max_rd_atomic=%d, supported=%d\n", 1942 attr->max_rd_atomic, 1943 dev->attr.max_qp_req_rd_atomic_resc); 1944 goto err; 1945 } 1946 1947 SET_FIELD(qp_params.modify_flags, 1948 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1); 1949 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 1950 } 1951 1952 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 1953 SET_FIELD(qp_params.modify_flags, 1954 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1); 1955 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 1956 } 1957 1958 if (attr_mask & IB_QP_SQ_PSN) { 1959 SET_FIELD(qp_params.modify_flags, 1960 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1); 1961 qp_params.sq_psn = attr->sq_psn; 1962 qp->sq_psn = attr->sq_psn; 1963 } 1964 1965 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1966 if (attr->max_dest_rd_atomic > 1967 dev->attr.max_qp_resp_rd_atomic_resc) { 1968 DP_ERR(dev, 1969 "unsupported max_dest_rd_atomic=%d, supported=%d\n", 1970 attr->max_dest_rd_atomic, 1971 dev->attr.max_qp_resp_rd_atomic_resc); 1972 1973 rc = -EINVAL; 1974 goto err; 1975 } 1976 1977 SET_FIELD(qp_params.modify_flags, 1978 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1); 1979 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 1980 } 1981 1982 if (attr_mask & IB_QP_DEST_QPN) { 1983 SET_FIELD(qp_params.modify_flags, 1984 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1); 1985 1986 qp_params.dest_qp = attr->dest_qp_num; 1987 qp->dest_qp_num = attr->dest_qp_num; 1988 } 1989 1990 if (qp->qp_type != IB_QPT_GSI) 1991 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx, 1992 qp->qed_qp, &qp_params); 1993 1994 if (attr_mask & IB_QP_STATE) { 1995 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 1996 rc = qedr_update_qp_state(dev, qp, qp_params.new_state); 1997 qp->state = qp_params.new_state; 1998 } 1999 2000 err: 2001 return rc; 2002 } 2003 2004 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params) 2005 { 2006 int ib_qp_acc_flags = 0; 2007 2008 if (params->incoming_rdma_write_en) 2009 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 2010 if (params->incoming_rdma_read_en) 2011 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 2012 if (params->incoming_atomic_en) 2013 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 2014 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 2015 return ib_qp_acc_flags; 2016 } 2017 2018 int qedr_query_qp(struct ib_qp *ibqp, 2019 struct ib_qp_attr *qp_attr, 2020 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 2021 { 2022 struct qed_rdma_query_qp_out_params params; 2023 struct qedr_qp *qp = get_qedr_qp(ibqp); 2024 struct qedr_dev *dev = qp->dev; 2025 int rc = 0; 2026 2027 memset(¶ms, 0, sizeof(params)); 2028 2029 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms); 2030 if (rc) 2031 goto err; 2032 2033 memset(qp_attr, 0, sizeof(*qp_attr)); 2034 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 2035 2036 qp_attr->qp_state = qedr_get_ibqp_state(params.state); 2037 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state); 2038 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu); 2039 qp_attr->path_mig_state = IB_MIG_MIGRATED; 2040 qp_attr->rq_psn = params.rq_psn; 2041 qp_attr->sq_psn = params.sq_psn; 2042 qp_attr->dest_qp_num = params.dest_qp; 2043 2044 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms); 2045 2046 qp_attr->cap.max_send_wr = qp->sq.max_wr; 2047 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 2048 qp_attr->cap.max_send_sge = qp->sq.max_sges; 2049 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 2050 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; 2051 qp_init_attr->cap = qp_attr->cap; 2052 2053 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2054 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 2055 params.flow_label, qp->sgid_idx, 2056 params.hop_limit_ttl, params.traffic_class_tos); 2057 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); 2058 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 2059 rdma_ah_set_sl(&qp_attr->ah_attr, 0); 2060 qp_attr->timeout = params.timeout; 2061 qp_attr->rnr_retry = params.rnr_retry; 2062 qp_attr->retry_cnt = params.retry_cnt; 2063 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 2064 qp_attr->pkey_index = params.pkey_index; 2065 qp_attr->port_num = 1; 2066 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 2067 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 2068 qp_attr->alt_pkey_index = 0; 2069 qp_attr->alt_port_num = 0; 2070 qp_attr->alt_timeout = 0; 2071 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 2072 2073 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0; 2074 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 2075 qp_attr->max_rd_atomic = params.max_rd_atomic; 2076 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0; 2077 2078 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n", 2079 qp_attr->cap.max_inline_data); 2080 2081 err: 2082 return rc; 2083 } 2084 2085 int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp) 2086 { 2087 int rc = 0; 2088 2089 if (qp->qp_type != IB_QPT_GSI) { 2090 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2091 if (rc) 2092 return rc; 2093 } 2094 2095 if (qp->ibqp.uobject && qp->ibqp.uobject->context) 2096 qedr_cleanup_user(dev, qp); 2097 else 2098 qedr_cleanup_kernel(dev, qp); 2099 2100 return 0; 2101 } 2102 2103 int qedr_destroy_qp(struct ib_qp *ibqp) 2104 { 2105 struct qedr_qp *qp = get_qedr_qp(ibqp); 2106 struct qedr_dev *dev = qp->dev; 2107 struct ib_qp_attr attr; 2108 int attr_mask = 0; 2109 int rc = 0; 2110 2111 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n", 2112 qp, qp->qp_type); 2113 2114 if ((qp->state != QED_ROCE_QP_STATE_RESET) && 2115 (qp->state != QED_ROCE_QP_STATE_ERR) && 2116 (qp->state != QED_ROCE_QP_STATE_INIT)) { 2117 2118 attr.qp_state = IB_QPS_ERR; 2119 attr_mask |= IB_QP_STATE; 2120 2121 /* Change the QP state to ERROR */ 2122 qedr_modify_qp(ibqp, &attr, attr_mask, NULL); 2123 } 2124 2125 if (qp->qp_type == IB_QPT_GSI) 2126 qedr_destroy_gsi_qp(dev); 2127 2128 qedr_free_qp_resources(dev, qp); 2129 2130 kfree(qp); 2131 2132 return rc; 2133 } 2134 2135 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr, 2136 struct ib_udata *udata) 2137 { 2138 struct qedr_ah *ah; 2139 2140 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 2141 if (!ah) 2142 return ERR_PTR(-ENOMEM); 2143 2144 ah->attr = *attr; 2145 2146 return &ah->ibah; 2147 } 2148 2149 int qedr_destroy_ah(struct ib_ah *ibah) 2150 { 2151 struct qedr_ah *ah = get_qedr_ah(ibah); 2152 2153 kfree(ah); 2154 return 0; 2155 } 2156 2157 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) 2158 { 2159 struct qedr_pbl *pbl, *tmp; 2160 2161 if (info->pbl_table) 2162 list_add_tail(&info->pbl_table->list_entry, 2163 &info->free_pbl_list); 2164 2165 if (!list_empty(&info->inuse_pbl_list)) 2166 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 2167 2168 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 2169 list_del(&pbl->list_entry); 2170 qedr_free_pbl(dev, &info->pbl_info, pbl); 2171 } 2172 } 2173 2174 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info, 2175 size_t page_list_len, bool two_layered) 2176 { 2177 struct qedr_pbl *tmp; 2178 int rc; 2179 2180 INIT_LIST_HEAD(&info->free_pbl_list); 2181 INIT_LIST_HEAD(&info->inuse_pbl_list); 2182 2183 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info, 2184 page_list_len, two_layered); 2185 if (rc) 2186 goto done; 2187 2188 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2189 if (IS_ERR(info->pbl_table)) { 2190 rc = PTR_ERR(info->pbl_table); 2191 goto done; 2192 } 2193 2194 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n", 2195 &info->pbl_table->pa); 2196 2197 /* in usual case we use 2 PBLs, so we add one to free 2198 * list and allocating another one 2199 */ 2200 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2201 if (IS_ERR(tmp)) { 2202 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n"); 2203 goto done; 2204 } 2205 2206 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 2207 2208 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa); 2209 2210 done: 2211 if (rc) 2212 free_mr_info(dev, info); 2213 2214 return rc; 2215 } 2216 2217 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 2218 u64 usr_addr, int acc, struct ib_udata *udata) 2219 { 2220 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2221 struct qedr_mr *mr; 2222 struct qedr_pd *pd; 2223 int rc = -ENOMEM; 2224 2225 pd = get_qedr_pd(ibpd); 2226 DP_DEBUG(dev, QEDR_MSG_MR, 2227 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 2228 pd->pd_id, start, len, usr_addr, acc); 2229 2230 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 2231 return ERR_PTR(-EINVAL); 2232 2233 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2234 if (!mr) 2235 return ERR_PTR(rc); 2236 2237 mr->type = QEDR_MR_USER; 2238 2239 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0); 2240 if (IS_ERR(mr->umem)) { 2241 rc = -EFAULT; 2242 goto err0; 2243 } 2244 2245 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); 2246 if (rc) 2247 goto err1; 2248 2249 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 2250 &mr->info.pbl_info, mr->umem->page_shift); 2251 2252 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2253 if (rc) { 2254 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2255 goto err1; 2256 } 2257 2258 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2259 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2260 mr->hw_mr.key = 0; 2261 mr->hw_mr.pd = pd->pd_id; 2262 mr->hw_mr.local_read = 1; 2263 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2264 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2265 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2266 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2267 mr->hw_mr.mw_bind = false; 2268 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 2269 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2270 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2271 mr->hw_mr.page_size_log = mr->umem->page_shift; 2272 mr->hw_mr.fbo = ib_umem_offset(mr->umem); 2273 mr->hw_mr.length = len; 2274 mr->hw_mr.vaddr = usr_addr; 2275 mr->hw_mr.zbva = false; 2276 mr->hw_mr.phy_mr = false; 2277 mr->hw_mr.dma_mr = false; 2278 2279 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2280 if (rc) { 2281 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2282 goto err2; 2283 } 2284 2285 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2286 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2287 mr->hw_mr.remote_atomic) 2288 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2289 2290 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n", 2291 mr->ibmr.lkey); 2292 return &mr->ibmr; 2293 2294 err2: 2295 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2296 err1: 2297 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2298 err0: 2299 kfree(mr); 2300 return ERR_PTR(rc); 2301 } 2302 2303 int qedr_dereg_mr(struct ib_mr *ib_mr) 2304 { 2305 struct qedr_mr *mr = get_qedr_mr(ib_mr); 2306 struct qedr_dev *dev = get_qedr_dev(ib_mr->device); 2307 int rc = 0; 2308 2309 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid); 2310 if (rc) 2311 return rc; 2312 2313 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2314 2315 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR)) 2316 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2317 2318 /* it could be user registered memory. */ 2319 if (mr->umem) 2320 ib_umem_release(mr->umem); 2321 2322 kfree(mr); 2323 2324 return rc; 2325 } 2326 2327 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, 2328 int max_page_list_len) 2329 { 2330 struct qedr_pd *pd = get_qedr_pd(ibpd); 2331 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2332 struct qedr_mr *mr; 2333 int rc = -ENOMEM; 2334 2335 DP_DEBUG(dev, QEDR_MSG_MR, 2336 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id, 2337 max_page_list_len); 2338 2339 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2340 if (!mr) 2341 return ERR_PTR(rc); 2342 2343 mr->dev = dev; 2344 mr->type = QEDR_MR_FRMR; 2345 2346 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1); 2347 if (rc) 2348 goto err0; 2349 2350 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2351 if (rc) { 2352 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2353 goto err0; 2354 } 2355 2356 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2357 mr->hw_mr.tid_type = QED_RDMA_TID_FMR; 2358 mr->hw_mr.key = 0; 2359 mr->hw_mr.pd = pd->pd_id; 2360 mr->hw_mr.local_read = 1; 2361 mr->hw_mr.local_write = 0; 2362 mr->hw_mr.remote_read = 0; 2363 mr->hw_mr.remote_write = 0; 2364 mr->hw_mr.remote_atomic = 0; 2365 mr->hw_mr.mw_bind = false; 2366 mr->hw_mr.pbl_ptr = 0; 2367 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2368 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2369 mr->hw_mr.fbo = 0; 2370 mr->hw_mr.length = 0; 2371 mr->hw_mr.vaddr = 0; 2372 mr->hw_mr.zbva = false; 2373 mr->hw_mr.phy_mr = true; 2374 mr->hw_mr.dma_mr = false; 2375 2376 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2377 if (rc) { 2378 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2379 goto err1; 2380 } 2381 2382 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2383 mr->ibmr.rkey = mr->ibmr.lkey; 2384 2385 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey); 2386 return mr; 2387 2388 err1: 2389 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2390 err0: 2391 kfree(mr); 2392 return ERR_PTR(rc); 2393 } 2394 2395 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, 2396 enum ib_mr_type mr_type, u32 max_num_sg) 2397 { 2398 struct qedr_dev *dev; 2399 struct qedr_mr *mr; 2400 2401 if (mr_type != IB_MR_TYPE_MEM_REG) 2402 return ERR_PTR(-EINVAL); 2403 2404 mr = __qedr_alloc_mr(ibpd, max_num_sg); 2405 2406 if (IS_ERR(mr)) 2407 return ERR_PTR(-EINVAL); 2408 2409 dev = mr->dev; 2410 2411 return &mr->ibmr; 2412 } 2413 2414 static int qedr_set_page(struct ib_mr *ibmr, u64 addr) 2415 { 2416 struct qedr_mr *mr = get_qedr_mr(ibmr); 2417 struct qedr_pbl *pbl_table; 2418 struct regpair *pbe; 2419 u32 pbes_in_page; 2420 2421 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 2422 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages); 2423 return -ENOMEM; 2424 } 2425 2426 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n", 2427 mr->npages, addr); 2428 2429 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 2430 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 2431 pbe = (struct regpair *)pbl_table->va; 2432 pbe += mr->npages % pbes_in_page; 2433 pbe->lo = cpu_to_le32((u32)addr); 2434 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 2435 2436 mr->npages++; 2437 2438 return 0; 2439 } 2440 2441 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info) 2442 { 2443 int work = info->completed - info->completed_handled - 1; 2444 2445 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work); 2446 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 2447 struct qedr_pbl *pbl; 2448 2449 /* Free all the page list that are possible to be freed 2450 * (all the ones that were invalidated), under the assumption 2451 * that if an FMR was completed successfully that means that 2452 * if there was an invalidate operation before it also ended 2453 */ 2454 pbl = list_first_entry(&info->inuse_pbl_list, 2455 struct qedr_pbl, list_entry); 2456 list_move_tail(&pbl->list_entry, &info->free_pbl_list); 2457 info->completed_handled++; 2458 } 2459 } 2460 2461 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 2462 int sg_nents, unsigned int *sg_offset) 2463 { 2464 struct qedr_mr *mr = get_qedr_mr(ibmr); 2465 2466 mr->npages = 0; 2467 2468 handle_completed_mrs(mr->dev, &mr->info); 2469 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page); 2470 } 2471 2472 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc) 2473 { 2474 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2475 struct qedr_pd *pd = get_qedr_pd(ibpd); 2476 struct qedr_mr *mr; 2477 int rc; 2478 2479 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2480 if (!mr) 2481 return ERR_PTR(-ENOMEM); 2482 2483 mr->type = QEDR_MR_DMA; 2484 2485 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2486 if (rc) { 2487 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2488 goto err1; 2489 } 2490 2491 /* index only, 18 bit long, lkey = itid << 8 | key */ 2492 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2493 mr->hw_mr.pd = pd->pd_id; 2494 mr->hw_mr.local_read = 1; 2495 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2496 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2497 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2498 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2499 mr->hw_mr.dma_mr = true; 2500 2501 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2502 if (rc) { 2503 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2504 goto err2; 2505 } 2506 2507 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2508 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2509 mr->hw_mr.remote_atomic) 2510 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2511 2512 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey); 2513 return &mr->ibmr; 2514 2515 err2: 2516 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2517 err1: 2518 kfree(mr); 2519 return ERR_PTR(rc); 2520 } 2521 2522 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq) 2523 { 2524 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 2525 } 2526 2527 static int sge_data_len(struct ib_sge *sg_list, int num_sge) 2528 { 2529 int i, len = 0; 2530 2531 for (i = 0; i < num_sge; i++) 2532 len += sg_list[i].length; 2533 2534 return len; 2535 } 2536 2537 static void swap_wqe_data64(u64 *p) 2538 { 2539 int i; 2540 2541 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 2542 *p = cpu_to_be64(cpu_to_le64(*p)); 2543 } 2544 2545 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev, 2546 struct qedr_qp *qp, u8 *wqe_size, 2547 struct ib_send_wr *wr, 2548 struct ib_send_wr **bad_wr, u8 *bits, 2549 u8 bit) 2550 { 2551 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 2552 char *seg_prt, *wqe; 2553 int i, seg_siz; 2554 2555 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 2556 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size); 2557 *bad_wr = wr; 2558 return 0; 2559 } 2560 2561 if (!data_size) 2562 return data_size; 2563 2564 *bits |= bit; 2565 2566 seg_prt = NULL; 2567 wqe = NULL; 2568 seg_siz = 0; 2569 2570 /* Copy data inline */ 2571 for (i = 0; i < wr->num_sge; i++) { 2572 u32 len = wr->sg_list[i].length; 2573 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 2574 2575 while (len > 0) { 2576 u32 cur; 2577 2578 /* New segment required */ 2579 if (!seg_siz) { 2580 wqe = (char *)qed_chain_produce(&qp->sq.pbl); 2581 seg_prt = wqe; 2582 seg_siz = sizeof(struct rdma_sq_common_wqe); 2583 (*wqe_size)++; 2584 } 2585 2586 /* Calculate currently allowed length */ 2587 cur = min_t(u32, len, seg_siz); 2588 memcpy(seg_prt, src, cur); 2589 2590 /* Update segment variables */ 2591 seg_prt += cur; 2592 seg_siz -= cur; 2593 2594 /* Update sge variables */ 2595 src += cur; 2596 len -= cur; 2597 2598 /* Swap fully-completed segments */ 2599 if (!seg_siz) 2600 swap_wqe_data64((u64 *)wqe); 2601 } 2602 } 2603 2604 /* swap last not completed segment */ 2605 if (seg_siz) 2606 swap_wqe_data64((u64 *)wqe); 2607 2608 return data_size; 2609 } 2610 2611 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 2612 do { \ 2613 DMA_REGPAIR_LE(sge->addr, vaddr); \ 2614 (sge)->length = cpu_to_le32(vlength); \ 2615 (sge)->flags = cpu_to_le32(vflags); \ 2616 } while (0) 2617 2618 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 2619 do { \ 2620 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \ 2621 (hdr)->num_sges = num_sge; \ 2622 } while (0) 2623 2624 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 2625 do { \ 2626 DMA_REGPAIR_LE(sge->addr, vaddr); \ 2627 (sge)->length = cpu_to_le32(vlength); \ 2628 (sge)->l_key = cpu_to_le32(vlkey); \ 2629 } while (0) 2630 2631 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, 2632 struct ib_send_wr *wr) 2633 { 2634 u32 data_size = 0; 2635 int i; 2636 2637 for (i = 0; i < wr->num_sge; i++) { 2638 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl); 2639 2640 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr); 2641 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 2642 sge->length = cpu_to_le32(wr->sg_list[i].length); 2643 data_size += wr->sg_list[i].length; 2644 } 2645 2646 if (wqe_size) 2647 *wqe_size += wr->num_sge; 2648 2649 return data_size; 2650 } 2651 2652 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev, 2653 struct qedr_qp *qp, 2654 struct rdma_sq_rdma_wqe_1st *rwqe, 2655 struct rdma_sq_rdma_wqe_2nd *rwqe2, 2656 struct ib_send_wr *wr, 2657 struct ib_send_wr **bad_wr) 2658 { 2659 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 2660 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr); 2661 2662 if (wr->send_flags & IB_SEND_INLINE && 2663 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 2664 wr->opcode == IB_WR_RDMA_WRITE)) { 2665 u8 flags = 0; 2666 2667 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 2668 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, 2669 bad_wr, &rwqe->flags, flags); 2670 } 2671 2672 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr); 2673 } 2674 2675 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev, 2676 struct qedr_qp *qp, 2677 struct rdma_sq_send_wqe_1st *swqe, 2678 struct rdma_sq_send_wqe_2st *swqe2, 2679 struct ib_send_wr *wr, 2680 struct ib_send_wr **bad_wr) 2681 { 2682 memset(swqe2, 0, sizeof(*swqe2)); 2683 if (wr->send_flags & IB_SEND_INLINE) { 2684 u8 flags = 0; 2685 2686 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 2687 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, 2688 bad_wr, &swqe->flags, flags); 2689 } 2690 2691 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr); 2692 } 2693 2694 static int qedr_prepare_reg(struct qedr_qp *qp, 2695 struct rdma_sq_fmr_wqe_1st *fwqe1, 2696 struct ib_reg_wr *wr) 2697 { 2698 struct qedr_mr *mr = get_qedr_mr(wr->mr); 2699 struct rdma_sq_fmr_wqe_2nd *fwqe2; 2700 2701 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl); 2702 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 2703 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 2704 fwqe1->l_key = wr->key; 2705 2706 fwqe2->access_ctrl = 0; 2707 2708 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 2709 !!(wr->access & IB_ACCESS_REMOTE_READ)); 2710 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 2711 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 2712 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 2713 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 2714 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 2715 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 2716 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 2717 fwqe2->fmr_ctrl = 0; 2718 2719 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 2720 ilog2(mr->ibmr.page_size) - 12); 2721 2722 fwqe2->length_hi = 0; 2723 fwqe2->length_lo = mr->ibmr.length; 2724 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 2725 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 2726 2727 qp->wqe_wr_id[qp->sq.prod].mr = mr; 2728 2729 return 0; 2730 } 2731 2732 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 2733 { 2734 switch (opcode) { 2735 case IB_WR_RDMA_WRITE: 2736 case IB_WR_RDMA_WRITE_WITH_IMM: 2737 return IB_WC_RDMA_WRITE; 2738 case IB_WR_SEND_WITH_IMM: 2739 case IB_WR_SEND: 2740 case IB_WR_SEND_WITH_INV: 2741 return IB_WC_SEND; 2742 case IB_WR_RDMA_READ: 2743 return IB_WC_RDMA_READ; 2744 case IB_WR_ATOMIC_CMP_AND_SWP: 2745 return IB_WC_COMP_SWAP; 2746 case IB_WR_ATOMIC_FETCH_AND_ADD: 2747 return IB_WC_FETCH_ADD; 2748 case IB_WR_REG_MR: 2749 return IB_WC_REG_MR; 2750 case IB_WR_LOCAL_INV: 2751 return IB_WC_LOCAL_INV; 2752 default: 2753 return IB_WC_SEND; 2754 } 2755 } 2756 2757 static inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr) 2758 { 2759 int wq_is_full, err_wr, pbl_is_full; 2760 struct qedr_dev *dev = qp->dev; 2761 2762 /* prevent SQ overflow and/or processing of a bad WR */ 2763 err_wr = wr->num_sge > qp->sq.max_sges; 2764 wq_is_full = qedr_wq_is_full(&qp->sq); 2765 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) < 2766 QEDR_MAX_SQE_ELEMENTS_PER_SQE; 2767 if (wq_is_full || err_wr || pbl_is_full) { 2768 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) { 2769 DP_ERR(dev, 2770 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n", 2771 qp); 2772 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL; 2773 } 2774 2775 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) { 2776 DP_ERR(dev, 2777 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n", 2778 qp); 2779 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR; 2780 } 2781 2782 if (pbl_is_full && 2783 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) { 2784 DP_ERR(dev, 2785 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n", 2786 qp); 2787 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL; 2788 } 2789 return false; 2790 } 2791 return true; 2792 } 2793 2794 static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 2795 struct ib_send_wr **bad_wr) 2796 { 2797 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 2798 struct qedr_qp *qp = get_qedr_qp(ibqp); 2799 struct rdma_sq_atomic_wqe_1st *awqe1; 2800 struct rdma_sq_atomic_wqe_2nd *awqe2; 2801 struct rdma_sq_atomic_wqe_3rd *awqe3; 2802 struct rdma_sq_send_wqe_2st *swqe2; 2803 struct rdma_sq_local_inv_wqe *iwqe; 2804 struct rdma_sq_rdma_wqe_2nd *rwqe2; 2805 struct rdma_sq_send_wqe_1st *swqe; 2806 struct rdma_sq_rdma_wqe_1st *rwqe; 2807 struct rdma_sq_fmr_wqe_1st *fwqe1; 2808 struct rdma_sq_common_wqe *wqe; 2809 u32 length; 2810 int rc = 0; 2811 bool comp; 2812 2813 if (!qedr_can_post_send(qp, wr)) { 2814 *bad_wr = wr; 2815 return -ENOMEM; 2816 } 2817 2818 wqe = qed_chain_produce(&qp->sq.pbl); 2819 qp->wqe_wr_id[qp->sq.prod].signaled = 2820 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 2821 2822 wqe->flags = 0; 2823 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, 2824 !!(wr->send_flags & IB_SEND_SOLICITED)); 2825 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled; 2826 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 2827 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, 2828 !!(wr->send_flags & IB_SEND_FENCE)); 2829 wqe->prev_wqe_size = qp->prev_wqe_size; 2830 2831 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode); 2832 2833 switch (wr->opcode) { 2834 case IB_WR_SEND_WITH_IMM: 2835 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 2836 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 2837 swqe->wqe_size = 2; 2838 swqe2 = qed_chain_produce(&qp->sq.pbl); 2839 2840 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data); 2841 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 2842 wr, bad_wr); 2843 swqe->length = cpu_to_le32(length); 2844 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 2845 qp->prev_wqe_size = swqe->wqe_size; 2846 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 2847 break; 2848 case IB_WR_SEND: 2849 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 2850 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 2851 2852 swqe->wqe_size = 2; 2853 swqe2 = qed_chain_produce(&qp->sq.pbl); 2854 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 2855 wr, bad_wr); 2856 swqe->length = cpu_to_le32(length); 2857 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 2858 qp->prev_wqe_size = swqe->wqe_size; 2859 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 2860 break; 2861 case IB_WR_SEND_WITH_INV: 2862 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 2863 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 2864 swqe2 = qed_chain_produce(&qp->sq.pbl); 2865 swqe->wqe_size = 2; 2866 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey); 2867 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 2868 wr, bad_wr); 2869 swqe->length = cpu_to_le32(length); 2870 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 2871 qp->prev_wqe_size = swqe->wqe_size; 2872 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 2873 break; 2874 2875 case IB_WR_RDMA_WRITE_WITH_IMM: 2876 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 2877 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 2878 2879 rwqe->wqe_size = 2; 2880 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 2881 rwqe2 = qed_chain_produce(&qp->sq.pbl); 2882 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 2883 wr, bad_wr); 2884 rwqe->length = cpu_to_le32(length); 2885 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 2886 qp->prev_wqe_size = rwqe->wqe_size; 2887 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 2888 break; 2889 case IB_WR_RDMA_WRITE: 2890 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 2891 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 2892 2893 rwqe->wqe_size = 2; 2894 rwqe2 = qed_chain_produce(&qp->sq.pbl); 2895 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 2896 wr, bad_wr); 2897 rwqe->length = cpu_to_le32(length); 2898 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 2899 qp->prev_wqe_size = rwqe->wqe_size; 2900 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 2901 break; 2902 case IB_WR_RDMA_READ_WITH_INV: 2903 DP_ERR(dev, 2904 "RDMA READ WITH INVALIDATE not supported\n"); 2905 *bad_wr = wr; 2906 rc = -EINVAL; 2907 break; 2908 2909 case IB_WR_RDMA_READ: 2910 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 2911 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 2912 2913 rwqe->wqe_size = 2; 2914 rwqe2 = qed_chain_produce(&qp->sq.pbl); 2915 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 2916 wr, bad_wr); 2917 rwqe->length = cpu_to_le32(length); 2918 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 2919 qp->prev_wqe_size = rwqe->wqe_size; 2920 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 2921 break; 2922 2923 case IB_WR_ATOMIC_CMP_AND_SWP: 2924 case IB_WR_ATOMIC_FETCH_AND_ADD: 2925 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe; 2926 awqe1->wqe_size = 4; 2927 2928 awqe2 = qed_chain_produce(&qp->sq.pbl); 2929 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr); 2930 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 2931 2932 awqe3 = qed_chain_produce(&qp->sq.pbl); 2933 2934 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 2935 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 2936 DMA_REGPAIR_LE(awqe3->swap_data, 2937 atomic_wr(wr)->compare_add); 2938 } else { 2939 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 2940 DMA_REGPAIR_LE(awqe3->swap_data, 2941 atomic_wr(wr)->swap); 2942 DMA_REGPAIR_LE(awqe3->cmp_data, 2943 atomic_wr(wr)->compare_add); 2944 } 2945 2946 qedr_prepare_sq_sges(qp, NULL, wr); 2947 2948 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size; 2949 qp->prev_wqe_size = awqe1->wqe_size; 2950 break; 2951 2952 case IB_WR_LOCAL_INV: 2953 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 2954 iwqe->wqe_size = 1; 2955 2956 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 2957 iwqe->inv_l_key = wr->ex.invalidate_rkey; 2958 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size; 2959 qp->prev_wqe_size = iwqe->wqe_size; 2960 break; 2961 case IB_WR_REG_MR: 2962 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n"); 2963 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 2964 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 2965 fwqe1->wqe_size = 2; 2966 2967 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr)); 2968 if (rc) { 2969 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc); 2970 *bad_wr = wr; 2971 break; 2972 } 2973 2974 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 2975 qp->prev_wqe_size = fwqe1->wqe_size; 2976 break; 2977 default: 2978 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode); 2979 rc = -EINVAL; 2980 *bad_wr = wr; 2981 break; 2982 } 2983 2984 if (*bad_wr) { 2985 u16 value; 2986 2987 /* Restore prod to its position before 2988 * this WR was processed 2989 */ 2990 value = le16_to_cpu(qp->sq.db_data.data.value); 2991 qed_chain_set_prod(&qp->sq.pbl, value, wqe); 2992 2993 /* Restore prev_wqe_size */ 2994 qp->prev_wqe_size = wqe->prev_wqe_size; 2995 rc = -EINVAL; 2996 DP_ERR(dev, "POST SEND FAILED\n"); 2997 } 2998 2999 return rc; 3000 } 3001 3002 int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 3003 struct ib_send_wr **bad_wr) 3004 { 3005 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3006 struct qedr_qp *qp = get_qedr_qp(ibqp); 3007 unsigned long flags; 3008 int rc = 0; 3009 3010 *bad_wr = NULL; 3011 3012 if (qp->qp_type == IB_QPT_GSI) 3013 return qedr_gsi_post_send(ibqp, wr, bad_wr); 3014 3015 spin_lock_irqsave(&qp->q_lock, flags); 3016 3017 if ((qp->state != QED_ROCE_QP_STATE_RTS) && 3018 (qp->state != QED_ROCE_QP_STATE_ERR) && 3019 (qp->state != QED_ROCE_QP_STATE_SQD)) { 3020 spin_unlock_irqrestore(&qp->q_lock, flags); 3021 *bad_wr = wr; 3022 DP_DEBUG(dev, QEDR_MSG_CQ, 3023 "QP in wrong state! QP icid=0x%x state %d\n", 3024 qp->icid, qp->state); 3025 return -EINVAL; 3026 } 3027 3028 while (wr) { 3029 rc = __qedr_post_send(ibqp, wr, bad_wr); 3030 if (rc) 3031 break; 3032 3033 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 3034 3035 qedr_inc_sw_prod(&qp->sq); 3036 3037 qp->sq.db_data.data.value++; 3038 3039 wr = wr->next; 3040 } 3041 3042 /* Trigger doorbell 3043 * If there was a failure in the first WR then it will be triggered in 3044 * vane. However this is not harmful (as long as the producer value is 3045 * unchanged). For performance reasons we avoid checking for this 3046 * redundant doorbell. 3047 */ 3048 wmb(); 3049 writel(qp->sq.db_data.raw, qp->sq.db); 3050 3051 /* Make sure write sticks */ 3052 mmiowb(); 3053 3054 spin_unlock_irqrestore(&qp->q_lock, flags); 3055 3056 return rc; 3057 } 3058 3059 int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, 3060 struct ib_recv_wr **bad_wr) 3061 { 3062 struct qedr_qp *qp = get_qedr_qp(ibqp); 3063 struct qedr_dev *dev = qp->dev; 3064 unsigned long flags; 3065 int status = 0; 3066 3067 if (qp->qp_type == IB_QPT_GSI) 3068 return qedr_gsi_post_recv(ibqp, wr, bad_wr); 3069 3070 spin_lock_irqsave(&qp->q_lock, flags); 3071 3072 if (qp->state == QED_ROCE_QP_STATE_RESET) { 3073 spin_unlock_irqrestore(&qp->q_lock, flags); 3074 *bad_wr = wr; 3075 return -EINVAL; 3076 } 3077 3078 while (wr) { 3079 int i; 3080 3081 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) < 3082 QEDR_MAX_RQE_ELEMENTS_PER_RQE || 3083 wr->num_sge > qp->rq.max_sges) { 3084 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n", 3085 qed_chain_get_elem_left_u32(&qp->rq.pbl), 3086 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge, 3087 qp->rq.max_sges); 3088 status = -ENOMEM; 3089 *bad_wr = wr; 3090 break; 3091 } 3092 for (i = 0; i < wr->num_sge; i++) { 3093 u32 flags = 0; 3094 struct rdma_rq_sge *rqe = 3095 qed_chain_produce(&qp->rq.pbl); 3096 3097 /* First one must include the number 3098 * of SGE in the list 3099 */ 3100 if (!i) 3101 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 3102 wr->num_sge); 3103 3104 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 3105 wr->sg_list[i].lkey); 3106 3107 RQ_SGE_SET(rqe, wr->sg_list[i].addr, 3108 wr->sg_list[i].length, flags); 3109 } 3110 3111 /* Special case of no sges. FW requires between 1-4 sges... 3112 * in this case we need to post 1 sge with length zero. this is 3113 * because rdma write with immediate consumes an RQ. 3114 */ 3115 if (!wr->num_sge) { 3116 u32 flags = 0; 3117 struct rdma_rq_sge *rqe = 3118 qed_chain_produce(&qp->rq.pbl); 3119 3120 /* First one must include the number 3121 * of SGE in the list 3122 */ 3123 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0); 3124 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 3125 3126 RQ_SGE_SET(rqe, 0, 0, flags); 3127 i = 1; 3128 } 3129 3130 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 3131 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 3132 3133 qedr_inc_sw_prod(&qp->rq); 3134 3135 /* Flush all the writes before signalling doorbell */ 3136 wmb(); 3137 3138 qp->rq.db_data.data.value++; 3139 3140 writel(qp->rq.db_data.raw, qp->rq.db); 3141 3142 /* Make sure write sticks */ 3143 mmiowb(); 3144 3145 wr = wr->next; 3146 } 3147 3148 spin_unlock_irqrestore(&qp->q_lock, flags); 3149 3150 return status; 3151 } 3152 3153 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe) 3154 { 3155 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3156 3157 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) == 3158 cq->pbl_toggle; 3159 } 3160 3161 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe) 3162 { 3163 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3164 struct qedr_qp *qp; 3165 3166 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi, 3167 resp_cqe->qp_handle.lo, 3168 u64); 3169 return qp; 3170 } 3171 3172 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe) 3173 { 3174 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3175 3176 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 3177 } 3178 3179 /* Return latest CQE (needs processing) */ 3180 static union rdma_cqe *get_cqe(struct qedr_cq *cq) 3181 { 3182 return cq->latest_cqe; 3183 } 3184 3185 /* In fmr we need to increase the number of fmr completed counter for the fmr 3186 * algorithm determining whether we can free a pbl or not. 3187 * we need to perform this whether the work request was signaled or not. for 3188 * this purpose we call this function from the condition that checks if a wr 3189 * should be skipped, to make sure we don't miss it ( possibly this fmr 3190 * operation was not signalted) 3191 */ 3192 static inline void qedr_chk_if_fmr(struct qedr_qp *qp) 3193 { 3194 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 3195 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3196 } 3197 3198 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, 3199 struct qedr_cq *cq, int num_entries, 3200 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, 3201 int force) 3202 { 3203 u16 cnt = 0; 3204 3205 while (num_entries && qp->sq.wqe_cons != hw_cons) { 3206 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 3207 qedr_chk_if_fmr(qp); 3208 /* skip WC */ 3209 goto next_cqe; 3210 } 3211 3212 /* fill WC */ 3213 wc->status = status; 3214 wc->vendor_err = 0; 3215 wc->wc_flags = 0; 3216 wc->src_qp = qp->id; 3217 wc->qp = &qp->ibqp; 3218 3219 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 3220 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 3221 3222 switch (wc->opcode) { 3223 case IB_WC_RDMA_WRITE: 3224 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3225 break; 3226 case IB_WC_COMP_SWAP: 3227 case IB_WC_FETCH_ADD: 3228 wc->byte_len = 8; 3229 break; 3230 case IB_WC_REG_MR: 3231 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3232 break; 3233 case IB_WC_RDMA_READ: 3234 case IB_WC_SEND: 3235 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3236 break; 3237 default: 3238 break; 3239 } 3240 3241 num_entries--; 3242 wc++; 3243 cnt++; 3244 next_cqe: 3245 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 3246 qed_chain_consume(&qp->sq.pbl); 3247 qedr_inc_sw_cons(&qp->sq); 3248 } 3249 3250 return cnt; 3251 } 3252 3253 static int qedr_poll_cq_req(struct qedr_dev *dev, 3254 struct qedr_qp *qp, struct qedr_cq *cq, 3255 int num_entries, struct ib_wc *wc, 3256 struct rdma_cqe_requester *req) 3257 { 3258 int cnt = 0; 3259 3260 switch (req->status) { 3261 case RDMA_CQE_REQ_STS_OK: 3262 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3263 IB_WC_SUCCESS, 0); 3264 break; 3265 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 3266 if (qp->state != QED_ROCE_QP_STATE_ERR) 3267 DP_ERR(dev, 3268 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3269 cq->icid, qp->icid); 3270 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3271 IB_WC_WR_FLUSH_ERR, 1); 3272 break; 3273 default: 3274 /* process all WQE before the cosumer */ 3275 qp->state = QED_ROCE_QP_STATE_ERR; 3276 cnt = process_req(dev, qp, cq, num_entries, wc, 3277 req->sq_cons - 1, IB_WC_SUCCESS, 0); 3278 wc += cnt; 3279 /* if we have extra WC fill it with actual error info */ 3280 if (cnt < num_entries) { 3281 enum ib_wc_status wc_status; 3282 3283 switch (req->status) { 3284 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 3285 DP_ERR(dev, 3286 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3287 cq->icid, qp->icid); 3288 wc_status = IB_WC_BAD_RESP_ERR; 3289 break; 3290 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 3291 DP_ERR(dev, 3292 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3293 cq->icid, qp->icid); 3294 wc_status = IB_WC_LOC_LEN_ERR; 3295 break; 3296 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 3297 DP_ERR(dev, 3298 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3299 cq->icid, qp->icid); 3300 wc_status = IB_WC_LOC_QP_OP_ERR; 3301 break; 3302 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 3303 DP_ERR(dev, 3304 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3305 cq->icid, qp->icid); 3306 wc_status = IB_WC_LOC_PROT_ERR; 3307 break; 3308 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 3309 DP_ERR(dev, 3310 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3311 cq->icid, qp->icid); 3312 wc_status = IB_WC_MW_BIND_ERR; 3313 break; 3314 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 3315 DP_ERR(dev, 3316 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3317 cq->icid, qp->icid); 3318 wc_status = IB_WC_REM_INV_REQ_ERR; 3319 break; 3320 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 3321 DP_ERR(dev, 3322 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3323 cq->icid, qp->icid); 3324 wc_status = IB_WC_REM_ACCESS_ERR; 3325 break; 3326 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 3327 DP_ERR(dev, 3328 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3329 cq->icid, qp->icid); 3330 wc_status = IB_WC_REM_OP_ERR; 3331 break; 3332 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 3333 DP_ERR(dev, 3334 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3335 cq->icid, qp->icid); 3336 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 3337 break; 3338 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 3339 DP_ERR(dev, 3340 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3341 cq->icid, qp->icid); 3342 wc_status = IB_WC_RETRY_EXC_ERR; 3343 break; 3344 default: 3345 DP_ERR(dev, 3346 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3347 cq->icid, qp->icid); 3348 wc_status = IB_WC_GENERAL_ERR; 3349 } 3350 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 3351 wc_status, 1); 3352 } 3353 } 3354 3355 return cnt; 3356 } 3357 3358 static inline int qedr_cqe_resp_status_to_ib(u8 status) 3359 { 3360 switch (status) { 3361 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 3362 return IB_WC_LOC_ACCESS_ERR; 3363 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 3364 return IB_WC_LOC_LEN_ERR; 3365 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 3366 return IB_WC_LOC_QP_OP_ERR; 3367 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 3368 return IB_WC_LOC_PROT_ERR; 3369 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 3370 return IB_WC_MW_BIND_ERR; 3371 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 3372 return IB_WC_REM_INV_RD_REQ_ERR; 3373 case RDMA_CQE_RESP_STS_OK: 3374 return IB_WC_SUCCESS; 3375 default: 3376 return IB_WC_GENERAL_ERR; 3377 } 3378 } 3379 3380 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp, 3381 struct ib_wc *wc) 3382 { 3383 wc->status = IB_WC_SUCCESS; 3384 wc->byte_len = le32_to_cpu(resp->length); 3385 3386 if (resp->flags & QEDR_RESP_IMM) { 3387 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 3388 wc->wc_flags |= IB_WC_WITH_IMM; 3389 3390 if (resp->flags & QEDR_RESP_RDMA) 3391 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 3392 3393 if (resp->flags & QEDR_RESP_INV) 3394 return -EINVAL; 3395 3396 } else if (resp->flags & QEDR_RESP_INV) { 3397 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 3398 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3399 3400 if (resp->flags & QEDR_RESP_RDMA) 3401 return -EINVAL; 3402 3403 } else if (resp->flags & QEDR_RESP_RDMA) { 3404 return -EINVAL; 3405 } 3406 3407 return 0; 3408 } 3409 3410 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 3411 struct qedr_cq *cq, struct ib_wc *wc, 3412 struct rdma_cqe_responder *resp, u64 wr_id) 3413 { 3414 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */ 3415 wc->opcode = IB_WC_RECV; 3416 wc->wc_flags = 0; 3417 3418 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) { 3419 if (qedr_set_ok_cqe_resp_wc(resp, wc)) 3420 DP_ERR(dev, 3421 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n", 3422 cq, cq->icid, resp->flags); 3423 3424 } else { 3425 wc->status = qedr_cqe_resp_status_to_ib(resp->status); 3426 if (wc->status == IB_WC_GENERAL_ERR) 3427 DP_ERR(dev, 3428 "CQ %p (icid=%d) contains an invalid CQE status %d\n", 3429 cq, cq->icid, resp->status); 3430 } 3431 3432 /* Fill the rest of the WC */ 3433 wc->vendor_err = 0; 3434 wc->src_qp = qp->id; 3435 wc->qp = &qp->ibqp; 3436 wc->wr_id = wr_id; 3437 } 3438 3439 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 3440 struct qedr_cq *cq, struct ib_wc *wc, 3441 struct rdma_cqe_responder *resp) 3442 { 3443 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 3444 3445 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 3446 3447 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 3448 qed_chain_consume(&qp->rq.pbl); 3449 qedr_inc_sw_cons(&qp->rq); 3450 3451 return 1; 3452 } 3453 3454 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, 3455 int num_entries, struct ib_wc *wc, u16 hw_cons) 3456 { 3457 u16 cnt = 0; 3458 3459 while (num_entries && qp->rq.wqe_cons != hw_cons) { 3460 /* fill WC */ 3461 wc->status = IB_WC_WR_FLUSH_ERR; 3462 wc->vendor_err = 0; 3463 wc->wc_flags = 0; 3464 wc->src_qp = qp->id; 3465 wc->byte_len = 0; 3466 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 3467 wc->qp = &qp->ibqp; 3468 num_entries--; 3469 wc++; 3470 cnt++; 3471 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 3472 qed_chain_consume(&qp->rq.pbl); 3473 qedr_inc_sw_cons(&qp->rq); 3474 } 3475 3476 return cnt; 3477 } 3478 3479 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 3480 struct rdma_cqe_responder *resp, int *update) 3481 { 3482 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) { 3483 consume_cqe(cq); 3484 *update |= 1; 3485 } 3486 } 3487 3488 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, 3489 struct qedr_cq *cq, int num_entries, 3490 struct ib_wc *wc, struct rdma_cqe_responder *resp, 3491 int *update) 3492 { 3493 int cnt; 3494 3495 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 3496 cnt = process_resp_flush(qp, cq, num_entries, wc, 3497 resp->rq_cons); 3498 try_consume_resp_cqe(cq, qp, resp, update); 3499 } else { 3500 cnt = process_resp_one(dev, qp, cq, wc, resp); 3501 consume_cqe(cq); 3502 *update |= 1; 3503 } 3504 3505 return cnt; 3506 } 3507 3508 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 3509 struct rdma_cqe_requester *req, int *update) 3510 { 3511 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 3512 consume_cqe(cq); 3513 *update |= 1; 3514 } 3515 } 3516 3517 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 3518 { 3519 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 3520 struct qedr_cq *cq = get_qedr_cq(ibcq); 3521 union rdma_cqe *cqe = cq->latest_cqe; 3522 u32 old_cons, new_cons; 3523 unsigned long flags; 3524 int update = 0; 3525 int done = 0; 3526 3527 if (cq->destroyed) { 3528 DP_ERR(dev, 3529 "warning: poll was invoked after destroy for cq %p (icid=%d)\n", 3530 cq, cq->icid); 3531 return 0; 3532 } 3533 3534 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 3535 return qedr_gsi_poll_cq(ibcq, num_entries, wc); 3536 3537 spin_lock_irqsave(&cq->cq_lock, flags); 3538 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 3539 while (num_entries && is_valid_cqe(cq, cqe)) { 3540 struct qedr_qp *qp; 3541 int cnt = 0; 3542 3543 /* prevent speculative reads of any field of CQE */ 3544 rmb(); 3545 3546 qp = cqe_get_qp(cqe); 3547 if (!qp) { 3548 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe); 3549 break; 3550 } 3551 3552 wc->qp = &qp->ibqp; 3553 3554 switch (cqe_get_type(cqe)) { 3555 case RDMA_CQE_TYPE_REQUESTER: 3556 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc, 3557 &cqe->req); 3558 try_consume_req_cqe(cq, qp, &cqe->req, &update); 3559 break; 3560 case RDMA_CQE_TYPE_RESPONDER_RQ: 3561 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc, 3562 &cqe->resp, &update); 3563 break; 3564 case RDMA_CQE_TYPE_INVALID: 3565 default: 3566 DP_ERR(dev, "Error: invalid CQE type = %d\n", 3567 cqe_get_type(cqe)); 3568 } 3569 num_entries -= cnt; 3570 wc += cnt; 3571 done += cnt; 3572 3573 cqe = get_cqe(cq); 3574 } 3575 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 3576 3577 cq->cq_cons += new_cons - old_cons; 3578 3579 if (update) 3580 /* doorbell notifies abount latest VALID entry, 3581 * but chain already point to the next INVALID one 3582 */ 3583 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 3584 3585 spin_unlock_irqrestore(&cq->cq_lock, flags); 3586 return done; 3587 } 3588 3589 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, 3590 u8 port_num, 3591 const struct ib_wc *in_wc, 3592 const struct ib_grh *in_grh, 3593 const struct ib_mad_hdr *mad_hdr, 3594 size_t in_mad_size, struct ib_mad_hdr *out_mad, 3595 size_t *out_mad_size, u16 *out_mad_pkey_index) 3596 { 3597 struct qedr_dev *dev = get_qedr_dev(ibdev); 3598 3599 DP_DEBUG(dev, QEDR_MSG_GSI, 3600 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n", 3601 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod, 3602 mad_hdr->class_specific, mad_hdr->class_version, 3603 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status); 3604 return IB_MAD_RESULT_SUCCESS; 3605 } 3606 3607 int qedr_port_immutable(struct ib_device *ibdev, u8 port_num, 3608 struct ib_port_immutable *immutable) 3609 { 3610 struct ib_port_attr attr; 3611 int err; 3612 3613 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE | 3614 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 3615 3616 err = ib_query_port(ibdev, port_num, &attr); 3617 if (err) 3618 return err; 3619 3620 immutable->pkey_tbl_len = attr.pkey_tbl_len; 3621 immutable->gid_tbl_len = attr.gid_tbl_len; 3622 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 3623 3624 return 0; 3625 } 3626