1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/dma-mapping.h> 33 #include <linux/crc32.h> 34 #include <net/ip.h> 35 #include <net/ipv6.h> 36 #include <net/udp.h> 37 #include <linux/iommu.h> 38 39 #include <rdma/ib_verbs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/iw_cm.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/ib_addr.h> 44 #include <rdma/ib_cache.h> 45 46 #include <linux/qed/common_hsi.h> 47 #include "qedr_hsi_rdma.h" 48 #include <linux/qed/qed_if.h> 49 #include "qedr.h" 50 #include "verbs.h" 51 #include <rdma/qedr-abi.h> 52 #include "qedr_cm.h" 53 54 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 55 56 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, 57 size_t len) 58 { 59 size_t min_len = min_t(size_t, len, udata->outlen); 60 61 return ib_copy_to_udata(udata, src, min_len); 62 } 63 64 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 65 { 66 if (index > QEDR_ROCE_PKEY_TABLE_LEN) 67 return -EINVAL; 68 69 *pkey = QEDR_ROCE_PKEY_DEFAULT; 70 return 0; 71 } 72 73 int qedr_query_gid(struct ib_device *ibdev, u8 port, int index, 74 union ib_gid *sgid) 75 { 76 struct qedr_dev *dev = get_qedr_dev(ibdev); 77 int rc = 0; 78 79 if (!rdma_cap_roce_gid_table(ibdev, port)) 80 return -ENODEV; 81 82 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL); 83 if (rc == -EAGAIN) { 84 memcpy(sgid, &zgid, sizeof(*sgid)); 85 return 0; 86 } 87 88 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index, 89 sgid->global.interface_id, sgid->global.subnet_prefix); 90 91 return rc; 92 } 93 94 int qedr_add_gid(struct ib_device *device, u8 port_num, 95 unsigned int index, const union ib_gid *gid, 96 const struct ib_gid_attr *attr, void **context) 97 { 98 if (!rdma_cap_roce_gid_table(device, port_num)) 99 return -EINVAL; 100 101 if (port_num > QEDR_MAX_PORT) 102 return -EINVAL; 103 104 if (!context) 105 return -EINVAL; 106 107 return 0; 108 } 109 110 int qedr_del_gid(struct ib_device *device, u8 port_num, 111 unsigned int index, void **context) 112 { 113 if (!rdma_cap_roce_gid_table(device, port_num)) 114 return -EINVAL; 115 116 if (port_num > QEDR_MAX_PORT) 117 return -EINVAL; 118 119 if (!context) 120 return -EINVAL; 121 122 return 0; 123 } 124 125 int qedr_query_device(struct ib_device *ibdev, 126 struct ib_device_attr *attr, struct ib_udata *udata) 127 { 128 struct qedr_dev *dev = get_qedr_dev(ibdev); 129 struct qedr_device_attr *qattr = &dev->attr; 130 131 if (!dev->rdma_ctx) { 132 DP_ERR(dev, 133 "qedr_query_device called with invalid params rdma_ctx=%p\n", 134 dev->rdma_ctx); 135 return -EINVAL; 136 } 137 138 memset(attr, 0, sizeof(*attr)); 139 140 attr->fw_ver = qattr->fw_ver; 141 attr->sys_image_guid = qattr->sys_image_guid; 142 attr->max_mr_size = qattr->max_mr_size; 143 attr->page_size_cap = qattr->page_size_caps; 144 attr->vendor_id = qattr->vendor_id; 145 attr->vendor_part_id = qattr->vendor_part_id; 146 attr->hw_ver = qattr->hw_ver; 147 attr->max_qp = qattr->max_qp; 148 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe); 149 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 150 IB_DEVICE_RC_RNR_NAK_GEN | 151 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS; 152 153 attr->max_sge = qattr->max_sge; 154 attr->max_sge_rd = qattr->max_sge; 155 attr->max_cq = qattr->max_cq; 156 attr->max_cqe = qattr->max_cqe; 157 attr->max_mr = qattr->max_mr; 158 attr->max_mw = qattr->max_mw; 159 attr->max_pd = qattr->max_pd; 160 attr->atomic_cap = dev->atomic_cap; 161 attr->max_fmr = qattr->max_fmr; 162 attr->max_map_per_fmr = 16; 163 attr->max_qp_init_rd_atom = 164 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 165 attr->max_qp_rd_atom = 166 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 167 attr->max_qp_init_rd_atom); 168 169 attr->max_srq = qattr->max_srq; 170 attr->max_srq_sge = qattr->max_srq_sge; 171 attr->max_srq_wr = qattr->max_srq_wr; 172 173 attr->local_ca_ack_delay = qattr->dev_ack_delay; 174 attr->max_fast_reg_page_list_len = qattr->max_mr / 8; 175 attr->max_pkeys = QEDR_ROCE_PKEY_MAX; 176 attr->max_ah = qattr->max_ah; 177 178 return 0; 179 } 180 181 #define QEDR_SPEED_SDR (1) 182 #define QEDR_SPEED_DDR (2) 183 #define QEDR_SPEED_QDR (4) 184 #define QEDR_SPEED_FDR10 (8) 185 #define QEDR_SPEED_FDR (16) 186 #define QEDR_SPEED_EDR (32) 187 188 static inline void get_link_speed_and_width(int speed, u8 *ib_speed, 189 u8 *ib_width) 190 { 191 switch (speed) { 192 case 1000: 193 *ib_speed = QEDR_SPEED_SDR; 194 *ib_width = IB_WIDTH_1X; 195 break; 196 case 10000: 197 *ib_speed = QEDR_SPEED_QDR; 198 *ib_width = IB_WIDTH_1X; 199 break; 200 201 case 20000: 202 *ib_speed = QEDR_SPEED_DDR; 203 *ib_width = IB_WIDTH_4X; 204 break; 205 206 case 25000: 207 *ib_speed = QEDR_SPEED_EDR; 208 *ib_width = IB_WIDTH_1X; 209 break; 210 211 case 40000: 212 *ib_speed = QEDR_SPEED_QDR; 213 *ib_width = IB_WIDTH_4X; 214 break; 215 216 case 50000: 217 *ib_speed = QEDR_SPEED_QDR; 218 *ib_width = IB_WIDTH_4X; 219 break; 220 221 case 100000: 222 *ib_speed = QEDR_SPEED_EDR; 223 *ib_width = IB_WIDTH_4X; 224 break; 225 226 default: 227 /* Unsupported */ 228 *ib_speed = QEDR_SPEED_SDR; 229 *ib_width = IB_WIDTH_1X; 230 } 231 } 232 233 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr) 234 { 235 struct qedr_dev *dev; 236 struct qed_rdma_port *rdma_port; 237 238 dev = get_qedr_dev(ibdev); 239 if (port > 1) { 240 DP_ERR(dev, "invalid_port=0x%x\n", port); 241 return -EINVAL; 242 } 243 244 if (!dev->rdma_ctx) { 245 DP_ERR(dev, "rdma_ctx is NULL\n"); 246 return -EINVAL; 247 } 248 249 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx); 250 251 /* *attr being zeroed by the caller, avoid zeroing it here */ 252 if (rdma_port->port_state == QED_RDMA_PORT_UP) { 253 attr->state = IB_PORT_ACTIVE; 254 attr->phys_state = 5; 255 } else { 256 attr->state = IB_PORT_DOWN; 257 attr->phys_state = 3; 258 } 259 attr->max_mtu = IB_MTU_4096; 260 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); 261 attr->lid = 0; 262 attr->lmc = 0; 263 attr->sm_lid = 0; 264 attr->sm_sl = 0; 265 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS; 266 attr->gid_tbl_len = QEDR_MAX_SGID; 267 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; 268 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 269 attr->qkey_viol_cntr = 0; 270 get_link_speed_and_width(rdma_port->link_speed, 271 &attr->active_speed, &attr->active_width); 272 attr->max_msg_sz = rdma_port->max_msg_size; 273 attr->max_vl_num = 4; 274 275 return 0; 276 } 277 278 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask, 279 struct ib_port_modify *props) 280 { 281 struct qedr_dev *dev; 282 283 dev = get_qedr_dev(ibdev); 284 if (port > 1) { 285 DP_ERR(dev, "invalid_port=0x%x\n", port); 286 return -EINVAL; 287 } 288 289 return 0; 290 } 291 292 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr, 293 unsigned long len) 294 { 295 struct qedr_mm *mm; 296 297 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 298 if (!mm) 299 return -ENOMEM; 300 301 mm->key.phy_addr = phy_addr; 302 /* This function might be called with a length which is not a multiple 303 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel 304 * forces this granularity by increasing the requested size if needed. 305 * When qedr_mmap is called, it will search the list with the updated 306 * length as a key. To prevent search failures, the length is rounded up 307 * in advance to PAGE_SIZE. 308 */ 309 mm->key.len = roundup(len, PAGE_SIZE); 310 INIT_LIST_HEAD(&mm->entry); 311 312 mutex_lock(&uctx->mm_list_lock); 313 list_add(&mm->entry, &uctx->mm_head); 314 mutex_unlock(&uctx->mm_list_lock); 315 316 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 317 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n", 318 (unsigned long long)mm->key.phy_addr, 319 (unsigned long)mm->key.len, uctx); 320 321 return 0; 322 } 323 324 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr, 325 unsigned long len) 326 { 327 bool found = false; 328 struct qedr_mm *mm; 329 330 mutex_lock(&uctx->mm_list_lock); 331 list_for_each_entry(mm, &uctx->mm_head, entry) { 332 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 333 continue; 334 335 found = true; 336 break; 337 } 338 mutex_unlock(&uctx->mm_list_lock); 339 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 340 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n", 341 mm->key.phy_addr, mm->key.len, uctx, found); 342 343 return found; 344 } 345 346 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev, 347 struct ib_udata *udata) 348 { 349 int rc; 350 struct qedr_ucontext *ctx; 351 struct qedr_alloc_ucontext_resp uresp; 352 struct qedr_dev *dev = get_qedr_dev(ibdev); 353 struct qed_rdma_add_user_out_params oparams; 354 355 if (!udata) 356 return ERR_PTR(-EFAULT); 357 358 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 359 if (!ctx) 360 return ERR_PTR(-ENOMEM); 361 362 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams); 363 if (rc) { 364 DP_ERR(dev, 365 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n", 366 rc); 367 goto err; 368 } 369 370 ctx->dpi = oparams.dpi; 371 ctx->dpi_addr = oparams.dpi_addr; 372 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 373 ctx->dpi_size = oparams.dpi_size; 374 INIT_LIST_HEAD(&ctx->mm_head); 375 mutex_init(&ctx->mm_list_lock); 376 377 memset(&uresp, 0, sizeof(uresp)); 378 379 uresp.db_pa = ctx->dpi_phys_addr; 380 uresp.db_size = ctx->dpi_size; 381 uresp.max_send_wr = dev->attr.max_sqe; 382 uresp.max_recv_wr = dev->attr.max_rqe; 383 uresp.max_srq_wr = dev->attr.max_srq_wr; 384 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE; 385 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE; 386 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 387 uresp.max_cqes = QEDR_MAX_CQES; 388 389 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 390 if (rc) 391 goto err; 392 393 ctx->dev = dev; 394 395 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size); 396 if (rc) 397 goto err; 398 399 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n", 400 &ctx->ibucontext); 401 return &ctx->ibucontext; 402 403 err: 404 kfree(ctx); 405 return ERR_PTR(rc); 406 } 407 408 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx) 409 { 410 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx); 411 struct qedr_mm *mm, *tmp; 412 int status = 0; 413 414 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n", 415 uctx); 416 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi); 417 418 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 419 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 420 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n", 421 mm->key.phy_addr, mm->key.len, uctx); 422 list_del(&mm->entry); 423 kfree(mm); 424 } 425 426 kfree(uctx); 427 return status; 428 } 429 430 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 431 { 432 struct qedr_ucontext *ucontext = get_qedr_ucontext(context); 433 struct qedr_dev *dev = get_qedr_dev(context->device); 434 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 435 u64 unmapped_db = dev->db_phys_addr; 436 unsigned long len = (vma->vm_end - vma->vm_start); 437 int rc = 0; 438 bool found; 439 440 DP_DEBUG(dev, QEDR_MSG_INIT, 441 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n", 442 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len); 443 if (vma->vm_start & (PAGE_SIZE - 1)) { 444 DP_ERR(dev, "Vma_start not page aligned = %ld\n", 445 vma->vm_start); 446 return -EINVAL; 447 } 448 449 found = qedr_search_mmap(ucontext, vm_page, len); 450 if (!found) { 451 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n", 452 vma->vm_pgoff); 453 return -EINVAL; 454 } 455 456 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n"); 457 458 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + 459 dev->db_size))) { 460 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n"); 461 if (vma->vm_flags & VM_READ) { 462 DP_ERR(dev, "Trying to map doorbell bar for read\n"); 463 return -EPERM; 464 } 465 466 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 467 468 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 469 PAGE_SIZE, vma->vm_page_prot); 470 } else { 471 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n"); 472 rc = remap_pfn_range(vma, vma->vm_start, 473 vma->vm_pgoff, len, vma->vm_page_prot); 474 } 475 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc); 476 return rc; 477 } 478 479 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev, 480 struct ib_ucontext *context, struct ib_udata *udata) 481 { 482 struct qedr_dev *dev = get_qedr_dev(ibdev); 483 struct qedr_pd *pd; 484 u16 pd_id; 485 int rc; 486 487 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n", 488 (udata && context) ? "User Lib" : "Kernel"); 489 490 if (!dev->rdma_ctx) { 491 DP_ERR(dev, "invlaid RDMA context\n"); 492 return ERR_PTR(-EINVAL); 493 } 494 495 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 496 if (!pd) 497 return ERR_PTR(-ENOMEM); 498 499 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id); 500 if (rc) 501 goto err; 502 503 pd->pd_id = pd_id; 504 505 if (udata && context) { 506 struct qedr_alloc_pd_uresp uresp; 507 508 uresp.pd_id = pd_id; 509 510 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 511 if (rc) { 512 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); 513 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); 514 goto err; 515 } 516 517 pd->uctx = get_qedr_ucontext(context); 518 pd->uctx->pd = pd; 519 } 520 521 return &pd->ibpd; 522 523 err: 524 kfree(pd); 525 return ERR_PTR(rc); 526 } 527 528 int qedr_dealloc_pd(struct ib_pd *ibpd) 529 { 530 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 531 struct qedr_pd *pd = get_qedr_pd(ibpd); 532 533 if (!pd) { 534 pr_err("Invalid PD received in dealloc_pd\n"); 535 return -EINVAL; 536 } 537 538 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id); 539 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id); 540 541 kfree(pd); 542 543 return 0; 544 } 545 546 static void qedr_free_pbl(struct qedr_dev *dev, 547 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) 548 { 549 struct pci_dev *pdev = dev->pdev; 550 int i; 551 552 for (i = 0; i < pbl_info->num_pbls; i++) { 553 if (!pbl[i].va) 554 continue; 555 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 556 pbl[i].va, pbl[i].pa); 557 } 558 559 kfree(pbl); 560 } 561 562 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024) 563 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024) 564 565 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 566 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 567 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE) 568 569 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev, 570 struct qedr_pbl_info *pbl_info, 571 gfp_t flags) 572 { 573 struct pci_dev *pdev = dev->pdev; 574 struct qedr_pbl *pbl_table; 575 dma_addr_t *pbl_main_tbl; 576 dma_addr_t pa; 577 void *va; 578 int i; 579 580 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags); 581 if (!pbl_table) 582 return ERR_PTR(-ENOMEM); 583 584 for (i = 0; i < pbl_info->num_pbls; i++) { 585 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, 586 &pa, flags); 587 if (!va) 588 goto err; 589 590 memset(va, 0, pbl_info->pbl_size); 591 pbl_table[i].va = va; 592 pbl_table[i].pa = pa; 593 } 594 595 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 596 * the first one with physical pointers to all of the rest 597 */ 598 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 599 for (i = 0; i < pbl_info->num_pbls - 1; i++) 600 pbl_main_tbl[i] = pbl_table[i + 1].pa; 601 602 return pbl_table; 603 604 err: 605 for (i--; i >= 0; i--) 606 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 607 pbl_table[i].va, pbl_table[i].pa); 608 609 qedr_free_pbl(dev, pbl_info, pbl_table); 610 611 return ERR_PTR(-ENOMEM); 612 } 613 614 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev, 615 struct qedr_pbl_info *pbl_info, 616 u32 num_pbes, int two_layer_capable) 617 { 618 u32 pbl_capacity; 619 u32 pbl_size; 620 u32 num_pbls; 621 622 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 623 if (num_pbes > MAX_PBES_TWO_LAYER) { 624 DP_ERR(dev, "prepare pbl table: too many pages %d\n", 625 num_pbes); 626 return -EINVAL; 627 } 628 629 /* calculate required pbl page size */ 630 pbl_size = MIN_FW_PBL_PAGE_SIZE; 631 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 632 NUM_PBES_ON_PAGE(pbl_size); 633 634 while (pbl_capacity < num_pbes) { 635 pbl_size *= 2; 636 pbl_capacity = pbl_size / sizeof(u64); 637 pbl_capacity = pbl_capacity * pbl_capacity; 638 } 639 640 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 641 num_pbls++; /* One for the layer0 ( points to the pbls) */ 642 pbl_info->two_layered = true; 643 } else { 644 /* One layered PBL */ 645 num_pbls = 1; 646 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, 647 roundup_pow_of_two((num_pbes * sizeof(u64)))); 648 pbl_info->two_layered = false; 649 } 650 651 pbl_info->num_pbls = num_pbls; 652 pbl_info->pbl_size = pbl_size; 653 pbl_info->num_pbes = num_pbes; 654 655 DP_DEBUG(dev, QEDR_MSG_MR, 656 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n", 657 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 658 659 return 0; 660 } 661 662 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, 663 struct qedr_pbl *pbl, 664 struct qedr_pbl_info *pbl_info, u32 pg_shift) 665 { 666 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 667 u32 fw_pg_cnt, fw_pg_per_umem_pg; 668 struct qedr_pbl *pbl_tbl; 669 struct scatterlist *sg; 670 struct regpair *pbe; 671 u64 pg_addr; 672 int entry; 673 674 if (!pbl_info->num_pbes) 675 return; 676 677 /* If we have a two layered pbl, the first pbl points to the rest 678 * of the pbls and the first entry lays on the second pbl in the table 679 */ 680 if (pbl_info->two_layered) 681 pbl_tbl = &pbl[1]; 682 else 683 pbl_tbl = pbl; 684 685 pbe = (struct regpair *)pbl_tbl->va; 686 if (!pbe) { 687 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n"); 688 return; 689 } 690 691 pbe_cnt = 0; 692 693 shift = umem->page_shift; 694 695 fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift); 696 697 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 698 pages = sg_dma_len(sg) >> shift; 699 pg_addr = sg_dma_address(sg); 700 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 701 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) { 702 pbe->lo = cpu_to_le32(pg_addr); 703 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); 704 705 pg_addr += BIT(pg_shift); 706 pbe_cnt++; 707 total_num_pbes++; 708 pbe++; 709 710 if (total_num_pbes == pbl_info->num_pbes) 711 return; 712 713 /* If the given pbl is full storing the pbes, 714 * move to next pbl. 715 */ 716 if (pbe_cnt == 717 (pbl_info->pbl_size / sizeof(u64))) { 718 pbl_tbl++; 719 pbe = (struct regpair *)pbl_tbl->va; 720 pbe_cnt = 0; 721 } 722 723 fw_pg_cnt++; 724 } 725 } 726 } 727 } 728 729 static int qedr_copy_cq_uresp(struct qedr_dev *dev, 730 struct qedr_cq *cq, struct ib_udata *udata) 731 { 732 struct qedr_create_cq_uresp uresp; 733 int rc; 734 735 memset(&uresp, 0, sizeof(uresp)); 736 737 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 738 uresp.icid = cq->icid; 739 740 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 741 if (rc) 742 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); 743 744 return rc; 745 } 746 747 static void consume_cqe(struct qedr_cq *cq) 748 { 749 if (cq->latest_cqe == cq->toggle_cqe) 750 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 751 752 cq->latest_cqe = qed_chain_consume(&cq->pbl); 753 } 754 755 static inline int qedr_align_cq_entries(int entries) 756 { 757 u64 size, aligned_size; 758 759 /* We allocate an extra entry that we don't report to the FW. */ 760 size = (entries + 1) * QEDR_CQE_SIZE; 761 aligned_size = ALIGN(size, PAGE_SIZE); 762 763 return aligned_size / QEDR_CQE_SIZE; 764 } 765 766 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx, 767 struct qedr_dev *dev, 768 struct qedr_userq *q, 769 u64 buf_addr, size_t buf_len, 770 int access, int dmasync) 771 { 772 u32 fw_pages; 773 int rc; 774 775 q->buf_addr = buf_addr; 776 q->buf_len = buf_len; 777 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync); 778 if (IS_ERR(q->umem)) { 779 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", 780 PTR_ERR(q->umem)); 781 return PTR_ERR(q->umem); 782 } 783 784 fw_pages = ib_umem_page_count(q->umem) << 785 (q->umem->page_shift - FW_PAGE_SHIFT); 786 787 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); 788 if (rc) 789 goto err0; 790 791 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 792 if (IS_ERR(q->pbl_tbl)) { 793 rc = PTR_ERR(q->pbl_tbl); 794 goto err0; 795 } 796 797 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info, 798 FW_PAGE_SHIFT); 799 800 return 0; 801 802 err0: 803 ib_umem_release(q->umem); 804 805 return rc; 806 } 807 808 static inline void qedr_init_cq_params(struct qedr_cq *cq, 809 struct qedr_ucontext *ctx, 810 struct qedr_dev *dev, int vector, 811 int chain_entries, int page_cnt, 812 u64 pbl_ptr, 813 struct qed_rdma_create_cq_in_params 814 *params) 815 { 816 memset(params, 0, sizeof(*params)); 817 params->cq_handle_hi = upper_32_bits((uintptr_t)cq); 818 params->cq_handle_lo = lower_32_bits((uintptr_t)cq); 819 params->cnq_id = vector; 820 params->cq_size = chain_entries - 1; 821 params->dpi = (ctx) ? ctx->dpi : dev->dpi; 822 params->pbl_num_pages = page_cnt; 823 params->pbl_ptr = pbl_ptr; 824 params->pbl_two_level = 0; 825 } 826 827 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags) 828 { 829 /* Flush data before signalling doorbell */ 830 wmb(); 831 cq->db.data.agg_flags = flags; 832 cq->db.data.value = cpu_to_le32(cons); 833 writeq(cq->db.raw, cq->db_addr); 834 835 /* Make sure write would stick */ 836 mmiowb(); 837 } 838 839 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 840 { 841 struct qedr_cq *cq = get_qedr_cq(ibcq); 842 unsigned long sflags; 843 struct qedr_dev *dev; 844 845 dev = get_qedr_dev(ibcq->device); 846 847 if (cq->destroyed) { 848 DP_ERR(dev, 849 "warning: arm was invoked after destroy for cq %p (icid=%d)\n", 850 cq, cq->icid); 851 return -EINVAL; 852 } 853 854 855 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 856 return 0; 857 858 spin_lock_irqsave(&cq->cq_lock, sflags); 859 860 cq->arm_flags = 0; 861 862 if (flags & IB_CQ_SOLICITED) 863 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 864 865 if (flags & IB_CQ_NEXT_COMP) 866 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 867 868 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 869 870 spin_unlock_irqrestore(&cq->cq_lock, sflags); 871 872 return 0; 873 } 874 875 struct ib_cq *qedr_create_cq(struct ib_device *ibdev, 876 const struct ib_cq_init_attr *attr, 877 struct ib_ucontext *ib_ctx, struct ib_udata *udata) 878 { 879 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx); 880 struct qed_rdma_destroy_cq_out_params destroy_oparams; 881 struct qed_rdma_destroy_cq_in_params destroy_iparams; 882 struct qedr_dev *dev = get_qedr_dev(ibdev); 883 struct qed_rdma_create_cq_in_params params; 884 struct qedr_create_cq_ureq ureq; 885 int vector = attr->comp_vector; 886 int entries = attr->cqe; 887 struct qedr_cq *cq; 888 int chain_entries; 889 int page_cnt; 890 u64 pbl_ptr; 891 u16 icid; 892 int rc; 893 894 DP_DEBUG(dev, QEDR_MSG_INIT, 895 "create_cq: called from %s. entries=%d, vector=%d\n", 896 udata ? "User Lib" : "Kernel", entries, vector); 897 898 if (entries > QEDR_MAX_CQES) { 899 DP_ERR(dev, 900 "create cq: the number of entries %d is too high. Must be equal or below %d.\n", 901 entries, QEDR_MAX_CQES); 902 return ERR_PTR(-EINVAL); 903 } 904 905 chain_entries = qedr_align_cq_entries(entries); 906 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); 907 908 cq = kzalloc(sizeof(*cq), GFP_KERNEL); 909 if (!cq) 910 return ERR_PTR(-ENOMEM); 911 912 if (udata) { 913 memset(&ureq, 0, sizeof(ureq)); 914 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { 915 DP_ERR(dev, 916 "create cq: problem copying data from user space\n"); 917 goto err0; 918 } 919 920 if (!ureq.len) { 921 DP_ERR(dev, 922 "create cq: cannot create a cq with 0 entries\n"); 923 goto err0; 924 } 925 926 cq->cq_type = QEDR_CQ_TYPE_USER; 927 928 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, 929 ureq.len, IB_ACCESS_LOCAL_WRITE, 1); 930 if (rc) 931 goto err0; 932 933 pbl_ptr = cq->q.pbl_tbl->pa; 934 page_cnt = cq->q.pbl_info.num_pbes; 935 936 cq->ibcq.cqe = chain_entries; 937 } else { 938 cq->cq_type = QEDR_CQ_TYPE_KERNEL; 939 940 rc = dev->ops->common->chain_alloc(dev->cdev, 941 QED_CHAIN_USE_TO_CONSUME, 942 QED_CHAIN_MODE_PBL, 943 QED_CHAIN_CNT_TYPE_U32, 944 chain_entries, 945 sizeof(union rdma_cqe), 946 &cq->pbl, NULL); 947 if (rc) 948 goto err1; 949 950 page_cnt = qed_chain_get_page_cnt(&cq->pbl); 951 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); 952 cq->ibcq.cqe = cq->pbl.capacity; 953 } 954 955 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, 956 pbl_ptr, ¶ms); 957 958 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 959 if (rc) 960 goto err2; 961 962 cq->icid = icid; 963 cq->sig = QEDR_CQ_MAGIC_NUMBER; 964 spin_lock_init(&cq->cq_lock); 965 966 if (ib_ctx) { 967 rc = qedr_copy_cq_uresp(dev, cq, udata); 968 if (rc) 969 goto err3; 970 } else { 971 /* Generate doorbell address. */ 972 cq->db_addr = dev->db_addr + 973 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 974 cq->db.data.icid = cq->icid; 975 cq->db.data.params = DB_AGG_CMD_SET << 976 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 977 978 /* point to the very last element, passing it we will toggle */ 979 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); 980 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 981 cq->latest_cqe = NULL; 982 consume_cqe(cq); 983 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 984 } 985 986 DP_DEBUG(dev, QEDR_MSG_CQ, 987 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", 988 cq->icid, cq, params.cq_size); 989 990 return &cq->ibcq; 991 992 err3: 993 destroy_iparams.icid = cq->icid; 994 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, 995 &destroy_oparams); 996 err2: 997 if (udata) 998 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 999 else 1000 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1001 err1: 1002 if (udata) 1003 ib_umem_release(cq->q.umem); 1004 err0: 1005 kfree(cq); 1006 return ERR_PTR(-EINVAL); 1007 } 1008 1009 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 1010 { 1011 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1012 struct qedr_cq *cq = get_qedr_cq(ibcq); 1013 1014 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq); 1015 1016 return 0; 1017 } 1018 1019 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) 1020 #define QEDR_DESTROY_CQ_ITER_DURATION (10) 1021 1022 int qedr_destroy_cq(struct ib_cq *ibcq) 1023 { 1024 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1025 struct qed_rdma_destroy_cq_out_params oparams; 1026 struct qed_rdma_destroy_cq_in_params iparams; 1027 struct qedr_cq *cq = get_qedr_cq(ibcq); 1028 int iter; 1029 int rc; 1030 1031 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); 1032 1033 cq->destroyed = 1; 1034 1035 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 1036 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 1037 goto done; 1038 1039 iparams.icid = cq->icid; 1040 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 1041 if (rc) 1042 return rc; 1043 1044 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1045 1046 if (ibcq->uobject && ibcq->uobject->context) { 1047 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1048 ib_umem_release(cq->q.umem); 1049 } 1050 1051 /* We don't want the IRQ handler to handle a non-existing CQ so we 1052 * wait until all CNQ interrupts, if any, are received. This will always 1053 * happen and will always happen very fast. If not, then a serious error 1054 * has occured. That is why we can use a long delay. 1055 * We spin for a short time so we don’t lose time on context switching 1056 * in case all the completions are handled in that span. Otherwise 1057 * we sleep for a while and check again. Since the CNQ may be 1058 * associated with (only) the current CPU we use msleep to allow the 1059 * current CPU to be freed. 1060 * The CNQ notification is increased in qedr_irq_handler(). 1061 */ 1062 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1063 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1064 udelay(QEDR_DESTROY_CQ_ITER_DURATION); 1065 iter--; 1066 } 1067 1068 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1069 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1070 msleep(QEDR_DESTROY_CQ_ITER_DURATION); 1071 iter--; 1072 } 1073 1074 if (oparams.num_cq_notif != cq->cnq_notif) 1075 goto err; 1076 1077 /* Note that we don't need to have explicit code to wait for the 1078 * completion of the event handler because it is invoked from the EQ. 1079 * Since the destroy CQ ramrod has also been received on the EQ we can 1080 * be certain that there's no event handler in process. 1081 */ 1082 done: 1083 cq->sig = ~cq->sig; 1084 1085 kfree(cq); 1086 1087 return 0; 1088 1089 err: 1090 DP_ERR(dev, 1091 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n", 1092 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif); 1093 1094 return -EINVAL; 1095 } 1096 1097 static inline int get_gid_info_from_table(struct ib_qp *ibqp, 1098 struct ib_qp_attr *attr, 1099 int attr_mask, 1100 struct qed_rdma_modify_qp_in_params 1101 *qp_params) 1102 { 1103 enum rdma_network_type nw_type; 1104 struct ib_gid_attr gid_attr; 1105 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1106 union ib_gid gid; 1107 u32 ipv4_addr; 1108 int rc = 0; 1109 int i; 1110 1111 rc = ib_get_cached_gid(ibqp->device, 1112 rdma_ah_get_port_num(&attr->ah_attr), 1113 grh->sgid_index, &gid, &gid_attr); 1114 if (rc) 1115 return rc; 1116 1117 if (!memcmp(&gid, &zgid, sizeof(gid))) 1118 return -ENOENT; 1119 1120 if (gid_attr.ndev) { 1121 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev); 1122 1123 dev_put(gid_attr.ndev); 1124 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid); 1125 switch (nw_type) { 1126 case RDMA_NETWORK_IPV6: 1127 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0], 1128 sizeof(qp_params->sgid)); 1129 memcpy(&qp_params->dgid.bytes[0], 1130 &grh->dgid, 1131 sizeof(qp_params->dgid)); 1132 qp_params->roce_mode = ROCE_V2_IPV6; 1133 SET_FIELD(qp_params->modify_flags, 1134 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1135 break; 1136 case RDMA_NETWORK_IB: 1137 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0], 1138 sizeof(qp_params->sgid)); 1139 memcpy(&qp_params->dgid.bytes[0], 1140 &grh->dgid, 1141 sizeof(qp_params->dgid)); 1142 qp_params->roce_mode = ROCE_V1; 1143 break; 1144 case RDMA_NETWORK_IPV4: 1145 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid)); 1146 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid)); 1147 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw); 1148 qp_params->sgid.ipv4_addr = ipv4_addr; 1149 ipv4_addr = 1150 qedr_get_ipv4_from_gid(grh->dgid.raw); 1151 qp_params->dgid.ipv4_addr = ipv4_addr; 1152 SET_FIELD(qp_params->modify_flags, 1153 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1154 qp_params->roce_mode = ROCE_V2_IPV4; 1155 break; 1156 } 1157 } 1158 1159 for (i = 0; i < 4; i++) { 1160 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 1161 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 1162 } 1163 1164 if (qp_params->vlan_id >= VLAN_CFI_MASK) 1165 qp_params->vlan_id = 0; 1166 1167 return 0; 1168 } 1169 1170 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, 1171 struct ib_qp_init_attr *attrs) 1172 { 1173 struct qedr_device_attr *qattr = &dev->attr; 1174 1175 /* QP0... attrs->qp_type == IB_QPT_GSI */ 1176 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { 1177 DP_DEBUG(dev, QEDR_MSG_QP, 1178 "create qp: unsupported qp type=0x%x requested\n", 1179 attrs->qp_type); 1180 return -EINVAL; 1181 } 1182 1183 if (attrs->cap.max_send_wr > qattr->max_sqe) { 1184 DP_ERR(dev, 1185 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n", 1186 attrs->cap.max_send_wr, qattr->max_sqe); 1187 return -EINVAL; 1188 } 1189 1190 if (attrs->cap.max_inline_data > qattr->max_inline) { 1191 DP_ERR(dev, 1192 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n", 1193 attrs->cap.max_inline_data, qattr->max_inline); 1194 return -EINVAL; 1195 } 1196 1197 if (attrs->cap.max_send_sge > qattr->max_sge) { 1198 DP_ERR(dev, 1199 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n", 1200 attrs->cap.max_send_sge, qattr->max_sge); 1201 return -EINVAL; 1202 } 1203 1204 if (attrs->cap.max_recv_sge > qattr->max_sge) { 1205 DP_ERR(dev, 1206 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n", 1207 attrs->cap.max_recv_sge, qattr->max_sge); 1208 return -EINVAL; 1209 } 1210 1211 /* Unprivileged user space cannot create special QP */ 1212 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) { 1213 DP_ERR(dev, 1214 "create qp: userspace can't create special QPs of type=0x%x\n", 1215 attrs->qp_type); 1216 return -EINVAL; 1217 } 1218 1219 return 0; 1220 } 1221 1222 static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp, 1223 struct qedr_qp *qp) 1224 { 1225 uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1226 uresp->rq_icid = qp->icid; 1227 } 1228 1229 static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp, 1230 struct qedr_qp *qp) 1231 { 1232 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1233 uresp->sq_icid = qp->icid + 1; 1234 } 1235 1236 static int qedr_copy_qp_uresp(struct qedr_dev *dev, 1237 struct qedr_qp *qp, struct ib_udata *udata) 1238 { 1239 struct qedr_create_qp_uresp uresp; 1240 int rc; 1241 1242 memset(&uresp, 0, sizeof(uresp)); 1243 qedr_copy_sq_uresp(&uresp, qp); 1244 qedr_copy_rq_uresp(&uresp, qp); 1245 1246 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 1247 uresp.qp_id = qp->qp_id; 1248 1249 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1250 if (rc) 1251 DP_ERR(dev, 1252 "create qp: failed a copy to user space with qp icid=0x%x.\n", 1253 qp->icid); 1254 1255 return rc; 1256 } 1257 1258 static void qedr_set_common_qp_params(struct qedr_dev *dev, 1259 struct qedr_qp *qp, 1260 struct qedr_pd *pd, 1261 struct ib_qp_init_attr *attrs) 1262 { 1263 spin_lock_init(&qp->q_lock); 1264 qp->pd = pd; 1265 qp->qp_type = attrs->qp_type; 1266 qp->max_inline_data = attrs->cap.max_inline_data; 1267 qp->sq.max_sges = attrs->cap.max_send_sge; 1268 qp->state = QED_ROCE_QP_STATE_RESET; 1269 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1270 qp->sq_cq = get_qedr_cq(attrs->send_cq); 1271 qp->rq_cq = get_qedr_cq(attrs->recv_cq); 1272 qp->dev = dev; 1273 qp->rq.max_sges = attrs->cap.max_recv_sge; 1274 1275 DP_DEBUG(dev, QEDR_MSG_QP, 1276 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 1277 qp->rq.max_sges, qp->rq_cq->icid); 1278 DP_DEBUG(dev, QEDR_MSG_QP, 1279 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n", 1280 pd->pd_id, qp->qp_type, qp->max_inline_data, 1281 qp->state, qp->signaled, (attrs->srq) ? 1 : 0); 1282 DP_DEBUG(dev, QEDR_MSG_QP, 1283 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1284 qp->sq.max_sges, qp->sq_cq->icid); 1285 } 1286 1287 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1288 { 1289 qp->sq.db = dev->db_addr + 1290 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1291 qp->sq.db_data.data.icid = qp->icid + 1; 1292 qp->rq.db = dev->db_addr + 1293 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1294 qp->rq.db_data.data.icid = qp->icid; 1295 } 1296 1297 static inline void 1298 qedr_init_common_qp_in_params(struct qedr_dev *dev, 1299 struct qedr_pd *pd, 1300 struct qedr_qp *qp, 1301 struct ib_qp_init_attr *attrs, 1302 bool fmr_and_reserved_lkey, 1303 struct qed_rdma_create_qp_in_params *params) 1304 { 1305 /* QP handle to be written in an async event */ 1306 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp); 1307 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp); 1308 1309 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 1310 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 1311 params->pd = pd->pd_id; 1312 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 1313 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; 1314 params->stats_queue = 0; 1315 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1316 params->srq_id = 0; 1317 params->use_srq = false; 1318 } 1319 1320 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) 1321 { 1322 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. " 1323 "qp=%p. " 1324 "sq_addr=0x%llx, " 1325 "sq_len=%zd, " 1326 "rq_addr=0x%llx, " 1327 "rq_len=%zd" 1328 "\n", 1329 qp, 1330 qp->usq.buf_addr, 1331 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len); 1332 } 1333 1334 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp) 1335 { 1336 if (qp->usq.umem) 1337 ib_umem_release(qp->usq.umem); 1338 qp->usq.umem = NULL; 1339 1340 if (qp->urq.umem) 1341 ib_umem_release(qp->urq.umem); 1342 qp->urq.umem = NULL; 1343 } 1344 1345 static int qedr_create_user_qp(struct qedr_dev *dev, 1346 struct qedr_qp *qp, 1347 struct ib_pd *ibpd, 1348 struct ib_udata *udata, 1349 struct ib_qp_init_attr *attrs) 1350 { 1351 struct qed_rdma_create_qp_in_params in_params; 1352 struct qed_rdma_create_qp_out_params out_params; 1353 struct qedr_pd *pd = get_qedr_pd(ibpd); 1354 struct ib_ucontext *ib_ctx = NULL; 1355 struct qedr_ucontext *ctx = NULL; 1356 struct qedr_create_qp_ureq ureq; 1357 int rc = -EINVAL; 1358 1359 ib_ctx = ibpd->uobject->context; 1360 ctx = get_qedr_ucontext(ib_ctx); 1361 1362 memset(&ureq, 0, sizeof(ureq)); 1363 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq)); 1364 if (rc) { 1365 DP_ERR(dev, "Problem copying data from user space\n"); 1366 return rc; 1367 } 1368 1369 /* SQ - read access only (0), dma sync not required (0) */ 1370 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr, 1371 ureq.sq_len, 0, 0); 1372 if (rc) 1373 return rc; 1374 1375 /* RQ - read access only (0), dma sync not required (0) */ 1376 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr, 1377 ureq.rq_len, 0, 0); 1378 1379 if (rc) 1380 return rc; 1381 1382 memset(&in_params, 0, sizeof(in_params)); 1383 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 1384 in_params.qp_handle_lo = ureq.qp_handle_lo; 1385 in_params.qp_handle_hi = ureq.qp_handle_hi; 1386 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 1387 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 1388 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 1389 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 1390 1391 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1392 &in_params, &out_params); 1393 1394 if (!qp->qed_qp) { 1395 rc = -ENOMEM; 1396 goto err1; 1397 } 1398 1399 qp->qp_id = out_params.qp_id; 1400 qp->icid = out_params.icid; 1401 1402 rc = qedr_copy_qp_uresp(dev, qp, udata); 1403 if (rc) 1404 goto err; 1405 1406 qedr_qp_user_print(dev, qp); 1407 1408 return 0; 1409 err: 1410 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1411 if (rc) 1412 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc); 1413 1414 err1: 1415 qedr_cleanup_user(dev, qp); 1416 return rc; 1417 } 1418 1419 static int 1420 qedr_roce_create_kernel_qp(struct qedr_dev *dev, 1421 struct qedr_qp *qp, 1422 struct qed_rdma_create_qp_in_params *in_params, 1423 u32 n_sq_elems, u32 n_rq_elems) 1424 { 1425 struct qed_rdma_create_qp_out_params out_params; 1426 int rc; 1427 1428 rc = dev->ops->common->chain_alloc(dev->cdev, 1429 QED_CHAIN_USE_TO_PRODUCE, 1430 QED_CHAIN_MODE_PBL, 1431 QED_CHAIN_CNT_TYPE_U32, 1432 n_sq_elems, 1433 QEDR_SQE_ELEMENT_SIZE, 1434 &qp->sq.pbl, NULL); 1435 1436 if (rc) 1437 return rc; 1438 1439 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl); 1440 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl); 1441 1442 rc = dev->ops->common->chain_alloc(dev->cdev, 1443 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1444 QED_CHAIN_MODE_PBL, 1445 QED_CHAIN_CNT_TYPE_U32, 1446 n_rq_elems, 1447 QEDR_RQE_ELEMENT_SIZE, 1448 &qp->rq.pbl, NULL); 1449 if (rc) 1450 return rc; 1451 1452 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl); 1453 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl); 1454 1455 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1456 in_params, &out_params); 1457 1458 if (!qp->qed_qp) 1459 return -EINVAL; 1460 1461 qp->qp_id = out_params.qp_id; 1462 qp->icid = out_params.icid; 1463 1464 qedr_set_roce_db_info(dev, qp); 1465 1466 return 0; 1467 } 1468 1469 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) 1470 { 1471 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl); 1472 kfree(qp->wqe_wr_id); 1473 1474 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl); 1475 kfree(qp->rqe_wr_id); 1476 } 1477 1478 static int qedr_create_kernel_qp(struct qedr_dev *dev, 1479 struct qedr_qp *qp, 1480 struct ib_pd *ibpd, 1481 struct ib_qp_init_attr *attrs) 1482 { 1483 struct qed_rdma_create_qp_in_params in_params; 1484 struct qedr_pd *pd = get_qedr_pd(ibpd); 1485 int rc = -EINVAL; 1486 u32 n_rq_elems; 1487 u32 n_sq_elems; 1488 u32 n_sq_entries; 1489 1490 memset(&in_params, 0, sizeof(in_params)); 1491 1492 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in 1493 * the ring. The ring should allow at least a single WR, even if the 1494 * user requested none, due to allocation issues. 1495 * We should add an extra WR since the prod and cons indices of 1496 * wqe_wr_id are managed in such a way that the WQ is considered full 1497 * when (prod+1)%max_wr==cons. We currently don't do that because we 1498 * double the number of entries due an iSER issue that pushes far more 1499 * WRs than indicated. If we decline its ib_post_send() then we get 1500 * error prints in the dmesg we'd like to avoid. 1501 */ 1502 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 1503 dev->attr.max_sqe); 1504 1505 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id), 1506 GFP_KERNEL); 1507 if (!qp->wqe_wr_id) { 1508 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n"); 1509 return -ENOMEM; 1510 } 1511 1512 /* QP handle to be written in CQE */ 1513 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp); 1514 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp); 1515 1516 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in 1517 * the ring. There ring should allow at least a single WR, even if the 1518 * user requested none, due to allocation issues. 1519 */ 1520 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1); 1521 1522 /* Allocate driver internal RQ array */ 1523 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id), 1524 GFP_KERNEL); 1525 if (!qp->rqe_wr_id) { 1526 DP_ERR(dev, 1527 "create qp: failed RQ shadow memory allocation\n"); 1528 kfree(qp->wqe_wr_id); 1529 return -ENOMEM; 1530 } 1531 1532 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 1533 1534 n_sq_entries = attrs->cap.max_send_wr; 1535 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe); 1536 n_sq_entries = max_t(u32, n_sq_entries, 1); 1537 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE; 1538 1539 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE; 1540 1541 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params, 1542 n_sq_elems, n_rq_elems); 1543 if (rc) 1544 qedr_cleanup_kernel(dev, qp); 1545 1546 return rc; 1547 } 1548 1549 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, 1550 struct ib_qp_init_attr *attrs, 1551 struct ib_udata *udata) 1552 { 1553 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 1554 struct qedr_pd *pd = get_qedr_pd(ibpd); 1555 struct qedr_qp *qp; 1556 struct ib_qp *ibqp; 1557 int rc = 0; 1558 1559 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", 1560 udata ? "user library" : "kernel", pd); 1561 1562 rc = qedr_check_qp_attrs(ibpd, dev, attrs); 1563 if (rc) 1564 return ERR_PTR(rc); 1565 1566 if (attrs->srq) 1567 return ERR_PTR(-EINVAL); 1568 1569 DP_DEBUG(dev, QEDR_MSG_QP, 1570 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 1571 udata ? "user library" : "kernel", attrs->event_handler, pd, 1572 get_qedr_cq(attrs->send_cq), 1573 get_qedr_cq(attrs->send_cq)->icid, 1574 get_qedr_cq(attrs->recv_cq), 1575 get_qedr_cq(attrs->recv_cq)->icid); 1576 1577 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1578 if (!qp) { 1579 DP_ERR(dev, "create qp: failed allocating memory\n"); 1580 return ERR_PTR(-ENOMEM); 1581 } 1582 1583 qedr_set_common_qp_params(dev, qp, pd, attrs); 1584 1585 if (attrs->qp_type == IB_QPT_GSI) { 1586 ibqp = qedr_create_gsi_qp(dev, attrs, qp); 1587 if (IS_ERR(ibqp)) 1588 kfree(qp); 1589 return ibqp; 1590 } 1591 1592 if (udata) 1593 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); 1594 else 1595 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); 1596 1597 if (rc) 1598 goto err; 1599 1600 qp->ibqp.qp_num = qp->qp_id; 1601 1602 return &qp->ibqp; 1603 1604 err: 1605 kfree(qp); 1606 1607 return ERR_PTR(-EFAULT); 1608 } 1609 1610 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) 1611 { 1612 switch (qp_state) { 1613 case QED_ROCE_QP_STATE_RESET: 1614 return IB_QPS_RESET; 1615 case QED_ROCE_QP_STATE_INIT: 1616 return IB_QPS_INIT; 1617 case QED_ROCE_QP_STATE_RTR: 1618 return IB_QPS_RTR; 1619 case QED_ROCE_QP_STATE_RTS: 1620 return IB_QPS_RTS; 1621 case QED_ROCE_QP_STATE_SQD: 1622 return IB_QPS_SQD; 1623 case QED_ROCE_QP_STATE_ERR: 1624 return IB_QPS_ERR; 1625 case QED_ROCE_QP_STATE_SQE: 1626 return IB_QPS_SQE; 1627 } 1628 return IB_QPS_ERR; 1629 } 1630 1631 static enum qed_roce_qp_state qedr_get_state_from_ibqp( 1632 enum ib_qp_state qp_state) 1633 { 1634 switch (qp_state) { 1635 case IB_QPS_RESET: 1636 return QED_ROCE_QP_STATE_RESET; 1637 case IB_QPS_INIT: 1638 return QED_ROCE_QP_STATE_INIT; 1639 case IB_QPS_RTR: 1640 return QED_ROCE_QP_STATE_RTR; 1641 case IB_QPS_RTS: 1642 return QED_ROCE_QP_STATE_RTS; 1643 case IB_QPS_SQD: 1644 return QED_ROCE_QP_STATE_SQD; 1645 case IB_QPS_ERR: 1646 return QED_ROCE_QP_STATE_ERR; 1647 default: 1648 return QED_ROCE_QP_STATE_ERR; 1649 } 1650 } 1651 1652 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) 1653 { 1654 qed_chain_reset(&qph->pbl); 1655 qph->prod = 0; 1656 qph->cons = 0; 1657 qph->wqe_cons = 0; 1658 qph->db_data.data.value = cpu_to_le16(0); 1659 } 1660 1661 static int qedr_update_qp_state(struct qedr_dev *dev, 1662 struct qedr_qp *qp, 1663 enum qed_roce_qp_state new_state) 1664 { 1665 int status = 0; 1666 1667 if (new_state == qp->state) 1668 return 0; 1669 1670 switch (qp->state) { 1671 case QED_ROCE_QP_STATE_RESET: 1672 switch (new_state) { 1673 case QED_ROCE_QP_STATE_INIT: 1674 qp->prev_wqe_size = 0; 1675 qedr_reset_qp_hwq_info(&qp->sq); 1676 qedr_reset_qp_hwq_info(&qp->rq); 1677 break; 1678 default: 1679 status = -EINVAL; 1680 break; 1681 }; 1682 break; 1683 case QED_ROCE_QP_STATE_INIT: 1684 switch (new_state) { 1685 case QED_ROCE_QP_STATE_RTR: 1686 /* Update doorbell (in case post_recv was 1687 * done before move to RTR) 1688 */ 1689 wmb(); 1690 writel(qp->rq.db_data.raw, qp->rq.db); 1691 /* Make sure write takes effect */ 1692 mmiowb(); 1693 break; 1694 case QED_ROCE_QP_STATE_ERR: 1695 break; 1696 default: 1697 /* Invalid state change. */ 1698 status = -EINVAL; 1699 break; 1700 }; 1701 break; 1702 case QED_ROCE_QP_STATE_RTR: 1703 /* RTR->XXX */ 1704 switch (new_state) { 1705 case QED_ROCE_QP_STATE_RTS: 1706 break; 1707 case QED_ROCE_QP_STATE_ERR: 1708 break; 1709 default: 1710 /* Invalid state change. */ 1711 status = -EINVAL; 1712 break; 1713 }; 1714 break; 1715 case QED_ROCE_QP_STATE_RTS: 1716 /* RTS->XXX */ 1717 switch (new_state) { 1718 case QED_ROCE_QP_STATE_SQD: 1719 break; 1720 case QED_ROCE_QP_STATE_ERR: 1721 break; 1722 default: 1723 /* Invalid state change. */ 1724 status = -EINVAL; 1725 break; 1726 }; 1727 break; 1728 case QED_ROCE_QP_STATE_SQD: 1729 /* SQD->XXX */ 1730 switch (new_state) { 1731 case QED_ROCE_QP_STATE_RTS: 1732 case QED_ROCE_QP_STATE_ERR: 1733 break; 1734 default: 1735 /* Invalid state change. */ 1736 status = -EINVAL; 1737 break; 1738 }; 1739 break; 1740 case QED_ROCE_QP_STATE_ERR: 1741 /* ERR->XXX */ 1742 switch (new_state) { 1743 case QED_ROCE_QP_STATE_RESET: 1744 if ((qp->rq.prod != qp->rq.cons) || 1745 (qp->sq.prod != qp->sq.cons)) { 1746 DP_NOTICE(dev, 1747 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n", 1748 qp->rq.prod, qp->rq.cons, qp->sq.prod, 1749 qp->sq.cons); 1750 status = -EINVAL; 1751 } 1752 break; 1753 default: 1754 status = -EINVAL; 1755 break; 1756 }; 1757 break; 1758 default: 1759 status = -EINVAL; 1760 break; 1761 }; 1762 1763 return status; 1764 } 1765 1766 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1767 int attr_mask, struct ib_udata *udata) 1768 { 1769 struct qedr_qp *qp = get_qedr_qp(ibqp); 1770 struct qed_rdma_modify_qp_in_params qp_params = { 0 }; 1771 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); 1772 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1773 enum ib_qp_state old_qp_state, new_qp_state; 1774 int rc = 0; 1775 1776 DP_DEBUG(dev, QEDR_MSG_QP, 1777 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask, 1778 attr->qp_state); 1779 1780 old_qp_state = qedr_get_ibqp_state(qp->state); 1781 if (attr_mask & IB_QP_STATE) 1782 new_qp_state = attr->qp_state; 1783 else 1784 new_qp_state = old_qp_state; 1785 1786 if (!ib_modify_qp_is_ok 1787 (old_qp_state, new_qp_state, ibqp->qp_type, attr_mask, 1788 IB_LINK_LAYER_ETHERNET)) { 1789 DP_ERR(dev, 1790 "modify qp: invalid attribute mask=0x%x specified for\n" 1791 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n", 1792 attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state, 1793 new_qp_state); 1794 rc = -EINVAL; 1795 goto err; 1796 } 1797 1798 /* Translate the masks... */ 1799 if (attr_mask & IB_QP_STATE) { 1800 SET_FIELD(qp_params.modify_flags, 1801 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 1802 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state); 1803 } 1804 1805 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 1806 qp_params.sqd_async = true; 1807 1808 if (attr_mask & IB_QP_PKEY_INDEX) { 1809 SET_FIELD(qp_params.modify_flags, 1810 QED_ROCE_MODIFY_QP_VALID_PKEY, 1); 1811 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) { 1812 rc = -EINVAL; 1813 goto err; 1814 } 1815 1816 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT; 1817 } 1818 1819 if (attr_mask & IB_QP_QKEY) 1820 qp->qkey = attr->qkey; 1821 1822 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1823 SET_FIELD(qp_params.modify_flags, 1824 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 1825 qp_params.incoming_rdma_read_en = attr->qp_access_flags & 1826 IB_ACCESS_REMOTE_READ; 1827 qp_params.incoming_rdma_write_en = attr->qp_access_flags & 1828 IB_ACCESS_REMOTE_WRITE; 1829 qp_params.incoming_atomic_en = attr->qp_access_flags & 1830 IB_ACCESS_REMOTE_ATOMIC; 1831 } 1832 1833 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 1834 if (attr_mask & IB_QP_PATH_MTU) { 1835 if (attr->path_mtu < IB_MTU_256 || 1836 attr->path_mtu > IB_MTU_4096) { 1837 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n"); 1838 rc = -EINVAL; 1839 goto err; 1840 } 1841 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 1842 ib_mtu_enum_to_int(iboe_get_mtu 1843 (dev->ndev->mtu))); 1844 } 1845 1846 if (!qp->mtu) { 1847 qp->mtu = 1848 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 1849 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu); 1850 } 1851 1852 SET_FIELD(qp_params.modify_flags, 1853 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1); 1854 1855 qp_params.traffic_class_tos = grh->traffic_class; 1856 qp_params.flow_label = grh->flow_label; 1857 qp_params.hop_limit_ttl = grh->hop_limit; 1858 1859 qp->sgid_idx = grh->sgid_index; 1860 1861 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params); 1862 if (rc) { 1863 DP_ERR(dev, 1864 "modify qp: problems with GID index %d (rc=%d)\n", 1865 grh->sgid_index, rc); 1866 return rc; 1867 } 1868 1869 rc = qedr_get_dmac(dev, &attr->ah_attr, 1870 qp_params.remote_mac_addr); 1871 if (rc) 1872 return rc; 1873 1874 qp_params.use_local_mac = true; 1875 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr); 1876 1877 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n", 1878 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 1879 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 1880 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n", 1881 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 1882 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 1883 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n", 1884 qp_params.remote_mac_addr); 1885 1886 qp_params.mtu = qp->mtu; 1887 qp_params.lb_indication = false; 1888 } 1889 1890 if (!qp_params.mtu) { 1891 /* Stay with current MTU */ 1892 if (qp->mtu) 1893 qp_params.mtu = qp->mtu; 1894 else 1895 qp_params.mtu = 1896 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 1897 } 1898 1899 if (attr_mask & IB_QP_TIMEOUT) { 1900 SET_FIELD(qp_params.modify_flags, 1901 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 1902 1903 qp_params.ack_timeout = attr->timeout; 1904 if (attr->timeout) { 1905 u32 temp; 1906 1907 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000; 1908 /* FW requires [msec] */ 1909 qp_params.ack_timeout = temp; 1910 } else { 1911 /* Infinite */ 1912 qp_params.ack_timeout = 0; 1913 } 1914 } 1915 if (attr_mask & IB_QP_RETRY_CNT) { 1916 SET_FIELD(qp_params.modify_flags, 1917 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 1918 qp_params.retry_cnt = attr->retry_cnt; 1919 } 1920 1921 if (attr_mask & IB_QP_RNR_RETRY) { 1922 SET_FIELD(qp_params.modify_flags, 1923 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1); 1924 qp_params.rnr_retry_cnt = attr->rnr_retry; 1925 } 1926 1927 if (attr_mask & IB_QP_RQ_PSN) { 1928 SET_FIELD(qp_params.modify_flags, 1929 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1); 1930 qp_params.rq_psn = attr->rq_psn; 1931 qp->rq_psn = attr->rq_psn; 1932 } 1933 1934 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1935 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) { 1936 rc = -EINVAL; 1937 DP_ERR(dev, 1938 "unsupported max_rd_atomic=%d, supported=%d\n", 1939 attr->max_rd_atomic, 1940 dev->attr.max_qp_req_rd_atomic_resc); 1941 goto err; 1942 } 1943 1944 SET_FIELD(qp_params.modify_flags, 1945 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1); 1946 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 1947 } 1948 1949 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 1950 SET_FIELD(qp_params.modify_flags, 1951 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1); 1952 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 1953 } 1954 1955 if (attr_mask & IB_QP_SQ_PSN) { 1956 SET_FIELD(qp_params.modify_flags, 1957 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1); 1958 qp_params.sq_psn = attr->sq_psn; 1959 qp->sq_psn = attr->sq_psn; 1960 } 1961 1962 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1963 if (attr->max_dest_rd_atomic > 1964 dev->attr.max_qp_resp_rd_atomic_resc) { 1965 DP_ERR(dev, 1966 "unsupported max_dest_rd_atomic=%d, supported=%d\n", 1967 attr->max_dest_rd_atomic, 1968 dev->attr.max_qp_resp_rd_atomic_resc); 1969 1970 rc = -EINVAL; 1971 goto err; 1972 } 1973 1974 SET_FIELD(qp_params.modify_flags, 1975 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1); 1976 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 1977 } 1978 1979 if (attr_mask & IB_QP_DEST_QPN) { 1980 SET_FIELD(qp_params.modify_flags, 1981 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1); 1982 1983 qp_params.dest_qp = attr->dest_qp_num; 1984 qp->dest_qp_num = attr->dest_qp_num; 1985 } 1986 1987 if (qp->qp_type != IB_QPT_GSI) 1988 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx, 1989 qp->qed_qp, &qp_params); 1990 1991 if (attr_mask & IB_QP_STATE) { 1992 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 1993 rc = qedr_update_qp_state(dev, qp, qp_params.new_state); 1994 qp->state = qp_params.new_state; 1995 } 1996 1997 err: 1998 return rc; 1999 } 2000 2001 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params) 2002 { 2003 int ib_qp_acc_flags = 0; 2004 2005 if (params->incoming_rdma_write_en) 2006 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 2007 if (params->incoming_rdma_read_en) 2008 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 2009 if (params->incoming_atomic_en) 2010 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 2011 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 2012 return ib_qp_acc_flags; 2013 } 2014 2015 int qedr_query_qp(struct ib_qp *ibqp, 2016 struct ib_qp_attr *qp_attr, 2017 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 2018 { 2019 struct qed_rdma_query_qp_out_params params; 2020 struct qedr_qp *qp = get_qedr_qp(ibqp); 2021 struct qedr_dev *dev = qp->dev; 2022 int rc = 0; 2023 2024 memset(¶ms, 0, sizeof(params)); 2025 2026 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms); 2027 if (rc) 2028 goto err; 2029 2030 memset(qp_attr, 0, sizeof(*qp_attr)); 2031 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 2032 2033 qp_attr->qp_state = qedr_get_ibqp_state(params.state); 2034 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state); 2035 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu); 2036 qp_attr->path_mig_state = IB_MIG_MIGRATED; 2037 qp_attr->rq_psn = params.rq_psn; 2038 qp_attr->sq_psn = params.sq_psn; 2039 qp_attr->dest_qp_num = params.dest_qp; 2040 2041 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms); 2042 2043 qp_attr->cap.max_send_wr = qp->sq.max_wr; 2044 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 2045 qp_attr->cap.max_send_sge = qp->sq.max_sges; 2046 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 2047 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; 2048 qp_init_attr->cap = qp_attr->cap; 2049 2050 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2051 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 2052 params.flow_label, qp->sgid_idx, 2053 params.hop_limit_ttl, params.traffic_class_tos); 2054 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); 2055 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 2056 rdma_ah_set_sl(&qp_attr->ah_attr, 0); 2057 qp_attr->timeout = params.timeout; 2058 qp_attr->rnr_retry = params.rnr_retry; 2059 qp_attr->retry_cnt = params.retry_cnt; 2060 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 2061 qp_attr->pkey_index = params.pkey_index; 2062 qp_attr->port_num = 1; 2063 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 2064 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 2065 qp_attr->alt_pkey_index = 0; 2066 qp_attr->alt_port_num = 0; 2067 qp_attr->alt_timeout = 0; 2068 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 2069 2070 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0; 2071 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 2072 qp_attr->max_rd_atomic = params.max_rd_atomic; 2073 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0; 2074 2075 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n", 2076 qp_attr->cap.max_inline_data); 2077 2078 err: 2079 return rc; 2080 } 2081 2082 int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp) 2083 { 2084 int rc = 0; 2085 2086 if (qp->qp_type != IB_QPT_GSI) { 2087 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2088 if (rc) 2089 return rc; 2090 } 2091 2092 if (qp->ibqp.uobject && qp->ibqp.uobject->context) 2093 qedr_cleanup_user(dev, qp); 2094 else 2095 qedr_cleanup_kernel(dev, qp); 2096 2097 return 0; 2098 } 2099 2100 int qedr_destroy_qp(struct ib_qp *ibqp) 2101 { 2102 struct qedr_qp *qp = get_qedr_qp(ibqp); 2103 struct qedr_dev *dev = qp->dev; 2104 struct ib_qp_attr attr; 2105 int attr_mask = 0; 2106 int rc = 0; 2107 2108 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n", 2109 qp, qp->qp_type); 2110 2111 if ((qp->state != QED_ROCE_QP_STATE_RESET) && 2112 (qp->state != QED_ROCE_QP_STATE_ERR) && 2113 (qp->state != QED_ROCE_QP_STATE_INIT)) { 2114 2115 attr.qp_state = IB_QPS_ERR; 2116 attr_mask |= IB_QP_STATE; 2117 2118 /* Change the QP state to ERROR */ 2119 qedr_modify_qp(ibqp, &attr, attr_mask, NULL); 2120 } 2121 2122 if (qp->qp_type == IB_QPT_GSI) 2123 qedr_destroy_gsi_qp(dev); 2124 2125 qedr_free_qp_resources(dev, qp); 2126 2127 kfree(qp); 2128 2129 return rc; 2130 } 2131 2132 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr, 2133 struct ib_udata *udata) 2134 { 2135 struct qedr_ah *ah; 2136 2137 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 2138 if (!ah) 2139 return ERR_PTR(-ENOMEM); 2140 2141 ah->attr = *attr; 2142 2143 return &ah->ibah; 2144 } 2145 2146 int qedr_destroy_ah(struct ib_ah *ibah) 2147 { 2148 struct qedr_ah *ah = get_qedr_ah(ibah); 2149 2150 kfree(ah); 2151 return 0; 2152 } 2153 2154 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) 2155 { 2156 struct qedr_pbl *pbl, *tmp; 2157 2158 if (info->pbl_table) 2159 list_add_tail(&info->pbl_table->list_entry, 2160 &info->free_pbl_list); 2161 2162 if (!list_empty(&info->inuse_pbl_list)) 2163 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 2164 2165 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 2166 list_del(&pbl->list_entry); 2167 qedr_free_pbl(dev, &info->pbl_info, pbl); 2168 } 2169 } 2170 2171 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info, 2172 size_t page_list_len, bool two_layered) 2173 { 2174 struct qedr_pbl *tmp; 2175 int rc; 2176 2177 INIT_LIST_HEAD(&info->free_pbl_list); 2178 INIT_LIST_HEAD(&info->inuse_pbl_list); 2179 2180 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info, 2181 page_list_len, two_layered); 2182 if (rc) 2183 goto done; 2184 2185 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2186 if (IS_ERR(info->pbl_table)) { 2187 rc = PTR_ERR(info->pbl_table); 2188 goto done; 2189 } 2190 2191 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n", 2192 &info->pbl_table->pa); 2193 2194 /* in usual case we use 2 PBLs, so we add one to free 2195 * list and allocating another one 2196 */ 2197 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2198 if (IS_ERR(tmp)) { 2199 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n"); 2200 goto done; 2201 } 2202 2203 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 2204 2205 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa); 2206 2207 done: 2208 if (rc) 2209 free_mr_info(dev, info); 2210 2211 return rc; 2212 } 2213 2214 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 2215 u64 usr_addr, int acc, struct ib_udata *udata) 2216 { 2217 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2218 struct qedr_mr *mr; 2219 struct qedr_pd *pd; 2220 int rc = -ENOMEM; 2221 2222 pd = get_qedr_pd(ibpd); 2223 DP_DEBUG(dev, QEDR_MSG_MR, 2224 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 2225 pd->pd_id, start, len, usr_addr, acc); 2226 2227 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 2228 return ERR_PTR(-EINVAL); 2229 2230 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2231 if (!mr) 2232 return ERR_PTR(rc); 2233 2234 mr->type = QEDR_MR_USER; 2235 2236 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0); 2237 if (IS_ERR(mr->umem)) { 2238 rc = -EFAULT; 2239 goto err0; 2240 } 2241 2242 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); 2243 if (rc) 2244 goto err1; 2245 2246 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 2247 &mr->info.pbl_info, mr->umem->page_shift); 2248 2249 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2250 if (rc) { 2251 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2252 goto err1; 2253 } 2254 2255 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2256 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2257 mr->hw_mr.key = 0; 2258 mr->hw_mr.pd = pd->pd_id; 2259 mr->hw_mr.local_read = 1; 2260 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2261 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2262 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2263 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2264 mr->hw_mr.mw_bind = false; 2265 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 2266 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2267 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2268 mr->hw_mr.page_size_log = mr->umem->page_shift; 2269 mr->hw_mr.fbo = ib_umem_offset(mr->umem); 2270 mr->hw_mr.length = len; 2271 mr->hw_mr.vaddr = usr_addr; 2272 mr->hw_mr.zbva = false; 2273 mr->hw_mr.phy_mr = false; 2274 mr->hw_mr.dma_mr = false; 2275 2276 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2277 if (rc) { 2278 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2279 goto err2; 2280 } 2281 2282 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2283 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2284 mr->hw_mr.remote_atomic) 2285 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2286 2287 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n", 2288 mr->ibmr.lkey); 2289 return &mr->ibmr; 2290 2291 err2: 2292 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2293 err1: 2294 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2295 err0: 2296 kfree(mr); 2297 return ERR_PTR(rc); 2298 } 2299 2300 int qedr_dereg_mr(struct ib_mr *ib_mr) 2301 { 2302 struct qedr_mr *mr = get_qedr_mr(ib_mr); 2303 struct qedr_dev *dev = get_qedr_dev(ib_mr->device); 2304 int rc = 0; 2305 2306 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid); 2307 if (rc) 2308 return rc; 2309 2310 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2311 2312 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR)) 2313 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2314 2315 /* it could be user registered memory. */ 2316 if (mr->umem) 2317 ib_umem_release(mr->umem); 2318 2319 kfree(mr); 2320 2321 return rc; 2322 } 2323 2324 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, 2325 int max_page_list_len) 2326 { 2327 struct qedr_pd *pd = get_qedr_pd(ibpd); 2328 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2329 struct qedr_mr *mr; 2330 int rc = -ENOMEM; 2331 2332 DP_DEBUG(dev, QEDR_MSG_MR, 2333 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id, 2334 max_page_list_len); 2335 2336 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2337 if (!mr) 2338 return ERR_PTR(rc); 2339 2340 mr->dev = dev; 2341 mr->type = QEDR_MR_FRMR; 2342 2343 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1); 2344 if (rc) 2345 goto err0; 2346 2347 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2348 if (rc) { 2349 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2350 goto err0; 2351 } 2352 2353 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2354 mr->hw_mr.tid_type = QED_RDMA_TID_FMR; 2355 mr->hw_mr.key = 0; 2356 mr->hw_mr.pd = pd->pd_id; 2357 mr->hw_mr.local_read = 1; 2358 mr->hw_mr.local_write = 0; 2359 mr->hw_mr.remote_read = 0; 2360 mr->hw_mr.remote_write = 0; 2361 mr->hw_mr.remote_atomic = 0; 2362 mr->hw_mr.mw_bind = false; 2363 mr->hw_mr.pbl_ptr = 0; 2364 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2365 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2366 mr->hw_mr.fbo = 0; 2367 mr->hw_mr.length = 0; 2368 mr->hw_mr.vaddr = 0; 2369 mr->hw_mr.zbva = false; 2370 mr->hw_mr.phy_mr = true; 2371 mr->hw_mr.dma_mr = false; 2372 2373 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2374 if (rc) { 2375 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2376 goto err1; 2377 } 2378 2379 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2380 mr->ibmr.rkey = mr->ibmr.lkey; 2381 2382 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey); 2383 return mr; 2384 2385 err1: 2386 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2387 err0: 2388 kfree(mr); 2389 return ERR_PTR(rc); 2390 } 2391 2392 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, 2393 enum ib_mr_type mr_type, u32 max_num_sg) 2394 { 2395 struct qedr_dev *dev; 2396 struct qedr_mr *mr; 2397 2398 if (mr_type != IB_MR_TYPE_MEM_REG) 2399 return ERR_PTR(-EINVAL); 2400 2401 mr = __qedr_alloc_mr(ibpd, max_num_sg); 2402 2403 if (IS_ERR(mr)) 2404 return ERR_PTR(-EINVAL); 2405 2406 dev = mr->dev; 2407 2408 return &mr->ibmr; 2409 } 2410 2411 static int qedr_set_page(struct ib_mr *ibmr, u64 addr) 2412 { 2413 struct qedr_mr *mr = get_qedr_mr(ibmr); 2414 struct qedr_pbl *pbl_table; 2415 struct regpair *pbe; 2416 u32 pbes_in_page; 2417 2418 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 2419 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages); 2420 return -ENOMEM; 2421 } 2422 2423 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n", 2424 mr->npages, addr); 2425 2426 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 2427 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 2428 pbe = (struct regpair *)pbl_table->va; 2429 pbe += mr->npages % pbes_in_page; 2430 pbe->lo = cpu_to_le32((u32)addr); 2431 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 2432 2433 mr->npages++; 2434 2435 return 0; 2436 } 2437 2438 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info) 2439 { 2440 int work = info->completed - info->completed_handled - 1; 2441 2442 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work); 2443 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 2444 struct qedr_pbl *pbl; 2445 2446 /* Free all the page list that are possible to be freed 2447 * (all the ones that were invalidated), under the assumption 2448 * that if an FMR was completed successfully that means that 2449 * if there was an invalidate operation before it also ended 2450 */ 2451 pbl = list_first_entry(&info->inuse_pbl_list, 2452 struct qedr_pbl, list_entry); 2453 list_move_tail(&pbl->list_entry, &info->free_pbl_list); 2454 info->completed_handled++; 2455 } 2456 } 2457 2458 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 2459 int sg_nents, unsigned int *sg_offset) 2460 { 2461 struct qedr_mr *mr = get_qedr_mr(ibmr); 2462 2463 mr->npages = 0; 2464 2465 handle_completed_mrs(mr->dev, &mr->info); 2466 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page); 2467 } 2468 2469 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc) 2470 { 2471 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2472 struct qedr_pd *pd = get_qedr_pd(ibpd); 2473 struct qedr_mr *mr; 2474 int rc; 2475 2476 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2477 if (!mr) 2478 return ERR_PTR(-ENOMEM); 2479 2480 mr->type = QEDR_MR_DMA; 2481 2482 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2483 if (rc) { 2484 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2485 goto err1; 2486 } 2487 2488 /* index only, 18 bit long, lkey = itid << 8 | key */ 2489 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2490 mr->hw_mr.pd = pd->pd_id; 2491 mr->hw_mr.local_read = 1; 2492 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2493 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2494 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2495 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2496 mr->hw_mr.dma_mr = true; 2497 2498 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2499 if (rc) { 2500 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2501 goto err2; 2502 } 2503 2504 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2505 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2506 mr->hw_mr.remote_atomic) 2507 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2508 2509 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey); 2510 return &mr->ibmr; 2511 2512 err2: 2513 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2514 err1: 2515 kfree(mr); 2516 return ERR_PTR(rc); 2517 } 2518 2519 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq) 2520 { 2521 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 2522 } 2523 2524 static int sge_data_len(struct ib_sge *sg_list, int num_sge) 2525 { 2526 int i, len = 0; 2527 2528 for (i = 0; i < num_sge; i++) 2529 len += sg_list[i].length; 2530 2531 return len; 2532 } 2533 2534 static void swap_wqe_data64(u64 *p) 2535 { 2536 int i; 2537 2538 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 2539 *p = cpu_to_be64(cpu_to_le64(*p)); 2540 } 2541 2542 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev, 2543 struct qedr_qp *qp, u8 *wqe_size, 2544 struct ib_send_wr *wr, 2545 struct ib_send_wr **bad_wr, u8 *bits, 2546 u8 bit) 2547 { 2548 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 2549 char *seg_prt, *wqe; 2550 int i, seg_siz; 2551 2552 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 2553 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size); 2554 *bad_wr = wr; 2555 return 0; 2556 } 2557 2558 if (!data_size) 2559 return data_size; 2560 2561 *bits |= bit; 2562 2563 seg_prt = NULL; 2564 wqe = NULL; 2565 seg_siz = 0; 2566 2567 /* Copy data inline */ 2568 for (i = 0; i < wr->num_sge; i++) { 2569 u32 len = wr->sg_list[i].length; 2570 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 2571 2572 while (len > 0) { 2573 u32 cur; 2574 2575 /* New segment required */ 2576 if (!seg_siz) { 2577 wqe = (char *)qed_chain_produce(&qp->sq.pbl); 2578 seg_prt = wqe; 2579 seg_siz = sizeof(struct rdma_sq_common_wqe); 2580 (*wqe_size)++; 2581 } 2582 2583 /* Calculate currently allowed length */ 2584 cur = min_t(u32, len, seg_siz); 2585 memcpy(seg_prt, src, cur); 2586 2587 /* Update segment variables */ 2588 seg_prt += cur; 2589 seg_siz -= cur; 2590 2591 /* Update sge variables */ 2592 src += cur; 2593 len -= cur; 2594 2595 /* Swap fully-completed segments */ 2596 if (!seg_siz) 2597 swap_wqe_data64((u64 *)wqe); 2598 } 2599 } 2600 2601 /* swap last not completed segment */ 2602 if (seg_siz) 2603 swap_wqe_data64((u64 *)wqe); 2604 2605 return data_size; 2606 } 2607 2608 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 2609 do { \ 2610 DMA_REGPAIR_LE(sge->addr, vaddr); \ 2611 (sge)->length = cpu_to_le32(vlength); \ 2612 (sge)->flags = cpu_to_le32(vflags); \ 2613 } while (0) 2614 2615 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 2616 do { \ 2617 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \ 2618 (hdr)->num_sges = num_sge; \ 2619 } while (0) 2620 2621 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 2622 do { \ 2623 DMA_REGPAIR_LE(sge->addr, vaddr); \ 2624 (sge)->length = cpu_to_le32(vlength); \ 2625 (sge)->l_key = cpu_to_le32(vlkey); \ 2626 } while (0) 2627 2628 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, 2629 struct ib_send_wr *wr) 2630 { 2631 u32 data_size = 0; 2632 int i; 2633 2634 for (i = 0; i < wr->num_sge; i++) { 2635 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl); 2636 2637 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr); 2638 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 2639 sge->length = cpu_to_le32(wr->sg_list[i].length); 2640 data_size += wr->sg_list[i].length; 2641 } 2642 2643 if (wqe_size) 2644 *wqe_size += wr->num_sge; 2645 2646 return data_size; 2647 } 2648 2649 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev, 2650 struct qedr_qp *qp, 2651 struct rdma_sq_rdma_wqe_1st *rwqe, 2652 struct rdma_sq_rdma_wqe_2nd *rwqe2, 2653 struct ib_send_wr *wr, 2654 struct ib_send_wr **bad_wr) 2655 { 2656 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 2657 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr); 2658 2659 if (wr->send_flags & IB_SEND_INLINE && 2660 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 2661 wr->opcode == IB_WR_RDMA_WRITE)) { 2662 u8 flags = 0; 2663 2664 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 2665 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, 2666 bad_wr, &rwqe->flags, flags); 2667 } 2668 2669 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr); 2670 } 2671 2672 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev, 2673 struct qedr_qp *qp, 2674 struct rdma_sq_send_wqe_1st *swqe, 2675 struct rdma_sq_send_wqe_2st *swqe2, 2676 struct ib_send_wr *wr, 2677 struct ib_send_wr **bad_wr) 2678 { 2679 memset(swqe2, 0, sizeof(*swqe2)); 2680 if (wr->send_flags & IB_SEND_INLINE) { 2681 u8 flags = 0; 2682 2683 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 2684 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, 2685 bad_wr, &swqe->flags, flags); 2686 } 2687 2688 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr); 2689 } 2690 2691 static int qedr_prepare_reg(struct qedr_qp *qp, 2692 struct rdma_sq_fmr_wqe_1st *fwqe1, 2693 struct ib_reg_wr *wr) 2694 { 2695 struct qedr_mr *mr = get_qedr_mr(wr->mr); 2696 struct rdma_sq_fmr_wqe_2nd *fwqe2; 2697 2698 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl); 2699 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 2700 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 2701 fwqe1->l_key = wr->key; 2702 2703 fwqe2->access_ctrl = 0; 2704 2705 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 2706 !!(wr->access & IB_ACCESS_REMOTE_READ)); 2707 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 2708 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 2709 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 2710 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 2711 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 2712 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 2713 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 2714 fwqe2->fmr_ctrl = 0; 2715 2716 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 2717 ilog2(mr->ibmr.page_size) - 12); 2718 2719 fwqe2->length_hi = 0; 2720 fwqe2->length_lo = mr->ibmr.length; 2721 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 2722 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 2723 2724 qp->wqe_wr_id[qp->sq.prod].mr = mr; 2725 2726 return 0; 2727 } 2728 2729 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 2730 { 2731 switch (opcode) { 2732 case IB_WR_RDMA_WRITE: 2733 case IB_WR_RDMA_WRITE_WITH_IMM: 2734 return IB_WC_RDMA_WRITE; 2735 case IB_WR_SEND_WITH_IMM: 2736 case IB_WR_SEND: 2737 case IB_WR_SEND_WITH_INV: 2738 return IB_WC_SEND; 2739 case IB_WR_RDMA_READ: 2740 return IB_WC_RDMA_READ; 2741 case IB_WR_ATOMIC_CMP_AND_SWP: 2742 return IB_WC_COMP_SWAP; 2743 case IB_WR_ATOMIC_FETCH_AND_ADD: 2744 return IB_WC_FETCH_ADD; 2745 case IB_WR_REG_MR: 2746 return IB_WC_REG_MR; 2747 case IB_WR_LOCAL_INV: 2748 return IB_WC_LOCAL_INV; 2749 default: 2750 return IB_WC_SEND; 2751 } 2752 } 2753 2754 static inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr) 2755 { 2756 int wq_is_full, err_wr, pbl_is_full; 2757 struct qedr_dev *dev = qp->dev; 2758 2759 /* prevent SQ overflow and/or processing of a bad WR */ 2760 err_wr = wr->num_sge > qp->sq.max_sges; 2761 wq_is_full = qedr_wq_is_full(&qp->sq); 2762 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) < 2763 QEDR_MAX_SQE_ELEMENTS_PER_SQE; 2764 if (wq_is_full || err_wr || pbl_is_full) { 2765 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) { 2766 DP_ERR(dev, 2767 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n", 2768 qp); 2769 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL; 2770 } 2771 2772 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) { 2773 DP_ERR(dev, 2774 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n", 2775 qp); 2776 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR; 2777 } 2778 2779 if (pbl_is_full && 2780 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) { 2781 DP_ERR(dev, 2782 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n", 2783 qp); 2784 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL; 2785 } 2786 return false; 2787 } 2788 return true; 2789 } 2790 2791 static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 2792 struct ib_send_wr **bad_wr) 2793 { 2794 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 2795 struct qedr_qp *qp = get_qedr_qp(ibqp); 2796 struct rdma_sq_atomic_wqe_1st *awqe1; 2797 struct rdma_sq_atomic_wqe_2nd *awqe2; 2798 struct rdma_sq_atomic_wqe_3rd *awqe3; 2799 struct rdma_sq_send_wqe_2st *swqe2; 2800 struct rdma_sq_local_inv_wqe *iwqe; 2801 struct rdma_sq_rdma_wqe_2nd *rwqe2; 2802 struct rdma_sq_send_wqe_1st *swqe; 2803 struct rdma_sq_rdma_wqe_1st *rwqe; 2804 struct rdma_sq_fmr_wqe_1st *fwqe1; 2805 struct rdma_sq_common_wqe *wqe; 2806 u32 length; 2807 int rc = 0; 2808 bool comp; 2809 2810 if (!qedr_can_post_send(qp, wr)) { 2811 *bad_wr = wr; 2812 return -ENOMEM; 2813 } 2814 2815 wqe = qed_chain_produce(&qp->sq.pbl); 2816 qp->wqe_wr_id[qp->sq.prod].signaled = 2817 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 2818 2819 wqe->flags = 0; 2820 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, 2821 !!(wr->send_flags & IB_SEND_SOLICITED)); 2822 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled; 2823 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 2824 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, 2825 !!(wr->send_flags & IB_SEND_FENCE)); 2826 wqe->prev_wqe_size = qp->prev_wqe_size; 2827 2828 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode); 2829 2830 switch (wr->opcode) { 2831 case IB_WR_SEND_WITH_IMM: 2832 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 2833 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 2834 swqe->wqe_size = 2; 2835 swqe2 = qed_chain_produce(&qp->sq.pbl); 2836 2837 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data); 2838 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 2839 wr, bad_wr); 2840 swqe->length = cpu_to_le32(length); 2841 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 2842 qp->prev_wqe_size = swqe->wqe_size; 2843 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 2844 break; 2845 case IB_WR_SEND: 2846 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 2847 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 2848 2849 swqe->wqe_size = 2; 2850 swqe2 = qed_chain_produce(&qp->sq.pbl); 2851 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 2852 wr, bad_wr); 2853 swqe->length = cpu_to_le32(length); 2854 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 2855 qp->prev_wqe_size = swqe->wqe_size; 2856 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 2857 break; 2858 case IB_WR_SEND_WITH_INV: 2859 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 2860 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 2861 swqe2 = qed_chain_produce(&qp->sq.pbl); 2862 swqe->wqe_size = 2; 2863 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey); 2864 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 2865 wr, bad_wr); 2866 swqe->length = cpu_to_le32(length); 2867 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 2868 qp->prev_wqe_size = swqe->wqe_size; 2869 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 2870 break; 2871 2872 case IB_WR_RDMA_WRITE_WITH_IMM: 2873 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 2874 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 2875 2876 rwqe->wqe_size = 2; 2877 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 2878 rwqe2 = qed_chain_produce(&qp->sq.pbl); 2879 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 2880 wr, bad_wr); 2881 rwqe->length = cpu_to_le32(length); 2882 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 2883 qp->prev_wqe_size = rwqe->wqe_size; 2884 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 2885 break; 2886 case IB_WR_RDMA_WRITE: 2887 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 2888 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 2889 2890 rwqe->wqe_size = 2; 2891 rwqe2 = qed_chain_produce(&qp->sq.pbl); 2892 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 2893 wr, bad_wr); 2894 rwqe->length = cpu_to_le32(length); 2895 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 2896 qp->prev_wqe_size = rwqe->wqe_size; 2897 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 2898 break; 2899 case IB_WR_RDMA_READ_WITH_INV: 2900 DP_ERR(dev, 2901 "RDMA READ WITH INVALIDATE not supported\n"); 2902 *bad_wr = wr; 2903 rc = -EINVAL; 2904 break; 2905 2906 case IB_WR_RDMA_READ: 2907 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 2908 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 2909 2910 rwqe->wqe_size = 2; 2911 rwqe2 = qed_chain_produce(&qp->sq.pbl); 2912 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 2913 wr, bad_wr); 2914 rwqe->length = cpu_to_le32(length); 2915 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 2916 qp->prev_wqe_size = rwqe->wqe_size; 2917 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 2918 break; 2919 2920 case IB_WR_ATOMIC_CMP_AND_SWP: 2921 case IB_WR_ATOMIC_FETCH_AND_ADD: 2922 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe; 2923 awqe1->wqe_size = 4; 2924 2925 awqe2 = qed_chain_produce(&qp->sq.pbl); 2926 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr); 2927 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 2928 2929 awqe3 = qed_chain_produce(&qp->sq.pbl); 2930 2931 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 2932 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 2933 DMA_REGPAIR_LE(awqe3->swap_data, 2934 atomic_wr(wr)->compare_add); 2935 } else { 2936 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 2937 DMA_REGPAIR_LE(awqe3->swap_data, 2938 atomic_wr(wr)->swap); 2939 DMA_REGPAIR_LE(awqe3->cmp_data, 2940 atomic_wr(wr)->compare_add); 2941 } 2942 2943 qedr_prepare_sq_sges(qp, NULL, wr); 2944 2945 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size; 2946 qp->prev_wqe_size = awqe1->wqe_size; 2947 break; 2948 2949 case IB_WR_LOCAL_INV: 2950 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 2951 iwqe->wqe_size = 1; 2952 2953 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 2954 iwqe->inv_l_key = wr->ex.invalidate_rkey; 2955 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size; 2956 qp->prev_wqe_size = iwqe->wqe_size; 2957 break; 2958 case IB_WR_REG_MR: 2959 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n"); 2960 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 2961 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 2962 fwqe1->wqe_size = 2; 2963 2964 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr)); 2965 if (rc) { 2966 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc); 2967 *bad_wr = wr; 2968 break; 2969 } 2970 2971 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 2972 qp->prev_wqe_size = fwqe1->wqe_size; 2973 break; 2974 default: 2975 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode); 2976 rc = -EINVAL; 2977 *bad_wr = wr; 2978 break; 2979 } 2980 2981 if (*bad_wr) { 2982 u16 value; 2983 2984 /* Restore prod to its position before 2985 * this WR was processed 2986 */ 2987 value = le16_to_cpu(qp->sq.db_data.data.value); 2988 qed_chain_set_prod(&qp->sq.pbl, value, wqe); 2989 2990 /* Restore prev_wqe_size */ 2991 qp->prev_wqe_size = wqe->prev_wqe_size; 2992 rc = -EINVAL; 2993 DP_ERR(dev, "POST SEND FAILED\n"); 2994 } 2995 2996 return rc; 2997 } 2998 2999 int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 3000 struct ib_send_wr **bad_wr) 3001 { 3002 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3003 struct qedr_qp *qp = get_qedr_qp(ibqp); 3004 unsigned long flags; 3005 int rc = 0; 3006 3007 *bad_wr = NULL; 3008 3009 if (qp->qp_type == IB_QPT_GSI) 3010 return qedr_gsi_post_send(ibqp, wr, bad_wr); 3011 3012 spin_lock_irqsave(&qp->q_lock, flags); 3013 3014 if ((qp->state != QED_ROCE_QP_STATE_RTS) && 3015 (qp->state != QED_ROCE_QP_STATE_ERR) && 3016 (qp->state != QED_ROCE_QP_STATE_SQD)) { 3017 spin_unlock_irqrestore(&qp->q_lock, flags); 3018 *bad_wr = wr; 3019 DP_DEBUG(dev, QEDR_MSG_CQ, 3020 "QP in wrong state! QP icid=0x%x state %d\n", 3021 qp->icid, qp->state); 3022 return -EINVAL; 3023 } 3024 3025 while (wr) { 3026 rc = __qedr_post_send(ibqp, wr, bad_wr); 3027 if (rc) 3028 break; 3029 3030 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 3031 3032 qedr_inc_sw_prod(&qp->sq); 3033 3034 qp->sq.db_data.data.value++; 3035 3036 wr = wr->next; 3037 } 3038 3039 /* Trigger doorbell 3040 * If there was a failure in the first WR then it will be triggered in 3041 * vane. However this is not harmful (as long as the producer value is 3042 * unchanged). For performance reasons we avoid checking for this 3043 * redundant doorbell. 3044 */ 3045 wmb(); 3046 writel(qp->sq.db_data.raw, qp->sq.db); 3047 3048 /* Make sure write sticks */ 3049 mmiowb(); 3050 3051 spin_unlock_irqrestore(&qp->q_lock, flags); 3052 3053 return rc; 3054 } 3055 3056 int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, 3057 struct ib_recv_wr **bad_wr) 3058 { 3059 struct qedr_qp *qp = get_qedr_qp(ibqp); 3060 struct qedr_dev *dev = qp->dev; 3061 unsigned long flags; 3062 int status = 0; 3063 3064 if (qp->qp_type == IB_QPT_GSI) 3065 return qedr_gsi_post_recv(ibqp, wr, bad_wr); 3066 3067 spin_lock_irqsave(&qp->q_lock, flags); 3068 3069 if (qp->state == QED_ROCE_QP_STATE_RESET) { 3070 spin_unlock_irqrestore(&qp->q_lock, flags); 3071 *bad_wr = wr; 3072 return -EINVAL; 3073 } 3074 3075 while (wr) { 3076 int i; 3077 3078 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) < 3079 QEDR_MAX_RQE_ELEMENTS_PER_RQE || 3080 wr->num_sge > qp->rq.max_sges) { 3081 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n", 3082 qed_chain_get_elem_left_u32(&qp->rq.pbl), 3083 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge, 3084 qp->rq.max_sges); 3085 status = -ENOMEM; 3086 *bad_wr = wr; 3087 break; 3088 } 3089 for (i = 0; i < wr->num_sge; i++) { 3090 u32 flags = 0; 3091 struct rdma_rq_sge *rqe = 3092 qed_chain_produce(&qp->rq.pbl); 3093 3094 /* First one must include the number 3095 * of SGE in the list 3096 */ 3097 if (!i) 3098 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 3099 wr->num_sge); 3100 3101 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 3102 wr->sg_list[i].lkey); 3103 3104 RQ_SGE_SET(rqe, wr->sg_list[i].addr, 3105 wr->sg_list[i].length, flags); 3106 } 3107 3108 /* Special case of no sges. FW requires between 1-4 sges... 3109 * in this case we need to post 1 sge with length zero. this is 3110 * because rdma write with immediate consumes an RQ. 3111 */ 3112 if (!wr->num_sge) { 3113 u32 flags = 0; 3114 struct rdma_rq_sge *rqe = 3115 qed_chain_produce(&qp->rq.pbl); 3116 3117 /* First one must include the number 3118 * of SGE in the list 3119 */ 3120 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0); 3121 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 3122 3123 RQ_SGE_SET(rqe, 0, 0, flags); 3124 i = 1; 3125 } 3126 3127 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 3128 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 3129 3130 qedr_inc_sw_prod(&qp->rq); 3131 3132 /* Flush all the writes before signalling doorbell */ 3133 wmb(); 3134 3135 qp->rq.db_data.data.value++; 3136 3137 writel(qp->rq.db_data.raw, qp->rq.db); 3138 3139 /* Make sure write sticks */ 3140 mmiowb(); 3141 3142 wr = wr->next; 3143 } 3144 3145 spin_unlock_irqrestore(&qp->q_lock, flags); 3146 3147 return status; 3148 } 3149 3150 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe) 3151 { 3152 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3153 3154 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) == 3155 cq->pbl_toggle; 3156 } 3157 3158 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe) 3159 { 3160 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3161 struct qedr_qp *qp; 3162 3163 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi, 3164 resp_cqe->qp_handle.lo, 3165 u64); 3166 return qp; 3167 } 3168 3169 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe) 3170 { 3171 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3172 3173 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 3174 } 3175 3176 /* Return latest CQE (needs processing) */ 3177 static union rdma_cqe *get_cqe(struct qedr_cq *cq) 3178 { 3179 return cq->latest_cqe; 3180 } 3181 3182 /* In fmr we need to increase the number of fmr completed counter for the fmr 3183 * algorithm determining whether we can free a pbl or not. 3184 * we need to perform this whether the work request was signaled or not. for 3185 * this purpose we call this function from the condition that checks if a wr 3186 * should be skipped, to make sure we don't miss it ( possibly this fmr 3187 * operation was not signalted) 3188 */ 3189 static inline void qedr_chk_if_fmr(struct qedr_qp *qp) 3190 { 3191 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 3192 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3193 } 3194 3195 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, 3196 struct qedr_cq *cq, int num_entries, 3197 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, 3198 int force) 3199 { 3200 u16 cnt = 0; 3201 3202 while (num_entries && qp->sq.wqe_cons != hw_cons) { 3203 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 3204 qedr_chk_if_fmr(qp); 3205 /* skip WC */ 3206 goto next_cqe; 3207 } 3208 3209 /* fill WC */ 3210 wc->status = status; 3211 wc->vendor_err = 0; 3212 wc->wc_flags = 0; 3213 wc->src_qp = qp->id; 3214 wc->qp = &qp->ibqp; 3215 3216 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 3217 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 3218 3219 switch (wc->opcode) { 3220 case IB_WC_RDMA_WRITE: 3221 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3222 break; 3223 case IB_WC_COMP_SWAP: 3224 case IB_WC_FETCH_ADD: 3225 wc->byte_len = 8; 3226 break; 3227 case IB_WC_REG_MR: 3228 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3229 break; 3230 case IB_WC_RDMA_READ: 3231 case IB_WC_SEND: 3232 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3233 break; 3234 default: 3235 break; 3236 } 3237 3238 num_entries--; 3239 wc++; 3240 cnt++; 3241 next_cqe: 3242 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 3243 qed_chain_consume(&qp->sq.pbl); 3244 qedr_inc_sw_cons(&qp->sq); 3245 } 3246 3247 return cnt; 3248 } 3249 3250 static int qedr_poll_cq_req(struct qedr_dev *dev, 3251 struct qedr_qp *qp, struct qedr_cq *cq, 3252 int num_entries, struct ib_wc *wc, 3253 struct rdma_cqe_requester *req) 3254 { 3255 int cnt = 0; 3256 3257 switch (req->status) { 3258 case RDMA_CQE_REQ_STS_OK: 3259 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3260 IB_WC_SUCCESS, 0); 3261 break; 3262 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 3263 if (qp->state != QED_ROCE_QP_STATE_ERR) 3264 DP_ERR(dev, 3265 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3266 cq->icid, qp->icid); 3267 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3268 IB_WC_WR_FLUSH_ERR, 1); 3269 break; 3270 default: 3271 /* process all WQE before the cosumer */ 3272 qp->state = QED_ROCE_QP_STATE_ERR; 3273 cnt = process_req(dev, qp, cq, num_entries, wc, 3274 req->sq_cons - 1, IB_WC_SUCCESS, 0); 3275 wc += cnt; 3276 /* if we have extra WC fill it with actual error info */ 3277 if (cnt < num_entries) { 3278 enum ib_wc_status wc_status; 3279 3280 switch (req->status) { 3281 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 3282 DP_ERR(dev, 3283 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3284 cq->icid, qp->icid); 3285 wc_status = IB_WC_BAD_RESP_ERR; 3286 break; 3287 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 3288 DP_ERR(dev, 3289 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3290 cq->icid, qp->icid); 3291 wc_status = IB_WC_LOC_LEN_ERR; 3292 break; 3293 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 3294 DP_ERR(dev, 3295 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3296 cq->icid, qp->icid); 3297 wc_status = IB_WC_LOC_QP_OP_ERR; 3298 break; 3299 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 3300 DP_ERR(dev, 3301 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3302 cq->icid, qp->icid); 3303 wc_status = IB_WC_LOC_PROT_ERR; 3304 break; 3305 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 3306 DP_ERR(dev, 3307 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3308 cq->icid, qp->icid); 3309 wc_status = IB_WC_MW_BIND_ERR; 3310 break; 3311 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 3312 DP_ERR(dev, 3313 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3314 cq->icid, qp->icid); 3315 wc_status = IB_WC_REM_INV_REQ_ERR; 3316 break; 3317 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 3318 DP_ERR(dev, 3319 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3320 cq->icid, qp->icid); 3321 wc_status = IB_WC_REM_ACCESS_ERR; 3322 break; 3323 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 3324 DP_ERR(dev, 3325 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3326 cq->icid, qp->icid); 3327 wc_status = IB_WC_REM_OP_ERR; 3328 break; 3329 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 3330 DP_ERR(dev, 3331 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3332 cq->icid, qp->icid); 3333 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 3334 break; 3335 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 3336 DP_ERR(dev, 3337 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3338 cq->icid, qp->icid); 3339 wc_status = IB_WC_RETRY_EXC_ERR; 3340 break; 3341 default: 3342 DP_ERR(dev, 3343 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3344 cq->icid, qp->icid); 3345 wc_status = IB_WC_GENERAL_ERR; 3346 } 3347 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 3348 wc_status, 1); 3349 } 3350 } 3351 3352 return cnt; 3353 } 3354 3355 static inline int qedr_cqe_resp_status_to_ib(u8 status) 3356 { 3357 switch (status) { 3358 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 3359 return IB_WC_LOC_ACCESS_ERR; 3360 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 3361 return IB_WC_LOC_LEN_ERR; 3362 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 3363 return IB_WC_LOC_QP_OP_ERR; 3364 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 3365 return IB_WC_LOC_PROT_ERR; 3366 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 3367 return IB_WC_MW_BIND_ERR; 3368 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 3369 return IB_WC_REM_INV_RD_REQ_ERR; 3370 case RDMA_CQE_RESP_STS_OK: 3371 return IB_WC_SUCCESS; 3372 default: 3373 return IB_WC_GENERAL_ERR; 3374 } 3375 } 3376 3377 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp, 3378 struct ib_wc *wc) 3379 { 3380 wc->status = IB_WC_SUCCESS; 3381 wc->byte_len = le32_to_cpu(resp->length); 3382 3383 if (resp->flags & QEDR_RESP_IMM) { 3384 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 3385 wc->wc_flags |= IB_WC_WITH_IMM; 3386 3387 if (resp->flags & QEDR_RESP_RDMA) 3388 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 3389 3390 if (resp->flags & QEDR_RESP_INV) 3391 return -EINVAL; 3392 3393 } else if (resp->flags & QEDR_RESP_INV) { 3394 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 3395 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3396 3397 if (resp->flags & QEDR_RESP_RDMA) 3398 return -EINVAL; 3399 3400 } else if (resp->flags & QEDR_RESP_RDMA) { 3401 return -EINVAL; 3402 } 3403 3404 return 0; 3405 } 3406 3407 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 3408 struct qedr_cq *cq, struct ib_wc *wc, 3409 struct rdma_cqe_responder *resp, u64 wr_id) 3410 { 3411 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */ 3412 wc->opcode = IB_WC_RECV; 3413 wc->wc_flags = 0; 3414 3415 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) { 3416 if (qedr_set_ok_cqe_resp_wc(resp, wc)) 3417 DP_ERR(dev, 3418 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n", 3419 cq, cq->icid, resp->flags); 3420 3421 } else { 3422 wc->status = qedr_cqe_resp_status_to_ib(resp->status); 3423 if (wc->status == IB_WC_GENERAL_ERR) 3424 DP_ERR(dev, 3425 "CQ %p (icid=%d) contains an invalid CQE status %d\n", 3426 cq, cq->icid, resp->status); 3427 } 3428 3429 /* Fill the rest of the WC */ 3430 wc->vendor_err = 0; 3431 wc->src_qp = qp->id; 3432 wc->qp = &qp->ibqp; 3433 wc->wr_id = wr_id; 3434 } 3435 3436 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 3437 struct qedr_cq *cq, struct ib_wc *wc, 3438 struct rdma_cqe_responder *resp) 3439 { 3440 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 3441 3442 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 3443 3444 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 3445 qed_chain_consume(&qp->rq.pbl); 3446 qedr_inc_sw_cons(&qp->rq); 3447 3448 return 1; 3449 } 3450 3451 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, 3452 int num_entries, struct ib_wc *wc, u16 hw_cons) 3453 { 3454 u16 cnt = 0; 3455 3456 while (num_entries && qp->rq.wqe_cons != hw_cons) { 3457 /* fill WC */ 3458 wc->status = IB_WC_WR_FLUSH_ERR; 3459 wc->vendor_err = 0; 3460 wc->wc_flags = 0; 3461 wc->src_qp = qp->id; 3462 wc->byte_len = 0; 3463 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 3464 wc->qp = &qp->ibqp; 3465 num_entries--; 3466 wc++; 3467 cnt++; 3468 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 3469 qed_chain_consume(&qp->rq.pbl); 3470 qedr_inc_sw_cons(&qp->rq); 3471 } 3472 3473 return cnt; 3474 } 3475 3476 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 3477 struct rdma_cqe_responder *resp, int *update) 3478 { 3479 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) { 3480 consume_cqe(cq); 3481 *update |= 1; 3482 } 3483 } 3484 3485 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, 3486 struct qedr_cq *cq, int num_entries, 3487 struct ib_wc *wc, struct rdma_cqe_responder *resp, 3488 int *update) 3489 { 3490 int cnt; 3491 3492 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 3493 cnt = process_resp_flush(qp, cq, num_entries, wc, 3494 resp->rq_cons); 3495 try_consume_resp_cqe(cq, qp, resp, update); 3496 } else { 3497 cnt = process_resp_one(dev, qp, cq, wc, resp); 3498 consume_cqe(cq); 3499 *update |= 1; 3500 } 3501 3502 return cnt; 3503 } 3504 3505 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 3506 struct rdma_cqe_requester *req, int *update) 3507 { 3508 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 3509 consume_cqe(cq); 3510 *update |= 1; 3511 } 3512 } 3513 3514 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 3515 { 3516 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 3517 struct qedr_cq *cq = get_qedr_cq(ibcq); 3518 union rdma_cqe *cqe = cq->latest_cqe; 3519 u32 old_cons, new_cons; 3520 unsigned long flags; 3521 int update = 0; 3522 int done = 0; 3523 3524 if (cq->destroyed) { 3525 DP_ERR(dev, 3526 "warning: poll was invoked after destroy for cq %p (icid=%d)\n", 3527 cq, cq->icid); 3528 return 0; 3529 } 3530 3531 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 3532 return qedr_gsi_poll_cq(ibcq, num_entries, wc); 3533 3534 spin_lock_irqsave(&cq->cq_lock, flags); 3535 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 3536 while (num_entries && is_valid_cqe(cq, cqe)) { 3537 struct qedr_qp *qp; 3538 int cnt = 0; 3539 3540 /* prevent speculative reads of any field of CQE */ 3541 rmb(); 3542 3543 qp = cqe_get_qp(cqe); 3544 if (!qp) { 3545 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe); 3546 break; 3547 } 3548 3549 wc->qp = &qp->ibqp; 3550 3551 switch (cqe_get_type(cqe)) { 3552 case RDMA_CQE_TYPE_REQUESTER: 3553 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc, 3554 &cqe->req); 3555 try_consume_req_cqe(cq, qp, &cqe->req, &update); 3556 break; 3557 case RDMA_CQE_TYPE_RESPONDER_RQ: 3558 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc, 3559 &cqe->resp, &update); 3560 break; 3561 case RDMA_CQE_TYPE_INVALID: 3562 default: 3563 DP_ERR(dev, "Error: invalid CQE type = %d\n", 3564 cqe_get_type(cqe)); 3565 } 3566 num_entries -= cnt; 3567 wc += cnt; 3568 done += cnt; 3569 3570 cqe = get_cqe(cq); 3571 } 3572 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 3573 3574 cq->cq_cons += new_cons - old_cons; 3575 3576 if (update) 3577 /* doorbell notifies abount latest VALID entry, 3578 * but chain already point to the next INVALID one 3579 */ 3580 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 3581 3582 spin_unlock_irqrestore(&cq->cq_lock, flags); 3583 return done; 3584 } 3585 3586 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, 3587 u8 port_num, 3588 const struct ib_wc *in_wc, 3589 const struct ib_grh *in_grh, 3590 const struct ib_mad_hdr *mad_hdr, 3591 size_t in_mad_size, struct ib_mad_hdr *out_mad, 3592 size_t *out_mad_size, u16 *out_mad_pkey_index) 3593 { 3594 struct qedr_dev *dev = get_qedr_dev(ibdev); 3595 3596 DP_DEBUG(dev, QEDR_MSG_GSI, 3597 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n", 3598 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod, 3599 mad_hdr->class_specific, mad_hdr->class_version, 3600 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status); 3601 return IB_MAD_RESULT_SUCCESS; 3602 } 3603 3604 int qedr_port_immutable(struct ib_device *ibdev, u8 port_num, 3605 struct ib_port_immutable *immutable) 3606 { 3607 struct ib_port_attr attr; 3608 int err; 3609 3610 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE | 3611 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 3612 3613 err = ib_query_port(ibdev, port_num, &attr); 3614 if (err) 3615 return err; 3616 3617 immutable->pkey_tbl_len = attr.pkey_tbl_len; 3618 immutable->gid_tbl_len = attr.gid_tbl_len; 3619 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 3620 3621 return 0; 3622 } 3623