1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/dma-mapping.h> 33 #include <linux/crc32.h> 34 #include <net/ip.h> 35 #include <net/ipv6.h> 36 #include <net/udp.h> 37 #include <linux/iommu.h> 38 39 #include <rdma/ib_verbs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/iw_cm.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/ib_addr.h> 44 #include <rdma/ib_cache.h> 45 46 #include <linux/qed/common_hsi.h> 47 #include "qedr_hsi_rdma.h" 48 #include <linux/qed/qed_if.h> 49 #include "qedr.h" 50 #include "verbs.h" 51 #include <rdma/qedr-abi.h> 52 #include "qedr_cm.h" 53 54 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 55 56 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 57 { 58 if (index > QEDR_ROCE_PKEY_TABLE_LEN) 59 return -EINVAL; 60 61 *pkey = QEDR_ROCE_PKEY_DEFAULT; 62 return 0; 63 } 64 65 int qedr_query_gid(struct ib_device *ibdev, u8 port, int index, 66 union ib_gid *sgid) 67 { 68 struct qedr_dev *dev = get_qedr_dev(ibdev); 69 int rc = 0; 70 71 if (!rdma_cap_roce_gid_table(ibdev, port)) 72 return -ENODEV; 73 74 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL); 75 if (rc == -EAGAIN) { 76 memcpy(sgid, &zgid, sizeof(*sgid)); 77 return 0; 78 } 79 80 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index, 81 sgid->global.interface_id, sgid->global.subnet_prefix); 82 83 return rc; 84 } 85 86 int qedr_add_gid(struct ib_device *device, u8 port_num, 87 unsigned int index, const union ib_gid *gid, 88 const struct ib_gid_attr *attr, void **context) 89 { 90 if (!rdma_cap_roce_gid_table(device, port_num)) 91 return -EINVAL; 92 93 if (port_num > QEDR_MAX_PORT) 94 return -EINVAL; 95 96 if (!context) 97 return -EINVAL; 98 99 return 0; 100 } 101 102 int qedr_del_gid(struct ib_device *device, u8 port_num, 103 unsigned int index, void **context) 104 { 105 if (!rdma_cap_roce_gid_table(device, port_num)) 106 return -EINVAL; 107 108 if (port_num > QEDR_MAX_PORT) 109 return -EINVAL; 110 111 if (!context) 112 return -EINVAL; 113 114 return 0; 115 } 116 117 int qedr_query_device(struct ib_device *ibdev, 118 struct ib_device_attr *attr, struct ib_udata *udata) 119 { 120 struct qedr_dev *dev = get_qedr_dev(ibdev); 121 struct qedr_device_attr *qattr = &dev->attr; 122 123 if (!dev->rdma_ctx) { 124 DP_ERR(dev, 125 "qedr_query_device called with invalid params rdma_ctx=%p\n", 126 dev->rdma_ctx); 127 return -EINVAL; 128 } 129 130 memset(attr, 0, sizeof(*attr)); 131 132 attr->fw_ver = qattr->fw_ver; 133 attr->sys_image_guid = qattr->sys_image_guid; 134 attr->max_mr_size = qattr->max_mr_size; 135 attr->page_size_cap = qattr->page_size_caps; 136 attr->vendor_id = qattr->vendor_id; 137 attr->vendor_part_id = qattr->vendor_part_id; 138 attr->hw_ver = qattr->hw_ver; 139 attr->max_qp = qattr->max_qp; 140 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe); 141 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 142 IB_DEVICE_RC_RNR_NAK_GEN | 143 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS; 144 145 attr->max_sge = qattr->max_sge; 146 attr->max_sge_rd = qattr->max_sge; 147 attr->max_cq = qattr->max_cq; 148 attr->max_cqe = qattr->max_cqe; 149 attr->max_mr = qattr->max_mr; 150 attr->max_mw = qattr->max_mw; 151 attr->max_pd = qattr->max_pd; 152 attr->atomic_cap = dev->atomic_cap; 153 attr->max_fmr = qattr->max_fmr; 154 attr->max_map_per_fmr = 16; 155 attr->max_qp_init_rd_atom = 156 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 157 attr->max_qp_rd_atom = 158 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 159 attr->max_qp_init_rd_atom); 160 161 attr->max_srq = qattr->max_srq; 162 attr->max_srq_sge = qattr->max_srq_sge; 163 attr->max_srq_wr = qattr->max_srq_wr; 164 165 attr->local_ca_ack_delay = qattr->dev_ack_delay; 166 attr->max_fast_reg_page_list_len = qattr->max_mr / 8; 167 attr->max_pkeys = QEDR_ROCE_PKEY_MAX; 168 attr->max_ah = qattr->max_ah; 169 170 return 0; 171 } 172 173 #define QEDR_SPEED_SDR (1) 174 #define QEDR_SPEED_DDR (2) 175 #define QEDR_SPEED_QDR (4) 176 #define QEDR_SPEED_FDR10 (8) 177 #define QEDR_SPEED_FDR (16) 178 #define QEDR_SPEED_EDR (32) 179 180 static inline void get_link_speed_and_width(int speed, u8 *ib_speed, 181 u8 *ib_width) 182 { 183 switch (speed) { 184 case 1000: 185 *ib_speed = QEDR_SPEED_SDR; 186 *ib_width = IB_WIDTH_1X; 187 break; 188 case 10000: 189 *ib_speed = QEDR_SPEED_QDR; 190 *ib_width = IB_WIDTH_1X; 191 break; 192 193 case 20000: 194 *ib_speed = QEDR_SPEED_DDR; 195 *ib_width = IB_WIDTH_4X; 196 break; 197 198 case 25000: 199 *ib_speed = QEDR_SPEED_EDR; 200 *ib_width = IB_WIDTH_1X; 201 break; 202 203 case 40000: 204 *ib_speed = QEDR_SPEED_QDR; 205 *ib_width = IB_WIDTH_4X; 206 break; 207 208 case 50000: 209 *ib_speed = QEDR_SPEED_QDR; 210 *ib_width = IB_WIDTH_4X; 211 break; 212 213 case 100000: 214 *ib_speed = QEDR_SPEED_EDR; 215 *ib_width = IB_WIDTH_4X; 216 break; 217 218 default: 219 /* Unsupported */ 220 *ib_speed = QEDR_SPEED_SDR; 221 *ib_width = IB_WIDTH_1X; 222 } 223 } 224 225 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr) 226 { 227 struct qedr_dev *dev; 228 struct qed_rdma_port *rdma_port; 229 230 dev = get_qedr_dev(ibdev); 231 if (port > 1) { 232 DP_ERR(dev, "invalid_port=0x%x\n", port); 233 return -EINVAL; 234 } 235 236 if (!dev->rdma_ctx) { 237 DP_ERR(dev, "rdma_ctx is NULL\n"); 238 return -EINVAL; 239 } 240 241 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx); 242 243 /* *attr being zeroed by the caller, avoid zeroing it here */ 244 if (rdma_port->port_state == QED_RDMA_PORT_UP) { 245 attr->state = IB_PORT_ACTIVE; 246 attr->phys_state = 5; 247 } else { 248 attr->state = IB_PORT_DOWN; 249 attr->phys_state = 3; 250 } 251 attr->max_mtu = IB_MTU_4096; 252 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); 253 attr->lid = 0; 254 attr->lmc = 0; 255 attr->sm_lid = 0; 256 attr->sm_sl = 0; 257 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS; 258 attr->gid_tbl_len = QEDR_MAX_SGID; 259 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; 260 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 261 attr->qkey_viol_cntr = 0; 262 get_link_speed_and_width(rdma_port->link_speed, 263 &attr->active_speed, &attr->active_width); 264 attr->max_msg_sz = rdma_port->max_msg_size; 265 attr->max_vl_num = 4; 266 267 return 0; 268 } 269 270 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask, 271 struct ib_port_modify *props) 272 { 273 struct qedr_dev *dev; 274 275 dev = get_qedr_dev(ibdev); 276 if (port > 1) { 277 DP_ERR(dev, "invalid_port=0x%x\n", port); 278 return -EINVAL; 279 } 280 281 return 0; 282 } 283 284 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr, 285 unsigned long len) 286 { 287 struct qedr_mm *mm; 288 289 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 290 if (!mm) 291 return -ENOMEM; 292 293 mm->key.phy_addr = phy_addr; 294 /* This function might be called with a length which is not a multiple 295 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel 296 * forces this granularity by increasing the requested size if needed. 297 * When qedr_mmap is called, it will search the list with the updated 298 * length as a key. To prevent search failures, the length is rounded up 299 * in advance to PAGE_SIZE. 300 */ 301 mm->key.len = roundup(len, PAGE_SIZE); 302 INIT_LIST_HEAD(&mm->entry); 303 304 mutex_lock(&uctx->mm_list_lock); 305 list_add(&mm->entry, &uctx->mm_head); 306 mutex_unlock(&uctx->mm_list_lock); 307 308 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 309 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n", 310 (unsigned long long)mm->key.phy_addr, 311 (unsigned long)mm->key.len, uctx); 312 313 return 0; 314 } 315 316 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr, 317 unsigned long len) 318 { 319 bool found = false; 320 struct qedr_mm *mm; 321 322 mutex_lock(&uctx->mm_list_lock); 323 list_for_each_entry(mm, &uctx->mm_head, entry) { 324 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 325 continue; 326 327 found = true; 328 break; 329 } 330 mutex_unlock(&uctx->mm_list_lock); 331 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 332 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n", 333 mm->key.phy_addr, mm->key.len, uctx, found); 334 335 return found; 336 } 337 338 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev, 339 struct ib_udata *udata) 340 { 341 int rc; 342 struct qedr_ucontext *ctx; 343 struct qedr_alloc_ucontext_resp uresp; 344 struct qedr_dev *dev = get_qedr_dev(ibdev); 345 struct qed_rdma_add_user_out_params oparams; 346 347 if (!udata) 348 return ERR_PTR(-EFAULT); 349 350 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 351 if (!ctx) 352 return ERR_PTR(-ENOMEM); 353 354 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams); 355 if (rc) { 356 DP_ERR(dev, 357 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n", 358 rc); 359 goto err; 360 } 361 362 ctx->dpi = oparams.dpi; 363 ctx->dpi_addr = oparams.dpi_addr; 364 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 365 ctx->dpi_size = oparams.dpi_size; 366 INIT_LIST_HEAD(&ctx->mm_head); 367 mutex_init(&ctx->mm_list_lock); 368 369 memset(&uresp, 0, sizeof(uresp)); 370 371 uresp.db_pa = ctx->dpi_phys_addr; 372 uresp.db_size = ctx->dpi_size; 373 uresp.max_send_wr = dev->attr.max_sqe; 374 uresp.max_recv_wr = dev->attr.max_rqe; 375 uresp.max_srq_wr = dev->attr.max_srq_wr; 376 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE; 377 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE; 378 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 379 uresp.max_cqes = QEDR_MAX_CQES; 380 381 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 382 if (rc) 383 goto err; 384 385 ctx->dev = dev; 386 387 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size); 388 if (rc) 389 goto err; 390 391 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n", 392 &ctx->ibucontext); 393 return &ctx->ibucontext; 394 395 err: 396 kfree(ctx); 397 return ERR_PTR(rc); 398 } 399 400 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx) 401 { 402 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx); 403 struct qedr_mm *mm, *tmp; 404 int status = 0; 405 406 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n", 407 uctx); 408 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi); 409 410 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 411 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 412 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n", 413 mm->key.phy_addr, mm->key.len, uctx); 414 list_del(&mm->entry); 415 kfree(mm); 416 } 417 418 kfree(uctx); 419 return status; 420 } 421 422 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 423 { 424 struct qedr_ucontext *ucontext = get_qedr_ucontext(context); 425 struct qedr_dev *dev = get_qedr_dev(context->device); 426 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 427 u64 unmapped_db = dev->db_phys_addr; 428 unsigned long len = (vma->vm_end - vma->vm_start); 429 int rc = 0; 430 bool found; 431 432 DP_DEBUG(dev, QEDR_MSG_INIT, 433 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n", 434 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len); 435 if (vma->vm_start & (PAGE_SIZE - 1)) { 436 DP_ERR(dev, "Vma_start not page aligned = %ld\n", 437 vma->vm_start); 438 return -EINVAL; 439 } 440 441 found = qedr_search_mmap(ucontext, vm_page, len); 442 if (!found) { 443 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n", 444 vma->vm_pgoff); 445 return -EINVAL; 446 } 447 448 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n"); 449 450 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + 451 dev->db_size))) { 452 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n"); 453 if (vma->vm_flags & VM_READ) { 454 DP_ERR(dev, "Trying to map doorbell bar for read\n"); 455 return -EPERM; 456 } 457 458 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 459 460 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 461 PAGE_SIZE, vma->vm_page_prot); 462 } else { 463 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n"); 464 rc = remap_pfn_range(vma, vma->vm_start, 465 vma->vm_pgoff, len, vma->vm_page_prot); 466 } 467 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc); 468 return rc; 469 } 470 471 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev, 472 struct ib_ucontext *context, struct ib_udata *udata) 473 { 474 struct qedr_dev *dev = get_qedr_dev(ibdev); 475 struct qedr_pd *pd; 476 u16 pd_id; 477 int rc; 478 479 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n", 480 (udata && context) ? "User Lib" : "Kernel"); 481 482 if (!dev->rdma_ctx) { 483 DP_ERR(dev, "invlaid RDMA context\n"); 484 return ERR_PTR(-EINVAL); 485 } 486 487 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 488 if (!pd) 489 return ERR_PTR(-ENOMEM); 490 491 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id); 492 if (rc) 493 goto err; 494 495 pd->pd_id = pd_id; 496 497 if (udata && context) { 498 struct qedr_alloc_pd_uresp uresp; 499 500 uresp.pd_id = pd_id; 501 502 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 503 if (rc) { 504 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); 505 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); 506 goto err; 507 } 508 509 pd->uctx = get_qedr_ucontext(context); 510 pd->uctx->pd = pd; 511 } 512 513 return &pd->ibpd; 514 515 err: 516 kfree(pd); 517 return ERR_PTR(rc); 518 } 519 520 int qedr_dealloc_pd(struct ib_pd *ibpd) 521 { 522 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 523 struct qedr_pd *pd = get_qedr_pd(ibpd); 524 525 if (!pd) { 526 pr_err("Invalid PD received in dealloc_pd\n"); 527 return -EINVAL; 528 } 529 530 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id); 531 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id); 532 533 kfree(pd); 534 535 return 0; 536 } 537 538 static void qedr_free_pbl(struct qedr_dev *dev, 539 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) 540 { 541 struct pci_dev *pdev = dev->pdev; 542 int i; 543 544 for (i = 0; i < pbl_info->num_pbls; i++) { 545 if (!pbl[i].va) 546 continue; 547 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 548 pbl[i].va, pbl[i].pa); 549 } 550 551 kfree(pbl); 552 } 553 554 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024) 555 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024) 556 557 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 558 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 559 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE) 560 561 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev, 562 struct qedr_pbl_info *pbl_info, 563 gfp_t flags) 564 { 565 struct pci_dev *pdev = dev->pdev; 566 struct qedr_pbl *pbl_table; 567 dma_addr_t *pbl_main_tbl; 568 dma_addr_t pa; 569 void *va; 570 int i; 571 572 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags); 573 if (!pbl_table) 574 return ERR_PTR(-ENOMEM); 575 576 for (i = 0; i < pbl_info->num_pbls; i++) { 577 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, 578 &pa, flags); 579 if (!va) 580 goto err; 581 582 memset(va, 0, pbl_info->pbl_size); 583 pbl_table[i].va = va; 584 pbl_table[i].pa = pa; 585 } 586 587 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 588 * the first one with physical pointers to all of the rest 589 */ 590 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 591 for (i = 0; i < pbl_info->num_pbls - 1; i++) 592 pbl_main_tbl[i] = pbl_table[i + 1].pa; 593 594 return pbl_table; 595 596 err: 597 for (i--; i >= 0; i--) 598 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 599 pbl_table[i].va, pbl_table[i].pa); 600 601 qedr_free_pbl(dev, pbl_info, pbl_table); 602 603 return ERR_PTR(-ENOMEM); 604 } 605 606 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev, 607 struct qedr_pbl_info *pbl_info, 608 u32 num_pbes, int two_layer_capable) 609 { 610 u32 pbl_capacity; 611 u32 pbl_size; 612 u32 num_pbls; 613 614 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 615 if (num_pbes > MAX_PBES_TWO_LAYER) { 616 DP_ERR(dev, "prepare pbl table: too many pages %d\n", 617 num_pbes); 618 return -EINVAL; 619 } 620 621 /* calculate required pbl page size */ 622 pbl_size = MIN_FW_PBL_PAGE_SIZE; 623 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 624 NUM_PBES_ON_PAGE(pbl_size); 625 626 while (pbl_capacity < num_pbes) { 627 pbl_size *= 2; 628 pbl_capacity = pbl_size / sizeof(u64); 629 pbl_capacity = pbl_capacity * pbl_capacity; 630 } 631 632 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 633 num_pbls++; /* One for the layer0 ( points to the pbls) */ 634 pbl_info->two_layered = true; 635 } else { 636 /* One layered PBL */ 637 num_pbls = 1; 638 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, 639 roundup_pow_of_two((num_pbes * sizeof(u64)))); 640 pbl_info->two_layered = false; 641 } 642 643 pbl_info->num_pbls = num_pbls; 644 pbl_info->pbl_size = pbl_size; 645 pbl_info->num_pbes = num_pbes; 646 647 DP_DEBUG(dev, QEDR_MSG_MR, 648 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n", 649 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 650 651 return 0; 652 } 653 654 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, 655 struct qedr_pbl *pbl, 656 struct qedr_pbl_info *pbl_info, u32 pg_shift) 657 { 658 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 659 u32 fw_pg_cnt, fw_pg_per_umem_pg; 660 struct qedr_pbl *pbl_tbl; 661 struct scatterlist *sg; 662 struct regpair *pbe; 663 u64 pg_addr; 664 int entry; 665 666 if (!pbl_info->num_pbes) 667 return; 668 669 /* If we have a two layered pbl, the first pbl points to the rest 670 * of the pbls and the first entry lays on the second pbl in the table 671 */ 672 if (pbl_info->two_layered) 673 pbl_tbl = &pbl[1]; 674 else 675 pbl_tbl = pbl; 676 677 pbe = (struct regpair *)pbl_tbl->va; 678 if (!pbe) { 679 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n"); 680 return; 681 } 682 683 pbe_cnt = 0; 684 685 shift = umem->page_shift; 686 687 fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift); 688 689 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 690 pages = sg_dma_len(sg) >> shift; 691 pg_addr = sg_dma_address(sg); 692 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 693 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) { 694 pbe->lo = cpu_to_le32(pg_addr); 695 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); 696 697 pg_addr += BIT(pg_shift); 698 pbe_cnt++; 699 total_num_pbes++; 700 pbe++; 701 702 if (total_num_pbes == pbl_info->num_pbes) 703 return; 704 705 /* If the given pbl is full storing the pbes, 706 * move to next pbl. 707 */ 708 if (pbe_cnt == 709 (pbl_info->pbl_size / sizeof(u64))) { 710 pbl_tbl++; 711 pbe = (struct regpair *)pbl_tbl->va; 712 pbe_cnt = 0; 713 } 714 715 fw_pg_cnt++; 716 } 717 } 718 } 719 } 720 721 static int qedr_copy_cq_uresp(struct qedr_dev *dev, 722 struct qedr_cq *cq, struct ib_udata *udata) 723 { 724 struct qedr_create_cq_uresp uresp; 725 int rc; 726 727 memset(&uresp, 0, sizeof(uresp)); 728 729 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 730 uresp.icid = cq->icid; 731 732 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 733 if (rc) 734 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); 735 736 return rc; 737 } 738 739 static void consume_cqe(struct qedr_cq *cq) 740 { 741 if (cq->latest_cqe == cq->toggle_cqe) 742 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 743 744 cq->latest_cqe = qed_chain_consume(&cq->pbl); 745 } 746 747 static inline int qedr_align_cq_entries(int entries) 748 { 749 u64 size, aligned_size; 750 751 /* We allocate an extra entry that we don't report to the FW. */ 752 size = (entries + 1) * QEDR_CQE_SIZE; 753 aligned_size = ALIGN(size, PAGE_SIZE); 754 755 return aligned_size / QEDR_CQE_SIZE; 756 } 757 758 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx, 759 struct qedr_dev *dev, 760 struct qedr_userq *q, 761 u64 buf_addr, size_t buf_len, 762 int access, int dmasync) 763 { 764 u32 fw_pages; 765 int rc; 766 767 q->buf_addr = buf_addr; 768 q->buf_len = buf_len; 769 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync); 770 if (IS_ERR(q->umem)) { 771 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", 772 PTR_ERR(q->umem)); 773 return PTR_ERR(q->umem); 774 } 775 776 fw_pages = ib_umem_page_count(q->umem) << 777 (q->umem->page_shift - FW_PAGE_SHIFT); 778 779 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); 780 if (rc) 781 goto err0; 782 783 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 784 if (IS_ERR(q->pbl_tbl)) { 785 rc = PTR_ERR(q->pbl_tbl); 786 goto err0; 787 } 788 789 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info, 790 FW_PAGE_SHIFT); 791 792 return 0; 793 794 err0: 795 ib_umem_release(q->umem); 796 797 return rc; 798 } 799 800 static inline void qedr_init_cq_params(struct qedr_cq *cq, 801 struct qedr_ucontext *ctx, 802 struct qedr_dev *dev, int vector, 803 int chain_entries, int page_cnt, 804 u64 pbl_ptr, 805 struct qed_rdma_create_cq_in_params 806 *params) 807 { 808 memset(params, 0, sizeof(*params)); 809 params->cq_handle_hi = upper_32_bits((uintptr_t)cq); 810 params->cq_handle_lo = lower_32_bits((uintptr_t)cq); 811 params->cnq_id = vector; 812 params->cq_size = chain_entries - 1; 813 params->dpi = (ctx) ? ctx->dpi : dev->dpi; 814 params->pbl_num_pages = page_cnt; 815 params->pbl_ptr = pbl_ptr; 816 params->pbl_two_level = 0; 817 } 818 819 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags) 820 { 821 /* Flush data before signalling doorbell */ 822 wmb(); 823 cq->db.data.agg_flags = flags; 824 cq->db.data.value = cpu_to_le32(cons); 825 writeq(cq->db.raw, cq->db_addr); 826 827 /* Make sure write would stick */ 828 mmiowb(); 829 } 830 831 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 832 { 833 struct qedr_cq *cq = get_qedr_cq(ibcq); 834 unsigned long sflags; 835 struct qedr_dev *dev; 836 837 dev = get_qedr_dev(ibcq->device); 838 839 if (cq->destroyed) { 840 DP_ERR(dev, 841 "warning: arm was invoked after destroy for cq %p (icid=%d)\n", 842 cq, cq->icid); 843 return -EINVAL; 844 } 845 846 847 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 848 return 0; 849 850 spin_lock_irqsave(&cq->cq_lock, sflags); 851 852 cq->arm_flags = 0; 853 854 if (flags & IB_CQ_SOLICITED) 855 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 856 857 if (flags & IB_CQ_NEXT_COMP) 858 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 859 860 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 861 862 spin_unlock_irqrestore(&cq->cq_lock, sflags); 863 864 return 0; 865 } 866 867 struct ib_cq *qedr_create_cq(struct ib_device *ibdev, 868 const struct ib_cq_init_attr *attr, 869 struct ib_ucontext *ib_ctx, struct ib_udata *udata) 870 { 871 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx); 872 struct qed_rdma_destroy_cq_out_params destroy_oparams; 873 struct qed_rdma_destroy_cq_in_params destroy_iparams; 874 struct qedr_dev *dev = get_qedr_dev(ibdev); 875 struct qed_rdma_create_cq_in_params params; 876 struct qedr_create_cq_ureq ureq; 877 int vector = attr->comp_vector; 878 int entries = attr->cqe; 879 struct qedr_cq *cq; 880 int chain_entries; 881 int page_cnt; 882 u64 pbl_ptr; 883 u16 icid; 884 int rc; 885 886 DP_DEBUG(dev, QEDR_MSG_INIT, 887 "create_cq: called from %s. entries=%d, vector=%d\n", 888 udata ? "User Lib" : "Kernel", entries, vector); 889 890 if (entries > QEDR_MAX_CQES) { 891 DP_ERR(dev, 892 "create cq: the number of entries %d is too high. Must be equal or below %d.\n", 893 entries, QEDR_MAX_CQES); 894 return ERR_PTR(-EINVAL); 895 } 896 897 chain_entries = qedr_align_cq_entries(entries); 898 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); 899 900 cq = kzalloc(sizeof(*cq), GFP_KERNEL); 901 if (!cq) 902 return ERR_PTR(-ENOMEM); 903 904 if (udata) { 905 memset(&ureq, 0, sizeof(ureq)); 906 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { 907 DP_ERR(dev, 908 "create cq: problem copying data from user space\n"); 909 goto err0; 910 } 911 912 if (!ureq.len) { 913 DP_ERR(dev, 914 "create cq: cannot create a cq with 0 entries\n"); 915 goto err0; 916 } 917 918 cq->cq_type = QEDR_CQ_TYPE_USER; 919 920 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, 921 ureq.len, IB_ACCESS_LOCAL_WRITE, 1); 922 if (rc) 923 goto err0; 924 925 pbl_ptr = cq->q.pbl_tbl->pa; 926 page_cnt = cq->q.pbl_info.num_pbes; 927 928 cq->ibcq.cqe = chain_entries; 929 } else { 930 cq->cq_type = QEDR_CQ_TYPE_KERNEL; 931 932 rc = dev->ops->common->chain_alloc(dev->cdev, 933 QED_CHAIN_USE_TO_CONSUME, 934 QED_CHAIN_MODE_PBL, 935 QED_CHAIN_CNT_TYPE_U32, 936 chain_entries, 937 sizeof(union rdma_cqe), 938 &cq->pbl); 939 if (rc) 940 goto err1; 941 942 page_cnt = qed_chain_get_page_cnt(&cq->pbl); 943 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); 944 cq->ibcq.cqe = cq->pbl.capacity; 945 } 946 947 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, 948 pbl_ptr, ¶ms); 949 950 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 951 if (rc) 952 goto err2; 953 954 cq->icid = icid; 955 cq->sig = QEDR_CQ_MAGIC_NUMBER; 956 spin_lock_init(&cq->cq_lock); 957 958 if (ib_ctx) { 959 rc = qedr_copy_cq_uresp(dev, cq, udata); 960 if (rc) 961 goto err3; 962 } else { 963 /* Generate doorbell address. */ 964 cq->db_addr = dev->db_addr + 965 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 966 cq->db.data.icid = cq->icid; 967 cq->db.data.params = DB_AGG_CMD_SET << 968 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 969 970 /* point to the very last element, passing it we will toggle */ 971 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); 972 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 973 cq->latest_cqe = NULL; 974 consume_cqe(cq); 975 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 976 } 977 978 DP_DEBUG(dev, QEDR_MSG_CQ, 979 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", 980 cq->icid, cq, params.cq_size); 981 982 return &cq->ibcq; 983 984 err3: 985 destroy_iparams.icid = cq->icid; 986 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, 987 &destroy_oparams); 988 err2: 989 if (udata) 990 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 991 else 992 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 993 err1: 994 if (udata) 995 ib_umem_release(cq->q.umem); 996 err0: 997 kfree(cq); 998 return ERR_PTR(-EINVAL); 999 } 1000 1001 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 1002 { 1003 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1004 struct qedr_cq *cq = get_qedr_cq(ibcq); 1005 1006 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq); 1007 1008 return 0; 1009 } 1010 1011 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) 1012 #define QEDR_DESTROY_CQ_ITER_DURATION (10) 1013 1014 int qedr_destroy_cq(struct ib_cq *ibcq) 1015 { 1016 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1017 struct qed_rdma_destroy_cq_out_params oparams; 1018 struct qed_rdma_destroy_cq_in_params iparams; 1019 struct qedr_cq *cq = get_qedr_cq(ibcq); 1020 int iter; 1021 int rc; 1022 1023 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); 1024 1025 cq->destroyed = 1; 1026 1027 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 1028 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 1029 goto done; 1030 1031 iparams.icid = cq->icid; 1032 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 1033 if (rc) 1034 return rc; 1035 1036 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1037 1038 if (ibcq->uobject && ibcq->uobject->context) { 1039 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1040 ib_umem_release(cq->q.umem); 1041 } 1042 1043 /* We don't want the IRQ handler to handle a non-existing CQ so we 1044 * wait until all CNQ interrupts, if any, are received. This will always 1045 * happen and will always happen very fast. If not, then a serious error 1046 * has occured. That is why we can use a long delay. 1047 * We spin for a short time so we don’t lose time on context switching 1048 * in case all the completions are handled in that span. Otherwise 1049 * we sleep for a while and check again. Since the CNQ may be 1050 * associated with (only) the current CPU we use msleep to allow the 1051 * current CPU to be freed. 1052 * The CNQ notification is increased in qedr_irq_handler(). 1053 */ 1054 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1055 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1056 udelay(QEDR_DESTROY_CQ_ITER_DURATION); 1057 iter--; 1058 } 1059 1060 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1061 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1062 msleep(QEDR_DESTROY_CQ_ITER_DURATION); 1063 iter--; 1064 } 1065 1066 if (oparams.num_cq_notif != cq->cnq_notif) 1067 goto err; 1068 1069 /* Note that we don't need to have explicit code to wait for the 1070 * completion of the event handler because it is invoked from the EQ. 1071 * Since the destroy CQ ramrod has also been received on the EQ we can 1072 * be certain that there's no event handler in process. 1073 */ 1074 done: 1075 cq->sig = ~cq->sig; 1076 1077 kfree(cq); 1078 1079 return 0; 1080 1081 err: 1082 DP_ERR(dev, 1083 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n", 1084 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif); 1085 1086 return -EINVAL; 1087 } 1088 1089 static inline int get_gid_info_from_table(struct ib_qp *ibqp, 1090 struct ib_qp_attr *attr, 1091 int attr_mask, 1092 struct qed_rdma_modify_qp_in_params 1093 *qp_params) 1094 { 1095 enum rdma_network_type nw_type; 1096 struct ib_gid_attr gid_attr; 1097 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1098 union ib_gid gid; 1099 u32 ipv4_addr; 1100 int rc = 0; 1101 int i; 1102 1103 rc = ib_get_cached_gid(ibqp->device, 1104 rdma_ah_get_port_num(&attr->ah_attr), 1105 grh->sgid_index, &gid, &gid_attr); 1106 if (rc) 1107 return rc; 1108 1109 if (!memcmp(&gid, &zgid, sizeof(gid))) 1110 return -ENOENT; 1111 1112 if (gid_attr.ndev) { 1113 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev); 1114 1115 dev_put(gid_attr.ndev); 1116 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid); 1117 switch (nw_type) { 1118 case RDMA_NETWORK_IPV6: 1119 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0], 1120 sizeof(qp_params->sgid)); 1121 memcpy(&qp_params->dgid.bytes[0], 1122 &grh->dgid, 1123 sizeof(qp_params->dgid)); 1124 qp_params->roce_mode = ROCE_V2_IPV6; 1125 SET_FIELD(qp_params->modify_flags, 1126 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1127 break; 1128 case RDMA_NETWORK_IB: 1129 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0], 1130 sizeof(qp_params->sgid)); 1131 memcpy(&qp_params->dgid.bytes[0], 1132 &grh->dgid, 1133 sizeof(qp_params->dgid)); 1134 qp_params->roce_mode = ROCE_V1; 1135 break; 1136 case RDMA_NETWORK_IPV4: 1137 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid)); 1138 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid)); 1139 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw); 1140 qp_params->sgid.ipv4_addr = ipv4_addr; 1141 ipv4_addr = 1142 qedr_get_ipv4_from_gid(grh->dgid.raw); 1143 qp_params->dgid.ipv4_addr = ipv4_addr; 1144 SET_FIELD(qp_params->modify_flags, 1145 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1146 qp_params->roce_mode = ROCE_V2_IPV4; 1147 break; 1148 } 1149 } 1150 1151 for (i = 0; i < 4; i++) { 1152 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 1153 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 1154 } 1155 1156 if (qp_params->vlan_id >= VLAN_CFI_MASK) 1157 qp_params->vlan_id = 0; 1158 1159 return 0; 1160 } 1161 1162 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, 1163 struct ib_qp_init_attr *attrs) 1164 { 1165 struct qedr_device_attr *qattr = &dev->attr; 1166 1167 /* QP0... attrs->qp_type == IB_QPT_GSI */ 1168 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { 1169 DP_DEBUG(dev, QEDR_MSG_QP, 1170 "create qp: unsupported qp type=0x%x requested\n", 1171 attrs->qp_type); 1172 return -EINVAL; 1173 } 1174 1175 if (attrs->cap.max_send_wr > qattr->max_sqe) { 1176 DP_ERR(dev, 1177 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n", 1178 attrs->cap.max_send_wr, qattr->max_sqe); 1179 return -EINVAL; 1180 } 1181 1182 if (attrs->cap.max_inline_data > qattr->max_inline) { 1183 DP_ERR(dev, 1184 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n", 1185 attrs->cap.max_inline_data, qattr->max_inline); 1186 return -EINVAL; 1187 } 1188 1189 if (attrs->cap.max_send_sge > qattr->max_sge) { 1190 DP_ERR(dev, 1191 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n", 1192 attrs->cap.max_send_sge, qattr->max_sge); 1193 return -EINVAL; 1194 } 1195 1196 if (attrs->cap.max_recv_sge > qattr->max_sge) { 1197 DP_ERR(dev, 1198 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n", 1199 attrs->cap.max_recv_sge, qattr->max_sge); 1200 return -EINVAL; 1201 } 1202 1203 /* Unprivileged user space cannot create special QP */ 1204 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) { 1205 DP_ERR(dev, 1206 "create qp: userspace can't create special QPs of type=0x%x\n", 1207 attrs->qp_type); 1208 return -EINVAL; 1209 } 1210 1211 return 0; 1212 } 1213 1214 static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp, 1215 struct qedr_qp *qp) 1216 { 1217 uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1218 uresp->rq_icid = qp->icid; 1219 } 1220 1221 static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp, 1222 struct qedr_qp *qp) 1223 { 1224 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1225 uresp->sq_icid = qp->icid + 1; 1226 } 1227 1228 static int qedr_copy_qp_uresp(struct qedr_dev *dev, 1229 struct qedr_qp *qp, struct ib_udata *udata) 1230 { 1231 struct qedr_create_qp_uresp uresp; 1232 int rc; 1233 1234 memset(&uresp, 0, sizeof(uresp)); 1235 qedr_copy_sq_uresp(&uresp, qp); 1236 qedr_copy_rq_uresp(&uresp, qp); 1237 1238 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 1239 uresp.qp_id = qp->qp_id; 1240 1241 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1242 if (rc) 1243 DP_ERR(dev, 1244 "create qp: failed a copy to user space with qp icid=0x%x.\n", 1245 qp->icid); 1246 1247 return rc; 1248 } 1249 1250 static void qedr_set_common_qp_params(struct qedr_dev *dev, 1251 struct qedr_qp *qp, 1252 struct qedr_pd *pd, 1253 struct ib_qp_init_attr *attrs) 1254 { 1255 spin_lock_init(&qp->q_lock); 1256 qp->pd = pd; 1257 qp->qp_type = attrs->qp_type; 1258 qp->max_inline_data = attrs->cap.max_inline_data; 1259 qp->sq.max_sges = attrs->cap.max_send_sge; 1260 qp->state = QED_ROCE_QP_STATE_RESET; 1261 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1262 qp->sq_cq = get_qedr_cq(attrs->send_cq); 1263 qp->rq_cq = get_qedr_cq(attrs->recv_cq); 1264 qp->dev = dev; 1265 qp->rq.max_sges = attrs->cap.max_recv_sge; 1266 1267 DP_DEBUG(dev, QEDR_MSG_QP, 1268 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 1269 qp->rq.max_sges, qp->rq_cq->icid); 1270 DP_DEBUG(dev, QEDR_MSG_QP, 1271 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n", 1272 pd->pd_id, qp->qp_type, qp->max_inline_data, 1273 qp->state, qp->signaled, (attrs->srq) ? 1 : 0); 1274 DP_DEBUG(dev, QEDR_MSG_QP, 1275 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1276 qp->sq.max_sges, qp->sq_cq->icid); 1277 } 1278 1279 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1280 { 1281 qp->sq.db = dev->db_addr + 1282 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1283 qp->sq.db_data.data.icid = qp->icid + 1; 1284 qp->rq.db = dev->db_addr + 1285 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1286 qp->rq.db_data.data.icid = qp->icid; 1287 } 1288 1289 static inline void 1290 qedr_init_common_qp_in_params(struct qedr_dev *dev, 1291 struct qedr_pd *pd, 1292 struct qedr_qp *qp, 1293 struct ib_qp_init_attr *attrs, 1294 bool fmr_and_reserved_lkey, 1295 struct qed_rdma_create_qp_in_params *params) 1296 { 1297 /* QP handle to be written in an async event */ 1298 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp); 1299 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp); 1300 1301 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 1302 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 1303 params->pd = pd->pd_id; 1304 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 1305 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; 1306 params->stats_queue = 0; 1307 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1308 params->srq_id = 0; 1309 params->use_srq = false; 1310 } 1311 1312 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) 1313 { 1314 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. " 1315 "qp=%p. " 1316 "sq_addr=0x%llx, " 1317 "sq_len=%zd, " 1318 "rq_addr=0x%llx, " 1319 "rq_len=%zd" 1320 "\n", 1321 qp, 1322 qp->usq.buf_addr, 1323 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len); 1324 } 1325 1326 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp) 1327 { 1328 if (qp->usq.umem) 1329 ib_umem_release(qp->usq.umem); 1330 qp->usq.umem = NULL; 1331 1332 if (qp->urq.umem) 1333 ib_umem_release(qp->urq.umem); 1334 qp->urq.umem = NULL; 1335 } 1336 1337 static int qedr_create_user_qp(struct qedr_dev *dev, 1338 struct qedr_qp *qp, 1339 struct ib_pd *ibpd, 1340 struct ib_udata *udata, 1341 struct ib_qp_init_attr *attrs) 1342 { 1343 struct qed_rdma_create_qp_in_params in_params; 1344 struct qed_rdma_create_qp_out_params out_params; 1345 struct qedr_pd *pd = get_qedr_pd(ibpd); 1346 struct ib_ucontext *ib_ctx = NULL; 1347 struct qedr_ucontext *ctx = NULL; 1348 struct qedr_create_qp_ureq ureq; 1349 int rc = -EINVAL; 1350 1351 ib_ctx = ibpd->uobject->context; 1352 ctx = get_qedr_ucontext(ib_ctx); 1353 1354 memset(&ureq, 0, sizeof(ureq)); 1355 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq)); 1356 if (rc) { 1357 DP_ERR(dev, "Problem copying data from user space\n"); 1358 return rc; 1359 } 1360 1361 /* SQ - read access only (0), dma sync not required (0) */ 1362 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr, 1363 ureq.sq_len, 0, 0); 1364 if (rc) 1365 return rc; 1366 1367 /* RQ - read access only (0), dma sync not required (0) */ 1368 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr, 1369 ureq.rq_len, 0, 0); 1370 1371 if (rc) 1372 return rc; 1373 1374 memset(&in_params, 0, sizeof(in_params)); 1375 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 1376 in_params.qp_handle_lo = ureq.qp_handle_lo; 1377 in_params.qp_handle_hi = ureq.qp_handle_hi; 1378 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 1379 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 1380 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 1381 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 1382 1383 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1384 &in_params, &out_params); 1385 1386 if (!qp->qed_qp) { 1387 rc = -ENOMEM; 1388 goto err1; 1389 } 1390 1391 qp->qp_id = out_params.qp_id; 1392 qp->icid = out_params.icid; 1393 1394 rc = qedr_copy_qp_uresp(dev, qp, udata); 1395 if (rc) 1396 goto err; 1397 1398 qedr_qp_user_print(dev, qp); 1399 1400 return 0; 1401 err: 1402 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1403 if (rc) 1404 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc); 1405 1406 err1: 1407 qedr_cleanup_user(dev, qp); 1408 return rc; 1409 } 1410 1411 static int 1412 qedr_roce_create_kernel_qp(struct qedr_dev *dev, 1413 struct qedr_qp *qp, 1414 struct qed_rdma_create_qp_in_params *in_params, 1415 u32 n_sq_elems, u32 n_rq_elems) 1416 { 1417 struct qed_rdma_create_qp_out_params out_params; 1418 int rc; 1419 1420 rc = dev->ops->common->chain_alloc(dev->cdev, 1421 QED_CHAIN_USE_TO_PRODUCE, 1422 QED_CHAIN_MODE_PBL, 1423 QED_CHAIN_CNT_TYPE_U32, 1424 n_sq_elems, 1425 QEDR_SQE_ELEMENT_SIZE, 1426 &qp->sq.pbl); 1427 1428 if (rc) 1429 return rc; 1430 1431 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl); 1432 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl); 1433 1434 rc = dev->ops->common->chain_alloc(dev->cdev, 1435 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1436 QED_CHAIN_MODE_PBL, 1437 QED_CHAIN_CNT_TYPE_U32, 1438 n_rq_elems, 1439 QEDR_RQE_ELEMENT_SIZE, 1440 &qp->rq.pbl); 1441 if (rc) 1442 return rc; 1443 1444 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl); 1445 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl); 1446 1447 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1448 in_params, &out_params); 1449 1450 if (!qp->qed_qp) 1451 return -EINVAL; 1452 1453 qp->qp_id = out_params.qp_id; 1454 qp->icid = out_params.icid; 1455 1456 qedr_set_roce_db_info(dev, qp); 1457 1458 return 0; 1459 } 1460 1461 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) 1462 { 1463 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl); 1464 kfree(qp->wqe_wr_id); 1465 1466 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl); 1467 kfree(qp->rqe_wr_id); 1468 } 1469 1470 static int qedr_create_kernel_qp(struct qedr_dev *dev, 1471 struct qedr_qp *qp, 1472 struct ib_pd *ibpd, 1473 struct ib_qp_init_attr *attrs) 1474 { 1475 struct qed_rdma_create_qp_in_params in_params; 1476 struct qedr_pd *pd = get_qedr_pd(ibpd); 1477 int rc = -EINVAL; 1478 u32 n_rq_elems; 1479 u32 n_sq_elems; 1480 u32 n_sq_entries; 1481 1482 memset(&in_params, 0, sizeof(in_params)); 1483 1484 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in 1485 * the ring. The ring should allow at least a single WR, even if the 1486 * user requested none, due to allocation issues. 1487 * We should add an extra WR since the prod and cons indices of 1488 * wqe_wr_id are managed in such a way that the WQ is considered full 1489 * when (prod+1)%max_wr==cons. We currently don't do that because we 1490 * double the number of entries due an iSER issue that pushes far more 1491 * WRs than indicated. If we decline its ib_post_send() then we get 1492 * error prints in the dmesg we'd like to avoid. 1493 */ 1494 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 1495 dev->attr.max_sqe); 1496 1497 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id), 1498 GFP_KERNEL); 1499 if (!qp->wqe_wr_id) { 1500 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n"); 1501 return -ENOMEM; 1502 } 1503 1504 /* QP handle to be written in CQE */ 1505 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp); 1506 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp); 1507 1508 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in 1509 * the ring. There ring should allow at least a single WR, even if the 1510 * user requested none, due to allocation issues. 1511 */ 1512 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1); 1513 1514 /* Allocate driver internal RQ array */ 1515 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id), 1516 GFP_KERNEL); 1517 if (!qp->rqe_wr_id) { 1518 DP_ERR(dev, 1519 "create qp: failed RQ shadow memory allocation\n"); 1520 kfree(qp->wqe_wr_id); 1521 return -ENOMEM; 1522 } 1523 1524 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 1525 1526 n_sq_entries = attrs->cap.max_send_wr; 1527 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe); 1528 n_sq_entries = max_t(u32, n_sq_entries, 1); 1529 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE; 1530 1531 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE; 1532 1533 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params, 1534 n_sq_elems, n_rq_elems); 1535 if (rc) 1536 qedr_cleanup_kernel(dev, qp); 1537 1538 return rc; 1539 } 1540 1541 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, 1542 struct ib_qp_init_attr *attrs, 1543 struct ib_udata *udata) 1544 { 1545 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 1546 struct qedr_pd *pd = get_qedr_pd(ibpd); 1547 struct qedr_qp *qp; 1548 struct ib_qp *ibqp; 1549 int rc = 0; 1550 1551 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", 1552 udata ? "user library" : "kernel", pd); 1553 1554 rc = qedr_check_qp_attrs(ibpd, dev, attrs); 1555 if (rc) 1556 return ERR_PTR(rc); 1557 1558 if (attrs->srq) 1559 return ERR_PTR(-EINVAL); 1560 1561 DP_DEBUG(dev, QEDR_MSG_QP, 1562 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 1563 udata ? "user library" : "kernel", attrs->event_handler, pd, 1564 get_qedr_cq(attrs->send_cq), 1565 get_qedr_cq(attrs->send_cq)->icid, 1566 get_qedr_cq(attrs->recv_cq), 1567 get_qedr_cq(attrs->recv_cq)->icid); 1568 1569 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1570 if (!qp) { 1571 DP_ERR(dev, "create qp: failed allocating memory\n"); 1572 return ERR_PTR(-ENOMEM); 1573 } 1574 1575 qedr_set_common_qp_params(dev, qp, pd, attrs); 1576 1577 if (attrs->qp_type == IB_QPT_GSI) { 1578 ibqp = qedr_create_gsi_qp(dev, attrs, qp); 1579 if (IS_ERR(ibqp)) 1580 kfree(qp); 1581 return ibqp; 1582 } 1583 1584 if (udata) 1585 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); 1586 else 1587 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); 1588 1589 if (rc) 1590 goto err; 1591 1592 qp->ibqp.qp_num = qp->qp_id; 1593 1594 return &qp->ibqp; 1595 1596 err: 1597 kfree(qp); 1598 1599 return ERR_PTR(-EFAULT); 1600 } 1601 1602 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) 1603 { 1604 switch (qp_state) { 1605 case QED_ROCE_QP_STATE_RESET: 1606 return IB_QPS_RESET; 1607 case QED_ROCE_QP_STATE_INIT: 1608 return IB_QPS_INIT; 1609 case QED_ROCE_QP_STATE_RTR: 1610 return IB_QPS_RTR; 1611 case QED_ROCE_QP_STATE_RTS: 1612 return IB_QPS_RTS; 1613 case QED_ROCE_QP_STATE_SQD: 1614 return IB_QPS_SQD; 1615 case QED_ROCE_QP_STATE_ERR: 1616 return IB_QPS_ERR; 1617 case QED_ROCE_QP_STATE_SQE: 1618 return IB_QPS_SQE; 1619 } 1620 return IB_QPS_ERR; 1621 } 1622 1623 static enum qed_roce_qp_state qedr_get_state_from_ibqp( 1624 enum ib_qp_state qp_state) 1625 { 1626 switch (qp_state) { 1627 case IB_QPS_RESET: 1628 return QED_ROCE_QP_STATE_RESET; 1629 case IB_QPS_INIT: 1630 return QED_ROCE_QP_STATE_INIT; 1631 case IB_QPS_RTR: 1632 return QED_ROCE_QP_STATE_RTR; 1633 case IB_QPS_RTS: 1634 return QED_ROCE_QP_STATE_RTS; 1635 case IB_QPS_SQD: 1636 return QED_ROCE_QP_STATE_SQD; 1637 case IB_QPS_ERR: 1638 return QED_ROCE_QP_STATE_ERR; 1639 default: 1640 return QED_ROCE_QP_STATE_ERR; 1641 } 1642 } 1643 1644 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) 1645 { 1646 qed_chain_reset(&qph->pbl); 1647 qph->prod = 0; 1648 qph->cons = 0; 1649 qph->wqe_cons = 0; 1650 qph->db_data.data.value = cpu_to_le16(0); 1651 } 1652 1653 static int qedr_update_qp_state(struct qedr_dev *dev, 1654 struct qedr_qp *qp, 1655 enum qed_roce_qp_state new_state) 1656 { 1657 int status = 0; 1658 1659 if (new_state == qp->state) 1660 return 0; 1661 1662 switch (qp->state) { 1663 case QED_ROCE_QP_STATE_RESET: 1664 switch (new_state) { 1665 case QED_ROCE_QP_STATE_INIT: 1666 qp->prev_wqe_size = 0; 1667 qedr_reset_qp_hwq_info(&qp->sq); 1668 qedr_reset_qp_hwq_info(&qp->rq); 1669 break; 1670 default: 1671 status = -EINVAL; 1672 break; 1673 }; 1674 break; 1675 case QED_ROCE_QP_STATE_INIT: 1676 switch (new_state) { 1677 case QED_ROCE_QP_STATE_RTR: 1678 /* Update doorbell (in case post_recv was 1679 * done before move to RTR) 1680 */ 1681 wmb(); 1682 writel(qp->rq.db_data.raw, qp->rq.db); 1683 /* Make sure write takes effect */ 1684 mmiowb(); 1685 break; 1686 case QED_ROCE_QP_STATE_ERR: 1687 break; 1688 default: 1689 /* Invalid state change. */ 1690 status = -EINVAL; 1691 break; 1692 }; 1693 break; 1694 case QED_ROCE_QP_STATE_RTR: 1695 /* RTR->XXX */ 1696 switch (new_state) { 1697 case QED_ROCE_QP_STATE_RTS: 1698 break; 1699 case QED_ROCE_QP_STATE_ERR: 1700 break; 1701 default: 1702 /* Invalid state change. */ 1703 status = -EINVAL; 1704 break; 1705 }; 1706 break; 1707 case QED_ROCE_QP_STATE_RTS: 1708 /* RTS->XXX */ 1709 switch (new_state) { 1710 case QED_ROCE_QP_STATE_SQD: 1711 break; 1712 case QED_ROCE_QP_STATE_ERR: 1713 break; 1714 default: 1715 /* Invalid state change. */ 1716 status = -EINVAL; 1717 break; 1718 }; 1719 break; 1720 case QED_ROCE_QP_STATE_SQD: 1721 /* SQD->XXX */ 1722 switch (new_state) { 1723 case QED_ROCE_QP_STATE_RTS: 1724 case QED_ROCE_QP_STATE_ERR: 1725 break; 1726 default: 1727 /* Invalid state change. */ 1728 status = -EINVAL; 1729 break; 1730 }; 1731 break; 1732 case QED_ROCE_QP_STATE_ERR: 1733 /* ERR->XXX */ 1734 switch (new_state) { 1735 case QED_ROCE_QP_STATE_RESET: 1736 if ((qp->rq.prod != qp->rq.cons) || 1737 (qp->sq.prod != qp->sq.cons)) { 1738 DP_NOTICE(dev, 1739 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n", 1740 qp->rq.prod, qp->rq.cons, qp->sq.prod, 1741 qp->sq.cons); 1742 status = -EINVAL; 1743 } 1744 break; 1745 default: 1746 status = -EINVAL; 1747 break; 1748 }; 1749 break; 1750 default: 1751 status = -EINVAL; 1752 break; 1753 }; 1754 1755 return status; 1756 } 1757 1758 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1759 int attr_mask, struct ib_udata *udata) 1760 { 1761 struct qedr_qp *qp = get_qedr_qp(ibqp); 1762 struct qed_rdma_modify_qp_in_params qp_params = { 0 }; 1763 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); 1764 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1765 enum ib_qp_state old_qp_state, new_qp_state; 1766 int rc = 0; 1767 1768 DP_DEBUG(dev, QEDR_MSG_QP, 1769 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask, 1770 attr->qp_state); 1771 1772 old_qp_state = qedr_get_ibqp_state(qp->state); 1773 if (attr_mask & IB_QP_STATE) 1774 new_qp_state = attr->qp_state; 1775 else 1776 new_qp_state = old_qp_state; 1777 1778 if (!ib_modify_qp_is_ok 1779 (old_qp_state, new_qp_state, ibqp->qp_type, attr_mask, 1780 IB_LINK_LAYER_ETHERNET)) { 1781 DP_ERR(dev, 1782 "modify qp: invalid attribute mask=0x%x specified for\n" 1783 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n", 1784 attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state, 1785 new_qp_state); 1786 rc = -EINVAL; 1787 goto err; 1788 } 1789 1790 /* Translate the masks... */ 1791 if (attr_mask & IB_QP_STATE) { 1792 SET_FIELD(qp_params.modify_flags, 1793 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 1794 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state); 1795 } 1796 1797 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 1798 qp_params.sqd_async = true; 1799 1800 if (attr_mask & IB_QP_PKEY_INDEX) { 1801 SET_FIELD(qp_params.modify_flags, 1802 QED_ROCE_MODIFY_QP_VALID_PKEY, 1); 1803 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) { 1804 rc = -EINVAL; 1805 goto err; 1806 } 1807 1808 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT; 1809 } 1810 1811 if (attr_mask & IB_QP_QKEY) 1812 qp->qkey = attr->qkey; 1813 1814 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1815 SET_FIELD(qp_params.modify_flags, 1816 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 1817 qp_params.incoming_rdma_read_en = attr->qp_access_flags & 1818 IB_ACCESS_REMOTE_READ; 1819 qp_params.incoming_rdma_write_en = attr->qp_access_flags & 1820 IB_ACCESS_REMOTE_WRITE; 1821 qp_params.incoming_atomic_en = attr->qp_access_flags & 1822 IB_ACCESS_REMOTE_ATOMIC; 1823 } 1824 1825 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 1826 if (attr_mask & IB_QP_PATH_MTU) { 1827 if (attr->path_mtu < IB_MTU_256 || 1828 attr->path_mtu > IB_MTU_4096) { 1829 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n"); 1830 rc = -EINVAL; 1831 goto err; 1832 } 1833 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 1834 ib_mtu_enum_to_int(iboe_get_mtu 1835 (dev->ndev->mtu))); 1836 } 1837 1838 if (!qp->mtu) { 1839 qp->mtu = 1840 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 1841 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu); 1842 } 1843 1844 SET_FIELD(qp_params.modify_flags, 1845 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1); 1846 1847 qp_params.traffic_class_tos = grh->traffic_class; 1848 qp_params.flow_label = grh->flow_label; 1849 qp_params.hop_limit_ttl = grh->hop_limit; 1850 1851 qp->sgid_idx = grh->sgid_index; 1852 1853 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params); 1854 if (rc) { 1855 DP_ERR(dev, 1856 "modify qp: problems with GID index %d (rc=%d)\n", 1857 grh->sgid_index, rc); 1858 return rc; 1859 } 1860 1861 rc = qedr_get_dmac(dev, &attr->ah_attr, 1862 qp_params.remote_mac_addr); 1863 if (rc) 1864 return rc; 1865 1866 qp_params.use_local_mac = true; 1867 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr); 1868 1869 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n", 1870 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 1871 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 1872 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n", 1873 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 1874 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 1875 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n", 1876 qp_params.remote_mac_addr); 1877 1878 qp_params.mtu = qp->mtu; 1879 qp_params.lb_indication = false; 1880 } 1881 1882 if (!qp_params.mtu) { 1883 /* Stay with current MTU */ 1884 if (qp->mtu) 1885 qp_params.mtu = qp->mtu; 1886 else 1887 qp_params.mtu = 1888 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 1889 } 1890 1891 if (attr_mask & IB_QP_TIMEOUT) { 1892 SET_FIELD(qp_params.modify_flags, 1893 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 1894 1895 qp_params.ack_timeout = attr->timeout; 1896 if (attr->timeout) { 1897 u32 temp; 1898 1899 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000; 1900 /* FW requires [msec] */ 1901 qp_params.ack_timeout = temp; 1902 } else { 1903 /* Infinite */ 1904 qp_params.ack_timeout = 0; 1905 } 1906 } 1907 if (attr_mask & IB_QP_RETRY_CNT) { 1908 SET_FIELD(qp_params.modify_flags, 1909 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 1910 qp_params.retry_cnt = attr->retry_cnt; 1911 } 1912 1913 if (attr_mask & IB_QP_RNR_RETRY) { 1914 SET_FIELD(qp_params.modify_flags, 1915 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1); 1916 qp_params.rnr_retry_cnt = attr->rnr_retry; 1917 } 1918 1919 if (attr_mask & IB_QP_RQ_PSN) { 1920 SET_FIELD(qp_params.modify_flags, 1921 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1); 1922 qp_params.rq_psn = attr->rq_psn; 1923 qp->rq_psn = attr->rq_psn; 1924 } 1925 1926 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1927 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) { 1928 rc = -EINVAL; 1929 DP_ERR(dev, 1930 "unsupported max_rd_atomic=%d, supported=%d\n", 1931 attr->max_rd_atomic, 1932 dev->attr.max_qp_req_rd_atomic_resc); 1933 goto err; 1934 } 1935 1936 SET_FIELD(qp_params.modify_flags, 1937 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1); 1938 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 1939 } 1940 1941 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 1942 SET_FIELD(qp_params.modify_flags, 1943 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1); 1944 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 1945 } 1946 1947 if (attr_mask & IB_QP_SQ_PSN) { 1948 SET_FIELD(qp_params.modify_flags, 1949 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1); 1950 qp_params.sq_psn = attr->sq_psn; 1951 qp->sq_psn = attr->sq_psn; 1952 } 1953 1954 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1955 if (attr->max_dest_rd_atomic > 1956 dev->attr.max_qp_resp_rd_atomic_resc) { 1957 DP_ERR(dev, 1958 "unsupported max_dest_rd_atomic=%d, supported=%d\n", 1959 attr->max_dest_rd_atomic, 1960 dev->attr.max_qp_resp_rd_atomic_resc); 1961 1962 rc = -EINVAL; 1963 goto err; 1964 } 1965 1966 SET_FIELD(qp_params.modify_flags, 1967 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1); 1968 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 1969 } 1970 1971 if (attr_mask & IB_QP_DEST_QPN) { 1972 SET_FIELD(qp_params.modify_flags, 1973 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1); 1974 1975 qp_params.dest_qp = attr->dest_qp_num; 1976 qp->dest_qp_num = attr->dest_qp_num; 1977 } 1978 1979 if (qp->qp_type != IB_QPT_GSI) 1980 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx, 1981 qp->qed_qp, &qp_params); 1982 1983 if (attr_mask & IB_QP_STATE) { 1984 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 1985 rc = qedr_update_qp_state(dev, qp, qp_params.new_state); 1986 qp->state = qp_params.new_state; 1987 } 1988 1989 err: 1990 return rc; 1991 } 1992 1993 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params) 1994 { 1995 int ib_qp_acc_flags = 0; 1996 1997 if (params->incoming_rdma_write_en) 1998 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 1999 if (params->incoming_rdma_read_en) 2000 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 2001 if (params->incoming_atomic_en) 2002 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 2003 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 2004 return ib_qp_acc_flags; 2005 } 2006 2007 int qedr_query_qp(struct ib_qp *ibqp, 2008 struct ib_qp_attr *qp_attr, 2009 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 2010 { 2011 struct qed_rdma_query_qp_out_params params; 2012 struct qedr_qp *qp = get_qedr_qp(ibqp); 2013 struct qedr_dev *dev = qp->dev; 2014 int rc = 0; 2015 2016 memset(¶ms, 0, sizeof(params)); 2017 2018 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms); 2019 if (rc) 2020 goto err; 2021 2022 memset(qp_attr, 0, sizeof(*qp_attr)); 2023 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 2024 2025 qp_attr->qp_state = qedr_get_ibqp_state(params.state); 2026 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state); 2027 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu); 2028 qp_attr->path_mig_state = IB_MIG_MIGRATED; 2029 qp_attr->rq_psn = params.rq_psn; 2030 qp_attr->sq_psn = params.sq_psn; 2031 qp_attr->dest_qp_num = params.dest_qp; 2032 2033 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms); 2034 2035 qp_attr->cap.max_send_wr = qp->sq.max_wr; 2036 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 2037 qp_attr->cap.max_send_sge = qp->sq.max_sges; 2038 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 2039 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; 2040 qp_init_attr->cap = qp_attr->cap; 2041 2042 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2043 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 2044 params.flow_label, qp->sgid_idx, 2045 params.hop_limit_ttl, params.traffic_class_tos); 2046 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); 2047 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 2048 rdma_ah_set_sl(&qp_attr->ah_attr, 0); 2049 qp_attr->timeout = params.timeout; 2050 qp_attr->rnr_retry = params.rnr_retry; 2051 qp_attr->retry_cnt = params.retry_cnt; 2052 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 2053 qp_attr->pkey_index = params.pkey_index; 2054 qp_attr->port_num = 1; 2055 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 2056 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 2057 qp_attr->alt_pkey_index = 0; 2058 qp_attr->alt_port_num = 0; 2059 qp_attr->alt_timeout = 0; 2060 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 2061 2062 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0; 2063 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 2064 qp_attr->max_rd_atomic = params.max_rd_atomic; 2065 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0; 2066 2067 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n", 2068 qp_attr->cap.max_inline_data); 2069 2070 err: 2071 return rc; 2072 } 2073 2074 int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp) 2075 { 2076 int rc = 0; 2077 2078 if (qp->qp_type != IB_QPT_GSI) { 2079 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2080 if (rc) 2081 return rc; 2082 } 2083 2084 if (qp->ibqp.uobject && qp->ibqp.uobject->context) 2085 qedr_cleanup_user(dev, qp); 2086 else 2087 qedr_cleanup_kernel(dev, qp); 2088 2089 return 0; 2090 } 2091 2092 int qedr_destroy_qp(struct ib_qp *ibqp) 2093 { 2094 struct qedr_qp *qp = get_qedr_qp(ibqp); 2095 struct qedr_dev *dev = qp->dev; 2096 struct ib_qp_attr attr; 2097 int attr_mask = 0; 2098 int rc = 0; 2099 2100 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n", 2101 qp, qp->qp_type); 2102 2103 if ((qp->state != QED_ROCE_QP_STATE_RESET) && 2104 (qp->state != QED_ROCE_QP_STATE_ERR) && 2105 (qp->state != QED_ROCE_QP_STATE_INIT)) { 2106 2107 attr.qp_state = IB_QPS_ERR; 2108 attr_mask |= IB_QP_STATE; 2109 2110 /* Change the QP state to ERROR */ 2111 qedr_modify_qp(ibqp, &attr, attr_mask, NULL); 2112 } 2113 2114 if (qp->qp_type == IB_QPT_GSI) 2115 qedr_destroy_gsi_qp(dev); 2116 2117 qedr_free_qp_resources(dev, qp); 2118 2119 kfree(qp); 2120 2121 return rc; 2122 } 2123 2124 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr, 2125 struct ib_udata *udata) 2126 { 2127 struct qedr_ah *ah; 2128 2129 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 2130 if (!ah) 2131 return ERR_PTR(-ENOMEM); 2132 2133 ah->attr = *attr; 2134 2135 return &ah->ibah; 2136 } 2137 2138 int qedr_destroy_ah(struct ib_ah *ibah) 2139 { 2140 struct qedr_ah *ah = get_qedr_ah(ibah); 2141 2142 kfree(ah); 2143 return 0; 2144 } 2145 2146 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) 2147 { 2148 struct qedr_pbl *pbl, *tmp; 2149 2150 if (info->pbl_table) 2151 list_add_tail(&info->pbl_table->list_entry, 2152 &info->free_pbl_list); 2153 2154 if (!list_empty(&info->inuse_pbl_list)) 2155 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 2156 2157 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 2158 list_del(&pbl->list_entry); 2159 qedr_free_pbl(dev, &info->pbl_info, pbl); 2160 } 2161 } 2162 2163 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info, 2164 size_t page_list_len, bool two_layered) 2165 { 2166 struct qedr_pbl *tmp; 2167 int rc; 2168 2169 INIT_LIST_HEAD(&info->free_pbl_list); 2170 INIT_LIST_HEAD(&info->inuse_pbl_list); 2171 2172 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info, 2173 page_list_len, two_layered); 2174 if (rc) 2175 goto done; 2176 2177 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2178 if (IS_ERR(info->pbl_table)) { 2179 rc = PTR_ERR(info->pbl_table); 2180 goto done; 2181 } 2182 2183 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n", 2184 &info->pbl_table->pa); 2185 2186 /* in usual case we use 2 PBLs, so we add one to free 2187 * list and allocating another one 2188 */ 2189 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2190 if (IS_ERR(tmp)) { 2191 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n"); 2192 goto done; 2193 } 2194 2195 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 2196 2197 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa); 2198 2199 done: 2200 if (rc) 2201 free_mr_info(dev, info); 2202 2203 return rc; 2204 } 2205 2206 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 2207 u64 usr_addr, int acc, struct ib_udata *udata) 2208 { 2209 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2210 struct qedr_mr *mr; 2211 struct qedr_pd *pd; 2212 int rc = -ENOMEM; 2213 2214 pd = get_qedr_pd(ibpd); 2215 DP_DEBUG(dev, QEDR_MSG_MR, 2216 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 2217 pd->pd_id, start, len, usr_addr, acc); 2218 2219 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 2220 return ERR_PTR(-EINVAL); 2221 2222 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2223 if (!mr) 2224 return ERR_PTR(rc); 2225 2226 mr->type = QEDR_MR_USER; 2227 2228 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0); 2229 if (IS_ERR(mr->umem)) { 2230 rc = -EFAULT; 2231 goto err0; 2232 } 2233 2234 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); 2235 if (rc) 2236 goto err1; 2237 2238 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 2239 &mr->info.pbl_info, mr->umem->page_shift); 2240 2241 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2242 if (rc) { 2243 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2244 goto err1; 2245 } 2246 2247 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2248 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2249 mr->hw_mr.key = 0; 2250 mr->hw_mr.pd = pd->pd_id; 2251 mr->hw_mr.local_read = 1; 2252 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2253 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2254 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2255 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2256 mr->hw_mr.mw_bind = false; 2257 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 2258 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2259 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2260 mr->hw_mr.page_size_log = mr->umem->page_shift; 2261 mr->hw_mr.fbo = ib_umem_offset(mr->umem); 2262 mr->hw_mr.length = len; 2263 mr->hw_mr.vaddr = usr_addr; 2264 mr->hw_mr.zbva = false; 2265 mr->hw_mr.phy_mr = false; 2266 mr->hw_mr.dma_mr = false; 2267 2268 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2269 if (rc) { 2270 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2271 goto err2; 2272 } 2273 2274 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2275 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2276 mr->hw_mr.remote_atomic) 2277 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2278 2279 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n", 2280 mr->ibmr.lkey); 2281 return &mr->ibmr; 2282 2283 err2: 2284 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2285 err1: 2286 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2287 err0: 2288 kfree(mr); 2289 return ERR_PTR(rc); 2290 } 2291 2292 int qedr_dereg_mr(struct ib_mr *ib_mr) 2293 { 2294 struct qedr_mr *mr = get_qedr_mr(ib_mr); 2295 struct qedr_dev *dev = get_qedr_dev(ib_mr->device); 2296 int rc = 0; 2297 2298 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid); 2299 if (rc) 2300 return rc; 2301 2302 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2303 2304 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR)) 2305 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2306 2307 /* it could be user registered memory. */ 2308 if (mr->umem) 2309 ib_umem_release(mr->umem); 2310 2311 kfree(mr); 2312 2313 return rc; 2314 } 2315 2316 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, 2317 int max_page_list_len) 2318 { 2319 struct qedr_pd *pd = get_qedr_pd(ibpd); 2320 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2321 struct qedr_mr *mr; 2322 int rc = -ENOMEM; 2323 2324 DP_DEBUG(dev, QEDR_MSG_MR, 2325 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id, 2326 max_page_list_len); 2327 2328 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2329 if (!mr) 2330 return ERR_PTR(rc); 2331 2332 mr->dev = dev; 2333 mr->type = QEDR_MR_FRMR; 2334 2335 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1); 2336 if (rc) 2337 goto err0; 2338 2339 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2340 if (rc) { 2341 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2342 goto err0; 2343 } 2344 2345 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2346 mr->hw_mr.tid_type = QED_RDMA_TID_FMR; 2347 mr->hw_mr.key = 0; 2348 mr->hw_mr.pd = pd->pd_id; 2349 mr->hw_mr.local_read = 1; 2350 mr->hw_mr.local_write = 0; 2351 mr->hw_mr.remote_read = 0; 2352 mr->hw_mr.remote_write = 0; 2353 mr->hw_mr.remote_atomic = 0; 2354 mr->hw_mr.mw_bind = false; 2355 mr->hw_mr.pbl_ptr = 0; 2356 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2357 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2358 mr->hw_mr.fbo = 0; 2359 mr->hw_mr.length = 0; 2360 mr->hw_mr.vaddr = 0; 2361 mr->hw_mr.zbva = false; 2362 mr->hw_mr.phy_mr = true; 2363 mr->hw_mr.dma_mr = false; 2364 2365 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2366 if (rc) { 2367 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2368 goto err1; 2369 } 2370 2371 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2372 mr->ibmr.rkey = mr->ibmr.lkey; 2373 2374 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey); 2375 return mr; 2376 2377 err1: 2378 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2379 err0: 2380 kfree(mr); 2381 return ERR_PTR(rc); 2382 } 2383 2384 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, 2385 enum ib_mr_type mr_type, u32 max_num_sg) 2386 { 2387 struct qedr_dev *dev; 2388 struct qedr_mr *mr; 2389 2390 if (mr_type != IB_MR_TYPE_MEM_REG) 2391 return ERR_PTR(-EINVAL); 2392 2393 mr = __qedr_alloc_mr(ibpd, max_num_sg); 2394 2395 if (IS_ERR(mr)) 2396 return ERR_PTR(-EINVAL); 2397 2398 dev = mr->dev; 2399 2400 return &mr->ibmr; 2401 } 2402 2403 static int qedr_set_page(struct ib_mr *ibmr, u64 addr) 2404 { 2405 struct qedr_mr *mr = get_qedr_mr(ibmr); 2406 struct qedr_pbl *pbl_table; 2407 struct regpair *pbe; 2408 u32 pbes_in_page; 2409 2410 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 2411 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages); 2412 return -ENOMEM; 2413 } 2414 2415 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n", 2416 mr->npages, addr); 2417 2418 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 2419 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 2420 pbe = (struct regpair *)pbl_table->va; 2421 pbe += mr->npages % pbes_in_page; 2422 pbe->lo = cpu_to_le32((u32)addr); 2423 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 2424 2425 mr->npages++; 2426 2427 return 0; 2428 } 2429 2430 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info) 2431 { 2432 int work = info->completed - info->completed_handled - 1; 2433 2434 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work); 2435 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 2436 struct qedr_pbl *pbl; 2437 2438 /* Free all the page list that are possible to be freed 2439 * (all the ones that were invalidated), under the assumption 2440 * that if an FMR was completed successfully that means that 2441 * if there was an invalidate operation before it also ended 2442 */ 2443 pbl = list_first_entry(&info->inuse_pbl_list, 2444 struct qedr_pbl, list_entry); 2445 list_move_tail(&pbl->list_entry, &info->free_pbl_list); 2446 info->completed_handled++; 2447 } 2448 } 2449 2450 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 2451 int sg_nents, unsigned int *sg_offset) 2452 { 2453 struct qedr_mr *mr = get_qedr_mr(ibmr); 2454 2455 mr->npages = 0; 2456 2457 handle_completed_mrs(mr->dev, &mr->info); 2458 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page); 2459 } 2460 2461 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc) 2462 { 2463 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2464 struct qedr_pd *pd = get_qedr_pd(ibpd); 2465 struct qedr_mr *mr; 2466 int rc; 2467 2468 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2469 if (!mr) 2470 return ERR_PTR(-ENOMEM); 2471 2472 mr->type = QEDR_MR_DMA; 2473 2474 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2475 if (rc) { 2476 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2477 goto err1; 2478 } 2479 2480 /* index only, 18 bit long, lkey = itid << 8 | key */ 2481 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2482 mr->hw_mr.pd = pd->pd_id; 2483 mr->hw_mr.local_read = 1; 2484 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2485 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2486 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2487 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2488 mr->hw_mr.dma_mr = true; 2489 2490 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2491 if (rc) { 2492 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2493 goto err2; 2494 } 2495 2496 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2497 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2498 mr->hw_mr.remote_atomic) 2499 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2500 2501 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey); 2502 return &mr->ibmr; 2503 2504 err2: 2505 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2506 err1: 2507 kfree(mr); 2508 return ERR_PTR(rc); 2509 } 2510 2511 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq) 2512 { 2513 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 2514 } 2515 2516 static int sge_data_len(struct ib_sge *sg_list, int num_sge) 2517 { 2518 int i, len = 0; 2519 2520 for (i = 0; i < num_sge; i++) 2521 len += sg_list[i].length; 2522 2523 return len; 2524 } 2525 2526 static void swap_wqe_data64(u64 *p) 2527 { 2528 int i; 2529 2530 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 2531 *p = cpu_to_be64(cpu_to_le64(*p)); 2532 } 2533 2534 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev, 2535 struct qedr_qp *qp, u8 *wqe_size, 2536 struct ib_send_wr *wr, 2537 struct ib_send_wr **bad_wr, u8 *bits, 2538 u8 bit) 2539 { 2540 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 2541 char *seg_prt, *wqe; 2542 int i, seg_siz; 2543 2544 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 2545 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size); 2546 *bad_wr = wr; 2547 return 0; 2548 } 2549 2550 if (!data_size) 2551 return data_size; 2552 2553 *bits |= bit; 2554 2555 seg_prt = NULL; 2556 wqe = NULL; 2557 seg_siz = 0; 2558 2559 /* Copy data inline */ 2560 for (i = 0; i < wr->num_sge; i++) { 2561 u32 len = wr->sg_list[i].length; 2562 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 2563 2564 while (len > 0) { 2565 u32 cur; 2566 2567 /* New segment required */ 2568 if (!seg_siz) { 2569 wqe = (char *)qed_chain_produce(&qp->sq.pbl); 2570 seg_prt = wqe; 2571 seg_siz = sizeof(struct rdma_sq_common_wqe); 2572 (*wqe_size)++; 2573 } 2574 2575 /* Calculate currently allowed length */ 2576 cur = min_t(u32, len, seg_siz); 2577 memcpy(seg_prt, src, cur); 2578 2579 /* Update segment variables */ 2580 seg_prt += cur; 2581 seg_siz -= cur; 2582 2583 /* Update sge variables */ 2584 src += cur; 2585 len -= cur; 2586 2587 /* Swap fully-completed segments */ 2588 if (!seg_siz) 2589 swap_wqe_data64((u64 *)wqe); 2590 } 2591 } 2592 2593 /* swap last not completed segment */ 2594 if (seg_siz) 2595 swap_wqe_data64((u64 *)wqe); 2596 2597 return data_size; 2598 } 2599 2600 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 2601 do { \ 2602 DMA_REGPAIR_LE(sge->addr, vaddr); \ 2603 (sge)->length = cpu_to_le32(vlength); \ 2604 (sge)->flags = cpu_to_le32(vflags); \ 2605 } while (0) 2606 2607 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 2608 do { \ 2609 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \ 2610 (hdr)->num_sges = num_sge; \ 2611 } while (0) 2612 2613 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 2614 do { \ 2615 DMA_REGPAIR_LE(sge->addr, vaddr); \ 2616 (sge)->length = cpu_to_le32(vlength); \ 2617 (sge)->l_key = cpu_to_le32(vlkey); \ 2618 } while (0) 2619 2620 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, 2621 struct ib_send_wr *wr) 2622 { 2623 u32 data_size = 0; 2624 int i; 2625 2626 for (i = 0; i < wr->num_sge; i++) { 2627 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl); 2628 2629 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr); 2630 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 2631 sge->length = cpu_to_le32(wr->sg_list[i].length); 2632 data_size += wr->sg_list[i].length; 2633 } 2634 2635 if (wqe_size) 2636 *wqe_size += wr->num_sge; 2637 2638 return data_size; 2639 } 2640 2641 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev, 2642 struct qedr_qp *qp, 2643 struct rdma_sq_rdma_wqe_1st *rwqe, 2644 struct rdma_sq_rdma_wqe_2nd *rwqe2, 2645 struct ib_send_wr *wr, 2646 struct ib_send_wr **bad_wr) 2647 { 2648 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 2649 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr); 2650 2651 if (wr->send_flags & IB_SEND_INLINE && 2652 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 2653 wr->opcode == IB_WR_RDMA_WRITE)) { 2654 u8 flags = 0; 2655 2656 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 2657 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, 2658 bad_wr, &rwqe->flags, flags); 2659 } 2660 2661 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr); 2662 } 2663 2664 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev, 2665 struct qedr_qp *qp, 2666 struct rdma_sq_send_wqe_1st *swqe, 2667 struct rdma_sq_send_wqe_2st *swqe2, 2668 struct ib_send_wr *wr, 2669 struct ib_send_wr **bad_wr) 2670 { 2671 memset(swqe2, 0, sizeof(*swqe2)); 2672 if (wr->send_flags & IB_SEND_INLINE) { 2673 u8 flags = 0; 2674 2675 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 2676 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, 2677 bad_wr, &swqe->flags, flags); 2678 } 2679 2680 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr); 2681 } 2682 2683 static int qedr_prepare_reg(struct qedr_qp *qp, 2684 struct rdma_sq_fmr_wqe_1st *fwqe1, 2685 struct ib_reg_wr *wr) 2686 { 2687 struct qedr_mr *mr = get_qedr_mr(wr->mr); 2688 struct rdma_sq_fmr_wqe_2nd *fwqe2; 2689 2690 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl); 2691 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 2692 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 2693 fwqe1->l_key = wr->key; 2694 2695 fwqe2->access_ctrl = 0; 2696 2697 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 2698 !!(wr->access & IB_ACCESS_REMOTE_READ)); 2699 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 2700 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 2701 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 2702 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 2703 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 2704 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 2705 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 2706 fwqe2->fmr_ctrl = 0; 2707 2708 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 2709 ilog2(mr->ibmr.page_size) - 12); 2710 2711 fwqe2->length_hi = 0; 2712 fwqe2->length_lo = mr->ibmr.length; 2713 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 2714 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 2715 2716 qp->wqe_wr_id[qp->sq.prod].mr = mr; 2717 2718 return 0; 2719 } 2720 2721 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 2722 { 2723 switch (opcode) { 2724 case IB_WR_RDMA_WRITE: 2725 case IB_WR_RDMA_WRITE_WITH_IMM: 2726 return IB_WC_RDMA_WRITE; 2727 case IB_WR_SEND_WITH_IMM: 2728 case IB_WR_SEND: 2729 case IB_WR_SEND_WITH_INV: 2730 return IB_WC_SEND; 2731 case IB_WR_RDMA_READ: 2732 return IB_WC_RDMA_READ; 2733 case IB_WR_ATOMIC_CMP_AND_SWP: 2734 return IB_WC_COMP_SWAP; 2735 case IB_WR_ATOMIC_FETCH_AND_ADD: 2736 return IB_WC_FETCH_ADD; 2737 case IB_WR_REG_MR: 2738 return IB_WC_REG_MR; 2739 case IB_WR_LOCAL_INV: 2740 return IB_WC_LOCAL_INV; 2741 default: 2742 return IB_WC_SEND; 2743 } 2744 } 2745 2746 static inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr) 2747 { 2748 int wq_is_full, err_wr, pbl_is_full; 2749 struct qedr_dev *dev = qp->dev; 2750 2751 /* prevent SQ overflow and/or processing of a bad WR */ 2752 err_wr = wr->num_sge > qp->sq.max_sges; 2753 wq_is_full = qedr_wq_is_full(&qp->sq); 2754 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) < 2755 QEDR_MAX_SQE_ELEMENTS_PER_SQE; 2756 if (wq_is_full || err_wr || pbl_is_full) { 2757 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) { 2758 DP_ERR(dev, 2759 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n", 2760 qp); 2761 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL; 2762 } 2763 2764 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) { 2765 DP_ERR(dev, 2766 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n", 2767 qp); 2768 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR; 2769 } 2770 2771 if (pbl_is_full && 2772 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) { 2773 DP_ERR(dev, 2774 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n", 2775 qp); 2776 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL; 2777 } 2778 return false; 2779 } 2780 return true; 2781 } 2782 2783 static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 2784 struct ib_send_wr **bad_wr) 2785 { 2786 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 2787 struct qedr_qp *qp = get_qedr_qp(ibqp); 2788 struct rdma_sq_atomic_wqe_1st *awqe1; 2789 struct rdma_sq_atomic_wqe_2nd *awqe2; 2790 struct rdma_sq_atomic_wqe_3rd *awqe3; 2791 struct rdma_sq_send_wqe_2st *swqe2; 2792 struct rdma_sq_local_inv_wqe *iwqe; 2793 struct rdma_sq_rdma_wqe_2nd *rwqe2; 2794 struct rdma_sq_send_wqe_1st *swqe; 2795 struct rdma_sq_rdma_wqe_1st *rwqe; 2796 struct rdma_sq_fmr_wqe_1st *fwqe1; 2797 struct rdma_sq_common_wqe *wqe; 2798 u32 length; 2799 int rc = 0; 2800 bool comp; 2801 2802 if (!qedr_can_post_send(qp, wr)) { 2803 *bad_wr = wr; 2804 return -ENOMEM; 2805 } 2806 2807 wqe = qed_chain_produce(&qp->sq.pbl); 2808 qp->wqe_wr_id[qp->sq.prod].signaled = 2809 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 2810 2811 wqe->flags = 0; 2812 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, 2813 !!(wr->send_flags & IB_SEND_SOLICITED)); 2814 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled; 2815 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 2816 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, 2817 !!(wr->send_flags & IB_SEND_FENCE)); 2818 wqe->prev_wqe_size = qp->prev_wqe_size; 2819 2820 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode); 2821 2822 switch (wr->opcode) { 2823 case IB_WR_SEND_WITH_IMM: 2824 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 2825 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 2826 swqe->wqe_size = 2; 2827 swqe2 = qed_chain_produce(&qp->sq.pbl); 2828 2829 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data); 2830 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 2831 wr, bad_wr); 2832 swqe->length = cpu_to_le32(length); 2833 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 2834 qp->prev_wqe_size = swqe->wqe_size; 2835 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 2836 break; 2837 case IB_WR_SEND: 2838 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 2839 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 2840 2841 swqe->wqe_size = 2; 2842 swqe2 = qed_chain_produce(&qp->sq.pbl); 2843 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 2844 wr, bad_wr); 2845 swqe->length = cpu_to_le32(length); 2846 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 2847 qp->prev_wqe_size = swqe->wqe_size; 2848 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 2849 break; 2850 case IB_WR_SEND_WITH_INV: 2851 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 2852 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 2853 swqe2 = qed_chain_produce(&qp->sq.pbl); 2854 swqe->wqe_size = 2; 2855 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey); 2856 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 2857 wr, bad_wr); 2858 swqe->length = cpu_to_le32(length); 2859 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 2860 qp->prev_wqe_size = swqe->wqe_size; 2861 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 2862 break; 2863 2864 case IB_WR_RDMA_WRITE_WITH_IMM: 2865 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 2866 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 2867 2868 rwqe->wqe_size = 2; 2869 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 2870 rwqe2 = qed_chain_produce(&qp->sq.pbl); 2871 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 2872 wr, bad_wr); 2873 rwqe->length = cpu_to_le32(length); 2874 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 2875 qp->prev_wqe_size = rwqe->wqe_size; 2876 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 2877 break; 2878 case IB_WR_RDMA_WRITE: 2879 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 2880 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 2881 2882 rwqe->wqe_size = 2; 2883 rwqe2 = qed_chain_produce(&qp->sq.pbl); 2884 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 2885 wr, bad_wr); 2886 rwqe->length = cpu_to_le32(length); 2887 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 2888 qp->prev_wqe_size = rwqe->wqe_size; 2889 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 2890 break; 2891 case IB_WR_RDMA_READ_WITH_INV: 2892 DP_ERR(dev, 2893 "RDMA READ WITH INVALIDATE not supported\n"); 2894 *bad_wr = wr; 2895 rc = -EINVAL; 2896 break; 2897 2898 case IB_WR_RDMA_READ: 2899 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 2900 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 2901 2902 rwqe->wqe_size = 2; 2903 rwqe2 = qed_chain_produce(&qp->sq.pbl); 2904 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 2905 wr, bad_wr); 2906 rwqe->length = cpu_to_le32(length); 2907 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 2908 qp->prev_wqe_size = rwqe->wqe_size; 2909 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 2910 break; 2911 2912 case IB_WR_ATOMIC_CMP_AND_SWP: 2913 case IB_WR_ATOMIC_FETCH_AND_ADD: 2914 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe; 2915 awqe1->wqe_size = 4; 2916 2917 awqe2 = qed_chain_produce(&qp->sq.pbl); 2918 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr); 2919 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 2920 2921 awqe3 = qed_chain_produce(&qp->sq.pbl); 2922 2923 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 2924 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 2925 DMA_REGPAIR_LE(awqe3->swap_data, 2926 atomic_wr(wr)->compare_add); 2927 } else { 2928 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 2929 DMA_REGPAIR_LE(awqe3->swap_data, 2930 atomic_wr(wr)->swap); 2931 DMA_REGPAIR_LE(awqe3->cmp_data, 2932 atomic_wr(wr)->compare_add); 2933 } 2934 2935 qedr_prepare_sq_sges(qp, NULL, wr); 2936 2937 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size; 2938 qp->prev_wqe_size = awqe1->wqe_size; 2939 break; 2940 2941 case IB_WR_LOCAL_INV: 2942 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 2943 iwqe->wqe_size = 1; 2944 2945 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 2946 iwqe->inv_l_key = wr->ex.invalidate_rkey; 2947 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size; 2948 qp->prev_wqe_size = iwqe->wqe_size; 2949 break; 2950 case IB_WR_REG_MR: 2951 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n"); 2952 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 2953 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 2954 fwqe1->wqe_size = 2; 2955 2956 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr)); 2957 if (rc) { 2958 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc); 2959 *bad_wr = wr; 2960 break; 2961 } 2962 2963 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 2964 qp->prev_wqe_size = fwqe1->wqe_size; 2965 break; 2966 default: 2967 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode); 2968 rc = -EINVAL; 2969 *bad_wr = wr; 2970 break; 2971 } 2972 2973 if (*bad_wr) { 2974 u16 value; 2975 2976 /* Restore prod to its position before 2977 * this WR was processed 2978 */ 2979 value = le16_to_cpu(qp->sq.db_data.data.value); 2980 qed_chain_set_prod(&qp->sq.pbl, value, wqe); 2981 2982 /* Restore prev_wqe_size */ 2983 qp->prev_wqe_size = wqe->prev_wqe_size; 2984 rc = -EINVAL; 2985 DP_ERR(dev, "POST SEND FAILED\n"); 2986 } 2987 2988 return rc; 2989 } 2990 2991 int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 2992 struct ib_send_wr **bad_wr) 2993 { 2994 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 2995 struct qedr_qp *qp = get_qedr_qp(ibqp); 2996 unsigned long flags; 2997 int rc = 0; 2998 2999 *bad_wr = NULL; 3000 3001 if (qp->qp_type == IB_QPT_GSI) 3002 return qedr_gsi_post_send(ibqp, wr, bad_wr); 3003 3004 spin_lock_irqsave(&qp->q_lock, flags); 3005 3006 if ((qp->state != QED_ROCE_QP_STATE_RTS) && 3007 (qp->state != QED_ROCE_QP_STATE_ERR) && 3008 (qp->state != QED_ROCE_QP_STATE_SQD)) { 3009 spin_unlock_irqrestore(&qp->q_lock, flags); 3010 *bad_wr = wr; 3011 DP_DEBUG(dev, QEDR_MSG_CQ, 3012 "QP in wrong state! QP icid=0x%x state %d\n", 3013 qp->icid, qp->state); 3014 return -EINVAL; 3015 } 3016 3017 while (wr) { 3018 rc = __qedr_post_send(ibqp, wr, bad_wr); 3019 if (rc) 3020 break; 3021 3022 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 3023 3024 qedr_inc_sw_prod(&qp->sq); 3025 3026 qp->sq.db_data.data.value++; 3027 3028 wr = wr->next; 3029 } 3030 3031 /* Trigger doorbell 3032 * If there was a failure in the first WR then it will be triggered in 3033 * vane. However this is not harmful (as long as the producer value is 3034 * unchanged). For performance reasons we avoid checking for this 3035 * redundant doorbell. 3036 */ 3037 wmb(); 3038 writel(qp->sq.db_data.raw, qp->sq.db); 3039 3040 /* Make sure write sticks */ 3041 mmiowb(); 3042 3043 spin_unlock_irqrestore(&qp->q_lock, flags); 3044 3045 return rc; 3046 } 3047 3048 int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, 3049 struct ib_recv_wr **bad_wr) 3050 { 3051 struct qedr_qp *qp = get_qedr_qp(ibqp); 3052 struct qedr_dev *dev = qp->dev; 3053 unsigned long flags; 3054 int status = 0; 3055 3056 if (qp->qp_type == IB_QPT_GSI) 3057 return qedr_gsi_post_recv(ibqp, wr, bad_wr); 3058 3059 spin_lock_irqsave(&qp->q_lock, flags); 3060 3061 if (qp->state == QED_ROCE_QP_STATE_RESET) { 3062 spin_unlock_irqrestore(&qp->q_lock, flags); 3063 *bad_wr = wr; 3064 return -EINVAL; 3065 } 3066 3067 while (wr) { 3068 int i; 3069 3070 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) < 3071 QEDR_MAX_RQE_ELEMENTS_PER_RQE || 3072 wr->num_sge > qp->rq.max_sges) { 3073 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n", 3074 qed_chain_get_elem_left_u32(&qp->rq.pbl), 3075 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge, 3076 qp->rq.max_sges); 3077 status = -ENOMEM; 3078 *bad_wr = wr; 3079 break; 3080 } 3081 for (i = 0; i < wr->num_sge; i++) { 3082 u32 flags = 0; 3083 struct rdma_rq_sge *rqe = 3084 qed_chain_produce(&qp->rq.pbl); 3085 3086 /* First one must include the number 3087 * of SGE in the list 3088 */ 3089 if (!i) 3090 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 3091 wr->num_sge); 3092 3093 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 3094 wr->sg_list[i].lkey); 3095 3096 RQ_SGE_SET(rqe, wr->sg_list[i].addr, 3097 wr->sg_list[i].length, flags); 3098 } 3099 3100 /* Special case of no sges. FW requires between 1-4 sges... 3101 * in this case we need to post 1 sge with length zero. this is 3102 * because rdma write with immediate consumes an RQ. 3103 */ 3104 if (!wr->num_sge) { 3105 u32 flags = 0; 3106 struct rdma_rq_sge *rqe = 3107 qed_chain_produce(&qp->rq.pbl); 3108 3109 /* First one must include the number 3110 * of SGE in the list 3111 */ 3112 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0); 3113 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 3114 3115 RQ_SGE_SET(rqe, 0, 0, flags); 3116 i = 1; 3117 } 3118 3119 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 3120 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 3121 3122 qedr_inc_sw_prod(&qp->rq); 3123 3124 /* Flush all the writes before signalling doorbell */ 3125 wmb(); 3126 3127 qp->rq.db_data.data.value++; 3128 3129 writel(qp->rq.db_data.raw, qp->rq.db); 3130 3131 /* Make sure write sticks */ 3132 mmiowb(); 3133 3134 wr = wr->next; 3135 } 3136 3137 spin_unlock_irqrestore(&qp->q_lock, flags); 3138 3139 return status; 3140 } 3141 3142 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe) 3143 { 3144 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3145 3146 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) == 3147 cq->pbl_toggle; 3148 } 3149 3150 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe) 3151 { 3152 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3153 struct qedr_qp *qp; 3154 3155 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi, 3156 resp_cqe->qp_handle.lo, 3157 u64); 3158 return qp; 3159 } 3160 3161 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe) 3162 { 3163 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3164 3165 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 3166 } 3167 3168 /* Return latest CQE (needs processing) */ 3169 static union rdma_cqe *get_cqe(struct qedr_cq *cq) 3170 { 3171 return cq->latest_cqe; 3172 } 3173 3174 /* In fmr we need to increase the number of fmr completed counter for the fmr 3175 * algorithm determining whether we can free a pbl or not. 3176 * we need to perform this whether the work request was signaled or not. for 3177 * this purpose we call this function from the condition that checks if a wr 3178 * should be skipped, to make sure we don't miss it ( possibly this fmr 3179 * operation was not signalted) 3180 */ 3181 static inline void qedr_chk_if_fmr(struct qedr_qp *qp) 3182 { 3183 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 3184 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3185 } 3186 3187 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, 3188 struct qedr_cq *cq, int num_entries, 3189 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, 3190 int force) 3191 { 3192 u16 cnt = 0; 3193 3194 while (num_entries && qp->sq.wqe_cons != hw_cons) { 3195 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 3196 qedr_chk_if_fmr(qp); 3197 /* skip WC */ 3198 goto next_cqe; 3199 } 3200 3201 /* fill WC */ 3202 wc->status = status; 3203 wc->vendor_err = 0; 3204 wc->wc_flags = 0; 3205 wc->src_qp = qp->id; 3206 wc->qp = &qp->ibqp; 3207 3208 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 3209 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 3210 3211 switch (wc->opcode) { 3212 case IB_WC_RDMA_WRITE: 3213 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3214 break; 3215 case IB_WC_COMP_SWAP: 3216 case IB_WC_FETCH_ADD: 3217 wc->byte_len = 8; 3218 break; 3219 case IB_WC_REG_MR: 3220 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3221 break; 3222 case IB_WC_RDMA_READ: 3223 case IB_WC_SEND: 3224 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3225 break; 3226 default: 3227 break; 3228 } 3229 3230 num_entries--; 3231 wc++; 3232 cnt++; 3233 next_cqe: 3234 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 3235 qed_chain_consume(&qp->sq.pbl); 3236 qedr_inc_sw_cons(&qp->sq); 3237 } 3238 3239 return cnt; 3240 } 3241 3242 static int qedr_poll_cq_req(struct qedr_dev *dev, 3243 struct qedr_qp *qp, struct qedr_cq *cq, 3244 int num_entries, struct ib_wc *wc, 3245 struct rdma_cqe_requester *req) 3246 { 3247 int cnt = 0; 3248 3249 switch (req->status) { 3250 case RDMA_CQE_REQ_STS_OK: 3251 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3252 IB_WC_SUCCESS, 0); 3253 break; 3254 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 3255 if (qp->state != QED_ROCE_QP_STATE_ERR) 3256 DP_ERR(dev, 3257 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3258 cq->icid, qp->icid); 3259 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3260 IB_WC_WR_FLUSH_ERR, 1); 3261 break; 3262 default: 3263 /* process all WQE before the cosumer */ 3264 qp->state = QED_ROCE_QP_STATE_ERR; 3265 cnt = process_req(dev, qp, cq, num_entries, wc, 3266 req->sq_cons - 1, IB_WC_SUCCESS, 0); 3267 wc += cnt; 3268 /* if we have extra WC fill it with actual error info */ 3269 if (cnt < num_entries) { 3270 enum ib_wc_status wc_status; 3271 3272 switch (req->status) { 3273 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 3274 DP_ERR(dev, 3275 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3276 cq->icid, qp->icid); 3277 wc_status = IB_WC_BAD_RESP_ERR; 3278 break; 3279 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 3280 DP_ERR(dev, 3281 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3282 cq->icid, qp->icid); 3283 wc_status = IB_WC_LOC_LEN_ERR; 3284 break; 3285 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 3286 DP_ERR(dev, 3287 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3288 cq->icid, qp->icid); 3289 wc_status = IB_WC_LOC_QP_OP_ERR; 3290 break; 3291 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 3292 DP_ERR(dev, 3293 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3294 cq->icid, qp->icid); 3295 wc_status = IB_WC_LOC_PROT_ERR; 3296 break; 3297 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 3298 DP_ERR(dev, 3299 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3300 cq->icid, qp->icid); 3301 wc_status = IB_WC_MW_BIND_ERR; 3302 break; 3303 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 3304 DP_ERR(dev, 3305 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3306 cq->icid, qp->icid); 3307 wc_status = IB_WC_REM_INV_REQ_ERR; 3308 break; 3309 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 3310 DP_ERR(dev, 3311 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3312 cq->icid, qp->icid); 3313 wc_status = IB_WC_REM_ACCESS_ERR; 3314 break; 3315 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 3316 DP_ERR(dev, 3317 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3318 cq->icid, qp->icid); 3319 wc_status = IB_WC_REM_OP_ERR; 3320 break; 3321 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 3322 DP_ERR(dev, 3323 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3324 cq->icid, qp->icid); 3325 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 3326 break; 3327 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 3328 DP_ERR(dev, 3329 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3330 cq->icid, qp->icid); 3331 wc_status = IB_WC_RETRY_EXC_ERR; 3332 break; 3333 default: 3334 DP_ERR(dev, 3335 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3336 cq->icid, qp->icid); 3337 wc_status = IB_WC_GENERAL_ERR; 3338 } 3339 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 3340 wc_status, 1); 3341 } 3342 } 3343 3344 return cnt; 3345 } 3346 3347 static inline int qedr_cqe_resp_status_to_ib(u8 status) 3348 { 3349 switch (status) { 3350 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 3351 return IB_WC_LOC_ACCESS_ERR; 3352 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 3353 return IB_WC_LOC_LEN_ERR; 3354 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 3355 return IB_WC_LOC_QP_OP_ERR; 3356 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 3357 return IB_WC_LOC_PROT_ERR; 3358 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 3359 return IB_WC_MW_BIND_ERR; 3360 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 3361 return IB_WC_REM_INV_RD_REQ_ERR; 3362 case RDMA_CQE_RESP_STS_OK: 3363 return IB_WC_SUCCESS; 3364 default: 3365 return IB_WC_GENERAL_ERR; 3366 } 3367 } 3368 3369 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp, 3370 struct ib_wc *wc) 3371 { 3372 wc->status = IB_WC_SUCCESS; 3373 wc->byte_len = le32_to_cpu(resp->length); 3374 3375 if (resp->flags & QEDR_RESP_IMM) { 3376 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 3377 wc->wc_flags |= IB_WC_WITH_IMM; 3378 3379 if (resp->flags & QEDR_RESP_RDMA) 3380 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 3381 3382 if (resp->flags & QEDR_RESP_INV) 3383 return -EINVAL; 3384 3385 } else if (resp->flags & QEDR_RESP_INV) { 3386 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 3387 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3388 3389 if (resp->flags & QEDR_RESP_RDMA) 3390 return -EINVAL; 3391 3392 } else if (resp->flags & QEDR_RESP_RDMA) { 3393 return -EINVAL; 3394 } 3395 3396 return 0; 3397 } 3398 3399 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 3400 struct qedr_cq *cq, struct ib_wc *wc, 3401 struct rdma_cqe_responder *resp, u64 wr_id) 3402 { 3403 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */ 3404 wc->opcode = IB_WC_RECV; 3405 wc->wc_flags = 0; 3406 3407 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) { 3408 if (qedr_set_ok_cqe_resp_wc(resp, wc)) 3409 DP_ERR(dev, 3410 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n", 3411 cq, cq->icid, resp->flags); 3412 3413 } else { 3414 wc->status = qedr_cqe_resp_status_to_ib(resp->status); 3415 if (wc->status == IB_WC_GENERAL_ERR) 3416 DP_ERR(dev, 3417 "CQ %p (icid=%d) contains an invalid CQE status %d\n", 3418 cq, cq->icid, resp->status); 3419 } 3420 3421 /* Fill the rest of the WC */ 3422 wc->vendor_err = 0; 3423 wc->src_qp = qp->id; 3424 wc->qp = &qp->ibqp; 3425 wc->wr_id = wr_id; 3426 } 3427 3428 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 3429 struct qedr_cq *cq, struct ib_wc *wc, 3430 struct rdma_cqe_responder *resp) 3431 { 3432 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 3433 3434 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 3435 3436 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 3437 qed_chain_consume(&qp->rq.pbl); 3438 qedr_inc_sw_cons(&qp->rq); 3439 3440 return 1; 3441 } 3442 3443 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, 3444 int num_entries, struct ib_wc *wc, u16 hw_cons) 3445 { 3446 u16 cnt = 0; 3447 3448 while (num_entries && qp->rq.wqe_cons != hw_cons) { 3449 /* fill WC */ 3450 wc->status = IB_WC_WR_FLUSH_ERR; 3451 wc->vendor_err = 0; 3452 wc->wc_flags = 0; 3453 wc->src_qp = qp->id; 3454 wc->byte_len = 0; 3455 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 3456 wc->qp = &qp->ibqp; 3457 num_entries--; 3458 wc++; 3459 cnt++; 3460 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 3461 qed_chain_consume(&qp->rq.pbl); 3462 qedr_inc_sw_cons(&qp->rq); 3463 } 3464 3465 return cnt; 3466 } 3467 3468 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 3469 struct rdma_cqe_responder *resp, int *update) 3470 { 3471 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) { 3472 consume_cqe(cq); 3473 *update |= 1; 3474 } 3475 } 3476 3477 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, 3478 struct qedr_cq *cq, int num_entries, 3479 struct ib_wc *wc, struct rdma_cqe_responder *resp, 3480 int *update) 3481 { 3482 int cnt; 3483 3484 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 3485 cnt = process_resp_flush(qp, cq, num_entries, wc, 3486 resp->rq_cons); 3487 try_consume_resp_cqe(cq, qp, resp, update); 3488 } else { 3489 cnt = process_resp_one(dev, qp, cq, wc, resp); 3490 consume_cqe(cq); 3491 *update |= 1; 3492 } 3493 3494 return cnt; 3495 } 3496 3497 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 3498 struct rdma_cqe_requester *req, int *update) 3499 { 3500 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 3501 consume_cqe(cq); 3502 *update |= 1; 3503 } 3504 } 3505 3506 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 3507 { 3508 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 3509 struct qedr_cq *cq = get_qedr_cq(ibcq); 3510 union rdma_cqe *cqe = cq->latest_cqe; 3511 u32 old_cons, new_cons; 3512 unsigned long flags; 3513 int update = 0; 3514 int done = 0; 3515 3516 if (cq->destroyed) { 3517 DP_ERR(dev, 3518 "warning: poll was invoked after destroy for cq %p (icid=%d)\n", 3519 cq, cq->icid); 3520 return 0; 3521 } 3522 3523 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 3524 return qedr_gsi_poll_cq(ibcq, num_entries, wc); 3525 3526 spin_lock_irqsave(&cq->cq_lock, flags); 3527 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 3528 while (num_entries && is_valid_cqe(cq, cqe)) { 3529 struct qedr_qp *qp; 3530 int cnt = 0; 3531 3532 /* prevent speculative reads of any field of CQE */ 3533 rmb(); 3534 3535 qp = cqe_get_qp(cqe); 3536 if (!qp) { 3537 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe); 3538 break; 3539 } 3540 3541 wc->qp = &qp->ibqp; 3542 3543 switch (cqe_get_type(cqe)) { 3544 case RDMA_CQE_TYPE_REQUESTER: 3545 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc, 3546 &cqe->req); 3547 try_consume_req_cqe(cq, qp, &cqe->req, &update); 3548 break; 3549 case RDMA_CQE_TYPE_RESPONDER_RQ: 3550 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc, 3551 &cqe->resp, &update); 3552 break; 3553 case RDMA_CQE_TYPE_INVALID: 3554 default: 3555 DP_ERR(dev, "Error: invalid CQE type = %d\n", 3556 cqe_get_type(cqe)); 3557 } 3558 num_entries -= cnt; 3559 wc += cnt; 3560 done += cnt; 3561 3562 cqe = get_cqe(cq); 3563 } 3564 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 3565 3566 cq->cq_cons += new_cons - old_cons; 3567 3568 if (update) 3569 /* doorbell notifies abount latest VALID entry, 3570 * but chain already point to the next INVALID one 3571 */ 3572 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 3573 3574 spin_unlock_irqrestore(&cq->cq_lock, flags); 3575 return done; 3576 } 3577 3578 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, 3579 u8 port_num, 3580 const struct ib_wc *in_wc, 3581 const struct ib_grh *in_grh, 3582 const struct ib_mad_hdr *mad_hdr, 3583 size_t in_mad_size, struct ib_mad_hdr *out_mad, 3584 size_t *out_mad_size, u16 *out_mad_pkey_index) 3585 { 3586 struct qedr_dev *dev = get_qedr_dev(ibdev); 3587 3588 DP_DEBUG(dev, QEDR_MSG_GSI, 3589 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n", 3590 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod, 3591 mad_hdr->class_specific, mad_hdr->class_version, 3592 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status); 3593 return IB_MAD_RESULT_SUCCESS; 3594 } 3595 3596 int qedr_port_immutable(struct ib_device *ibdev, u8 port_num, 3597 struct ib_port_immutable *immutable) 3598 { 3599 struct ib_port_attr attr; 3600 int err; 3601 3602 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE | 3603 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 3604 3605 err = ib_query_port(ibdev, port_num, &attr); 3606 if (err) 3607 return err; 3608 3609 immutable->pkey_tbl_len = attr.pkey_tbl_len; 3610 immutable->gid_tbl_len = attr.gid_tbl_len; 3611 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 3612 3613 return 0; 3614 } 3615