1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/dma-mapping.h> 33 #include <linux/crc32.h> 34 #include <net/ip.h> 35 #include <net/ipv6.h> 36 #include <net/udp.h> 37 #include <linux/iommu.h> 38 39 #include <rdma/ib_verbs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/iw_cm.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/ib_addr.h> 44 #include <rdma/ib_cache.h> 45 46 #include <linux/qed/common_hsi.h> 47 #include "qedr_hsi_rdma.h" 48 #include <linux/qed/qed_if.h> 49 #include "qedr.h" 50 #include "verbs.h" 51 #include <rdma/qedr-abi.h> 52 #include "qedr_roce_cm.h" 53 54 #define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm) 55 #define RDMA_MAX_SGE_PER_SRQ (4) 56 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) 57 58 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 59 60 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, 61 size_t len) 62 { 63 size_t min_len = min_t(size_t, len, udata->outlen); 64 65 return ib_copy_to_udata(udata, src, min_len); 66 } 67 68 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 69 { 70 if (index >= QEDR_ROCE_PKEY_TABLE_LEN) 71 return -EINVAL; 72 73 *pkey = QEDR_ROCE_PKEY_DEFAULT; 74 return 0; 75 } 76 77 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port, 78 int index, union ib_gid *sgid) 79 { 80 struct qedr_dev *dev = get_qedr_dev(ibdev); 81 82 memset(sgid->raw, 0, sizeof(sgid->raw)); 83 ether_addr_copy(sgid->raw, dev->ndev->dev_addr); 84 85 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index, 86 sgid->global.interface_id, sgid->global.subnet_prefix); 87 88 return 0; 89 } 90 91 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 92 { 93 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 94 struct qedr_device_attr *qattr = &dev->attr; 95 struct qedr_srq *srq = get_qedr_srq(ibsrq); 96 97 srq_attr->srq_limit = srq->srq_limit; 98 srq_attr->max_wr = qattr->max_srq_wr; 99 srq_attr->max_sge = qattr->max_sge; 100 101 return 0; 102 } 103 104 int qedr_query_device(struct ib_device *ibdev, 105 struct ib_device_attr *attr, struct ib_udata *udata) 106 { 107 struct qedr_dev *dev = get_qedr_dev(ibdev); 108 struct qedr_device_attr *qattr = &dev->attr; 109 110 if (!dev->rdma_ctx) { 111 DP_ERR(dev, 112 "qedr_query_device called with invalid params rdma_ctx=%p\n", 113 dev->rdma_ctx); 114 return -EINVAL; 115 } 116 117 memset(attr, 0, sizeof(*attr)); 118 119 attr->fw_ver = qattr->fw_ver; 120 attr->sys_image_guid = qattr->sys_image_guid; 121 attr->max_mr_size = qattr->max_mr_size; 122 attr->page_size_cap = qattr->page_size_caps; 123 attr->vendor_id = qattr->vendor_id; 124 attr->vendor_part_id = qattr->vendor_part_id; 125 attr->hw_ver = qattr->hw_ver; 126 attr->max_qp = qattr->max_qp; 127 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe); 128 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 129 IB_DEVICE_RC_RNR_NAK_GEN | 130 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS; 131 132 attr->max_send_sge = qattr->max_sge; 133 attr->max_recv_sge = qattr->max_sge; 134 attr->max_sge_rd = qattr->max_sge; 135 attr->max_cq = qattr->max_cq; 136 attr->max_cqe = qattr->max_cqe; 137 attr->max_mr = qattr->max_mr; 138 attr->max_mw = qattr->max_mw; 139 attr->max_pd = qattr->max_pd; 140 attr->atomic_cap = dev->atomic_cap; 141 attr->max_fmr = qattr->max_fmr; 142 attr->max_map_per_fmr = 16; 143 attr->max_qp_init_rd_atom = 144 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 145 attr->max_qp_rd_atom = 146 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 147 attr->max_qp_init_rd_atom); 148 149 attr->max_srq = qattr->max_srq; 150 attr->max_srq_sge = qattr->max_srq_sge; 151 attr->max_srq_wr = qattr->max_srq_wr; 152 153 attr->local_ca_ack_delay = qattr->dev_ack_delay; 154 attr->max_fast_reg_page_list_len = qattr->max_mr / 8; 155 attr->max_pkeys = QEDR_ROCE_PKEY_MAX; 156 attr->max_ah = qattr->max_ah; 157 158 return 0; 159 } 160 161 #define QEDR_SPEED_SDR (1) 162 #define QEDR_SPEED_DDR (2) 163 #define QEDR_SPEED_QDR (4) 164 #define QEDR_SPEED_FDR10 (8) 165 #define QEDR_SPEED_FDR (16) 166 #define QEDR_SPEED_EDR (32) 167 168 static inline void get_link_speed_and_width(int speed, u8 *ib_speed, 169 u8 *ib_width) 170 { 171 switch (speed) { 172 case 1000: 173 *ib_speed = QEDR_SPEED_SDR; 174 *ib_width = IB_WIDTH_1X; 175 break; 176 case 10000: 177 *ib_speed = QEDR_SPEED_QDR; 178 *ib_width = IB_WIDTH_1X; 179 break; 180 181 case 20000: 182 *ib_speed = QEDR_SPEED_DDR; 183 *ib_width = IB_WIDTH_4X; 184 break; 185 186 case 25000: 187 *ib_speed = QEDR_SPEED_EDR; 188 *ib_width = IB_WIDTH_1X; 189 break; 190 191 case 40000: 192 *ib_speed = QEDR_SPEED_QDR; 193 *ib_width = IB_WIDTH_4X; 194 break; 195 196 case 50000: 197 *ib_speed = QEDR_SPEED_QDR; 198 *ib_width = IB_WIDTH_4X; 199 break; 200 201 case 100000: 202 *ib_speed = QEDR_SPEED_EDR; 203 *ib_width = IB_WIDTH_4X; 204 break; 205 206 default: 207 /* Unsupported */ 208 *ib_speed = QEDR_SPEED_SDR; 209 *ib_width = IB_WIDTH_1X; 210 } 211 } 212 213 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr) 214 { 215 struct qedr_dev *dev; 216 struct qed_rdma_port *rdma_port; 217 218 dev = get_qedr_dev(ibdev); 219 220 if (!dev->rdma_ctx) { 221 DP_ERR(dev, "rdma_ctx is NULL\n"); 222 return -EINVAL; 223 } 224 225 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx); 226 227 /* *attr being zeroed by the caller, avoid zeroing it here */ 228 if (rdma_port->port_state == QED_RDMA_PORT_UP) { 229 attr->state = IB_PORT_ACTIVE; 230 attr->phys_state = 5; 231 } else { 232 attr->state = IB_PORT_DOWN; 233 attr->phys_state = 3; 234 } 235 attr->max_mtu = IB_MTU_4096; 236 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); 237 attr->lid = 0; 238 attr->lmc = 0; 239 attr->sm_lid = 0; 240 attr->sm_sl = 0; 241 attr->ip_gids = true; 242 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 243 attr->gid_tbl_len = 1; 244 attr->pkey_tbl_len = 1; 245 } else { 246 attr->gid_tbl_len = QEDR_MAX_SGID; 247 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; 248 } 249 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 250 attr->qkey_viol_cntr = 0; 251 get_link_speed_and_width(rdma_port->link_speed, 252 &attr->active_speed, &attr->active_width); 253 attr->max_msg_sz = rdma_port->max_msg_size; 254 attr->max_vl_num = 4; 255 256 return 0; 257 } 258 259 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask, 260 struct ib_port_modify *props) 261 { 262 return 0; 263 } 264 265 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr, 266 unsigned long len) 267 { 268 struct qedr_mm *mm; 269 270 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 271 if (!mm) 272 return -ENOMEM; 273 274 mm->key.phy_addr = phy_addr; 275 /* This function might be called with a length which is not a multiple 276 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel 277 * forces this granularity by increasing the requested size if needed. 278 * When qedr_mmap is called, it will search the list with the updated 279 * length as a key. To prevent search failures, the length is rounded up 280 * in advance to PAGE_SIZE. 281 */ 282 mm->key.len = roundup(len, PAGE_SIZE); 283 INIT_LIST_HEAD(&mm->entry); 284 285 mutex_lock(&uctx->mm_list_lock); 286 list_add(&mm->entry, &uctx->mm_head); 287 mutex_unlock(&uctx->mm_list_lock); 288 289 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 290 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n", 291 (unsigned long long)mm->key.phy_addr, 292 (unsigned long)mm->key.len, uctx); 293 294 return 0; 295 } 296 297 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr, 298 unsigned long len) 299 { 300 bool found = false; 301 struct qedr_mm *mm; 302 303 mutex_lock(&uctx->mm_list_lock); 304 list_for_each_entry(mm, &uctx->mm_head, entry) { 305 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 306 continue; 307 308 found = true; 309 break; 310 } 311 mutex_unlock(&uctx->mm_list_lock); 312 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 313 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n", 314 mm->key.phy_addr, mm->key.len, uctx, found); 315 316 return found; 317 } 318 319 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 320 { 321 struct ib_device *ibdev = uctx->device; 322 int rc; 323 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx); 324 struct qedr_alloc_ucontext_resp uresp = {}; 325 struct qedr_dev *dev = get_qedr_dev(ibdev); 326 struct qed_rdma_add_user_out_params oparams; 327 328 if (!udata) 329 return -EFAULT; 330 331 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams); 332 if (rc) { 333 DP_ERR(dev, 334 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n", 335 rc); 336 return rc; 337 } 338 339 ctx->dpi = oparams.dpi; 340 ctx->dpi_addr = oparams.dpi_addr; 341 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 342 ctx->dpi_size = oparams.dpi_size; 343 INIT_LIST_HEAD(&ctx->mm_head); 344 mutex_init(&ctx->mm_list_lock); 345 346 uresp.dpm_enabled = dev->user_dpm_enabled; 347 uresp.wids_enabled = 1; 348 uresp.wid_count = oparams.wid_count; 349 uresp.db_pa = ctx->dpi_phys_addr; 350 uresp.db_size = ctx->dpi_size; 351 uresp.max_send_wr = dev->attr.max_sqe; 352 uresp.max_recv_wr = dev->attr.max_rqe; 353 uresp.max_srq_wr = dev->attr.max_srq_wr; 354 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE; 355 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE; 356 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 357 uresp.max_cqes = QEDR_MAX_CQES; 358 359 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 360 if (rc) 361 return rc; 362 363 ctx->dev = dev; 364 365 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size); 366 if (rc) 367 return rc; 368 369 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n", 370 &ctx->ibucontext); 371 return 0; 372 } 373 374 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx) 375 { 376 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx); 377 struct qedr_mm *mm, *tmp; 378 379 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n", 380 uctx); 381 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi); 382 383 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 384 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 385 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n", 386 mm->key.phy_addr, mm->key.len, uctx); 387 list_del(&mm->entry); 388 kfree(mm); 389 } 390 } 391 392 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 393 { 394 struct qedr_ucontext *ucontext = get_qedr_ucontext(context); 395 struct qedr_dev *dev = get_qedr_dev(context->device); 396 unsigned long phys_addr = vma->vm_pgoff << PAGE_SHIFT; 397 unsigned long len = (vma->vm_end - vma->vm_start); 398 unsigned long dpi_start; 399 400 dpi_start = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size); 401 402 DP_DEBUG(dev, QEDR_MSG_INIT, 403 "mmap invoked with vm_start=0x%pK, vm_end=0x%pK,vm_pgoff=0x%pK; dpi_start=0x%pK dpi_size=0x%x\n", 404 (void *)vma->vm_start, (void *)vma->vm_end, 405 (void *)vma->vm_pgoff, (void *)dpi_start, ucontext->dpi_size); 406 407 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) { 408 DP_ERR(dev, 409 "failed mmap, addresses must be page aligned: start=0x%pK, end=0x%pK\n", 410 (void *)vma->vm_start, (void *)vma->vm_end); 411 return -EINVAL; 412 } 413 414 if (!qedr_search_mmap(ucontext, phys_addr, len)) { 415 DP_ERR(dev, "failed mmap, vm_pgoff=0x%lx is not authorized\n", 416 vma->vm_pgoff); 417 return -EINVAL; 418 } 419 420 if (phys_addr < dpi_start || 421 ((phys_addr + len) > (dpi_start + ucontext->dpi_size))) { 422 DP_ERR(dev, 423 "failed mmap, pages are outside of dpi; page address=0x%pK, dpi_start=0x%pK, dpi_size=0x%x\n", 424 (void *)phys_addr, (void *)dpi_start, 425 ucontext->dpi_size); 426 return -EINVAL; 427 } 428 429 if (vma->vm_flags & VM_READ) { 430 DP_ERR(dev, "failed mmap, cannot map doorbell bar for read\n"); 431 return -EINVAL; 432 } 433 434 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 435 return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len, 436 vma->vm_page_prot); 437 } 438 439 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context, 440 struct ib_udata *udata) 441 { 442 struct ib_device *ibdev = ibpd->device; 443 struct qedr_dev *dev = get_qedr_dev(ibdev); 444 struct qedr_pd *pd = get_qedr_pd(ibpd); 445 u16 pd_id; 446 int rc; 447 448 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n", 449 (udata && context) ? "User Lib" : "Kernel"); 450 451 if (!dev->rdma_ctx) { 452 DP_ERR(dev, "invalid RDMA context\n"); 453 return -EINVAL; 454 } 455 456 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id); 457 if (rc) 458 return rc; 459 460 pd->pd_id = pd_id; 461 462 if (udata && context) { 463 struct qedr_alloc_pd_uresp uresp = { 464 .pd_id = pd_id, 465 }; 466 467 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 468 if (rc) { 469 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); 470 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); 471 return rc; 472 } 473 474 pd->uctx = get_qedr_ucontext(context); 475 pd->uctx->pd = pd; 476 } 477 478 return 0; 479 } 480 481 void qedr_dealloc_pd(struct ib_pd *ibpd) 482 { 483 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 484 struct qedr_pd *pd = get_qedr_pd(ibpd); 485 486 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id); 487 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id); 488 } 489 490 static void qedr_free_pbl(struct qedr_dev *dev, 491 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) 492 { 493 struct pci_dev *pdev = dev->pdev; 494 int i; 495 496 for (i = 0; i < pbl_info->num_pbls; i++) { 497 if (!pbl[i].va) 498 continue; 499 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 500 pbl[i].va, pbl[i].pa); 501 } 502 503 kfree(pbl); 504 } 505 506 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024) 507 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024) 508 509 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 510 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 511 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE) 512 513 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev, 514 struct qedr_pbl_info *pbl_info, 515 gfp_t flags) 516 { 517 struct pci_dev *pdev = dev->pdev; 518 struct qedr_pbl *pbl_table; 519 dma_addr_t *pbl_main_tbl; 520 dma_addr_t pa; 521 void *va; 522 int i; 523 524 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags); 525 if (!pbl_table) 526 return ERR_PTR(-ENOMEM); 527 528 for (i = 0; i < pbl_info->num_pbls; i++) { 529 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa, 530 flags); 531 if (!va) 532 goto err; 533 534 pbl_table[i].va = va; 535 pbl_table[i].pa = pa; 536 } 537 538 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 539 * the first one with physical pointers to all of the rest 540 */ 541 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 542 for (i = 0; i < pbl_info->num_pbls - 1; i++) 543 pbl_main_tbl[i] = pbl_table[i + 1].pa; 544 545 return pbl_table; 546 547 err: 548 for (i--; i >= 0; i--) 549 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 550 pbl_table[i].va, pbl_table[i].pa); 551 552 qedr_free_pbl(dev, pbl_info, pbl_table); 553 554 return ERR_PTR(-ENOMEM); 555 } 556 557 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev, 558 struct qedr_pbl_info *pbl_info, 559 u32 num_pbes, int two_layer_capable) 560 { 561 u32 pbl_capacity; 562 u32 pbl_size; 563 u32 num_pbls; 564 565 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 566 if (num_pbes > MAX_PBES_TWO_LAYER) { 567 DP_ERR(dev, "prepare pbl table: too many pages %d\n", 568 num_pbes); 569 return -EINVAL; 570 } 571 572 /* calculate required pbl page size */ 573 pbl_size = MIN_FW_PBL_PAGE_SIZE; 574 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 575 NUM_PBES_ON_PAGE(pbl_size); 576 577 while (pbl_capacity < num_pbes) { 578 pbl_size *= 2; 579 pbl_capacity = pbl_size / sizeof(u64); 580 pbl_capacity = pbl_capacity * pbl_capacity; 581 } 582 583 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 584 num_pbls++; /* One for the layer0 ( points to the pbls) */ 585 pbl_info->two_layered = true; 586 } else { 587 /* One layered PBL */ 588 num_pbls = 1; 589 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, 590 roundup_pow_of_two((num_pbes * sizeof(u64)))); 591 pbl_info->two_layered = false; 592 } 593 594 pbl_info->num_pbls = num_pbls; 595 pbl_info->pbl_size = pbl_size; 596 pbl_info->num_pbes = num_pbes; 597 598 DP_DEBUG(dev, QEDR_MSG_MR, 599 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n", 600 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 601 602 return 0; 603 } 604 605 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, 606 struct qedr_pbl *pbl, 607 struct qedr_pbl_info *pbl_info, u32 pg_shift) 608 { 609 int pbe_cnt, total_num_pbes = 0; 610 u32 fw_pg_cnt, fw_pg_per_umem_pg; 611 struct qedr_pbl *pbl_tbl; 612 struct sg_dma_page_iter sg_iter; 613 struct regpair *pbe; 614 u64 pg_addr; 615 616 if (!pbl_info->num_pbes) 617 return; 618 619 /* If we have a two layered pbl, the first pbl points to the rest 620 * of the pbls and the first entry lays on the second pbl in the table 621 */ 622 if (pbl_info->two_layered) 623 pbl_tbl = &pbl[1]; 624 else 625 pbl_tbl = pbl; 626 627 pbe = (struct regpair *)pbl_tbl->va; 628 if (!pbe) { 629 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n"); 630 return; 631 } 632 633 pbe_cnt = 0; 634 635 fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift); 636 637 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { 638 pg_addr = sg_page_iter_dma_address(&sg_iter); 639 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) { 640 pbe->lo = cpu_to_le32(pg_addr); 641 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); 642 643 pg_addr += BIT(pg_shift); 644 pbe_cnt++; 645 total_num_pbes++; 646 pbe++; 647 648 if (total_num_pbes == pbl_info->num_pbes) 649 return; 650 651 /* If the given pbl is full storing the pbes, 652 * move to next pbl. 653 */ 654 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { 655 pbl_tbl++; 656 pbe = (struct regpair *)pbl_tbl->va; 657 pbe_cnt = 0; 658 } 659 660 fw_pg_cnt++; 661 } 662 } 663 } 664 665 static int qedr_copy_cq_uresp(struct qedr_dev *dev, 666 struct qedr_cq *cq, struct ib_udata *udata) 667 { 668 struct qedr_create_cq_uresp uresp; 669 int rc; 670 671 memset(&uresp, 0, sizeof(uresp)); 672 673 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 674 uresp.icid = cq->icid; 675 676 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 677 if (rc) 678 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); 679 680 return rc; 681 } 682 683 static void consume_cqe(struct qedr_cq *cq) 684 { 685 if (cq->latest_cqe == cq->toggle_cqe) 686 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 687 688 cq->latest_cqe = qed_chain_consume(&cq->pbl); 689 } 690 691 static inline int qedr_align_cq_entries(int entries) 692 { 693 u64 size, aligned_size; 694 695 /* We allocate an extra entry that we don't report to the FW. */ 696 size = (entries + 1) * QEDR_CQE_SIZE; 697 aligned_size = ALIGN(size, PAGE_SIZE); 698 699 return aligned_size / QEDR_CQE_SIZE; 700 } 701 702 static inline int qedr_init_user_queue(struct ib_udata *udata, 703 struct qedr_dev *dev, 704 struct qedr_userq *q, u64 buf_addr, 705 size_t buf_len, int access, int dmasync, 706 int alloc_and_init) 707 { 708 u32 fw_pages; 709 int rc; 710 711 q->buf_addr = buf_addr; 712 q->buf_len = buf_len; 713 q->umem = ib_umem_get(udata, q->buf_addr, q->buf_len, access, dmasync); 714 if (IS_ERR(q->umem)) { 715 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", 716 PTR_ERR(q->umem)); 717 return PTR_ERR(q->umem); 718 } 719 720 fw_pages = ib_umem_page_count(q->umem) << 721 (PAGE_SHIFT - FW_PAGE_SHIFT); 722 723 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); 724 if (rc) 725 goto err0; 726 727 if (alloc_and_init) { 728 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 729 if (IS_ERR(q->pbl_tbl)) { 730 rc = PTR_ERR(q->pbl_tbl); 731 goto err0; 732 } 733 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info, 734 FW_PAGE_SHIFT); 735 } else { 736 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); 737 if (!q->pbl_tbl) { 738 rc = -ENOMEM; 739 goto err0; 740 } 741 } 742 743 return 0; 744 745 err0: 746 ib_umem_release(q->umem); 747 q->umem = NULL; 748 749 return rc; 750 } 751 752 static inline void qedr_init_cq_params(struct qedr_cq *cq, 753 struct qedr_ucontext *ctx, 754 struct qedr_dev *dev, int vector, 755 int chain_entries, int page_cnt, 756 u64 pbl_ptr, 757 struct qed_rdma_create_cq_in_params 758 *params) 759 { 760 memset(params, 0, sizeof(*params)); 761 params->cq_handle_hi = upper_32_bits((uintptr_t)cq); 762 params->cq_handle_lo = lower_32_bits((uintptr_t)cq); 763 params->cnq_id = vector; 764 params->cq_size = chain_entries - 1; 765 params->dpi = (ctx) ? ctx->dpi : dev->dpi; 766 params->pbl_num_pages = page_cnt; 767 params->pbl_ptr = pbl_ptr; 768 params->pbl_two_level = 0; 769 } 770 771 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags) 772 { 773 cq->db.data.agg_flags = flags; 774 cq->db.data.value = cpu_to_le32(cons); 775 writeq(cq->db.raw, cq->db_addr); 776 777 /* Make sure write would stick */ 778 mmiowb(); 779 } 780 781 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 782 { 783 struct qedr_cq *cq = get_qedr_cq(ibcq); 784 unsigned long sflags; 785 struct qedr_dev *dev; 786 787 dev = get_qedr_dev(ibcq->device); 788 789 if (cq->destroyed) { 790 DP_ERR(dev, 791 "warning: arm was invoked after destroy for cq %p (icid=%d)\n", 792 cq, cq->icid); 793 return -EINVAL; 794 } 795 796 797 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 798 return 0; 799 800 spin_lock_irqsave(&cq->cq_lock, sflags); 801 802 cq->arm_flags = 0; 803 804 if (flags & IB_CQ_SOLICITED) 805 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 806 807 if (flags & IB_CQ_NEXT_COMP) 808 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 809 810 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 811 812 spin_unlock_irqrestore(&cq->cq_lock, sflags); 813 814 return 0; 815 } 816 817 struct ib_cq *qedr_create_cq(struct ib_device *ibdev, 818 const struct ib_cq_init_attr *attr, 819 struct ib_ucontext *ib_ctx, struct ib_udata *udata) 820 { 821 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx); 822 struct qed_rdma_destroy_cq_out_params destroy_oparams; 823 struct qed_rdma_destroy_cq_in_params destroy_iparams; 824 struct qedr_dev *dev = get_qedr_dev(ibdev); 825 struct qed_rdma_create_cq_in_params params; 826 struct qedr_create_cq_ureq ureq; 827 int vector = attr->comp_vector; 828 int entries = attr->cqe; 829 struct qedr_cq *cq; 830 int chain_entries; 831 int page_cnt; 832 u64 pbl_ptr; 833 u16 icid; 834 int rc; 835 836 DP_DEBUG(dev, QEDR_MSG_INIT, 837 "create_cq: called from %s. entries=%d, vector=%d\n", 838 udata ? "User Lib" : "Kernel", entries, vector); 839 840 if (entries > QEDR_MAX_CQES) { 841 DP_ERR(dev, 842 "create cq: the number of entries %d is too high. Must be equal or below %d.\n", 843 entries, QEDR_MAX_CQES); 844 return ERR_PTR(-EINVAL); 845 } 846 847 chain_entries = qedr_align_cq_entries(entries); 848 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); 849 850 cq = kzalloc(sizeof(*cq), GFP_KERNEL); 851 if (!cq) 852 return ERR_PTR(-ENOMEM); 853 854 if (udata) { 855 memset(&ureq, 0, sizeof(ureq)); 856 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { 857 DP_ERR(dev, 858 "create cq: problem copying data from user space\n"); 859 goto err0; 860 } 861 862 if (!ureq.len) { 863 DP_ERR(dev, 864 "create cq: cannot create a cq with 0 entries\n"); 865 goto err0; 866 } 867 868 cq->cq_type = QEDR_CQ_TYPE_USER; 869 870 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr, 871 ureq.len, IB_ACCESS_LOCAL_WRITE, 1, 872 1); 873 if (rc) 874 goto err0; 875 876 pbl_ptr = cq->q.pbl_tbl->pa; 877 page_cnt = cq->q.pbl_info.num_pbes; 878 879 cq->ibcq.cqe = chain_entries; 880 } else { 881 cq->cq_type = QEDR_CQ_TYPE_KERNEL; 882 883 rc = dev->ops->common->chain_alloc(dev->cdev, 884 QED_CHAIN_USE_TO_CONSUME, 885 QED_CHAIN_MODE_PBL, 886 QED_CHAIN_CNT_TYPE_U32, 887 chain_entries, 888 sizeof(union rdma_cqe), 889 &cq->pbl, NULL); 890 if (rc) 891 goto err1; 892 893 page_cnt = qed_chain_get_page_cnt(&cq->pbl); 894 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); 895 cq->ibcq.cqe = cq->pbl.capacity; 896 } 897 898 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, 899 pbl_ptr, ¶ms); 900 901 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 902 if (rc) 903 goto err2; 904 905 cq->icid = icid; 906 cq->sig = QEDR_CQ_MAGIC_NUMBER; 907 spin_lock_init(&cq->cq_lock); 908 909 if (ib_ctx) { 910 rc = qedr_copy_cq_uresp(dev, cq, udata); 911 if (rc) 912 goto err3; 913 } else { 914 /* Generate doorbell address. */ 915 cq->db_addr = dev->db_addr + 916 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 917 cq->db.data.icid = cq->icid; 918 cq->db.data.params = DB_AGG_CMD_SET << 919 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 920 921 /* point to the very last element, passing it we will toggle */ 922 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); 923 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 924 cq->latest_cqe = NULL; 925 consume_cqe(cq); 926 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 927 } 928 929 DP_DEBUG(dev, QEDR_MSG_CQ, 930 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", 931 cq->icid, cq, params.cq_size); 932 933 return &cq->ibcq; 934 935 err3: 936 destroy_iparams.icid = cq->icid; 937 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, 938 &destroy_oparams); 939 err2: 940 if (udata) 941 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 942 else 943 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 944 err1: 945 if (udata) 946 ib_umem_release(cq->q.umem); 947 err0: 948 kfree(cq); 949 return ERR_PTR(-EINVAL); 950 } 951 952 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 953 { 954 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 955 struct qedr_cq *cq = get_qedr_cq(ibcq); 956 957 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq); 958 959 return 0; 960 } 961 962 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) 963 #define QEDR_DESTROY_CQ_ITER_DURATION (10) 964 965 int qedr_destroy_cq(struct ib_cq *ibcq) 966 { 967 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 968 struct qed_rdma_destroy_cq_out_params oparams; 969 struct qed_rdma_destroy_cq_in_params iparams; 970 struct qedr_cq *cq = get_qedr_cq(ibcq); 971 int iter; 972 int rc; 973 974 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); 975 976 cq->destroyed = 1; 977 978 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 979 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 980 goto done; 981 982 iparams.icid = cq->icid; 983 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 984 if (rc) 985 return rc; 986 987 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 988 989 if (ibcq->uobject && ibcq->uobject->context) { 990 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 991 ib_umem_release(cq->q.umem); 992 } 993 994 /* We don't want the IRQ handler to handle a non-existing CQ so we 995 * wait until all CNQ interrupts, if any, are received. This will always 996 * happen and will always happen very fast. If not, then a serious error 997 * has occured. That is why we can use a long delay. 998 * We spin for a short time so we don’t lose time on context switching 999 * in case all the completions are handled in that span. Otherwise 1000 * we sleep for a while and check again. Since the CNQ may be 1001 * associated with (only) the current CPU we use msleep to allow the 1002 * current CPU to be freed. 1003 * The CNQ notification is increased in qedr_irq_handler(). 1004 */ 1005 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1006 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1007 udelay(QEDR_DESTROY_CQ_ITER_DURATION); 1008 iter--; 1009 } 1010 1011 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1012 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1013 msleep(QEDR_DESTROY_CQ_ITER_DURATION); 1014 iter--; 1015 } 1016 1017 if (oparams.num_cq_notif != cq->cnq_notif) 1018 goto err; 1019 1020 /* Note that we don't need to have explicit code to wait for the 1021 * completion of the event handler because it is invoked from the EQ. 1022 * Since the destroy CQ ramrod has also been received on the EQ we can 1023 * be certain that there's no event handler in process. 1024 */ 1025 done: 1026 cq->sig = ~cq->sig; 1027 1028 kfree(cq); 1029 1030 return 0; 1031 1032 err: 1033 DP_ERR(dev, 1034 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n", 1035 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif); 1036 1037 return -EINVAL; 1038 } 1039 1040 static inline int get_gid_info_from_table(struct ib_qp *ibqp, 1041 struct ib_qp_attr *attr, 1042 int attr_mask, 1043 struct qed_rdma_modify_qp_in_params 1044 *qp_params) 1045 { 1046 const struct ib_gid_attr *gid_attr; 1047 enum rdma_network_type nw_type; 1048 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1049 u32 ipv4_addr; 1050 int i; 1051 1052 gid_attr = grh->sgid_attr; 1053 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr->ndev); 1054 1055 nw_type = rdma_gid_attr_network_type(gid_attr); 1056 switch (nw_type) { 1057 case RDMA_NETWORK_IPV6: 1058 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1059 sizeof(qp_params->sgid)); 1060 memcpy(&qp_params->dgid.bytes[0], 1061 &grh->dgid, 1062 sizeof(qp_params->dgid)); 1063 qp_params->roce_mode = ROCE_V2_IPV6; 1064 SET_FIELD(qp_params->modify_flags, 1065 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1066 break; 1067 case RDMA_NETWORK_IB: 1068 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1069 sizeof(qp_params->sgid)); 1070 memcpy(&qp_params->dgid.bytes[0], 1071 &grh->dgid, 1072 sizeof(qp_params->dgid)); 1073 qp_params->roce_mode = ROCE_V1; 1074 break; 1075 case RDMA_NETWORK_IPV4: 1076 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid)); 1077 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid)); 1078 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw); 1079 qp_params->sgid.ipv4_addr = ipv4_addr; 1080 ipv4_addr = 1081 qedr_get_ipv4_from_gid(grh->dgid.raw); 1082 qp_params->dgid.ipv4_addr = ipv4_addr; 1083 SET_FIELD(qp_params->modify_flags, 1084 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1085 qp_params->roce_mode = ROCE_V2_IPV4; 1086 break; 1087 } 1088 1089 for (i = 0; i < 4; i++) { 1090 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 1091 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 1092 } 1093 1094 if (qp_params->vlan_id >= VLAN_CFI_MASK) 1095 qp_params->vlan_id = 0; 1096 1097 return 0; 1098 } 1099 1100 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, 1101 struct ib_qp_init_attr *attrs, 1102 struct ib_udata *udata) 1103 { 1104 struct qedr_device_attr *qattr = &dev->attr; 1105 1106 /* QP0... attrs->qp_type == IB_QPT_GSI */ 1107 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { 1108 DP_DEBUG(dev, QEDR_MSG_QP, 1109 "create qp: unsupported qp type=0x%x requested\n", 1110 attrs->qp_type); 1111 return -EINVAL; 1112 } 1113 1114 if (attrs->cap.max_send_wr > qattr->max_sqe) { 1115 DP_ERR(dev, 1116 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n", 1117 attrs->cap.max_send_wr, qattr->max_sqe); 1118 return -EINVAL; 1119 } 1120 1121 if (attrs->cap.max_inline_data > qattr->max_inline) { 1122 DP_ERR(dev, 1123 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n", 1124 attrs->cap.max_inline_data, qattr->max_inline); 1125 return -EINVAL; 1126 } 1127 1128 if (attrs->cap.max_send_sge > qattr->max_sge) { 1129 DP_ERR(dev, 1130 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n", 1131 attrs->cap.max_send_sge, qattr->max_sge); 1132 return -EINVAL; 1133 } 1134 1135 if (attrs->cap.max_recv_sge > qattr->max_sge) { 1136 DP_ERR(dev, 1137 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n", 1138 attrs->cap.max_recv_sge, qattr->max_sge); 1139 return -EINVAL; 1140 } 1141 1142 /* Unprivileged user space cannot create special QP */ 1143 if (udata && attrs->qp_type == IB_QPT_GSI) { 1144 DP_ERR(dev, 1145 "create qp: userspace can't create special QPs of type=0x%x\n", 1146 attrs->qp_type); 1147 return -EINVAL; 1148 } 1149 1150 return 0; 1151 } 1152 1153 static int qedr_copy_srq_uresp(struct qedr_dev *dev, 1154 struct qedr_srq *srq, struct ib_udata *udata) 1155 { 1156 struct qedr_create_srq_uresp uresp = {}; 1157 int rc; 1158 1159 uresp.srq_id = srq->srq_id; 1160 1161 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1162 if (rc) 1163 DP_ERR(dev, "create srq: problem copying data to user space\n"); 1164 1165 return rc; 1166 } 1167 1168 static void qedr_copy_rq_uresp(struct qedr_dev *dev, 1169 struct qedr_create_qp_uresp *uresp, 1170 struct qedr_qp *qp) 1171 { 1172 /* iWARP requires two doorbells per RQ. */ 1173 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1174 uresp->rq_db_offset = 1175 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1176 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1177 } else { 1178 uresp->rq_db_offset = 1179 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1180 } 1181 1182 uresp->rq_icid = qp->icid; 1183 } 1184 1185 static void qedr_copy_sq_uresp(struct qedr_dev *dev, 1186 struct qedr_create_qp_uresp *uresp, 1187 struct qedr_qp *qp) 1188 { 1189 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1190 1191 /* iWARP uses the same cid for rq and sq */ 1192 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1193 uresp->sq_icid = qp->icid; 1194 else 1195 uresp->sq_icid = qp->icid + 1; 1196 } 1197 1198 static int qedr_copy_qp_uresp(struct qedr_dev *dev, 1199 struct qedr_qp *qp, struct ib_udata *udata) 1200 { 1201 struct qedr_create_qp_uresp uresp; 1202 int rc; 1203 1204 memset(&uresp, 0, sizeof(uresp)); 1205 qedr_copy_sq_uresp(dev, &uresp, qp); 1206 qedr_copy_rq_uresp(dev, &uresp, qp); 1207 1208 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 1209 uresp.qp_id = qp->qp_id; 1210 1211 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1212 if (rc) 1213 DP_ERR(dev, 1214 "create qp: failed a copy to user space with qp icid=0x%x.\n", 1215 qp->icid); 1216 1217 return rc; 1218 } 1219 1220 static void qedr_set_common_qp_params(struct qedr_dev *dev, 1221 struct qedr_qp *qp, 1222 struct qedr_pd *pd, 1223 struct ib_qp_init_attr *attrs) 1224 { 1225 spin_lock_init(&qp->q_lock); 1226 atomic_set(&qp->refcnt, 1); 1227 qp->pd = pd; 1228 qp->qp_type = attrs->qp_type; 1229 qp->max_inline_data = attrs->cap.max_inline_data; 1230 qp->sq.max_sges = attrs->cap.max_send_sge; 1231 qp->state = QED_ROCE_QP_STATE_RESET; 1232 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1233 qp->sq_cq = get_qedr_cq(attrs->send_cq); 1234 qp->dev = dev; 1235 1236 if (attrs->srq) { 1237 qp->srq = get_qedr_srq(attrs->srq); 1238 } else { 1239 qp->rq_cq = get_qedr_cq(attrs->recv_cq); 1240 qp->rq.max_sges = attrs->cap.max_recv_sge; 1241 DP_DEBUG(dev, QEDR_MSG_QP, 1242 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 1243 qp->rq.max_sges, qp->rq_cq->icid); 1244 } 1245 1246 DP_DEBUG(dev, QEDR_MSG_QP, 1247 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n", 1248 pd->pd_id, qp->qp_type, qp->max_inline_data, 1249 qp->state, qp->signaled, (attrs->srq) ? 1 : 0); 1250 DP_DEBUG(dev, QEDR_MSG_QP, 1251 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1252 qp->sq.max_sges, qp->sq_cq->icid); 1253 } 1254 1255 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1256 { 1257 qp->sq.db = dev->db_addr + 1258 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1259 qp->sq.db_data.data.icid = qp->icid + 1; 1260 if (!qp->srq) { 1261 qp->rq.db = dev->db_addr + 1262 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1263 qp->rq.db_data.data.icid = qp->icid; 1264 } 1265 } 1266 1267 static int qedr_check_srq_params(struct ib_pd *ibpd, struct qedr_dev *dev, 1268 struct ib_srq_init_attr *attrs, 1269 struct ib_udata *udata) 1270 { 1271 struct qedr_device_attr *qattr = &dev->attr; 1272 1273 if (attrs->attr.max_wr > qattr->max_srq_wr) { 1274 DP_ERR(dev, 1275 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n", 1276 attrs->attr.max_wr, qattr->max_srq_wr); 1277 return -EINVAL; 1278 } 1279 1280 if (attrs->attr.max_sge > qattr->max_sge) { 1281 DP_ERR(dev, 1282 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", 1283 attrs->attr.max_sge, qattr->max_sge); 1284 return -EINVAL; 1285 } 1286 1287 return 0; 1288 } 1289 1290 static void qedr_free_srq_user_params(struct qedr_srq *srq) 1291 { 1292 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1293 ib_umem_release(srq->usrq.umem); 1294 ib_umem_release(srq->prod_umem); 1295 } 1296 1297 static void qedr_free_srq_kernel_params(struct qedr_srq *srq) 1298 { 1299 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1300 struct qedr_dev *dev = srq->dev; 1301 1302 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl); 1303 1304 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1305 hw_srq->virt_prod_pair_addr, 1306 hw_srq->phy_prod_pair_addr); 1307 } 1308 1309 static int qedr_init_srq_user_params(struct ib_udata *udata, 1310 struct qedr_srq *srq, 1311 struct qedr_create_srq_ureq *ureq, 1312 int access, int dmasync) 1313 { 1314 struct scatterlist *sg; 1315 int rc; 1316 1317 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr, 1318 ureq->srq_len, access, dmasync, 1); 1319 if (rc) 1320 return rc; 1321 1322 srq->prod_umem = 1323 ib_umem_get(udata, ureq->prod_pair_addr, 1324 sizeof(struct rdma_srq_producers), access, dmasync); 1325 if (IS_ERR(srq->prod_umem)) { 1326 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1327 ib_umem_release(srq->usrq.umem); 1328 DP_ERR(srq->dev, 1329 "create srq: failed ib_umem_get for producer, got %ld\n", 1330 PTR_ERR(srq->prod_umem)); 1331 return PTR_ERR(srq->prod_umem); 1332 } 1333 1334 sg = srq->prod_umem->sg_head.sgl; 1335 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); 1336 1337 return 0; 1338 } 1339 1340 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq, 1341 struct qedr_dev *dev, 1342 struct ib_srq_init_attr *init_attr) 1343 { 1344 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1345 dma_addr_t phy_prod_pair_addr; 1346 u32 num_elems; 1347 void *va; 1348 int rc; 1349 1350 va = dma_alloc_coherent(&dev->pdev->dev, 1351 sizeof(struct rdma_srq_producers), 1352 &phy_prod_pair_addr, GFP_KERNEL); 1353 if (!va) { 1354 DP_ERR(dev, 1355 "create srq: failed to allocate dma memory for producer\n"); 1356 return -ENOMEM; 1357 } 1358 1359 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; 1360 hw_srq->virt_prod_pair_addr = va; 1361 1362 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE; 1363 rc = dev->ops->common->chain_alloc(dev->cdev, 1364 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1365 QED_CHAIN_MODE_PBL, 1366 QED_CHAIN_CNT_TYPE_U32, 1367 num_elems, 1368 QEDR_SRQ_WQE_ELEM_SIZE, 1369 &hw_srq->pbl, NULL); 1370 if (rc) 1371 goto err0; 1372 1373 hw_srq->num_elems = num_elems; 1374 1375 return 0; 1376 1377 err0: 1378 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1379 va, phy_prod_pair_addr); 1380 return rc; 1381 } 1382 1383 static int qedr_idr_add(struct qedr_dev *dev, struct qedr_idr *qidr, 1384 void *ptr, u32 id); 1385 static void qedr_idr_remove(struct qedr_dev *dev, 1386 struct qedr_idr *qidr, u32 id); 1387 1388 struct ib_srq *qedr_create_srq(struct ib_pd *ibpd, 1389 struct ib_srq_init_attr *init_attr, 1390 struct ib_udata *udata) 1391 { 1392 struct qed_rdma_destroy_srq_in_params destroy_in_params; 1393 struct qed_rdma_create_srq_in_params in_params = {}; 1394 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 1395 struct qed_rdma_create_srq_out_params out_params; 1396 struct qedr_pd *pd = get_qedr_pd(ibpd); 1397 struct qedr_create_srq_ureq ureq = {}; 1398 u64 pbl_base_addr, phy_prod_pair_addr; 1399 struct qedr_srq_hwq_info *hw_srq; 1400 u32 page_cnt, page_size; 1401 struct qedr_srq *srq; 1402 int rc = 0; 1403 1404 DP_DEBUG(dev, QEDR_MSG_QP, 1405 "create SRQ called from %s (pd %p)\n", 1406 (udata) ? "User lib" : "kernel", pd); 1407 1408 rc = qedr_check_srq_params(ibpd, dev, init_attr, udata); 1409 if (rc) 1410 return ERR_PTR(-EINVAL); 1411 1412 srq = kzalloc(sizeof(*srq), GFP_KERNEL); 1413 if (!srq) 1414 return ERR_PTR(-ENOMEM); 1415 1416 srq->dev = dev; 1417 hw_srq = &srq->hw_srq; 1418 spin_lock_init(&srq->lock); 1419 1420 hw_srq->max_wr = init_attr->attr.max_wr; 1421 hw_srq->max_sges = init_attr->attr.max_sge; 1422 1423 if (udata) { 1424 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { 1425 DP_ERR(dev, 1426 "create srq: problem copying data from user space\n"); 1427 goto err0; 1428 } 1429 1430 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0, 0); 1431 if (rc) 1432 goto err0; 1433 1434 page_cnt = srq->usrq.pbl_info.num_pbes; 1435 pbl_base_addr = srq->usrq.pbl_tbl->pa; 1436 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1437 page_size = PAGE_SIZE; 1438 } else { 1439 struct qed_chain *pbl; 1440 1441 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr); 1442 if (rc) 1443 goto err0; 1444 1445 pbl = &hw_srq->pbl; 1446 page_cnt = qed_chain_get_page_cnt(pbl); 1447 pbl_base_addr = qed_chain_get_pbl_phys(pbl); 1448 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1449 page_size = QED_CHAIN_PAGE_SIZE; 1450 } 1451 1452 in_params.pd_id = pd->pd_id; 1453 in_params.pbl_base_addr = pbl_base_addr; 1454 in_params.prod_pair_addr = phy_prod_pair_addr; 1455 in_params.num_pages = page_cnt; 1456 in_params.page_size = page_size; 1457 1458 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); 1459 if (rc) 1460 goto err1; 1461 1462 srq->srq_id = out_params.srq_id; 1463 1464 if (udata) { 1465 rc = qedr_copy_srq_uresp(dev, srq, udata); 1466 if (rc) 1467 goto err2; 1468 } 1469 1470 rc = qedr_idr_add(dev, &dev->srqidr, srq, srq->srq_id); 1471 if (rc) 1472 goto err2; 1473 1474 DP_DEBUG(dev, QEDR_MSG_SRQ, 1475 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id); 1476 return &srq->ibsrq; 1477 1478 err2: 1479 destroy_in_params.srq_id = srq->srq_id; 1480 1481 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); 1482 err1: 1483 if (udata) 1484 qedr_free_srq_user_params(srq); 1485 else 1486 qedr_free_srq_kernel_params(srq); 1487 err0: 1488 kfree(srq); 1489 1490 return ERR_PTR(-EFAULT); 1491 } 1492 1493 int qedr_destroy_srq(struct ib_srq *ibsrq) 1494 { 1495 struct qed_rdma_destroy_srq_in_params in_params = {}; 1496 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1497 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1498 1499 qedr_idr_remove(dev, &dev->srqidr, srq->srq_id); 1500 in_params.srq_id = srq->srq_id; 1501 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params); 1502 1503 if (ibsrq->uobject) 1504 qedr_free_srq_user_params(srq); 1505 else 1506 qedr_free_srq_kernel_params(srq); 1507 1508 DP_DEBUG(dev, QEDR_MSG_SRQ, 1509 "destroy srq: destroyed srq with srq_id=0x%0x\n", 1510 srq->srq_id); 1511 kfree(srq); 1512 1513 return 0; 1514 } 1515 1516 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 1517 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 1518 { 1519 struct qed_rdma_modify_srq_in_params in_params = {}; 1520 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1521 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1522 int rc; 1523 1524 if (attr_mask & IB_SRQ_MAX_WR) { 1525 DP_ERR(dev, 1526 "modify srq: invalid attribute mask=0x%x specified for %p\n", 1527 attr_mask, srq); 1528 return -EINVAL; 1529 } 1530 1531 if (attr_mask & IB_SRQ_LIMIT) { 1532 if (attr->srq_limit >= srq->hw_srq.max_wr) { 1533 DP_ERR(dev, 1534 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n", 1535 attr->srq_limit, srq->hw_srq.max_wr); 1536 return -EINVAL; 1537 } 1538 1539 in_params.srq_id = srq->srq_id; 1540 in_params.wqe_limit = attr->srq_limit; 1541 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params); 1542 if (rc) 1543 return rc; 1544 } 1545 1546 srq->srq_limit = attr->srq_limit; 1547 1548 DP_DEBUG(dev, QEDR_MSG_SRQ, 1549 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id); 1550 1551 return 0; 1552 } 1553 1554 static inline void 1555 qedr_init_common_qp_in_params(struct qedr_dev *dev, 1556 struct qedr_pd *pd, 1557 struct qedr_qp *qp, 1558 struct ib_qp_init_attr *attrs, 1559 bool fmr_and_reserved_lkey, 1560 struct qed_rdma_create_qp_in_params *params) 1561 { 1562 /* QP handle to be written in an async event */ 1563 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp); 1564 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp); 1565 1566 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 1567 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 1568 params->pd = pd->pd_id; 1569 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 1570 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; 1571 params->stats_queue = 0; 1572 params->srq_id = 0; 1573 params->use_srq = false; 1574 1575 if (!qp->srq) { 1576 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1577 1578 } else { 1579 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1580 params->srq_id = qp->srq->srq_id; 1581 params->use_srq = true; 1582 } 1583 } 1584 1585 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) 1586 { 1587 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. " 1588 "qp=%p. " 1589 "sq_addr=0x%llx, " 1590 "sq_len=%zd, " 1591 "rq_addr=0x%llx, " 1592 "rq_len=%zd" 1593 "\n", 1594 qp, 1595 qp->usq.buf_addr, 1596 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len); 1597 } 1598 1599 static int qedr_idr_add(struct qedr_dev *dev, struct qedr_idr *qidr, 1600 void *ptr, u32 id) 1601 { 1602 int rc; 1603 1604 idr_preload(GFP_KERNEL); 1605 spin_lock_irq(&qidr->idr_lock); 1606 1607 rc = idr_alloc(&qidr->idr, ptr, id, id + 1, GFP_ATOMIC); 1608 1609 spin_unlock_irq(&qidr->idr_lock); 1610 idr_preload_end(); 1611 1612 return rc < 0 ? rc : 0; 1613 } 1614 1615 static void qedr_idr_remove(struct qedr_dev *dev, struct qedr_idr *qidr, u32 id) 1616 { 1617 spin_lock_irq(&qidr->idr_lock); 1618 idr_remove(&qidr->idr, id); 1619 spin_unlock_irq(&qidr->idr_lock); 1620 } 1621 1622 static inline void 1623 qedr_iwarp_populate_user_qp(struct qedr_dev *dev, 1624 struct qedr_qp *qp, 1625 struct qed_rdma_create_qp_out_params *out_params) 1626 { 1627 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 1628 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 1629 1630 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 1631 &qp->usq.pbl_info, FW_PAGE_SHIFT); 1632 if (!qp->srq) { 1633 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 1634 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 1635 } 1636 1637 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 1638 &qp->urq.pbl_info, FW_PAGE_SHIFT); 1639 } 1640 1641 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp) 1642 { 1643 if (qp->usq.umem) 1644 ib_umem_release(qp->usq.umem); 1645 qp->usq.umem = NULL; 1646 1647 if (qp->urq.umem) 1648 ib_umem_release(qp->urq.umem); 1649 qp->urq.umem = NULL; 1650 } 1651 1652 static int qedr_create_user_qp(struct qedr_dev *dev, 1653 struct qedr_qp *qp, 1654 struct ib_pd *ibpd, 1655 struct ib_udata *udata, 1656 struct ib_qp_init_attr *attrs) 1657 { 1658 struct qed_rdma_create_qp_in_params in_params; 1659 struct qed_rdma_create_qp_out_params out_params; 1660 struct qedr_pd *pd = get_qedr_pd(ibpd); 1661 struct qedr_create_qp_ureq ureq; 1662 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1); 1663 int rc = -EINVAL; 1664 1665 memset(&ureq, 0, sizeof(ureq)); 1666 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq)); 1667 if (rc) { 1668 DP_ERR(dev, "Problem copying data from user space\n"); 1669 return rc; 1670 } 1671 1672 /* SQ - read access only (0), dma sync not required (0) */ 1673 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr, 1674 ureq.sq_len, 0, 0, alloc_and_init); 1675 if (rc) 1676 return rc; 1677 1678 if (!qp->srq) { 1679 /* RQ - read access only (0), dma sync not required (0) */ 1680 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr, 1681 ureq.rq_len, 0, 0, alloc_and_init); 1682 if (rc) 1683 return rc; 1684 } 1685 1686 memset(&in_params, 0, sizeof(in_params)); 1687 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 1688 in_params.qp_handle_lo = ureq.qp_handle_lo; 1689 in_params.qp_handle_hi = ureq.qp_handle_hi; 1690 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 1691 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 1692 if (!qp->srq) { 1693 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 1694 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 1695 } 1696 1697 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1698 &in_params, &out_params); 1699 1700 if (!qp->qed_qp) { 1701 rc = -ENOMEM; 1702 goto err1; 1703 } 1704 1705 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1706 qedr_iwarp_populate_user_qp(dev, qp, &out_params); 1707 1708 qp->qp_id = out_params.qp_id; 1709 qp->icid = out_params.icid; 1710 1711 rc = qedr_copy_qp_uresp(dev, qp, udata); 1712 if (rc) 1713 goto err; 1714 1715 qedr_qp_user_print(dev, qp); 1716 1717 return 0; 1718 err: 1719 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1720 if (rc) 1721 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc); 1722 1723 err1: 1724 qedr_cleanup_user(dev, qp); 1725 return rc; 1726 } 1727 1728 static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1729 { 1730 qp->sq.db = dev->db_addr + 1731 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1732 qp->sq.db_data.data.icid = qp->icid; 1733 1734 qp->rq.db = dev->db_addr + 1735 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1736 qp->rq.db_data.data.icid = qp->icid; 1737 qp->rq.iwarp_db2 = dev->db_addr + 1738 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1739 qp->rq.iwarp_db2_data.data.icid = qp->icid; 1740 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 1741 } 1742 1743 static int 1744 qedr_roce_create_kernel_qp(struct qedr_dev *dev, 1745 struct qedr_qp *qp, 1746 struct qed_rdma_create_qp_in_params *in_params, 1747 u32 n_sq_elems, u32 n_rq_elems) 1748 { 1749 struct qed_rdma_create_qp_out_params out_params; 1750 int rc; 1751 1752 rc = dev->ops->common->chain_alloc(dev->cdev, 1753 QED_CHAIN_USE_TO_PRODUCE, 1754 QED_CHAIN_MODE_PBL, 1755 QED_CHAIN_CNT_TYPE_U32, 1756 n_sq_elems, 1757 QEDR_SQE_ELEMENT_SIZE, 1758 &qp->sq.pbl, NULL); 1759 1760 if (rc) 1761 return rc; 1762 1763 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl); 1764 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl); 1765 1766 rc = dev->ops->common->chain_alloc(dev->cdev, 1767 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1768 QED_CHAIN_MODE_PBL, 1769 QED_CHAIN_CNT_TYPE_U32, 1770 n_rq_elems, 1771 QEDR_RQE_ELEMENT_SIZE, 1772 &qp->rq.pbl, NULL); 1773 if (rc) 1774 return rc; 1775 1776 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl); 1777 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl); 1778 1779 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1780 in_params, &out_params); 1781 1782 if (!qp->qed_qp) 1783 return -EINVAL; 1784 1785 qp->qp_id = out_params.qp_id; 1786 qp->icid = out_params.icid; 1787 1788 qedr_set_roce_db_info(dev, qp); 1789 return rc; 1790 } 1791 1792 static int 1793 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev, 1794 struct qedr_qp *qp, 1795 struct qed_rdma_create_qp_in_params *in_params, 1796 u32 n_sq_elems, u32 n_rq_elems) 1797 { 1798 struct qed_rdma_create_qp_out_params out_params; 1799 struct qed_chain_ext_pbl ext_pbl; 1800 int rc; 1801 1802 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems, 1803 QEDR_SQE_ELEMENT_SIZE, 1804 QED_CHAIN_MODE_PBL); 1805 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems, 1806 QEDR_RQE_ELEMENT_SIZE, 1807 QED_CHAIN_MODE_PBL); 1808 1809 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1810 in_params, &out_params); 1811 1812 if (!qp->qed_qp) 1813 return -EINVAL; 1814 1815 /* Now we allocate the chain */ 1816 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt; 1817 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys; 1818 1819 rc = dev->ops->common->chain_alloc(dev->cdev, 1820 QED_CHAIN_USE_TO_PRODUCE, 1821 QED_CHAIN_MODE_PBL, 1822 QED_CHAIN_CNT_TYPE_U32, 1823 n_sq_elems, 1824 QEDR_SQE_ELEMENT_SIZE, 1825 &qp->sq.pbl, &ext_pbl); 1826 1827 if (rc) 1828 goto err; 1829 1830 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt; 1831 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys; 1832 1833 rc = dev->ops->common->chain_alloc(dev->cdev, 1834 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1835 QED_CHAIN_MODE_PBL, 1836 QED_CHAIN_CNT_TYPE_U32, 1837 n_rq_elems, 1838 QEDR_RQE_ELEMENT_SIZE, 1839 &qp->rq.pbl, &ext_pbl); 1840 1841 if (rc) 1842 goto err; 1843 1844 qp->qp_id = out_params.qp_id; 1845 qp->icid = out_params.icid; 1846 1847 qedr_set_iwarp_db_info(dev, qp); 1848 return rc; 1849 1850 err: 1851 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1852 1853 return rc; 1854 } 1855 1856 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) 1857 { 1858 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl); 1859 kfree(qp->wqe_wr_id); 1860 1861 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl); 1862 kfree(qp->rqe_wr_id); 1863 } 1864 1865 static int qedr_create_kernel_qp(struct qedr_dev *dev, 1866 struct qedr_qp *qp, 1867 struct ib_pd *ibpd, 1868 struct ib_qp_init_attr *attrs) 1869 { 1870 struct qed_rdma_create_qp_in_params in_params; 1871 struct qedr_pd *pd = get_qedr_pd(ibpd); 1872 int rc = -EINVAL; 1873 u32 n_rq_elems; 1874 u32 n_sq_elems; 1875 u32 n_sq_entries; 1876 1877 memset(&in_params, 0, sizeof(in_params)); 1878 1879 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in 1880 * the ring. The ring should allow at least a single WR, even if the 1881 * user requested none, due to allocation issues. 1882 * We should add an extra WR since the prod and cons indices of 1883 * wqe_wr_id are managed in such a way that the WQ is considered full 1884 * when (prod+1)%max_wr==cons. We currently don't do that because we 1885 * double the number of entries due an iSER issue that pushes far more 1886 * WRs than indicated. If we decline its ib_post_send() then we get 1887 * error prints in the dmesg we'd like to avoid. 1888 */ 1889 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 1890 dev->attr.max_sqe); 1891 1892 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id), 1893 GFP_KERNEL); 1894 if (!qp->wqe_wr_id) { 1895 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n"); 1896 return -ENOMEM; 1897 } 1898 1899 /* QP handle to be written in CQE */ 1900 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp); 1901 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp); 1902 1903 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in 1904 * the ring. There ring should allow at least a single WR, even if the 1905 * user requested none, due to allocation issues. 1906 */ 1907 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1); 1908 1909 /* Allocate driver internal RQ array */ 1910 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id), 1911 GFP_KERNEL); 1912 if (!qp->rqe_wr_id) { 1913 DP_ERR(dev, 1914 "create qp: failed RQ shadow memory allocation\n"); 1915 kfree(qp->wqe_wr_id); 1916 return -ENOMEM; 1917 } 1918 1919 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 1920 1921 n_sq_entries = attrs->cap.max_send_wr; 1922 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe); 1923 n_sq_entries = max_t(u32, n_sq_entries, 1); 1924 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE; 1925 1926 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE; 1927 1928 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1929 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params, 1930 n_sq_elems, n_rq_elems); 1931 else 1932 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params, 1933 n_sq_elems, n_rq_elems); 1934 if (rc) 1935 qedr_cleanup_kernel(dev, qp); 1936 1937 return rc; 1938 } 1939 1940 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, 1941 struct ib_qp_init_attr *attrs, 1942 struct ib_udata *udata) 1943 { 1944 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 1945 struct qedr_pd *pd = get_qedr_pd(ibpd); 1946 struct qedr_qp *qp; 1947 struct ib_qp *ibqp; 1948 int rc = 0; 1949 1950 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", 1951 udata ? "user library" : "kernel", pd); 1952 1953 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata); 1954 if (rc) 1955 return ERR_PTR(rc); 1956 1957 DP_DEBUG(dev, QEDR_MSG_QP, 1958 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 1959 udata ? "user library" : "kernel", attrs->event_handler, pd, 1960 get_qedr_cq(attrs->send_cq), 1961 get_qedr_cq(attrs->send_cq)->icid, 1962 get_qedr_cq(attrs->recv_cq), 1963 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0); 1964 1965 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1966 if (!qp) { 1967 DP_ERR(dev, "create qp: failed allocating memory\n"); 1968 return ERR_PTR(-ENOMEM); 1969 } 1970 1971 qedr_set_common_qp_params(dev, qp, pd, attrs); 1972 1973 if (attrs->qp_type == IB_QPT_GSI) { 1974 ibqp = qedr_create_gsi_qp(dev, attrs, qp); 1975 if (IS_ERR(ibqp)) 1976 kfree(qp); 1977 return ibqp; 1978 } 1979 1980 if (udata) 1981 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); 1982 else 1983 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); 1984 1985 if (rc) 1986 goto err; 1987 1988 qp->ibqp.qp_num = qp->qp_id; 1989 1990 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1991 rc = qedr_idr_add(dev, &dev->qpidr, qp, qp->qp_id); 1992 if (rc) 1993 goto err; 1994 } 1995 1996 return &qp->ibqp; 1997 1998 err: 1999 kfree(qp); 2000 2001 return ERR_PTR(-EFAULT); 2002 } 2003 2004 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) 2005 { 2006 switch (qp_state) { 2007 case QED_ROCE_QP_STATE_RESET: 2008 return IB_QPS_RESET; 2009 case QED_ROCE_QP_STATE_INIT: 2010 return IB_QPS_INIT; 2011 case QED_ROCE_QP_STATE_RTR: 2012 return IB_QPS_RTR; 2013 case QED_ROCE_QP_STATE_RTS: 2014 return IB_QPS_RTS; 2015 case QED_ROCE_QP_STATE_SQD: 2016 return IB_QPS_SQD; 2017 case QED_ROCE_QP_STATE_ERR: 2018 return IB_QPS_ERR; 2019 case QED_ROCE_QP_STATE_SQE: 2020 return IB_QPS_SQE; 2021 } 2022 return IB_QPS_ERR; 2023 } 2024 2025 static enum qed_roce_qp_state qedr_get_state_from_ibqp( 2026 enum ib_qp_state qp_state) 2027 { 2028 switch (qp_state) { 2029 case IB_QPS_RESET: 2030 return QED_ROCE_QP_STATE_RESET; 2031 case IB_QPS_INIT: 2032 return QED_ROCE_QP_STATE_INIT; 2033 case IB_QPS_RTR: 2034 return QED_ROCE_QP_STATE_RTR; 2035 case IB_QPS_RTS: 2036 return QED_ROCE_QP_STATE_RTS; 2037 case IB_QPS_SQD: 2038 return QED_ROCE_QP_STATE_SQD; 2039 case IB_QPS_ERR: 2040 return QED_ROCE_QP_STATE_ERR; 2041 default: 2042 return QED_ROCE_QP_STATE_ERR; 2043 } 2044 } 2045 2046 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) 2047 { 2048 qed_chain_reset(&qph->pbl); 2049 qph->prod = 0; 2050 qph->cons = 0; 2051 qph->wqe_cons = 0; 2052 qph->db_data.data.value = cpu_to_le16(0); 2053 } 2054 2055 static int qedr_update_qp_state(struct qedr_dev *dev, 2056 struct qedr_qp *qp, 2057 enum qed_roce_qp_state cur_state, 2058 enum qed_roce_qp_state new_state) 2059 { 2060 int status = 0; 2061 2062 if (new_state == cur_state) 2063 return 0; 2064 2065 switch (cur_state) { 2066 case QED_ROCE_QP_STATE_RESET: 2067 switch (new_state) { 2068 case QED_ROCE_QP_STATE_INIT: 2069 qp->prev_wqe_size = 0; 2070 qedr_reset_qp_hwq_info(&qp->sq); 2071 qedr_reset_qp_hwq_info(&qp->rq); 2072 break; 2073 default: 2074 status = -EINVAL; 2075 break; 2076 } 2077 break; 2078 case QED_ROCE_QP_STATE_INIT: 2079 switch (new_state) { 2080 case QED_ROCE_QP_STATE_RTR: 2081 /* Update doorbell (in case post_recv was 2082 * done before move to RTR) 2083 */ 2084 2085 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2086 writel(qp->rq.db_data.raw, qp->rq.db); 2087 /* Make sure write takes effect */ 2088 mmiowb(); 2089 } 2090 break; 2091 case QED_ROCE_QP_STATE_ERR: 2092 break; 2093 default: 2094 /* Invalid state change. */ 2095 status = -EINVAL; 2096 break; 2097 } 2098 break; 2099 case QED_ROCE_QP_STATE_RTR: 2100 /* RTR->XXX */ 2101 switch (new_state) { 2102 case QED_ROCE_QP_STATE_RTS: 2103 break; 2104 case QED_ROCE_QP_STATE_ERR: 2105 break; 2106 default: 2107 /* Invalid state change. */ 2108 status = -EINVAL; 2109 break; 2110 } 2111 break; 2112 case QED_ROCE_QP_STATE_RTS: 2113 /* RTS->XXX */ 2114 switch (new_state) { 2115 case QED_ROCE_QP_STATE_SQD: 2116 break; 2117 case QED_ROCE_QP_STATE_ERR: 2118 break; 2119 default: 2120 /* Invalid state change. */ 2121 status = -EINVAL; 2122 break; 2123 } 2124 break; 2125 case QED_ROCE_QP_STATE_SQD: 2126 /* SQD->XXX */ 2127 switch (new_state) { 2128 case QED_ROCE_QP_STATE_RTS: 2129 case QED_ROCE_QP_STATE_ERR: 2130 break; 2131 default: 2132 /* Invalid state change. */ 2133 status = -EINVAL; 2134 break; 2135 } 2136 break; 2137 case QED_ROCE_QP_STATE_ERR: 2138 /* ERR->XXX */ 2139 switch (new_state) { 2140 case QED_ROCE_QP_STATE_RESET: 2141 if ((qp->rq.prod != qp->rq.cons) || 2142 (qp->sq.prod != qp->sq.cons)) { 2143 DP_NOTICE(dev, 2144 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n", 2145 qp->rq.prod, qp->rq.cons, qp->sq.prod, 2146 qp->sq.cons); 2147 status = -EINVAL; 2148 } 2149 break; 2150 default: 2151 status = -EINVAL; 2152 break; 2153 } 2154 break; 2155 default: 2156 status = -EINVAL; 2157 break; 2158 } 2159 2160 return status; 2161 } 2162 2163 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2164 int attr_mask, struct ib_udata *udata) 2165 { 2166 struct qedr_qp *qp = get_qedr_qp(ibqp); 2167 struct qed_rdma_modify_qp_in_params qp_params = { 0 }; 2168 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); 2169 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 2170 enum ib_qp_state old_qp_state, new_qp_state; 2171 enum qed_roce_qp_state cur_state; 2172 int rc = 0; 2173 2174 DP_DEBUG(dev, QEDR_MSG_QP, 2175 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask, 2176 attr->qp_state); 2177 2178 old_qp_state = qedr_get_ibqp_state(qp->state); 2179 if (attr_mask & IB_QP_STATE) 2180 new_qp_state = attr->qp_state; 2181 else 2182 new_qp_state = old_qp_state; 2183 2184 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2185 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state, 2186 ibqp->qp_type, attr_mask)) { 2187 DP_ERR(dev, 2188 "modify qp: invalid attribute mask=0x%x specified for\n" 2189 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n", 2190 attr_mask, qp->qp_id, ibqp->qp_type, 2191 old_qp_state, new_qp_state); 2192 rc = -EINVAL; 2193 goto err; 2194 } 2195 } 2196 2197 /* Translate the masks... */ 2198 if (attr_mask & IB_QP_STATE) { 2199 SET_FIELD(qp_params.modify_flags, 2200 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 2201 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state); 2202 } 2203 2204 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 2205 qp_params.sqd_async = true; 2206 2207 if (attr_mask & IB_QP_PKEY_INDEX) { 2208 SET_FIELD(qp_params.modify_flags, 2209 QED_ROCE_MODIFY_QP_VALID_PKEY, 1); 2210 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) { 2211 rc = -EINVAL; 2212 goto err; 2213 } 2214 2215 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT; 2216 } 2217 2218 if (attr_mask & IB_QP_QKEY) 2219 qp->qkey = attr->qkey; 2220 2221 if (attr_mask & IB_QP_ACCESS_FLAGS) { 2222 SET_FIELD(qp_params.modify_flags, 2223 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 2224 qp_params.incoming_rdma_read_en = attr->qp_access_flags & 2225 IB_ACCESS_REMOTE_READ; 2226 qp_params.incoming_rdma_write_en = attr->qp_access_flags & 2227 IB_ACCESS_REMOTE_WRITE; 2228 qp_params.incoming_atomic_en = attr->qp_access_flags & 2229 IB_ACCESS_REMOTE_ATOMIC; 2230 } 2231 2232 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 2233 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2234 return -EINVAL; 2235 2236 if (attr_mask & IB_QP_PATH_MTU) { 2237 if (attr->path_mtu < IB_MTU_256 || 2238 attr->path_mtu > IB_MTU_4096) { 2239 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n"); 2240 rc = -EINVAL; 2241 goto err; 2242 } 2243 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 2244 ib_mtu_enum_to_int(iboe_get_mtu 2245 (dev->ndev->mtu))); 2246 } 2247 2248 if (!qp->mtu) { 2249 qp->mtu = 2250 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2251 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu); 2252 } 2253 2254 SET_FIELD(qp_params.modify_flags, 2255 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1); 2256 2257 qp_params.traffic_class_tos = grh->traffic_class; 2258 qp_params.flow_label = grh->flow_label; 2259 qp_params.hop_limit_ttl = grh->hop_limit; 2260 2261 qp->sgid_idx = grh->sgid_index; 2262 2263 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params); 2264 if (rc) { 2265 DP_ERR(dev, 2266 "modify qp: problems with GID index %d (rc=%d)\n", 2267 grh->sgid_index, rc); 2268 return rc; 2269 } 2270 2271 rc = qedr_get_dmac(dev, &attr->ah_attr, 2272 qp_params.remote_mac_addr); 2273 if (rc) 2274 return rc; 2275 2276 qp_params.use_local_mac = true; 2277 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr); 2278 2279 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n", 2280 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 2281 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 2282 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n", 2283 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 2284 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 2285 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n", 2286 qp_params.remote_mac_addr); 2287 2288 qp_params.mtu = qp->mtu; 2289 qp_params.lb_indication = false; 2290 } 2291 2292 if (!qp_params.mtu) { 2293 /* Stay with current MTU */ 2294 if (qp->mtu) 2295 qp_params.mtu = qp->mtu; 2296 else 2297 qp_params.mtu = 2298 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2299 } 2300 2301 if (attr_mask & IB_QP_TIMEOUT) { 2302 SET_FIELD(qp_params.modify_flags, 2303 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 2304 2305 /* The received timeout value is an exponent used like this: 2306 * "12.7.34 LOCAL ACK TIMEOUT 2307 * Value representing the transport (ACK) timeout for use by 2308 * the remote, expressed as: 4.096 * 2^timeout [usec]" 2309 * The FW expects timeout in msec so we need to divide the usec 2310 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2, 2311 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8). 2312 * The value of zero means infinite so we use a 'max_t' to make 2313 * sure that sub 1 msec values will be configured as 1 msec. 2314 */ 2315 if (attr->timeout) 2316 qp_params.ack_timeout = 2317 1 << max_t(int, attr->timeout - 8, 0); 2318 else 2319 qp_params.ack_timeout = 0; 2320 } 2321 2322 if (attr_mask & IB_QP_RETRY_CNT) { 2323 SET_FIELD(qp_params.modify_flags, 2324 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 2325 qp_params.retry_cnt = attr->retry_cnt; 2326 } 2327 2328 if (attr_mask & IB_QP_RNR_RETRY) { 2329 SET_FIELD(qp_params.modify_flags, 2330 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1); 2331 qp_params.rnr_retry_cnt = attr->rnr_retry; 2332 } 2333 2334 if (attr_mask & IB_QP_RQ_PSN) { 2335 SET_FIELD(qp_params.modify_flags, 2336 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1); 2337 qp_params.rq_psn = attr->rq_psn; 2338 qp->rq_psn = attr->rq_psn; 2339 } 2340 2341 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 2342 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) { 2343 rc = -EINVAL; 2344 DP_ERR(dev, 2345 "unsupported max_rd_atomic=%d, supported=%d\n", 2346 attr->max_rd_atomic, 2347 dev->attr.max_qp_req_rd_atomic_resc); 2348 goto err; 2349 } 2350 2351 SET_FIELD(qp_params.modify_flags, 2352 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1); 2353 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 2354 } 2355 2356 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 2357 SET_FIELD(qp_params.modify_flags, 2358 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1); 2359 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 2360 } 2361 2362 if (attr_mask & IB_QP_SQ_PSN) { 2363 SET_FIELD(qp_params.modify_flags, 2364 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1); 2365 qp_params.sq_psn = attr->sq_psn; 2366 qp->sq_psn = attr->sq_psn; 2367 } 2368 2369 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 2370 if (attr->max_dest_rd_atomic > 2371 dev->attr.max_qp_resp_rd_atomic_resc) { 2372 DP_ERR(dev, 2373 "unsupported max_dest_rd_atomic=%d, supported=%d\n", 2374 attr->max_dest_rd_atomic, 2375 dev->attr.max_qp_resp_rd_atomic_resc); 2376 2377 rc = -EINVAL; 2378 goto err; 2379 } 2380 2381 SET_FIELD(qp_params.modify_flags, 2382 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1); 2383 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 2384 } 2385 2386 if (attr_mask & IB_QP_DEST_QPN) { 2387 SET_FIELD(qp_params.modify_flags, 2388 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1); 2389 2390 qp_params.dest_qp = attr->dest_qp_num; 2391 qp->dest_qp_num = attr->dest_qp_num; 2392 } 2393 2394 cur_state = qp->state; 2395 2396 /* Update the QP state before the actual ramrod to prevent a race with 2397 * fast path. Modifying the QP state to error will cause the device to 2398 * flush the CQEs and while polling the flushed CQEs will considered as 2399 * a potential issue if the QP isn't in error state. 2400 */ 2401 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI && 2402 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR) 2403 qp->state = QED_ROCE_QP_STATE_ERR; 2404 2405 if (qp->qp_type != IB_QPT_GSI) 2406 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx, 2407 qp->qed_qp, &qp_params); 2408 2409 if (attr_mask & IB_QP_STATE) { 2410 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 2411 rc = qedr_update_qp_state(dev, qp, cur_state, 2412 qp_params.new_state); 2413 qp->state = qp_params.new_state; 2414 } 2415 2416 err: 2417 return rc; 2418 } 2419 2420 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params) 2421 { 2422 int ib_qp_acc_flags = 0; 2423 2424 if (params->incoming_rdma_write_en) 2425 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 2426 if (params->incoming_rdma_read_en) 2427 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 2428 if (params->incoming_atomic_en) 2429 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 2430 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 2431 return ib_qp_acc_flags; 2432 } 2433 2434 int qedr_query_qp(struct ib_qp *ibqp, 2435 struct ib_qp_attr *qp_attr, 2436 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 2437 { 2438 struct qed_rdma_query_qp_out_params params; 2439 struct qedr_qp *qp = get_qedr_qp(ibqp); 2440 struct qedr_dev *dev = qp->dev; 2441 int rc = 0; 2442 2443 memset(¶ms, 0, sizeof(params)); 2444 2445 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms); 2446 if (rc) 2447 goto err; 2448 2449 memset(qp_attr, 0, sizeof(*qp_attr)); 2450 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 2451 2452 qp_attr->qp_state = qedr_get_ibqp_state(params.state); 2453 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state); 2454 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu); 2455 qp_attr->path_mig_state = IB_MIG_MIGRATED; 2456 qp_attr->rq_psn = params.rq_psn; 2457 qp_attr->sq_psn = params.sq_psn; 2458 qp_attr->dest_qp_num = params.dest_qp; 2459 2460 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms); 2461 2462 qp_attr->cap.max_send_wr = qp->sq.max_wr; 2463 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 2464 qp_attr->cap.max_send_sge = qp->sq.max_sges; 2465 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 2466 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; 2467 qp_init_attr->cap = qp_attr->cap; 2468 2469 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2470 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 2471 params.flow_label, qp->sgid_idx, 2472 params.hop_limit_ttl, params.traffic_class_tos); 2473 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); 2474 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 2475 rdma_ah_set_sl(&qp_attr->ah_attr, 0); 2476 qp_attr->timeout = params.timeout; 2477 qp_attr->rnr_retry = params.rnr_retry; 2478 qp_attr->retry_cnt = params.retry_cnt; 2479 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 2480 qp_attr->pkey_index = params.pkey_index; 2481 qp_attr->port_num = 1; 2482 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 2483 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 2484 qp_attr->alt_pkey_index = 0; 2485 qp_attr->alt_port_num = 0; 2486 qp_attr->alt_timeout = 0; 2487 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 2488 2489 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0; 2490 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 2491 qp_attr->max_rd_atomic = params.max_rd_atomic; 2492 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0; 2493 2494 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n", 2495 qp_attr->cap.max_inline_data); 2496 2497 err: 2498 return rc; 2499 } 2500 2501 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp) 2502 { 2503 int rc = 0; 2504 2505 if (qp->qp_type != IB_QPT_GSI) { 2506 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2507 if (rc) 2508 return rc; 2509 } 2510 2511 if (qp->ibqp.uobject && qp->ibqp.uobject->context) 2512 qedr_cleanup_user(dev, qp); 2513 else 2514 qedr_cleanup_kernel(dev, qp); 2515 2516 return 0; 2517 } 2518 2519 int qedr_destroy_qp(struct ib_qp *ibqp) 2520 { 2521 struct qedr_qp *qp = get_qedr_qp(ibqp); 2522 struct qedr_dev *dev = qp->dev; 2523 struct ib_qp_attr attr; 2524 int attr_mask = 0; 2525 int rc = 0; 2526 2527 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n", 2528 qp, qp->qp_type); 2529 2530 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2531 if ((qp->state != QED_ROCE_QP_STATE_RESET) && 2532 (qp->state != QED_ROCE_QP_STATE_ERR) && 2533 (qp->state != QED_ROCE_QP_STATE_INIT)) { 2534 2535 attr.qp_state = IB_QPS_ERR; 2536 attr_mask |= IB_QP_STATE; 2537 2538 /* Change the QP state to ERROR */ 2539 qedr_modify_qp(ibqp, &attr, attr_mask, NULL); 2540 } 2541 } else { 2542 /* Wait for the connect/accept to complete */ 2543 if (qp->ep) { 2544 int wait_count = 1; 2545 2546 while (qp->ep->during_connect) { 2547 DP_DEBUG(dev, QEDR_MSG_QP, 2548 "Still in during connect/accept\n"); 2549 2550 msleep(100); 2551 if (wait_count++ > 200) { 2552 DP_NOTICE(dev, 2553 "during connect timeout\n"); 2554 break; 2555 } 2556 } 2557 } 2558 } 2559 2560 if (qp->qp_type == IB_QPT_GSI) 2561 qedr_destroy_gsi_qp(dev); 2562 2563 qedr_free_qp_resources(dev, qp); 2564 2565 if (atomic_dec_and_test(&qp->refcnt) && 2566 rdma_protocol_iwarp(&dev->ibdev, 1)) { 2567 qedr_idr_remove(dev, &dev->qpidr, qp->qp_id); 2568 kfree(qp); 2569 } 2570 return rc; 2571 } 2572 2573 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr, 2574 u32 flags, struct ib_udata *udata) 2575 { 2576 struct qedr_ah *ah; 2577 2578 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 2579 if (!ah) 2580 return ERR_PTR(-ENOMEM); 2581 2582 rdma_copy_ah_attr(&ah->attr, attr); 2583 2584 return &ah->ibah; 2585 } 2586 2587 int qedr_destroy_ah(struct ib_ah *ibah, u32 flags) 2588 { 2589 struct qedr_ah *ah = get_qedr_ah(ibah); 2590 2591 rdma_destroy_ah_attr(&ah->attr); 2592 kfree(ah); 2593 return 0; 2594 } 2595 2596 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) 2597 { 2598 struct qedr_pbl *pbl, *tmp; 2599 2600 if (info->pbl_table) 2601 list_add_tail(&info->pbl_table->list_entry, 2602 &info->free_pbl_list); 2603 2604 if (!list_empty(&info->inuse_pbl_list)) 2605 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 2606 2607 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 2608 list_del(&pbl->list_entry); 2609 qedr_free_pbl(dev, &info->pbl_info, pbl); 2610 } 2611 } 2612 2613 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info, 2614 size_t page_list_len, bool two_layered) 2615 { 2616 struct qedr_pbl *tmp; 2617 int rc; 2618 2619 INIT_LIST_HEAD(&info->free_pbl_list); 2620 INIT_LIST_HEAD(&info->inuse_pbl_list); 2621 2622 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info, 2623 page_list_len, two_layered); 2624 if (rc) 2625 goto done; 2626 2627 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2628 if (IS_ERR(info->pbl_table)) { 2629 rc = PTR_ERR(info->pbl_table); 2630 goto done; 2631 } 2632 2633 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n", 2634 &info->pbl_table->pa); 2635 2636 /* in usual case we use 2 PBLs, so we add one to free 2637 * list and allocating another one 2638 */ 2639 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2640 if (IS_ERR(tmp)) { 2641 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n"); 2642 goto done; 2643 } 2644 2645 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 2646 2647 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa); 2648 2649 done: 2650 if (rc) 2651 free_mr_info(dev, info); 2652 2653 return rc; 2654 } 2655 2656 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 2657 u64 usr_addr, int acc, struct ib_udata *udata) 2658 { 2659 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2660 struct qedr_mr *mr; 2661 struct qedr_pd *pd; 2662 int rc = -ENOMEM; 2663 2664 pd = get_qedr_pd(ibpd); 2665 DP_DEBUG(dev, QEDR_MSG_MR, 2666 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 2667 pd->pd_id, start, len, usr_addr, acc); 2668 2669 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 2670 return ERR_PTR(-EINVAL); 2671 2672 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2673 if (!mr) 2674 return ERR_PTR(rc); 2675 2676 mr->type = QEDR_MR_USER; 2677 2678 mr->umem = ib_umem_get(udata, start, len, acc, 0); 2679 if (IS_ERR(mr->umem)) { 2680 rc = -EFAULT; 2681 goto err0; 2682 } 2683 2684 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); 2685 if (rc) 2686 goto err1; 2687 2688 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 2689 &mr->info.pbl_info, PAGE_SHIFT); 2690 2691 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2692 if (rc) { 2693 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2694 goto err1; 2695 } 2696 2697 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2698 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2699 mr->hw_mr.key = 0; 2700 mr->hw_mr.pd = pd->pd_id; 2701 mr->hw_mr.local_read = 1; 2702 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2703 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2704 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2705 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2706 mr->hw_mr.mw_bind = false; 2707 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 2708 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2709 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2710 mr->hw_mr.page_size_log = PAGE_SHIFT; 2711 mr->hw_mr.fbo = ib_umem_offset(mr->umem); 2712 mr->hw_mr.length = len; 2713 mr->hw_mr.vaddr = usr_addr; 2714 mr->hw_mr.zbva = false; 2715 mr->hw_mr.phy_mr = false; 2716 mr->hw_mr.dma_mr = false; 2717 2718 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2719 if (rc) { 2720 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2721 goto err2; 2722 } 2723 2724 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2725 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2726 mr->hw_mr.remote_atomic) 2727 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2728 2729 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n", 2730 mr->ibmr.lkey); 2731 return &mr->ibmr; 2732 2733 err2: 2734 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2735 err1: 2736 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2737 err0: 2738 kfree(mr); 2739 return ERR_PTR(rc); 2740 } 2741 2742 int qedr_dereg_mr(struct ib_mr *ib_mr) 2743 { 2744 struct qedr_mr *mr = get_qedr_mr(ib_mr); 2745 struct qedr_dev *dev = get_qedr_dev(ib_mr->device); 2746 int rc = 0; 2747 2748 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid); 2749 if (rc) 2750 return rc; 2751 2752 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2753 2754 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR)) 2755 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2756 2757 /* it could be user registered memory. */ 2758 if (mr->umem) 2759 ib_umem_release(mr->umem); 2760 2761 kfree(mr); 2762 2763 return rc; 2764 } 2765 2766 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, 2767 int max_page_list_len) 2768 { 2769 struct qedr_pd *pd = get_qedr_pd(ibpd); 2770 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2771 struct qedr_mr *mr; 2772 int rc = -ENOMEM; 2773 2774 DP_DEBUG(dev, QEDR_MSG_MR, 2775 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id, 2776 max_page_list_len); 2777 2778 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2779 if (!mr) 2780 return ERR_PTR(rc); 2781 2782 mr->dev = dev; 2783 mr->type = QEDR_MR_FRMR; 2784 2785 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1); 2786 if (rc) 2787 goto err0; 2788 2789 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2790 if (rc) { 2791 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2792 goto err0; 2793 } 2794 2795 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2796 mr->hw_mr.tid_type = QED_RDMA_TID_FMR; 2797 mr->hw_mr.key = 0; 2798 mr->hw_mr.pd = pd->pd_id; 2799 mr->hw_mr.local_read = 1; 2800 mr->hw_mr.local_write = 0; 2801 mr->hw_mr.remote_read = 0; 2802 mr->hw_mr.remote_write = 0; 2803 mr->hw_mr.remote_atomic = 0; 2804 mr->hw_mr.mw_bind = false; 2805 mr->hw_mr.pbl_ptr = 0; 2806 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2807 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2808 mr->hw_mr.fbo = 0; 2809 mr->hw_mr.length = 0; 2810 mr->hw_mr.vaddr = 0; 2811 mr->hw_mr.zbva = false; 2812 mr->hw_mr.phy_mr = true; 2813 mr->hw_mr.dma_mr = false; 2814 2815 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2816 if (rc) { 2817 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2818 goto err1; 2819 } 2820 2821 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2822 mr->ibmr.rkey = mr->ibmr.lkey; 2823 2824 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey); 2825 return mr; 2826 2827 err1: 2828 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2829 err0: 2830 kfree(mr); 2831 return ERR_PTR(rc); 2832 } 2833 2834 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, 2835 enum ib_mr_type mr_type, u32 max_num_sg) 2836 { 2837 struct qedr_mr *mr; 2838 2839 if (mr_type != IB_MR_TYPE_MEM_REG) 2840 return ERR_PTR(-EINVAL); 2841 2842 mr = __qedr_alloc_mr(ibpd, max_num_sg); 2843 2844 if (IS_ERR(mr)) 2845 return ERR_PTR(-EINVAL); 2846 2847 return &mr->ibmr; 2848 } 2849 2850 static int qedr_set_page(struct ib_mr *ibmr, u64 addr) 2851 { 2852 struct qedr_mr *mr = get_qedr_mr(ibmr); 2853 struct qedr_pbl *pbl_table; 2854 struct regpair *pbe; 2855 u32 pbes_in_page; 2856 2857 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 2858 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages); 2859 return -ENOMEM; 2860 } 2861 2862 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n", 2863 mr->npages, addr); 2864 2865 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 2866 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 2867 pbe = (struct regpair *)pbl_table->va; 2868 pbe += mr->npages % pbes_in_page; 2869 pbe->lo = cpu_to_le32((u32)addr); 2870 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 2871 2872 mr->npages++; 2873 2874 return 0; 2875 } 2876 2877 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info) 2878 { 2879 int work = info->completed - info->completed_handled - 1; 2880 2881 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work); 2882 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 2883 struct qedr_pbl *pbl; 2884 2885 /* Free all the page list that are possible to be freed 2886 * (all the ones that were invalidated), under the assumption 2887 * that if an FMR was completed successfully that means that 2888 * if there was an invalidate operation before it also ended 2889 */ 2890 pbl = list_first_entry(&info->inuse_pbl_list, 2891 struct qedr_pbl, list_entry); 2892 list_move_tail(&pbl->list_entry, &info->free_pbl_list); 2893 info->completed_handled++; 2894 } 2895 } 2896 2897 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 2898 int sg_nents, unsigned int *sg_offset) 2899 { 2900 struct qedr_mr *mr = get_qedr_mr(ibmr); 2901 2902 mr->npages = 0; 2903 2904 handle_completed_mrs(mr->dev, &mr->info); 2905 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page); 2906 } 2907 2908 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc) 2909 { 2910 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2911 struct qedr_pd *pd = get_qedr_pd(ibpd); 2912 struct qedr_mr *mr; 2913 int rc; 2914 2915 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2916 if (!mr) 2917 return ERR_PTR(-ENOMEM); 2918 2919 mr->type = QEDR_MR_DMA; 2920 2921 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2922 if (rc) { 2923 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2924 goto err1; 2925 } 2926 2927 /* index only, 18 bit long, lkey = itid << 8 | key */ 2928 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2929 mr->hw_mr.pd = pd->pd_id; 2930 mr->hw_mr.local_read = 1; 2931 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2932 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2933 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2934 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2935 mr->hw_mr.dma_mr = true; 2936 2937 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2938 if (rc) { 2939 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2940 goto err2; 2941 } 2942 2943 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2944 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2945 mr->hw_mr.remote_atomic) 2946 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2947 2948 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey); 2949 return &mr->ibmr; 2950 2951 err2: 2952 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2953 err1: 2954 kfree(mr); 2955 return ERR_PTR(rc); 2956 } 2957 2958 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq) 2959 { 2960 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 2961 } 2962 2963 static int sge_data_len(struct ib_sge *sg_list, int num_sge) 2964 { 2965 int i, len = 0; 2966 2967 for (i = 0; i < num_sge; i++) 2968 len += sg_list[i].length; 2969 2970 return len; 2971 } 2972 2973 static void swap_wqe_data64(u64 *p) 2974 { 2975 int i; 2976 2977 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 2978 *p = cpu_to_be64(cpu_to_le64(*p)); 2979 } 2980 2981 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev, 2982 struct qedr_qp *qp, u8 *wqe_size, 2983 const struct ib_send_wr *wr, 2984 const struct ib_send_wr **bad_wr, 2985 u8 *bits, u8 bit) 2986 { 2987 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 2988 char *seg_prt, *wqe; 2989 int i, seg_siz; 2990 2991 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 2992 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size); 2993 *bad_wr = wr; 2994 return 0; 2995 } 2996 2997 if (!data_size) 2998 return data_size; 2999 3000 *bits |= bit; 3001 3002 seg_prt = NULL; 3003 wqe = NULL; 3004 seg_siz = 0; 3005 3006 /* Copy data inline */ 3007 for (i = 0; i < wr->num_sge; i++) { 3008 u32 len = wr->sg_list[i].length; 3009 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 3010 3011 while (len > 0) { 3012 u32 cur; 3013 3014 /* New segment required */ 3015 if (!seg_siz) { 3016 wqe = (char *)qed_chain_produce(&qp->sq.pbl); 3017 seg_prt = wqe; 3018 seg_siz = sizeof(struct rdma_sq_common_wqe); 3019 (*wqe_size)++; 3020 } 3021 3022 /* Calculate currently allowed length */ 3023 cur = min_t(u32, len, seg_siz); 3024 memcpy(seg_prt, src, cur); 3025 3026 /* Update segment variables */ 3027 seg_prt += cur; 3028 seg_siz -= cur; 3029 3030 /* Update sge variables */ 3031 src += cur; 3032 len -= cur; 3033 3034 /* Swap fully-completed segments */ 3035 if (!seg_siz) 3036 swap_wqe_data64((u64 *)wqe); 3037 } 3038 } 3039 3040 /* swap last not completed segment */ 3041 if (seg_siz) 3042 swap_wqe_data64((u64 *)wqe); 3043 3044 return data_size; 3045 } 3046 3047 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 3048 do { \ 3049 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3050 (sge)->length = cpu_to_le32(vlength); \ 3051 (sge)->flags = cpu_to_le32(vflags); \ 3052 } while (0) 3053 3054 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 3055 do { \ 3056 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \ 3057 (hdr)->num_sges = num_sge; \ 3058 } while (0) 3059 3060 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 3061 do { \ 3062 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3063 (sge)->length = cpu_to_le32(vlength); \ 3064 (sge)->l_key = cpu_to_le32(vlkey); \ 3065 } while (0) 3066 3067 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, 3068 const struct ib_send_wr *wr) 3069 { 3070 u32 data_size = 0; 3071 int i; 3072 3073 for (i = 0; i < wr->num_sge; i++) { 3074 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl); 3075 3076 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr); 3077 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 3078 sge->length = cpu_to_le32(wr->sg_list[i].length); 3079 data_size += wr->sg_list[i].length; 3080 } 3081 3082 if (wqe_size) 3083 *wqe_size += wr->num_sge; 3084 3085 return data_size; 3086 } 3087 3088 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev, 3089 struct qedr_qp *qp, 3090 struct rdma_sq_rdma_wqe_1st *rwqe, 3091 struct rdma_sq_rdma_wqe_2nd *rwqe2, 3092 const struct ib_send_wr *wr, 3093 const struct ib_send_wr **bad_wr) 3094 { 3095 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 3096 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr); 3097 3098 if (wr->send_flags & IB_SEND_INLINE && 3099 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 3100 wr->opcode == IB_WR_RDMA_WRITE)) { 3101 u8 flags = 0; 3102 3103 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 3104 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, 3105 bad_wr, &rwqe->flags, flags); 3106 } 3107 3108 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr); 3109 } 3110 3111 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev, 3112 struct qedr_qp *qp, 3113 struct rdma_sq_send_wqe_1st *swqe, 3114 struct rdma_sq_send_wqe_2st *swqe2, 3115 const struct ib_send_wr *wr, 3116 const struct ib_send_wr **bad_wr) 3117 { 3118 memset(swqe2, 0, sizeof(*swqe2)); 3119 if (wr->send_flags & IB_SEND_INLINE) { 3120 u8 flags = 0; 3121 3122 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 3123 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, 3124 bad_wr, &swqe->flags, flags); 3125 } 3126 3127 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr); 3128 } 3129 3130 static int qedr_prepare_reg(struct qedr_qp *qp, 3131 struct rdma_sq_fmr_wqe_1st *fwqe1, 3132 const struct ib_reg_wr *wr) 3133 { 3134 struct qedr_mr *mr = get_qedr_mr(wr->mr); 3135 struct rdma_sq_fmr_wqe_2nd *fwqe2; 3136 3137 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl); 3138 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 3139 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 3140 fwqe1->l_key = wr->key; 3141 3142 fwqe2->access_ctrl = 0; 3143 3144 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 3145 !!(wr->access & IB_ACCESS_REMOTE_READ)); 3146 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 3147 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 3148 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 3149 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 3150 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 3151 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 3152 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 3153 fwqe2->fmr_ctrl = 0; 3154 3155 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 3156 ilog2(mr->ibmr.page_size) - 12); 3157 3158 fwqe2->length_hi = 0; 3159 fwqe2->length_lo = mr->ibmr.length; 3160 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 3161 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 3162 3163 qp->wqe_wr_id[qp->sq.prod].mr = mr; 3164 3165 return 0; 3166 } 3167 3168 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 3169 { 3170 switch (opcode) { 3171 case IB_WR_RDMA_WRITE: 3172 case IB_WR_RDMA_WRITE_WITH_IMM: 3173 return IB_WC_RDMA_WRITE; 3174 case IB_WR_SEND_WITH_IMM: 3175 case IB_WR_SEND: 3176 case IB_WR_SEND_WITH_INV: 3177 return IB_WC_SEND; 3178 case IB_WR_RDMA_READ: 3179 case IB_WR_RDMA_READ_WITH_INV: 3180 return IB_WC_RDMA_READ; 3181 case IB_WR_ATOMIC_CMP_AND_SWP: 3182 return IB_WC_COMP_SWAP; 3183 case IB_WR_ATOMIC_FETCH_AND_ADD: 3184 return IB_WC_FETCH_ADD; 3185 case IB_WR_REG_MR: 3186 return IB_WC_REG_MR; 3187 case IB_WR_LOCAL_INV: 3188 return IB_WC_LOCAL_INV; 3189 default: 3190 return IB_WC_SEND; 3191 } 3192 } 3193 3194 static inline bool qedr_can_post_send(struct qedr_qp *qp, 3195 const struct ib_send_wr *wr) 3196 { 3197 int wq_is_full, err_wr, pbl_is_full; 3198 struct qedr_dev *dev = qp->dev; 3199 3200 /* prevent SQ overflow and/or processing of a bad WR */ 3201 err_wr = wr->num_sge > qp->sq.max_sges; 3202 wq_is_full = qedr_wq_is_full(&qp->sq); 3203 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) < 3204 QEDR_MAX_SQE_ELEMENTS_PER_SQE; 3205 if (wq_is_full || err_wr || pbl_is_full) { 3206 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) { 3207 DP_ERR(dev, 3208 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n", 3209 qp); 3210 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL; 3211 } 3212 3213 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) { 3214 DP_ERR(dev, 3215 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n", 3216 qp); 3217 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR; 3218 } 3219 3220 if (pbl_is_full && 3221 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) { 3222 DP_ERR(dev, 3223 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n", 3224 qp); 3225 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL; 3226 } 3227 return false; 3228 } 3229 return true; 3230 } 3231 3232 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3233 const struct ib_send_wr **bad_wr) 3234 { 3235 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3236 struct qedr_qp *qp = get_qedr_qp(ibqp); 3237 struct rdma_sq_atomic_wqe_1st *awqe1; 3238 struct rdma_sq_atomic_wqe_2nd *awqe2; 3239 struct rdma_sq_atomic_wqe_3rd *awqe3; 3240 struct rdma_sq_send_wqe_2st *swqe2; 3241 struct rdma_sq_local_inv_wqe *iwqe; 3242 struct rdma_sq_rdma_wqe_2nd *rwqe2; 3243 struct rdma_sq_send_wqe_1st *swqe; 3244 struct rdma_sq_rdma_wqe_1st *rwqe; 3245 struct rdma_sq_fmr_wqe_1st *fwqe1; 3246 struct rdma_sq_common_wqe *wqe; 3247 u32 length; 3248 int rc = 0; 3249 bool comp; 3250 3251 if (!qedr_can_post_send(qp, wr)) { 3252 *bad_wr = wr; 3253 return -ENOMEM; 3254 } 3255 3256 wqe = qed_chain_produce(&qp->sq.pbl); 3257 qp->wqe_wr_id[qp->sq.prod].signaled = 3258 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 3259 3260 wqe->flags = 0; 3261 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, 3262 !!(wr->send_flags & IB_SEND_SOLICITED)); 3263 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled; 3264 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 3265 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, 3266 !!(wr->send_flags & IB_SEND_FENCE)); 3267 wqe->prev_wqe_size = qp->prev_wqe_size; 3268 3269 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode); 3270 3271 switch (wr->opcode) { 3272 case IB_WR_SEND_WITH_IMM: 3273 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3274 rc = -EINVAL; 3275 *bad_wr = wr; 3276 break; 3277 } 3278 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 3279 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3280 swqe->wqe_size = 2; 3281 swqe2 = qed_chain_produce(&qp->sq.pbl); 3282 3283 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 3284 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3285 wr, bad_wr); 3286 swqe->length = cpu_to_le32(length); 3287 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3288 qp->prev_wqe_size = swqe->wqe_size; 3289 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3290 break; 3291 case IB_WR_SEND: 3292 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 3293 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3294 3295 swqe->wqe_size = 2; 3296 swqe2 = qed_chain_produce(&qp->sq.pbl); 3297 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3298 wr, bad_wr); 3299 swqe->length = cpu_to_le32(length); 3300 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3301 qp->prev_wqe_size = swqe->wqe_size; 3302 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3303 break; 3304 case IB_WR_SEND_WITH_INV: 3305 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 3306 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3307 swqe2 = qed_chain_produce(&qp->sq.pbl); 3308 swqe->wqe_size = 2; 3309 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey); 3310 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3311 wr, bad_wr); 3312 swqe->length = cpu_to_le32(length); 3313 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3314 qp->prev_wqe_size = swqe->wqe_size; 3315 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3316 break; 3317 3318 case IB_WR_RDMA_WRITE_WITH_IMM: 3319 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3320 rc = -EINVAL; 3321 *bad_wr = wr; 3322 break; 3323 } 3324 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 3325 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3326 3327 rwqe->wqe_size = 2; 3328 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 3329 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3330 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3331 wr, bad_wr); 3332 rwqe->length = cpu_to_le32(length); 3333 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3334 qp->prev_wqe_size = rwqe->wqe_size; 3335 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3336 break; 3337 case IB_WR_RDMA_WRITE: 3338 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 3339 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3340 3341 rwqe->wqe_size = 2; 3342 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3343 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3344 wr, bad_wr); 3345 rwqe->length = cpu_to_le32(length); 3346 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3347 qp->prev_wqe_size = rwqe->wqe_size; 3348 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3349 break; 3350 case IB_WR_RDMA_READ_WITH_INV: 3351 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1); 3352 /* fallthrough -- same is identical to RDMA READ */ 3353 3354 case IB_WR_RDMA_READ: 3355 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 3356 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3357 3358 rwqe->wqe_size = 2; 3359 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3360 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3361 wr, bad_wr); 3362 rwqe->length = cpu_to_le32(length); 3363 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3364 qp->prev_wqe_size = rwqe->wqe_size; 3365 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3366 break; 3367 3368 case IB_WR_ATOMIC_CMP_AND_SWP: 3369 case IB_WR_ATOMIC_FETCH_AND_ADD: 3370 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe; 3371 awqe1->wqe_size = 4; 3372 3373 awqe2 = qed_chain_produce(&qp->sq.pbl); 3374 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr); 3375 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 3376 3377 awqe3 = qed_chain_produce(&qp->sq.pbl); 3378 3379 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 3380 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 3381 DMA_REGPAIR_LE(awqe3->swap_data, 3382 atomic_wr(wr)->compare_add); 3383 } else { 3384 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 3385 DMA_REGPAIR_LE(awqe3->swap_data, 3386 atomic_wr(wr)->swap); 3387 DMA_REGPAIR_LE(awqe3->cmp_data, 3388 atomic_wr(wr)->compare_add); 3389 } 3390 3391 qedr_prepare_sq_sges(qp, NULL, wr); 3392 3393 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size; 3394 qp->prev_wqe_size = awqe1->wqe_size; 3395 break; 3396 3397 case IB_WR_LOCAL_INV: 3398 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 3399 iwqe->wqe_size = 1; 3400 3401 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 3402 iwqe->inv_l_key = wr->ex.invalidate_rkey; 3403 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size; 3404 qp->prev_wqe_size = iwqe->wqe_size; 3405 break; 3406 case IB_WR_REG_MR: 3407 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n"); 3408 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 3409 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 3410 fwqe1->wqe_size = 2; 3411 3412 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr)); 3413 if (rc) { 3414 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc); 3415 *bad_wr = wr; 3416 break; 3417 } 3418 3419 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 3420 qp->prev_wqe_size = fwqe1->wqe_size; 3421 break; 3422 default: 3423 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode); 3424 rc = -EINVAL; 3425 *bad_wr = wr; 3426 break; 3427 } 3428 3429 if (*bad_wr) { 3430 u16 value; 3431 3432 /* Restore prod to its position before 3433 * this WR was processed 3434 */ 3435 value = le16_to_cpu(qp->sq.db_data.data.value); 3436 qed_chain_set_prod(&qp->sq.pbl, value, wqe); 3437 3438 /* Restore prev_wqe_size */ 3439 qp->prev_wqe_size = wqe->prev_wqe_size; 3440 rc = -EINVAL; 3441 DP_ERR(dev, "POST SEND FAILED\n"); 3442 } 3443 3444 return rc; 3445 } 3446 3447 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3448 const struct ib_send_wr **bad_wr) 3449 { 3450 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3451 struct qedr_qp *qp = get_qedr_qp(ibqp); 3452 unsigned long flags; 3453 int rc = 0; 3454 3455 *bad_wr = NULL; 3456 3457 if (qp->qp_type == IB_QPT_GSI) 3458 return qedr_gsi_post_send(ibqp, wr, bad_wr); 3459 3460 spin_lock_irqsave(&qp->q_lock, flags); 3461 3462 if (rdma_protocol_roce(&dev->ibdev, 1)) { 3463 if ((qp->state != QED_ROCE_QP_STATE_RTS) && 3464 (qp->state != QED_ROCE_QP_STATE_ERR) && 3465 (qp->state != QED_ROCE_QP_STATE_SQD)) { 3466 spin_unlock_irqrestore(&qp->q_lock, flags); 3467 *bad_wr = wr; 3468 DP_DEBUG(dev, QEDR_MSG_CQ, 3469 "QP in wrong state! QP icid=0x%x state %d\n", 3470 qp->icid, qp->state); 3471 return -EINVAL; 3472 } 3473 } 3474 3475 while (wr) { 3476 rc = __qedr_post_send(ibqp, wr, bad_wr); 3477 if (rc) 3478 break; 3479 3480 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 3481 3482 qedr_inc_sw_prod(&qp->sq); 3483 3484 qp->sq.db_data.data.value++; 3485 3486 wr = wr->next; 3487 } 3488 3489 /* Trigger doorbell 3490 * If there was a failure in the first WR then it will be triggered in 3491 * vane. However this is not harmful (as long as the producer value is 3492 * unchanged). For performance reasons we avoid checking for this 3493 * redundant doorbell. 3494 * 3495 * qp->wqe_wr_id is accessed during qedr_poll_cq, as 3496 * soon as we give the doorbell, we could get a completion 3497 * for this wr, therefore we need to make sure that the 3498 * memory is updated before giving the doorbell. 3499 * During qedr_poll_cq, rmb is called before accessing the 3500 * cqe. This covers for the smp_rmb as well. 3501 */ 3502 smp_wmb(); 3503 writel(qp->sq.db_data.raw, qp->sq.db); 3504 3505 /* Make sure write sticks */ 3506 mmiowb(); 3507 3508 spin_unlock_irqrestore(&qp->q_lock, flags); 3509 3510 return rc; 3511 } 3512 3513 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq) 3514 { 3515 u32 used; 3516 3517 /* Calculate number of elements used based on producer 3518 * count and consumer count and subtract it from max 3519 * work request supported so that we get elements left. 3520 */ 3521 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt; 3522 3523 return hw_srq->max_wr - used; 3524 } 3525 3526 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 3527 const struct ib_recv_wr **bad_wr) 3528 { 3529 struct qedr_srq *srq = get_qedr_srq(ibsrq); 3530 struct qedr_srq_hwq_info *hw_srq; 3531 struct qedr_dev *dev = srq->dev; 3532 struct qed_chain *pbl; 3533 unsigned long flags; 3534 int status = 0; 3535 u32 num_sge; 3536 u32 offset; 3537 3538 spin_lock_irqsave(&srq->lock, flags); 3539 3540 hw_srq = &srq->hw_srq; 3541 pbl = &srq->hw_srq.pbl; 3542 while (wr) { 3543 struct rdma_srq_wqe_header *hdr; 3544 int i; 3545 3546 if (!qedr_srq_elem_left(hw_srq) || 3547 wr->num_sge > srq->hw_srq.max_sges) { 3548 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n", 3549 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt, 3550 wr->num_sge, srq->hw_srq.max_sges); 3551 status = -ENOMEM; 3552 *bad_wr = wr; 3553 break; 3554 } 3555 3556 hdr = qed_chain_produce(pbl); 3557 num_sge = wr->num_sge; 3558 /* Set number of sge and work request id in header */ 3559 SRQ_HDR_SET(hdr, wr->wr_id, num_sge); 3560 3561 srq->hw_srq.wr_prod_cnt++; 3562 hw_srq->wqe_prod++; 3563 hw_srq->sge_prod++; 3564 3565 DP_DEBUG(dev, QEDR_MSG_SRQ, 3566 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n", 3567 wr->num_sge, hw_srq->wqe_prod, wr->wr_id); 3568 3569 for (i = 0; i < wr->num_sge; i++) { 3570 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl); 3571 3572 /* Set SGE length, lkey and address */ 3573 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, 3574 wr->sg_list[i].length, wr->sg_list[i].lkey); 3575 3576 DP_DEBUG(dev, QEDR_MSG_SRQ, 3577 "[%d]: len %d key %x addr %x:%x\n", 3578 i, srq_sge->length, srq_sge->l_key, 3579 srq_sge->addr.hi, srq_sge->addr.lo); 3580 hw_srq->sge_prod++; 3581 } 3582 3583 /* Flush WQE and SGE information before 3584 * updating producer. 3585 */ 3586 wmb(); 3587 3588 /* SRQ producer is 8 bytes. Need to update SGE producer index 3589 * in first 4 bytes and need to update WQE producer in 3590 * next 4 bytes. 3591 */ 3592 *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod; 3593 offset = offsetof(struct rdma_srq_producers, wqe_prod); 3594 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) = 3595 hw_srq->wqe_prod; 3596 3597 /* Flush producer after updating it. */ 3598 wmb(); 3599 wr = wr->next; 3600 } 3601 3602 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n", 3603 qed_chain_get_elem_left(pbl)); 3604 spin_unlock_irqrestore(&srq->lock, flags); 3605 3606 return status; 3607 } 3608 3609 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 3610 const struct ib_recv_wr **bad_wr) 3611 { 3612 struct qedr_qp *qp = get_qedr_qp(ibqp); 3613 struct qedr_dev *dev = qp->dev; 3614 unsigned long flags; 3615 int status = 0; 3616 3617 if (qp->qp_type == IB_QPT_GSI) 3618 return qedr_gsi_post_recv(ibqp, wr, bad_wr); 3619 3620 spin_lock_irqsave(&qp->q_lock, flags); 3621 3622 if (qp->state == QED_ROCE_QP_STATE_RESET) { 3623 spin_unlock_irqrestore(&qp->q_lock, flags); 3624 *bad_wr = wr; 3625 return -EINVAL; 3626 } 3627 3628 while (wr) { 3629 int i; 3630 3631 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) < 3632 QEDR_MAX_RQE_ELEMENTS_PER_RQE || 3633 wr->num_sge > qp->rq.max_sges) { 3634 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n", 3635 qed_chain_get_elem_left_u32(&qp->rq.pbl), 3636 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge, 3637 qp->rq.max_sges); 3638 status = -ENOMEM; 3639 *bad_wr = wr; 3640 break; 3641 } 3642 for (i = 0; i < wr->num_sge; i++) { 3643 u32 flags = 0; 3644 struct rdma_rq_sge *rqe = 3645 qed_chain_produce(&qp->rq.pbl); 3646 3647 /* First one must include the number 3648 * of SGE in the list 3649 */ 3650 if (!i) 3651 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 3652 wr->num_sge); 3653 3654 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 3655 wr->sg_list[i].lkey); 3656 3657 RQ_SGE_SET(rqe, wr->sg_list[i].addr, 3658 wr->sg_list[i].length, flags); 3659 } 3660 3661 /* Special case of no sges. FW requires between 1-4 sges... 3662 * in this case we need to post 1 sge with length zero. this is 3663 * because rdma write with immediate consumes an RQ. 3664 */ 3665 if (!wr->num_sge) { 3666 u32 flags = 0; 3667 struct rdma_rq_sge *rqe = 3668 qed_chain_produce(&qp->rq.pbl); 3669 3670 /* First one must include the number 3671 * of SGE in the list 3672 */ 3673 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0); 3674 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 3675 3676 RQ_SGE_SET(rqe, 0, 0, flags); 3677 i = 1; 3678 } 3679 3680 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 3681 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 3682 3683 qedr_inc_sw_prod(&qp->rq); 3684 3685 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as 3686 * soon as we give the doorbell, we could get a completion 3687 * for this wr, therefore we need to make sure that the 3688 * memory is update before giving the doorbell. 3689 * During qedr_poll_cq, rmb is called before accessing the 3690 * cqe. This covers for the smp_rmb as well. 3691 */ 3692 smp_wmb(); 3693 3694 qp->rq.db_data.data.value++; 3695 3696 writel(qp->rq.db_data.raw, qp->rq.db); 3697 3698 /* Make sure write sticks */ 3699 mmiowb(); 3700 3701 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 3702 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 3703 mmiowb(); /* for second doorbell */ 3704 } 3705 3706 wr = wr->next; 3707 } 3708 3709 spin_unlock_irqrestore(&qp->q_lock, flags); 3710 3711 return status; 3712 } 3713 3714 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe) 3715 { 3716 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3717 3718 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) == 3719 cq->pbl_toggle; 3720 } 3721 3722 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe) 3723 { 3724 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3725 struct qedr_qp *qp; 3726 3727 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi, 3728 resp_cqe->qp_handle.lo, 3729 u64); 3730 return qp; 3731 } 3732 3733 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe) 3734 { 3735 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3736 3737 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 3738 } 3739 3740 /* Return latest CQE (needs processing) */ 3741 static union rdma_cqe *get_cqe(struct qedr_cq *cq) 3742 { 3743 return cq->latest_cqe; 3744 } 3745 3746 /* In fmr we need to increase the number of fmr completed counter for the fmr 3747 * algorithm determining whether we can free a pbl or not. 3748 * we need to perform this whether the work request was signaled or not. for 3749 * this purpose we call this function from the condition that checks if a wr 3750 * should be skipped, to make sure we don't miss it ( possibly this fmr 3751 * operation was not signalted) 3752 */ 3753 static inline void qedr_chk_if_fmr(struct qedr_qp *qp) 3754 { 3755 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 3756 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3757 } 3758 3759 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, 3760 struct qedr_cq *cq, int num_entries, 3761 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, 3762 int force) 3763 { 3764 u16 cnt = 0; 3765 3766 while (num_entries && qp->sq.wqe_cons != hw_cons) { 3767 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 3768 qedr_chk_if_fmr(qp); 3769 /* skip WC */ 3770 goto next_cqe; 3771 } 3772 3773 /* fill WC */ 3774 wc->status = status; 3775 wc->vendor_err = 0; 3776 wc->wc_flags = 0; 3777 wc->src_qp = qp->id; 3778 wc->qp = &qp->ibqp; 3779 3780 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 3781 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 3782 3783 switch (wc->opcode) { 3784 case IB_WC_RDMA_WRITE: 3785 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3786 break; 3787 case IB_WC_COMP_SWAP: 3788 case IB_WC_FETCH_ADD: 3789 wc->byte_len = 8; 3790 break; 3791 case IB_WC_REG_MR: 3792 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3793 break; 3794 case IB_WC_RDMA_READ: 3795 case IB_WC_SEND: 3796 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3797 break; 3798 default: 3799 break; 3800 } 3801 3802 num_entries--; 3803 wc++; 3804 cnt++; 3805 next_cqe: 3806 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 3807 qed_chain_consume(&qp->sq.pbl); 3808 qedr_inc_sw_cons(&qp->sq); 3809 } 3810 3811 return cnt; 3812 } 3813 3814 static int qedr_poll_cq_req(struct qedr_dev *dev, 3815 struct qedr_qp *qp, struct qedr_cq *cq, 3816 int num_entries, struct ib_wc *wc, 3817 struct rdma_cqe_requester *req) 3818 { 3819 int cnt = 0; 3820 3821 switch (req->status) { 3822 case RDMA_CQE_REQ_STS_OK: 3823 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3824 IB_WC_SUCCESS, 0); 3825 break; 3826 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 3827 if (qp->state != QED_ROCE_QP_STATE_ERR) 3828 DP_DEBUG(dev, QEDR_MSG_CQ, 3829 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3830 cq->icid, qp->icid); 3831 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3832 IB_WC_WR_FLUSH_ERR, 1); 3833 break; 3834 default: 3835 /* process all WQE before the cosumer */ 3836 qp->state = QED_ROCE_QP_STATE_ERR; 3837 cnt = process_req(dev, qp, cq, num_entries, wc, 3838 req->sq_cons - 1, IB_WC_SUCCESS, 0); 3839 wc += cnt; 3840 /* if we have extra WC fill it with actual error info */ 3841 if (cnt < num_entries) { 3842 enum ib_wc_status wc_status; 3843 3844 switch (req->status) { 3845 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 3846 DP_ERR(dev, 3847 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3848 cq->icid, qp->icid); 3849 wc_status = IB_WC_BAD_RESP_ERR; 3850 break; 3851 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 3852 DP_ERR(dev, 3853 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3854 cq->icid, qp->icid); 3855 wc_status = IB_WC_LOC_LEN_ERR; 3856 break; 3857 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 3858 DP_ERR(dev, 3859 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3860 cq->icid, qp->icid); 3861 wc_status = IB_WC_LOC_QP_OP_ERR; 3862 break; 3863 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 3864 DP_ERR(dev, 3865 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3866 cq->icid, qp->icid); 3867 wc_status = IB_WC_LOC_PROT_ERR; 3868 break; 3869 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 3870 DP_ERR(dev, 3871 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3872 cq->icid, qp->icid); 3873 wc_status = IB_WC_MW_BIND_ERR; 3874 break; 3875 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 3876 DP_ERR(dev, 3877 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3878 cq->icid, qp->icid); 3879 wc_status = IB_WC_REM_INV_REQ_ERR; 3880 break; 3881 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 3882 DP_ERR(dev, 3883 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3884 cq->icid, qp->icid); 3885 wc_status = IB_WC_REM_ACCESS_ERR; 3886 break; 3887 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 3888 DP_ERR(dev, 3889 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3890 cq->icid, qp->icid); 3891 wc_status = IB_WC_REM_OP_ERR; 3892 break; 3893 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 3894 DP_ERR(dev, 3895 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3896 cq->icid, qp->icid); 3897 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 3898 break; 3899 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 3900 DP_ERR(dev, 3901 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3902 cq->icid, qp->icid); 3903 wc_status = IB_WC_RETRY_EXC_ERR; 3904 break; 3905 default: 3906 DP_ERR(dev, 3907 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3908 cq->icid, qp->icid); 3909 wc_status = IB_WC_GENERAL_ERR; 3910 } 3911 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 3912 wc_status, 1); 3913 } 3914 } 3915 3916 return cnt; 3917 } 3918 3919 static inline int qedr_cqe_resp_status_to_ib(u8 status) 3920 { 3921 switch (status) { 3922 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 3923 return IB_WC_LOC_ACCESS_ERR; 3924 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 3925 return IB_WC_LOC_LEN_ERR; 3926 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 3927 return IB_WC_LOC_QP_OP_ERR; 3928 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 3929 return IB_WC_LOC_PROT_ERR; 3930 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 3931 return IB_WC_MW_BIND_ERR; 3932 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 3933 return IB_WC_REM_INV_RD_REQ_ERR; 3934 case RDMA_CQE_RESP_STS_OK: 3935 return IB_WC_SUCCESS; 3936 default: 3937 return IB_WC_GENERAL_ERR; 3938 } 3939 } 3940 3941 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp, 3942 struct ib_wc *wc) 3943 { 3944 wc->status = IB_WC_SUCCESS; 3945 wc->byte_len = le32_to_cpu(resp->length); 3946 3947 if (resp->flags & QEDR_RESP_IMM) { 3948 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key)); 3949 wc->wc_flags |= IB_WC_WITH_IMM; 3950 3951 if (resp->flags & QEDR_RESP_RDMA) 3952 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 3953 3954 if (resp->flags & QEDR_RESP_INV) 3955 return -EINVAL; 3956 3957 } else if (resp->flags & QEDR_RESP_INV) { 3958 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 3959 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3960 3961 if (resp->flags & QEDR_RESP_RDMA) 3962 return -EINVAL; 3963 3964 } else if (resp->flags & QEDR_RESP_RDMA) { 3965 return -EINVAL; 3966 } 3967 3968 return 0; 3969 } 3970 3971 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 3972 struct qedr_cq *cq, struct ib_wc *wc, 3973 struct rdma_cqe_responder *resp, u64 wr_id) 3974 { 3975 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */ 3976 wc->opcode = IB_WC_RECV; 3977 wc->wc_flags = 0; 3978 3979 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) { 3980 if (qedr_set_ok_cqe_resp_wc(resp, wc)) 3981 DP_ERR(dev, 3982 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n", 3983 cq, cq->icid, resp->flags); 3984 3985 } else { 3986 wc->status = qedr_cqe_resp_status_to_ib(resp->status); 3987 if (wc->status == IB_WC_GENERAL_ERR) 3988 DP_ERR(dev, 3989 "CQ %p (icid=%d) contains an invalid CQE status %d\n", 3990 cq, cq->icid, resp->status); 3991 } 3992 3993 /* Fill the rest of the WC */ 3994 wc->vendor_err = 0; 3995 wc->src_qp = qp->id; 3996 wc->qp = &qp->ibqp; 3997 wc->wr_id = wr_id; 3998 } 3999 4000 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4001 struct qedr_cq *cq, struct ib_wc *wc, 4002 struct rdma_cqe_responder *resp) 4003 { 4004 struct qedr_srq *srq = qp->srq; 4005 u64 wr_id; 4006 4007 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi), 4008 le32_to_cpu(resp->srq_wr_id.lo), u64); 4009 4010 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4011 wc->status = IB_WC_WR_FLUSH_ERR; 4012 wc->vendor_err = 0; 4013 wc->wr_id = wr_id; 4014 wc->byte_len = 0; 4015 wc->src_qp = qp->id; 4016 wc->qp = &qp->ibqp; 4017 wc->wr_id = wr_id; 4018 } else { 4019 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4020 } 4021 srq->hw_srq.wr_cons_cnt++; 4022 4023 return 1; 4024 } 4025 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4026 struct qedr_cq *cq, struct ib_wc *wc, 4027 struct rdma_cqe_responder *resp) 4028 { 4029 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4030 4031 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4032 4033 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4034 qed_chain_consume(&qp->rq.pbl); 4035 qedr_inc_sw_cons(&qp->rq); 4036 4037 return 1; 4038 } 4039 4040 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, 4041 int num_entries, struct ib_wc *wc, u16 hw_cons) 4042 { 4043 u16 cnt = 0; 4044 4045 while (num_entries && qp->rq.wqe_cons != hw_cons) { 4046 /* fill WC */ 4047 wc->status = IB_WC_WR_FLUSH_ERR; 4048 wc->vendor_err = 0; 4049 wc->wc_flags = 0; 4050 wc->src_qp = qp->id; 4051 wc->byte_len = 0; 4052 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4053 wc->qp = &qp->ibqp; 4054 num_entries--; 4055 wc++; 4056 cnt++; 4057 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4058 qed_chain_consume(&qp->rq.pbl); 4059 qedr_inc_sw_cons(&qp->rq); 4060 } 4061 4062 return cnt; 4063 } 4064 4065 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4066 struct rdma_cqe_responder *resp, int *update) 4067 { 4068 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) { 4069 consume_cqe(cq); 4070 *update |= 1; 4071 } 4072 } 4073 4074 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4075 struct qedr_cq *cq, int num_entries, 4076 struct ib_wc *wc, 4077 struct rdma_cqe_responder *resp) 4078 { 4079 int cnt; 4080 4081 cnt = process_resp_one_srq(dev, qp, cq, wc, resp); 4082 consume_cqe(cq); 4083 4084 return cnt; 4085 } 4086 4087 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, 4088 struct qedr_cq *cq, int num_entries, 4089 struct ib_wc *wc, struct rdma_cqe_responder *resp, 4090 int *update) 4091 { 4092 int cnt; 4093 4094 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4095 cnt = process_resp_flush(qp, cq, num_entries, wc, 4096 resp->rq_cons_or_srq_id); 4097 try_consume_resp_cqe(cq, qp, resp, update); 4098 } else { 4099 cnt = process_resp_one(dev, qp, cq, wc, resp); 4100 consume_cqe(cq); 4101 *update |= 1; 4102 } 4103 4104 return cnt; 4105 } 4106 4107 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4108 struct rdma_cqe_requester *req, int *update) 4109 { 4110 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 4111 consume_cqe(cq); 4112 *update |= 1; 4113 } 4114 } 4115 4116 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 4117 { 4118 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 4119 struct qedr_cq *cq = get_qedr_cq(ibcq); 4120 union rdma_cqe *cqe; 4121 u32 old_cons, new_cons; 4122 unsigned long flags; 4123 int update = 0; 4124 int done = 0; 4125 4126 if (cq->destroyed) { 4127 DP_ERR(dev, 4128 "warning: poll was invoked after destroy for cq %p (icid=%d)\n", 4129 cq, cq->icid); 4130 return 0; 4131 } 4132 4133 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 4134 return qedr_gsi_poll_cq(ibcq, num_entries, wc); 4135 4136 spin_lock_irqsave(&cq->cq_lock, flags); 4137 cqe = cq->latest_cqe; 4138 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4139 while (num_entries && is_valid_cqe(cq, cqe)) { 4140 struct qedr_qp *qp; 4141 int cnt = 0; 4142 4143 /* prevent speculative reads of any field of CQE */ 4144 rmb(); 4145 4146 qp = cqe_get_qp(cqe); 4147 if (!qp) { 4148 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe); 4149 break; 4150 } 4151 4152 wc->qp = &qp->ibqp; 4153 4154 switch (cqe_get_type(cqe)) { 4155 case RDMA_CQE_TYPE_REQUESTER: 4156 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc, 4157 &cqe->req); 4158 try_consume_req_cqe(cq, qp, &cqe->req, &update); 4159 break; 4160 case RDMA_CQE_TYPE_RESPONDER_RQ: 4161 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc, 4162 &cqe->resp, &update); 4163 break; 4164 case RDMA_CQE_TYPE_RESPONDER_SRQ: 4165 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries, 4166 wc, &cqe->resp); 4167 update = 1; 4168 break; 4169 case RDMA_CQE_TYPE_INVALID: 4170 default: 4171 DP_ERR(dev, "Error: invalid CQE type = %d\n", 4172 cqe_get_type(cqe)); 4173 } 4174 num_entries -= cnt; 4175 wc += cnt; 4176 done += cnt; 4177 4178 cqe = get_cqe(cq); 4179 } 4180 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4181 4182 cq->cq_cons += new_cons - old_cons; 4183 4184 if (update) 4185 /* doorbell notifies abount latest VALID entry, 4186 * but chain already point to the next INVALID one 4187 */ 4188 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 4189 4190 spin_unlock_irqrestore(&cq->cq_lock, flags); 4191 return done; 4192 } 4193 4194 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, 4195 u8 port_num, 4196 const struct ib_wc *in_wc, 4197 const struct ib_grh *in_grh, 4198 const struct ib_mad_hdr *mad_hdr, 4199 size_t in_mad_size, struct ib_mad_hdr *out_mad, 4200 size_t *out_mad_size, u16 *out_mad_pkey_index) 4201 { 4202 struct qedr_dev *dev = get_qedr_dev(ibdev); 4203 4204 DP_DEBUG(dev, QEDR_MSG_GSI, 4205 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n", 4206 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod, 4207 mad_hdr->class_specific, mad_hdr->class_version, 4208 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status); 4209 return IB_MAD_RESULT_SUCCESS; 4210 } 4211