1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/dma-mapping.h> 33 #include <linux/crc32.h> 34 #include <net/ip.h> 35 #include <net/ipv6.h> 36 #include <net/udp.h> 37 #include <linux/iommu.h> 38 39 #include <rdma/ib_verbs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/iw_cm.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/ib_addr.h> 44 #include <rdma/ib_cache.h> 45 #include <rdma/uverbs_ioctl.h> 46 47 #include <linux/qed/common_hsi.h> 48 #include "qedr_hsi_rdma.h" 49 #include <linux/qed/qed_if.h> 50 #include "qedr.h" 51 #include "verbs.h" 52 #include <rdma/qedr-abi.h> 53 #include "qedr_roce_cm.h" 54 55 #define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm) 56 #define RDMA_MAX_SGE_PER_SRQ (4) 57 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) 58 59 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 60 61 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, 62 size_t len) 63 { 64 size_t min_len = min_t(size_t, len, udata->outlen); 65 66 return ib_copy_to_udata(udata, src, min_len); 67 } 68 69 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 70 { 71 if (index >= QEDR_ROCE_PKEY_TABLE_LEN) 72 return -EINVAL; 73 74 *pkey = QEDR_ROCE_PKEY_DEFAULT; 75 return 0; 76 } 77 78 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port, 79 int index, union ib_gid *sgid) 80 { 81 struct qedr_dev *dev = get_qedr_dev(ibdev); 82 83 memset(sgid->raw, 0, sizeof(sgid->raw)); 84 ether_addr_copy(sgid->raw, dev->ndev->dev_addr); 85 86 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index, 87 sgid->global.interface_id, sgid->global.subnet_prefix); 88 89 return 0; 90 } 91 92 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 93 { 94 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 95 struct qedr_device_attr *qattr = &dev->attr; 96 struct qedr_srq *srq = get_qedr_srq(ibsrq); 97 98 srq_attr->srq_limit = srq->srq_limit; 99 srq_attr->max_wr = qattr->max_srq_wr; 100 srq_attr->max_sge = qattr->max_sge; 101 102 return 0; 103 } 104 105 int qedr_query_device(struct ib_device *ibdev, 106 struct ib_device_attr *attr, struct ib_udata *udata) 107 { 108 struct qedr_dev *dev = get_qedr_dev(ibdev); 109 struct qedr_device_attr *qattr = &dev->attr; 110 111 if (!dev->rdma_ctx) { 112 DP_ERR(dev, 113 "qedr_query_device called with invalid params rdma_ctx=%p\n", 114 dev->rdma_ctx); 115 return -EINVAL; 116 } 117 118 memset(attr, 0, sizeof(*attr)); 119 120 attr->fw_ver = qattr->fw_ver; 121 attr->sys_image_guid = qattr->sys_image_guid; 122 attr->max_mr_size = qattr->max_mr_size; 123 attr->page_size_cap = qattr->page_size_caps; 124 attr->vendor_id = qattr->vendor_id; 125 attr->vendor_part_id = qattr->vendor_part_id; 126 attr->hw_ver = qattr->hw_ver; 127 attr->max_qp = qattr->max_qp; 128 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe); 129 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 130 IB_DEVICE_RC_RNR_NAK_GEN | 131 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS; 132 133 attr->max_send_sge = qattr->max_sge; 134 attr->max_recv_sge = qattr->max_sge; 135 attr->max_sge_rd = qattr->max_sge; 136 attr->max_cq = qattr->max_cq; 137 attr->max_cqe = qattr->max_cqe; 138 attr->max_mr = qattr->max_mr; 139 attr->max_mw = qattr->max_mw; 140 attr->max_pd = qattr->max_pd; 141 attr->atomic_cap = dev->atomic_cap; 142 attr->max_fmr = qattr->max_fmr; 143 attr->max_map_per_fmr = 16; 144 attr->max_qp_init_rd_atom = 145 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 146 attr->max_qp_rd_atom = 147 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 148 attr->max_qp_init_rd_atom); 149 150 attr->max_srq = qattr->max_srq; 151 attr->max_srq_sge = qattr->max_srq_sge; 152 attr->max_srq_wr = qattr->max_srq_wr; 153 154 attr->local_ca_ack_delay = qattr->dev_ack_delay; 155 attr->max_fast_reg_page_list_len = qattr->max_mr / 8; 156 attr->max_pkeys = QEDR_ROCE_PKEY_MAX; 157 attr->max_ah = qattr->max_ah; 158 159 return 0; 160 } 161 162 #define QEDR_SPEED_SDR (1) 163 #define QEDR_SPEED_DDR (2) 164 #define QEDR_SPEED_QDR (4) 165 #define QEDR_SPEED_FDR10 (8) 166 #define QEDR_SPEED_FDR (16) 167 #define QEDR_SPEED_EDR (32) 168 169 static inline void get_link_speed_and_width(int speed, u8 *ib_speed, 170 u8 *ib_width) 171 { 172 switch (speed) { 173 case 1000: 174 *ib_speed = QEDR_SPEED_SDR; 175 *ib_width = IB_WIDTH_1X; 176 break; 177 case 10000: 178 *ib_speed = QEDR_SPEED_QDR; 179 *ib_width = IB_WIDTH_1X; 180 break; 181 182 case 20000: 183 *ib_speed = QEDR_SPEED_DDR; 184 *ib_width = IB_WIDTH_4X; 185 break; 186 187 case 25000: 188 *ib_speed = QEDR_SPEED_EDR; 189 *ib_width = IB_WIDTH_1X; 190 break; 191 192 case 40000: 193 *ib_speed = QEDR_SPEED_QDR; 194 *ib_width = IB_WIDTH_4X; 195 break; 196 197 case 50000: 198 *ib_speed = QEDR_SPEED_QDR; 199 *ib_width = IB_WIDTH_4X; 200 break; 201 202 case 100000: 203 *ib_speed = QEDR_SPEED_EDR; 204 *ib_width = IB_WIDTH_4X; 205 break; 206 207 default: 208 /* Unsupported */ 209 *ib_speed = QEDR_SPEED_SDR; 210 *ib_width = IB_WIDTH_1X; 211 } 212 } 213 214 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr) 215 { 216 struct qedr_dev *dev; 217 struct qed_rdma_port *rdma_port; 218 219 dev = get_qedr_dev(ibdev); 220 221 if (!dev->rdma_ctx) { 222 DP_ERR(dev, "rdma_ctx is NULL\n"); 223 return -EINVAL; 224 } 225 226 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx); 227 228 /* *attr being zeroed by the caller, avoid zeroing it here */ 229 if (rdma_port->port_state == QED_RDMA_PORT_UP) { 230 attr->state = IB_PORT_ACTIVE; 231 attr->phys_state = 5; 232 } else { 233 attr->state = IB_PORT_DOWN; 234 attr->phys_state = 3; 235 } 236 attr->max_mtu = IB_MTU_4096; 237 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); 238 attr->lid = 0; 239 attr->lmc = 0; 240 attr->sm_lid = 0; 241 attr->sm_sl = 0; 242 attr->ip_gids = true; 243 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 244 attr->gid_tbl_len = 1; 245 attr->pkey_tbl_len = 1; 246 } else { 247 attr->gid_tbl_len = QEDR_MAX_SGID; 248 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; 249 } 250 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 251 attr->qkey_viol_cntr = 0; 252 get_link_speed_and_width(rdma_port->link_speed, 253 &attr->active_speed, &attr->active_width); 254 attr->max_msg_sz = rdma_port->max_msg_size; 255 attr->max_vl_num = 4; 256 257 return 0; 258 } 259 260 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask, 261 struct ib_port_modify *props) 262 { 263 return 0; 264 } 265 266 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr, 267 unsigned long len) 268 { 269 struct qedr_mm *mm; 270 271 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 272 if (!mm) 273 return -ENOMEM; 274 275 mm->key.phy_addr = phy_addr; 276 /* This function might be called with a length which is not a multiple 277 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel 278 * forces this granularity by increasing the requested size if needed. 279 * When qedr_mmap is called, it will search the list with the updated 280 * length as a key. To prevent search failures, the length is rounded up 281 * in advance to PAGE_SIZE. 282 */ 283 mm->key.len = roundup(len, PAGE_SIZE); 284 INIT_LIST_HEAD(&mm->entry); 285 286 mutex_lock(&uctx->mm_list_lock); 287 list_add(&mm->entry, &uctx->mm_head); 288 mutex_unlock(&uctx->mm_list_lock); 289 290 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 291 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n", 292 (unsigned long long)mm->key.phy_addr, 293 (unsigned long)mm->key.len, uctx); 294 295 return 0; 296 } 297 298 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr, 299 unsigned long len) 300 { 301 bool found = false; 302 struct qedr_mm *mm; 303 304 mutex_lock(&uctx->mm_list_lock); 305 list_for_each_entry(mm, &uctx->mm_head, entry) { 306 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 307 continue; 308 309 found = true; 310 break; 311 } 312 mutex_unlock(&uctx->mm_list_lock); 313 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 314 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n", 315 mm->key.phy_addr, mm->key.len, uctx, found); 316 317 return found; 318 } 319 320 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 321 { 322 struct ib_device *ibdev = uctx->device; 323 int rc; 324 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx); 325 struct qedr_alloc_ucontext_resp uresp = {}; 326 struct qedr_dev *dev = get_qedr_dev(ibdev); 327 struct qed_rdma_add_user_out_params oparams; 328 329 if (!udata) 330 return -EFAULT; 331 332 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams); 333 if (rc) { 334 DP_ERR(dev, 335 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n", 336 rc); 337 return rc; 338 } 339 340 ctx->dpi = oparams.dpi; 341 ctx->dpi_addr = oparams.dpi_addr; 342 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 343 ctx->dpi_size = oparams.dpi_size; 344 INIT_LIST_HEAD(&ctx->mm_head); 345 mutex_init(&ctx->mm_list_lock); 346 347 uresp.dpm_enabled = dev->user_dpm_enabled; 348 uresp.wids_enabled = 1; 349 uresp.wid_count = oparams.wid_count; 350 uresp.db_pa = ctx->dpi_phys_addr; 351 uresp.db_size = ctx->dpi_size; 352 uresp.max_send_wr = dev->attr.max_sqe; 353 uresp.max_recv_wr = dev->attr.max_rqe; 354 uresp.max_srq_wr = dev->attr.max_srq_wr; 355 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE; 356 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE; 357 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 358 uresp.max_cqes = QEDR_MAX_CQES; 359 360 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 361 if (rc) 362 return rc; 363 364 ctx->dev = dev; 365 366 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size); 367 if (rc) 368 return rc; 369 370 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n", 371 &ctx->ibucontext); 372 return 0; 373 } 374 375 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx) 376 { 377 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx); 378 struct qedr_mm *mm, *tmp; 379 380 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n", 381 uctx); 382 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi); 383 384 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 385 DP_DEBUG(uctx->dev, QEDR_MSG_MISC, 386 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n", 387 mm->key.phy_addr, mm->key.len, uctx); 388 list_del(&mm->entry); 389 kfree(mm); 390 } 391 } 392 393 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 394 { 395 struct qedr_ucontext *ucontext = get_qedr_ucontext(context); 396 struct qedr_dev *dev = get_qedr_dev(context->device); 397 unsigned long phys_addr = vma->vm_pgoff << PAGE_SHIFT; 398 unsigned long len = (vma->vm_end - vma->vm_start); 399 unsigned long dpi_start; 400 401 dpi_start = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size); 402 403 DP_DEBUG(dev, QEDR_MSG_INIT, 404 "mmap invoked with vm_start=0x%pK, vm_end=0x%pK,vm_pgoff=0x%pK; dpi_start=0x%pK dpi_size=0x%x\n", 405 (void *)vma->vm_start, (void *)vma->vm_end, 406 (void *)vma->vm_pgoff, (void *)dpi_start, ucontext->dpi_size); 407 408 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) { 409 DP_ERR(dev, 410 "failed mmap, addresses must be page aligned: start=0x%pK, end=0x%pK\n", 411 (void *)vma->vm_start, (void *)vma->vm_end); 412 return -EINVAL; 413 } 414 415 if (!qedr_search_mmap(ucontext, phys_addr, len)) { 416 DP_ERR(dev, "failed mmap, vm_pgoff=0x%lx is not authorized\n", 417 vma->vm_pgoff); 418 return -EINVAL; 419 } 420 421 if (phys_addr < dpi_start || 422 ((phys_addr + len) > (dpi_start + ucontext->dpi_size))) { 423 DP_ERR(dev, 424 "failed mmap, pages are outside of dpi; page address=0x%pK, dpi_start=0x%pK, dpi_size=0x%x\n", 425 (void *)phys_addr, (void *)dpi_start, 426 ucontext->dpi_size); 427 return -EINVAL; 428 } 429 430 if (vma->vm_flags & VM_READ) { 431 DP_ERR(dev, "failed mmap, cannot map doorbell bar for read\n"); 432 return -EINVAL; 433 } 434 435 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 436 return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len, 437 vma->vm_page_prot); 438 } 439 440 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 441 { 442 struct ib_device *ibdev = ibpd->device; 443 struct qedr_dev *dev = get_qedr_dev(ibdev); 444 struct qedr_pd *pd = get_qedr_pd(ibpd); 445 u16 pd_id; 446 int rc; 447 448 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n", 449 udata ? "User Lib" : "Kernel"); 450 451 if (!dev->rdma_ctx) { 452 DP_ERR(dev, "invalid RDMA context\n"); 453 return -EINVAL; 454 } 455 456 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id); 457 if (rc) 458 return rc; 459 460 pd->pd_id = pd_id; 461 462 if (udata) { 463 struct qedr_alloc_pd_uresp uresp = { 464 .pd_id = pd_id, 465 }; 466 struct qedr_ucontext *context = rdma_udata_to_drv_context( 467 udata, struct qedr_ucontext, ibucontext); 468 469 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 470 if (rc) { 471 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); 472 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); 473 return rc; 474 } 475 476 pd->uctx = context; 477 pd->uctx->pd = pd; 478 } 479 480 return 0; 481 } 482 483 void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 484 { 485 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 486 struct qedr_pd *pd = get_qedr_pd(ibpd); 487 488 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id); 489 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id); 490 } 491 492 static void qedr_free_pbl(struct qedr_dev *dev, 493 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) 494 { 495 struct pci_dev *pdev = dev->pdev; 496 int i; 497 498 for (i = 0; i < pbl_info->num_pbls; i++) { 499 if (!pbl[i].va) 500 continue; 501 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 502 pbl[i].va, pbl[i].pa); 503 } 504 505 kfree(pbl); 506 } 507 508 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024) 509 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024) 510 511 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 512 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 513 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE) 514 515 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev, 516 struct qedr_pbl_info *pbl_info, 517 gfp_t flags) 518 { 519 struct pci_dev *pdev = dev->pdev; 520 struct qedr_pbl *pbl_table; 521 dma_addr_t *pbl_main_tbl; 522 dma_addr_t pa; 523 void *va; 524 int i; 525 526 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags); 527 if (!pbl_table) 528 return ERR_PTR(-ENOMEM); 529 530 for (i = 0; i < pbl_info->num_pbls; i++) { 531 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa, 532 flags); 533 if (!va) 534 goto err; 535 536 pbl_table[i].va = va; 537 pbl_table[i].pa = pa; 538 } 539 540 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 541 * the first one with physical pointers to all of the rest 542 */ 543 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 544 for (i = 0; i < pbl_info->num_pbls - 1; i++) 545 pbl_main_tbl[i] = pbl_table[i + 1].pa; 546 547 return pbl_table; 548 549 err: 550 for (i--; i >= 0; i--) 551 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 552 pbl_table[i].va, pbl_table[i].pa); 553 554 qedr_free_pbl(dev, pbl_info, pbl_table); 555 556 return ERR_PTR(-ENOMEM); 557 } 558 559 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev, 560 struct qedr_pbl_info *pbl_info, 561 u32 num_pbes, int two_layer_capable) 562 { 563 u32 pbl_capacity; 564 u32 pbl_size; 565 u32 num_pbls; 566 567 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 568 if (num_pbes > MAX_PBES_TWO_LAYER) { 569 DP_ERR(dev, "prepare pbl table: too many pages %d\n", 570 num_pbes); 571 return -EINVAL; 572 } 573 574 /* calculate required pbl page size */ 575 pbl_size = MIN_FW_PBL_PAGE_SIZE; 576 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 577 NUM_PBES_ON_PAGE(pbl_size); 578 579 while (pbl_capacity < num_pbes) { 580 pbl_size *= 2; 581 pbl_capacity = pbl_size / sizeof(u64); 582 pbl_capacity = pbl_capacity * pbl_capacity; 583 } 584 585 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 586 num_pbls++; /* One for the layer0 ( points to the pbls) */ 587 pbl_info->two_layered = true; 588 } else { 589 /* One layered PBL */ 590 num_pbls = 1; 591 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, 592 roundup_pow_of_two((num_pbes * sizeof(u64)))); 593 pbl_info->two_layered = false; 594 } 595 596 pbl_info->num_pbls = num_pbls; 597 pbl_info->pbl_size = pbl_size; 598 pbl_info->num_pbes = num_pbes; 599 600 DP_DEBUG(dev, QEDR_MSG_MR, 601 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n", 602 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 603 604 return 0; 605 } 606 607 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, 608 struct qedr_pbl *pbl, 609 struct qedr_pbl_info *pbl_info, u32 pg_shift) 610 { 611 int pbe_cnt, total_num_pbes = 0; 612 u32 fw_pg_cnt, fw_pg_per_umem_pg; 613 struct qedr_pbl *pbl_tbl; 614 struct sg_dma_page_iter sg_iter; 615 struct regpair *pbe; 616 u64 pg_addr; 617 618 if (!pbl_info->num_pbes) 619 return; 620 621 /* If we have a two layered pbl, the first pbl points to the rest 622 * of the pbls and the first entry lays on the second pbl in the table 623 */ 624 if (pbl_info->two_layered) 625 pbl_tbl = &pbl[1]; 626 else 627 pbl_tbl = pbl; 628 629 pbe = (struct regpair *)pbl_tbl->va; 630 if (!pbe) { 631 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n"); 632 return; 633 } 634 635 pbe_cnt = 0; 636 637 fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift); 638 639 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { 640 pg_addr = sg_page_iter_dma_address(&sg_iter); 641 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) { 642 pbe->lo = cpu_to_le32(pg_addr); 643 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); 644 645 pg_addr += BIT(pg_shift); 646 pbe_cnt++; 647 total_num_pbes++; 648 pbe++; 649 650 if (total_num_pbes == pbl_info->num_pbes) 651 return; 652 653 /* If the given pbl is full storing the pbes, 654 * move to next pbl. 655 */ 656 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { 657 pbl_tbl++; 658 pbe = (struct regpair *)pbl_tbl->va; 659 pbe_cnt = 0; 660 } 661 662 fw_pg_cnt++; 663 } 664 } 665 } 666 667 static int qedr_copy_cq_uresp(struct qedr_dev *dev, 668 struct qedr_cq *cq, struct ib_udata *udata) 669 { 670 struct qedr_create_cq_uresp uresp; 671 int rc; 672 673 memset(&uresp, 0, sizeof(uresp)); 674 675 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 676 uresp.icid = cq->icid; 677 678 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 679 if (rc) 680 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); 681 682 return rc; 683 } 684 685 static void consume_cqe(struct qedr_cq *cq) 686 { 687 if (cq->latest_cqe == cq->toggle_cqe) 688 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 689 690 cq->latest_cqe = qed_chain_consume(&cq->pbl); 691 } 692 693 static inline int qedr_align_cq_entries(int entries) 694 { 695 u64 size, aligned_size; 696 697 /* We allocate an extra entry that we don't report to the FW. */ 698 size = (entries + 1) * QEDR_CQE_SIZE; 699 aligned_size = ALIGN(size, PAGE_SIZE); 700 701 return aligned_size / QEDR_CQE_SIZE; 702 } 703 704 static inline int qedr_init_user_queue(struct ib_udata *udata, 705 struct qedr_dev *dev, 706 struct qedr_userq *q, u64 buf_addr, 707 size_t buf_len, int access, int dmasync, 708 int alloc_and_init) 709 { 710 u32 fw_pages; 711 int rc; 712 713 q->buf_addr = buf_addr; 714 q->buf_len = buf_len; 715 q->umem = ib_umem_get(udata, q->buf_addr, q->buf_len, access, dmasync); 716 if (IS_ERR(q->umem)) { 717 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", 718 PTR_ERR(q->umem)); 719 return PTR_ERR(q->umem); 720 } 721 722 fw_pages = ib_umem_page_count(q->umem) << 723 (PAGE_SHIFT - FW_PAGE_SHIFT); 724 725 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); 726 if (rc) 727 goto err0; 728 729 if (alloc_and_init) { 730 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 731 if (IS_ERR(q->pbl_tbl)) { 732 rc = PTR_ERR(q->pbl_tbl); 733 goto err0; 734 } 735 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info, 736 FW_PAGE_SHIFT); 737 } else { 738 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); 739 if (!q->pbl_tbl) { 740 rc = -ENOMEM; 741 goto err0; 742 } 743 } 744 745 return 0; 746 747 err0: 748 ib_umem_release(q->umem); 749 q->umem = NULL; 750 751 return rc; 752 } 753 754 static inline void qedr_init_cq_params(struct qedr_cq *cq, 755 struct qedr_ucontext *ctx, 756 struct qedr_dev *dev, int vector, 757 int chain_entries, int page_cnt, 758 u64 pbl_ptr, 759 struct qed_rdma_create_cq_in_params 760 *params) 761 { 762 memset(params, 0, sizeof(*params)); 763 params->cq_handle_hi = upper_32_bits((uintptr_t)cq); 764 params->cq_handle_lo = lower_32_bits((uintptr_t)cq); 765 params->cnq_id = vector; 766 params->cq_size = chain_entries - 1; 767 params->dpi = (ctx) ? ctx->dpi : dev->dpi; 768 params->pbl_num_pages = page_cnt; 769 params->pbl_ptr = pbl_ptr; 770 params->pbl_two_level = 0; 771 } 772 773 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags) 774 { 775 cq->db.data.agg_flags = flags; 776 cq->db.data.value = cpu_to_le32(cons); 777 writeq(cq->db.raw, cq->db_addr); 778 } 779 780 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 781 { 782 struct qedr_cq *cq = get_qedr_cq(ibcq); 783 unsigned long sflags; 784 struct qedr_dev *dev; 785 786 dev = get_qedr_dev(ibcq->device); 787 788 if (cq->destroyed) { 789 DP_ERR(dev, 790 "warning: arm was invoked after destroy for cq %p (icid=%d)\n", 791 cq, cq->icid); 792 return -EINVAL; 793 } 794 795 796 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 797 return 0; 798 799 spin_lock_irqsave(&cq->cq_lock, sflags); 800 801 cq->arm_flags = 0; 802 803 if (flags & IB_CQ_SOLICITED) 804 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 805 806 if (flags & IB_CQ_NEXT_COMP) 807 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 808 809 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 810 811 spin_unlock_irqrestore(&cq->cq_lock, sflags); 812 813 return 0; 814 } 815 816 struct ib_cq *qedr_create_cq(struct ib_device *ibdev, 817 const struct ib_cq_init_attr *attr, 818 struct ib_udata *udata) 819 { 820 struct qedr_ucontext *ctx = rdma_udata_to_drv_context( 821 udata, struct qedr_ucontext, ibucontext); 822 struct qed_rdma_destroy_cq_out_params destroy_oparams; 823 struct qed_rdma_destroy_cq_in_params destroy_iparams; 824 struct qedr_dev *dev = get_qedr_dev(ibdev); 825 struct qed_rdma_create_cq_in_params params; 826 struct qedr_create_cq_ureq ureq; 827 int vector = attr->comp_vector; 828 int entries = attr->cqe; 829 struct qedr_cq *cq; 830 int chain_entries; 831 int page_cnt; 832 u64 pbl_ptr; 833 u16 icid; 834 int rc; 835 836 DP_DEBUG(dev, QEDR_MSG_INIT, 837 "create_cq: called from %s. entries=%d, vector=%d\n", 838 udata ? "User Lib" : "Kernel", entries, vector); 839 840 if (entries > QEDR_MAX_CQES) { 841 DP_ERR(dev, 842 "create cq: the number of entries %d is too high. Must be equal or below %d.\n", 843 entries, QEDR_MAX_CQES); 844 return ERR_PTR(-EINVAL); 845 } 846 847 chain_entries = qedr_align_cq_entries(entries); 848 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); 849 850 cq = kzalloc(sizeof(*cq), GFP_KERNEL); 851 if (!cq) 852 return ERR_PTR(-ENOMEM); 853 854 if (udata) { 855 memset(&ureq, 0, sizeof(ureq)); 856 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { 857 DP_ERR(dev, 858 "create cq: problem copying data from user space\n"); 859 goto err0; 860 } 861 862 if (!ureq.len) { 863 DP_ERR(dev, 864 "create cq: cannot create a cq with 0 entries\n"); 865 goto err0; 866 } 867 868 cq->cq_type = QEDR_CQ_TYPE_USER; 869 870 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr, 871 ureq.len, IB_ACCESS_LOCAL_WRITE, 1, 872 1); 873 if (rc) 874 goto err0; 875 876 pbl_ptr = cq->q.pbl_tbl->pa; 877 page_cnt = cq->q.pbl_info.num_pbes; 878 879 cq->ibcq.cqe = chain_entries; 880 } else { 881 cq->cq_type = QEDR_CQ_TYPE_KERNEL; 882 883 rc = dev->ops->common->chain_alloc(dev->cdev, 884 QED_CHAIN_USE_TO_CONSUME, 885 QED_CHAIN_MODE_PBL, 886 QED_CHAIN_CNT_TYPE_U32, 887 chain_entries, 888 sizeof(union rdma_cqe), 889 &cq->pbl, NULL); 890 if (rc) 891 goto err1; 892 893 page_cnt = qed_chain_get_page_cnt(&cq->pbl); 894 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); 895 cq->ibcq.cqe = cq->pbl.capacity; 896 } 897 898 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, 899 pbl_ptr, ¶ms); 900 901 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 902 if (rc) 903 goto err2; 904 905 cq->icid = icid; 906 cq->sig = QEDR_CQ_MAGIC_NUMBER; 907 spin_lock_init(&cq->cq_lock); 908 909 if (udata) { 910 rc = qedr_copy_cq_uresp(dev, cq, udata); 911 if (rc) 912 goto err3; 913 } else { 914 /* Generate doorbell address. */ 915 cq->db_addr = dev->db_addr + 916 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 917 cq->db.data.icid = cq->icid; 918 cq->db.data.params = DB_AGG_CMD_SET << 919 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 920 921 /* point to the very last element, passing it we will toggle */ 922 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); 923 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 924 cq->latest_cqe = NULL; 925 consume_cqe(cq); 926 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 927 } 928 929 DP_DEBUG(dev, QEDR_MSG_CQ, 930 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", 931 cq->icid, cq, params.cq_size); 932 933 return &cq->ibcq; 934 935 err3: 936 destroy_iparams.icid = cq->icid; 937 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, 938 &destroy_oparams); 939 err2: 940 if (udata) 941 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 942 else 943 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 944 err1: 945 if (udata) 946 ib_umem_release(cq->q.umem); 947 err0: 948 kfree(cq); 949 return ERR_PTR(-EINVAL); 950 } 951 952 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 953 { 954 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 955 struct qedr_cq *cq = get_qedr_cq(ibcq); 956 957 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq); 958 959 return 0; 960 } 961 962 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) 963 #define QEDR_DESTROY_CQ_ITER_DURATION (10) 964 965 int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 966 { 967 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 968 struct qed_rdma_destroy_cq_out_params oparams; 969 struct qed_rdma_destroy_cq_in_params iparams; 970 struct qedr_cq *cq = get_qedr_cq(ibcq); 971 int iter; 972 int rc; 973 974 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); 975 976 cq->destroyed = 1; 977 978 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 979 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 980 goto done; 981 982 iparams.icid = cq->icid; 983 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 984 if (rc) 985 return rc; 986 987 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 988 989 if (udata) { 990 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 991 ib_umem_release(cq->q.umem); 992 } 993 994 /* We don't want the IRQ handler to handle a non-existing CQ so we 995 * wait until all CNQ interrupts, if any, are received. This will always 996 * happen and will always happen very fast. If not, then a serious error 997 * has occured. That is why we can use a long delay. 998 * We spin for a short time so we don’t lose time on context switching 999 * in case all the completions are handled in that span. Otherwise 1000 * we sleep for a while and check again. Since the CNQ may be 1001 * associated with (only) the current CPU we use msleep to allow the 1002 * current CPU to be freed. 1003 * The CNQ notification is increased in qedr_irq_handler(). 1004 */ 1005 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1006 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1007 udelay(QEDR_DESTROY_CQ_ITER_DURATION); 1008 iter--; 1009 } 1010 1011 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1012 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1013 msleep(QEDR_DESTROY_CQ_ITER_DURATION); 1014 iter--; 1015 } 1016 1017 if (oparams.num_cq_notif != cq->cnq_notif) 1018 goto err; 1019 1020 /* Note that we don't need to have explicit code to wait for the 1021 * completion of the event handler because it is invoked from the EQ. 1022 * Since the destroy CQ ramrod has also been received on the EQ we can 1023 * be certain that there's no event handler in process. 1024 */ 1025 done: 1026 cq->sig = ~cq->sig; 1027 1028 kfree(cq); 1029 1030 return 0; 1031 1032 err: 1033 DP_ERR(dev, 1034 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n", 1035 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif); 1036 1037 return -EINVAL; 1038 } 1039 1040 static inline int get_gid_info_from_table(struct ib_qp *ibqp, 1041 struct ib_qp_attr *attr, 1042 int attr_mask, 1043 struct qed_rdma_modify_qp_in_params 1044 *qp_params) 1045 { 1046 const struct ib_gid_attr *gid_attr; 1047 enum rdma_network_type nw_type; 1048 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1049 u32 ipv4_addr; 1050 int ret; 1051 int i; 1052 1053 gid_attr = grh->sgid_attr; 1054 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL); 1055 if (ret) 1056 return ret; 1057 1058 nw_type = rdma_gid_attr_network_type(gid_attr); 1059 switch (nw_type) { 1060 case RDMA_NETWORK_IPV6: 1061 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1062 sizeof(qp_params->sgid)); 1063 memcpy(&qp_params->dgid.bytes[0], 1064 &grh->dgid, 1065 sizeof(qp_params->dgid)); 1066 qp_params->roce_mode = ROCE_V2_IPV6; 1067 SET_FIELD(qp_params->modify_flags, 1068 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1069 break; 1070 case RDMA_NETWORK_IB: 1071 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1072 sizeof(qp_params->sgid)); 1073 memcpy(&qp_params->dgid.bytes[0], 1074 &grh->dgid, 1075 sizeof(qp_params->dgid)); 1076 qp_params->roce_mode = ROCE_V1; 1077 break; 1078 case RDMA_NETWORK_IPV4: 1079 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid)); 1080 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid)); 1081 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw); 1082 qp_params->sgid.ipv4_addr = ipv4_addr; 1083 ipv4_addr = 1084 qedr_get_ipv4_from_gid(grh->dgid.raw); 1085 qp_params->dgid.ipv4_addr = ipv4_addr; 1086 SET_FIELD(qp_params->modify_flags, 1087 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1088 qp_params->roce_mode = ROCE_V2_IPV4; 1089 break; 1090 } 1091 1092 for (i = 0; i < 4; i++) { 1093 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 1094 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 1095 } 1096 1097 if (qp_params->vlan_id >= VLAN_CFI_MASK) 1098 qp_params->vlan_id = 0; 1099 1100 return 0; 1101 } 1102 1103 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, 1104 struct ib_qp_init_attr *attrs, 1105 struct ib_udata *udata) 1106 { 1107 struct qedr_device_attr *qattr = &dev->attr; 1108 1109 /* QP0... attrs->qp_type == IB_QPT_GSI */ 1110 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { 1111 DP_DEBUG(dev, QEDR_MSG_QP, 1112 "create qp: unsupported qp type=0x%x requested\n", 1113 attrs->qp_type); 1114 return -EINVAL; 1115 } 1116 1117 if (attrs->cap.max_send_wr > qattr->max_sqe) { 1118 DP_ERR(dev, 1119 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n", 1120 attrs->cap.max_send_wr, qattr->max_sqe); 1121 return -EINVAL; 1122 } 1123 1124 if (attrs->cap.max_inline_data > qattr->max_inline) { 1125 DP_ERR(dev, 1126 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n", 1127 attrs->cap.max_inline_data, qattr->max_inline); 1128 return -EINVAL; 1129 } 1130 1131 if (attrs->cap.max_send_sge > qattr->max_sge) { 1132 DP_ERR(dev, 1133 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n", 1134 attrs->cap.max_send_sge, qattr->max_sge); 1135 return -EINVAL; 1136 } 1137 1138 if (attrs->cap.max_recv_sge > qattr->max_sge) { 1139 DP_ERR(dev, 1140 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n", 1141 attrs->cap.max_recv_sge, qattr->max_sge); 1142 return -EINVAL; 1143 } 1144 1145 /* Unprivileged user space cannot create special QP */ 1146 if (udata && attrs->qp_type == IB_QPT_GSI) { 1147 DP_ERR(dev, 1148 "create qp: userspace can't create special QPs of type=0x%x\n", 1149 attrs->qp_type); 1150 return -EINVAL; 1151 } 1152 1153 return 0; 1154 } 1155 1156 static int qedr_copy_srq_uresp(struct qedr_dev *dev, 1157 struct qedr_srq *srq, struct ib_udata *udata) 1158 { 1159 struct qedr_create_srq_uresp uresp = {}; 1160 int rc; 1161 1162 uresp.srq_id = srq->srq_id; 1163 1164 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1165 if (rc) 1166 DP_ERR(dev, "create srq: problem copying data to user space\n"); 1167 1168 return rc; 1169 } 1170 1171 static void qedr_copy_rq_uresp(struct qedr_dev *dev, 1172 struct qedr_create_qp_uresp *uresp, 1173 struct qedr_qp *qp) 1174 { 1175 /* iWARP requires two doorbells per RQ. */ 1176 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1177 uresp->rq_db_offset = 1178 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1179 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1180 } else { 1181 uresp->rq_db_offset = 1182 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1183 } 1184 1185 uresp->rq_icid = qp->icid; 1186 } 1187 1188 static void qedr_copy_sq_uresp(struct qedr_dev *dev, 1189 struct qedr_create_qp_uresp *uresp, 1190 struct qedr_qp *qp) 1191 { 1192 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1193 1194 /* iWARP uses the same cid for rq and sq */ 1195 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1196 uresp->sq_icid = qp->icid; 1197 else 1198 uresp->sq_icid = qp->icid + 1; 1199 } 1200 1201 static int qedr_copy_qp_uresp(struct qedr_dev *dev, 1202 struct qedr_qp *qp, struct ib_udata *udata) 1203 { 1204 struct qedr_create_qp_uresp uresp; 1205 int rc; 1206 1207 memset(&uresp, 0, sizeof(uresp)); 1208 qedr_copy_sq_uresp(dev, &uresp, qp); 1209 qedr_copy_rq_uresp(dev, &uresp, qp); 1210 1211 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 1212 uresp.qp_id = qp->qp_id; 1213 1214 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1215 if (rc) 1216 DP_ERR(dev, 1217 "create qp: failed a copy to user space with qp icid=0x%x.\n", 1218 qp->icid); 1219 1220 return rc; 1221 } 1222 1223 static void qedr_set_common_qp_params(struct qedr_dev *dev, 1224 struct qedr_qp *qp, 1225 struct qedr_pd *pd, 1226 struct ib_qp_init_attr *attrs) 1227 { 1228 spin_lock_init(&qp->q_lock); 1229 atomic_set(&qp->refcnt, 1); 1230 qp->pd = pd; 1231 qp->qp_type = attrs->qp_type; 1232 qp->max_inline_data = attrs->cap.max_inline_data; 1233 qp->sq.max_sges = attrs->cap.max_send_sge; 1234 qp->state = QED_ROCE_QP_STATE_RESET; 1235 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1236 qp->sq_cq = get_qedr_cq(attrs->send_cq); 1237 qp->dev = dev; 1238 1239 if (attrs->srq) { 1240 qp->srq = get_qedr_srq(attrs->srq); 1241 } else { 1242 qp->rq_cq = get_qedr_cq(attrs->recv_cq); 1243 qp->rq.max_sges = attrs->cap.max_recv_sge; 1244 DP_DEBUG(dev, QEDR_MSG_QP, 1245 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 1246 qp->rq.max_sges, qp->rq_cq->icid); 1247 } 1248 1249 DP_DEBUG(dev, QEDR_MSG_QP, 1250 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n", 1251 pd->pd_id, qp->qp_type, qp->max_inline_data, 1252 qp->state, qp->signaled, (attrs->srq) ? 1 : 0); 1253 DP_DEBUG(dev, QEDR_MSG_QP, 1254 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1255 qp->sq.max_sges, qp->sq_cq->icid); 1256 } 1257 1258 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1259 { 1260 qp->sq.db = dev->db_addr + 1261 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1262 qp->sq.db_data.data.icid = qp->icid + 1; 1263 if (!qp->srq) { 1264 qp->rq.db = dev->db_addr + 1265 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1266 qp->rq.db_data.data.icid = qp->icid; 1267 } 1268 } 1269 1270 static int qedr_check_srq_params(struct qedr_dev *dev, 1271 struct ib_srq_init_attr *attrs, 1272 struct ib_udata *udata) 1273 { 1274 struct qedr_device_attr *qattr = &dev->attr; 1275 1276 if (attrs->attr.max_wr > qattr->max_srq_wr) { 1277 DP_ERR(dev, 1278 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n", 1279 attrs->attr.max_wr, qattr->max_srq_wr); 1280 return -EINVAL; 1281 } 1282 1283 if (attrs->attr.max_sge > qattr->max_sge) { 1284 DP_ERR(dev, 1285 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", 1286 attrs->attr.max_sge, qattr->max_sge); 1287 return -EINVAL; 1288 } 1289 1290 return 0; 1291 } 1292 1293 static void qedr_free_srq_user_params(struct qedr_srq *srq) 1294 { 1295 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1296 ib_umem_release(srq->usrq.umem); 1297 ib_umem_release(srq->prod_umem); 1298 } 1299 1300 static void qedr_free_srq_kernel_params(struct qedr_srq *srq) 1301 { 1302 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1303 struct qedr_dev *dev = srq->dev; 1304 1305 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl); 1306 1307 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1308 hw_srq->virt_prod_pair_addr, 1309 hw_srq->phy_prod_pair_addr); 1310 } 1311 1312 static int qedr_init_srq_user_params(struct ib_udata *udata, 1313 struct qedr_srq *srq, 1314 struct qedr_create_srq_ureq *ureq, 1315 int access, int dmasync) 1316 { 1317 struct scatterlist *sg; 1318 int rc; 1319 1320 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr, 1321 ureq->srq_len, access, dmasync, 1); 1322 if (rc) 1323 return rc; 1324 1325 srq->prod_umem = 1326 ib_umem_get(udata, ureq->prod_pair_addr, 1327 sizeof(struct rdma_srq_producers), access, dmasync); 1328 if (IS_ERR(srq->prod_umem)) { 1329 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1330 ib_umem_release(srq->usrq.umem); 1331 DP_ERR(srq->dev, 1332 "create srq: failed ib_umem_get for producer, got %ld\n", 1333 PTR_ERR(srq->prod_umem)); 1334 return PTR_ERR(srq->prod_umem); 1335 } 1336 1337 sg = srq->prod_umem->sg_head.sgl; 1338 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); 1339 1340 return 0; 1341 } 1342 1343 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq, 1344 struct qedr_dev *dev, 1345 struct ib_srq_init_attr *init_attr) 1346 { 1347 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1348 dma_addr_t phy_prod_pair_addr; 1349 u32 num_elems; 1350 void *va; 1351 int rc; 1352 1353 va = dma_alloc_coherent(&dev->pdev->dev, 1354 sizeof(struct rdma_srq_producers), 1355 &phy_prod_pair_addr, GFP_KERNEL); 1356 if (!va) { 1357 DP_ERR(dev, 1358 "create srq: failed to allocate dma memory for producer\n"); 1359 return -ENOMEM; 1360 } 1361 1362 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; 1363 hw_srq->virt_prod_pair_addr = va; 1364 1365 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE; 1366 rc = dev->ops->common->chain_alloc(dev->cdev, 1367 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1368 QED_CHAIN_MODE_PBL, 1369 QED_CHAIN_CNT_TYPE_U32, 1370 num_elems, 1371 QEDR_SRQ_WQE_ELEM_SIZE, 1372 &hw_srq->pbl, NULL); 1373 if (rc) 1374 goto err0; 1375 1376 hw_srq->num_elems = num_elems; 1377 1378 return 0; 1379 1380 err0: 1381 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1382 va, phy_prod_pair_addr); 1383 return rc; 1384 } 1385 1386 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, 1387 struct ib_udata *udata) 1388 { 1389 struct qed_rdma_destroy_srq_in_params destroy_in_params; 1390 struct qed_rdma_create_srq_in_params in_params = {}; 1391 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1392 struct qed_rdma_create_srq_out_params out_params; 1393 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd); 1394 struct qedr_create_srq_ureq ureq = {}; 1395 u64 pbl_base_addr, phy_prod_pair_addr; 1396 struct qedr_srq_hwq_info *hw_srq; 1397 u32 page_cnt, page_size; 1398 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1399 int rc = 0; 1400 1401 DP_DEBUG(dev, QEDR_MSG_QP, 1402 "create SRQ called from %s (pd %p)\n", 1403 (udata) ? "User lib" : "kernel", pd); 1404 1405 rc = qedr_check_srq_params(dev, init_attr, udata); 1406 if (rc) 1407 return -EINVAL; 1408 1409 srq->dev = dev; 1410 hw_srq = &srq->hw_srq; 1411 spin_lock_init(&srq->lock); 1412 1413 hw_srq->max_wr = init_attr->attr.max_wr; 1414 hw_srq->max_sges = init_attr->attr.max_sge; 1415 1416 if (udata) { 1417 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { 1418 DP_ERR(dev, 1419 "create srq: problem copying data from user space\n"); 1420 goto err0; 1421 } 1422 1423 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0, 0); 1424 if (rc) 1425 goto err0; 1426 1427 page_cnt = srq->usrq.pbl_info.num_pbes; 1428 pbl_base_addr = srq->usrq.pbl_tbl->pa; 1429 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1430 page_size = PAGE_SIZE; 1431 } else { 1432 struct qed_chain *pbl; 1433 1434 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr); 1435 if (rc) 1436 goto err0; 1437 1438 pbl = &hw_srq->pbl; 1439 page_cnt = qed_chain_get_page_cnt(pbl); 1440 pbl_base_addr = qed_chain_get_pbl_phys(pbl); 1441 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1442 page_size = QED_CHAIN_PAGE_SIZE; 1443 } 1444 1445 in_params.pd_id = pd->pd_id; 1446 in_params.pbl_base_addr = pbl_base_addr; 1447 in_params.prod_pair_addr = phy_prod_pair_addr; 1448 in_params.num_pages = page_cnt; 1449 in_params.page_size = page_size; 1450 1451 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); 1452 if (rc) 1453 goto err1; 1454 1455 srq->srq_id = out_params.srq_id; 1456 1457 if (udata) { 1458 rc = qedr_copy_srq_uresp(dev, srq, udata); 1459 if (rc) 1460 goto err2; 1461 } 1462 1463 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL); 1464 if (rc) 1465 goto err2; 1466 1467 DP_DEBUG(dev, QEDR_MSG_SRQ, 1468 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id); 1469 return 0; 1470 1471 err2: 1472 destroy_in_params.srq_id = srq->srq_id; 1473 1474 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); 1475 err1: 1476 if (udata) 1477 qedr_free_srq_user_params(srq); 1478 else 1479 qedr_free_srq_kernel_params(srq); 1480 err0: 1481 return -EFAULT; 1482 } 1483 1484 void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 1485 { 1486 struct qed_rdma_destroy_srq_in_params in_params = {}; 1487 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1488 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1489 1490 xa_erase_irq(&dev->srqs, srq->srq_id); 1491 in_params.srq_id = srq->srq_id; 1492 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params); 1493 1494 if (ibsrq->uobject) 1495 qedr_free_srq_user_params(srq); 1496 else 1497 qedr_free_srq_kernel_params(srq); 1498 1499 DP_DEBUG(dev, QEDR_MSG_SRQ, 1500 "destroy srq: destroyed srq with srq_id=0x%0x\n", 1501 srq->srq_id); 1502 } 1503 1504 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 1505 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 1506 { 1507 struct qed_rdma_modify_srq_in_params in_params = {}; 1508 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1509 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1510 int rc; 1511 1512 if (attr_mask & IB_SRQ_MAX_WR) { 1513 DP_ERR(dev, 1514 "modify srq: invalid attribute mask=0x%x specified for %p\n", 1515 attr_mask, srq); 1516 return -EINVAL; 1517 } 1518 1519 if (attr_mask & IB_SRQ_LIMIT) { 1520 if (attr->srq_limit >= srq->hw_srq.max_wr) { 1521 DP_ERR(dev, 1522 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n", 1523 attr->srq_limit, srq->hw_srq.max_wr); 1524 return -EINVAL; 1525 } 1526 1527 in_params.srq_id = srq->srq_id; 1528 in_params.wqe_limit = attr->srq_limit; 1529 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params); 1530 if (rc) 1531 return rc; 1532 } 1533 1534 srq->srq_limit = attr->srq_limit; 1535 1536 DP_DEBUG(dev, QEDR_MSG_SRQ, 1537 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id); 1538 1539 return 0; 1540 } 1541 1542 static inline void 1543 qedr_init_common_qp_in_params(struct qedr_dev *dev, 1544 struct qedr_pd *pd, 1545 struct qedr_qp *qp, 1546 struct ib_qp_init_attr *attrs, 1547 bool fmr_and_reserved_lkey, 1548 struct qed_rdma_create_qp_in_params *params) 1549 { 1550 /* QP handle to be written in an async event */ 1551 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp); 1552 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp); 1553 1554 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 1555 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 1556 params->pd = pd->pd_id; 1557 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 1558 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; 1559 params->stats_queue = 0; 1560 params->srq_id = 0; 1561 params->use_srq = false; 1562 1563 if (!qp->srq) { 1564 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1565 1566 } else { 1567 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1568 params->srq_id = qp->srq->srq_id; 1569 params->use_srq = true; 1570 } 1571 } 1572 1573 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) 1574 { 1575 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. " 1576 "qp=%p. " 1577 "sq_addr=0x%llx, " 1578 "sq_len=%zd, " 1579 "rq_addr=0x%llx, " 1580 "rq_len=%zd" 1581 "\n", 1582 qp, 1583 qp->usq.buf_addr, 1584 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len); 1585 } 1586 1587 static inline void 1588 qedr_iwarp_populate_user_qp(struct qedr_dev *dev, 1589 struct qedr_qp *qp, 1590 struct qed_rdma_create_qp_out_params *out_params) 1591 { 1592 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 1593 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 1594 1595 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 1596 &qp->usq.pbl_info, FW_PAGE_SHIFT); 1597 if (!qp->srq) { 1598 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 1599 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 1600 } 1601 1602 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 1603 &qp->urq.pbl_info, FW_PAGE_SHIFT); 1604 } 1605 1606 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp) 1607 { 1608 if (qp->usq.umem) 1609 ib_umem_release(qp->usq.umem); 1610 qp->usq.umem = NULL; 1611 1612 if (qp->urq.umem) 1613 ib_umem_release(qp->urq.umem); 1614 qp->urq.umem = NULL; 1615 } 1616 1617 static int qedr_create_user_qp(struct qedr_dev *dev, 1618 struct qedr_qp *qp, 1619 struct ib_pd *ibpd, 1620 struct ib_udata *udata, 1621 struct ib_qp_init_attr *attrs) 1622 { 1623 struct qed_rdma_create_qp_in_params in_params; 1624 struct qed_rdma_create_qp_out_params out_params; 1625 struct qedr_pd *pd = get_qedr_pd(ibpd); 1626 struct qedr_create_qp_ureq ureq; 1627 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1); 1628 int rc = -EINVAL; 1629 1630 memset(&ureq, 0, sizeof(ureq)); 1631 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq)); 1632 if (rc) { 1633 DP_ERR(dev, "Problem copying data from user space\n"); 1634 return rc; 1635 } 1636 1637 /* SQ - read access only (0), dma sync not required (0) */ 1638 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr, 1639 ureq.sq_len, 0, 0, alloc_and_init); 1640 if (rc) 1641 return rc; 1642 1643 if (!qp->srq) { 1644 /* RQ - read access only (0), dma sync not required (0) */ 1645 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr, 1646 ureq.rq_len, 0, 0, alloc_and_init); 1647 if (rc) 1648 return rc; 1649 } 1650 1651 memset(&in_params, 0, sizeof(in_params)); 1652 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 1653 in_params.qp_handle_lo = ureq.qp_handle_lo; 1654 in_params.qp_handle_hi = ureq.qp_handle_hi; 1655 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 1656 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 1657 if (!qp->srq) { 1658 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 1659 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 1660 } 1661 1662 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1663 &in_params, &out_params); 1664 1665 if (!qp->qed_qp) { 1666 rc = -ENOMEM; 1667 goto err1; 1668 } 1669 1670 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1671 qedr_iwarp_populate_user_qp(dev, qp, &out_params); 1672 1673 qp->qp_id = out_params.qp_id; 1674 qp->icid = out_params.icid; 1675 1676 rc = qedr_copy_qp_uresp(dev, qp, udata); 1677 if (rc) 1678 goto err; 1679 1680 qedr_qp_user_print(dev, qp); 1681 1682 return 0; 1683 err: 1684 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1685 if (rc) 1686 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc); 1687 1688 err1: 1689 qedr_cleanup_user(dev, qp); 1690 return rc; 1691 } 1692 1693 static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1694 { 1695 qp->sq.db = dev->db_addr + 1696 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1697 qp->sq.db_data.data.icid = qp->icid; 1698 1699 qp->rq.db = dev->db_addr + 1700 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1701 qp->rq.db_data.data.icid = qp->icid; 1702 qp->rq.iwarp_db2 = dev->db_addr + 1703 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1704 qp->rq.iwarp_db2_data.data.icid = qp->icid; 1705 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 1706 } 1707 1708 static int 1709 qedr_roce_create_kernel_qp(struct qedr_dev *dev, 1710 struct qedr_qp *qp, 1711 struct qed_rdma_create_qp_in_params *in_params, 1712 u32 n_sq_elems, u32 n_rq_elems) 1713 { 1714 struct qed_rdma_create_qp_out_params out_params; 1715 int rc; 1716 1717 rc = dev->ops->common->chain_alloc(dev->cdev, 1718 QED_CHAIN_USE_TO_PRODUCE, 1719 QED_CHAIN_MODE_PBL, 1720 QED_CHAIN_CNT_TYPE_U32, 1721 n_sq_elems, 1722 QEDR_SQE_ELEMENT_SIZE, 1723 &qp->sq.pbl, NULL); 1724 1725 if (rc) 1726 return rc; 1727 1728 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl); 1729 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl); 1730 1731 rc = dev->ops->common->chain_alloc(dev->cdev, 1732 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1733 QED_CHAIN_MODE_PBL, 1734 QED_CHAIN_CNT_TYPE_U32, 1735 n_rq_elems, 1736 QEDR_RQE_ELEMENT_SIZE, 1737 &qp->rq.pbl, NULL); 1738 if (rc) 1739 return rc; 1740 1741 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl); 1742 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl); 1743 1744 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1745 in_params, &out_params); 1746 1747 if (!qp->qed_qp) 1748 return -EINVAL; 1749 1750 qp->qp_id = out_params.qp_id; 1751 qp->icid = out_params.icid; 1752 1753 qedr_set_roce_db_info(dev, qp); 1754 return rc; 1755 } 1756 1757 static int 1758 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev, 1759 struct qedr_qp *qp, 1760 struct qed_rdma_create_qp_in_params *in_params, 1761 u32 n_sq_elems, u32 n_rq_elems) 1762 { 1763 struct qed_rdma_create_qp_out_params out_params; 1764 struct qed_chain_ext_pbl ext_pbl; 1765 int rc; 1766 1767 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems, 1768 QEDR_SQE_ELEMENT_SIZE, 1769 QED_CHAIN_MODE_PBL); 1770 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems, 1771 QEDR_RQE_ELEMENT_SIZE, 1772 QED_CHAIN_MODE_PBL); 1773 1774 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1775 in_params, &out_params); 1776 1777 if (!qp->qed_qp) 1778 return -EINVAL; 1779 1780 /* Now we allocate the chain */ 1781 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt; 1782 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys; 1783 1784 rc = dev->ops->common->chain_alloc(dev->cdev, 1785 QED_CHAIN_USE_TO_PRODUCE, 1786 QED_CHAIN_MODE_PBL, 1787 QED_CHAIN_CNT_TYPE_U32, 1788 n_sq_elems, 1789 QEDR_SQE_ELEMENT_SIZE, 1790 &qp->sq.pbl, &ext_pbl); 1791 1792 if (rc) 1793 goto err; 1794 1795 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt; 1796 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys; 1797 1798 rc = dev->ops->common->chain_alloc(dev->cdev, 1799 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1800 QED_CHAIN_MODE_PBL, 1801 QED_CHAIN_CNT_TYPE_U32, 1802 n_rq_elems, 1803 QEDR_RQE_ELEMENT_SIZE, 1804 &qp->rq.pbl, &ext_pbl); 1805 1806 if (rc) 1807 goto err; 1808 1809 qp->qp_id = out_params.qp_id; 1810 qp->icid = out_params.icid; 1811 1812 qedr_set_iwarp_db_info(dev, qp); 1813 return rc; 1814 1815 err: 1816 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1817 1818 return rc; 1819 } 1820 1821 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) 1822 { 1823 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl); 1824 kfree(qp->wqe_wr_id); 1825 1826 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl); 1827 kfree(qp->rqe_wr_id); 1828 } 1829 1830 static int qedr_create_kernel_qp(struct qedr_dev *dev, 1831 struct qedr_qp *qp, 1832 struct ib_pd *ibpd, 1833 struct ib_qp_init_attr *attrs) 1834 { 1835 struct qed_rdma_create_qp_in_params in_params; 1836 struct qedr_pd *pd = get_qedr_pd(ibpd); 1837 int rc = -EINVAL; 1838 u32 n_rq_elems; 1839 u32 n_sq_elems; 1840 u32 n_sq_entries; 1841 1842 memset(&in_params, 0, sizeof(in_params)); 1843 1844 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in 1845 * the ring. The ring should allow at least a single WR, even if the 1846 * user requested none, due to allocation issues. 1847 * We should add an extra WR since the prod and cons indices of 1848 * wqe_wr_id are managed in such a way that the WQ is considered full 1849 * when (prod+1)%max_wr==cons. We currently don't do that because we 1850 * double the number of entries due an iSER issue that pushes far more 1851 * WRs than indicated. If we decline its ib_post_send() then we get 1852 * error prints in the dmesg we'd like to avoid. 1853 */ 1854 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 1855 dev->attr.max_sqe); 1856 1857 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id), 1858 GFP_KERNEL); 1859 if (!qp->wqe_wr_id) { 1860 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n"); 1861 return -ENOMEM; 1862 } 1863 1864 /* QP handle to be written in CQE */ 1865 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp); 1866 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp); 1867 1868 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in 1869 * the ring. There ring should allow at least a single WR, even if the 1870 * user requested none, due to allocation issues. 1871 */ 1872 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1); 1873 1874 /* Allocate driver internal RQ array */ 1875 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id), 1876 GFP_KERNEL); 1877 if (!qp->rqe_wr_id) { 1878 DP_ERR(dev, 1879 "create qp: failed RQ shadow memory allocation\n"); 1880 kfree(qp->wqe_wr_id); 1881 return -ENOMEM; 1882 } 1883 1884 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 1885 1886 n_sq_entries = attrs->cap.max_send_wr; 1887 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe); 1888 n_sq_entries = max_t(u32, n_sq_entries, 1); 1889 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE; 1890 1891 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE; 1892 1893 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1894 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params, 1895 n_sq_elems, n_rq_elems); 1896 else 1897 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params, 1898 n_sq_elems, n_rq_elems); 1899 if (rc) 1900 qedr_cleanup_kernel(dev, qp); 1901 1902 return rc; 1903 } 1904 1905 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, 1906 struct ib_qp_init_attr *attrs, 1907 struct ib_udata *udata) 1908 { 1909 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 1910 struct qedr_pd *pd = get_qedr_pd(ibpd); 1911 struct qedr_qp *qp; 1912 struct ib_qp *ibqp; 1913 int rc = 0; 1914 1915 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", 1916 udata ? "user library" : "kernel", pd); 1917 1918 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata); 1919 if (rc) 1920 return ERR_PTR(rc); 1921 1922 DP_DEBUG(dev, QEDR_MSG_QP, 1923 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 1924 udata ? "user library" : "kernel", attrs->event_handler, pd, 1925 get_qedr_cq(attrs->send_cq), 1926 get_qedr_cq(attrs->send_cq)->icid, 1927 get_qedr_cq(attrs->recv_cq), 1928 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0); 1929 1930 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1931 if (!qp) { 1932 DP_ERR(dev, "create qp: failed allocating memory\n"); 1933 return ERR_PTR(-ENOMEM); 1934 } 1935 1936 qedr_set_common_qp_params(dev, qp, pd, attrs); 1937 1938 if (attrs->qp_type == IB_QPT_GSI) { 1939 ibqp = qedr_create_gsi_qp(dev, attrs, qp); 1940 if (IS_ERR(ibqp)) 1941 kfree(qp); 1942 return ibqp; 1943 } 1944 1945 if (udata) 1946 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); 1947 else 1948 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); 1949 1950 if (rc) 1951 goto err; 1952 1953 qp->ibqp.qp_num = qp->qp_id; 1954 1955 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1956 rc = xa_insert_irq(&dev->qps, qp->qp_id, qp, GFP_KERNEL); 1957 if (rc) 1958 goto err; 1959 } 1960 1961 return &qp->ibqp; 1962 1963 err: 1964 kfree(qp); 1965 1966 return ERR_PTR(-EFAULT); 1967 } 1968 1969 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) 1970 { 1971 switch (qp_state) { 1972 case QED_ROCE_QP_STATE_RESET: 1973 return IB_QPS_RESET; 1974 case QED_ROCE_QP_STATE_INIT: 1975 return IB_QPS_INIT; 1976 case QED_ROCE_QP_STATE_RTR: 1977 return IB_QPS_RTR; 1978 case QED_ROCE_QP_STATE_RTS: 1979 return IB_QPS_RTS; 1980 case QED_ROCE_QP_STATE_SQD: 1981 return IB_QPS_SQD; 1982 case QED_ROCE_QP_STATE_ERR: 1983 return IB_QPS_ERR; 1984 case QED_ROCE_QP_STATE_SQE: 1985 return IB_QPS_SQE; 1986 } 1987 return IB_QPS_ERR; 1988 } 1989 1990 static enum qed_roce_qp_state qedr_get_state_from_ibqp( 1991 enum ib_qp_state qp_state) 1992 { 1993 switch (qp_state) { 1994 case IB_QPS_RESET: 1995 return QED_ROCE_QP_STATE_RESET; 1996 case IB_QPS_INIT: 1997 return QED_ROCE_QP_STATE_INIT; 1998 case IB_QPS_RTR: 1999 return QED_ROCE_QP_STATE_RTR; 2000 case IB_QPS_RTS: 2001 return QED_ROCE_QP_STATE_RTS; 2002 case IB_QPS_SQD: 2003 return QED_ROCE_QP_STATE_SQD; 2004 case IB_QPS_ERR: 2005 return QED_ROCE_QP_STATE_ERR; 2006 default: 2007 return QED_ROCE_QP_STATE_ERR; 2008 } 2009 } 2010 2011 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) 2012 { 2013 qed_chain_reset(&qph->pbl); 2014 qph->prod = 0; 2015 qph->cons = 0; 2016 qph->wqe_cons = 0; 2017 qph->db_data.data.value = cpu_to_le16(0); 2018 } 2019 2020 static int qedr_update_qp_state(struct qedr_dev *dev, 2021 struct qedr_qp *qp, 2022 enum qed_roce_qp_state cur_state, 2023 enum qed_roce_qp_state new_state) 2024 { 2025 int status = 0; 2026 2027 if (new_state == cur_state) 2028 return 0; 2029 2030 switch (cur_state) { 2031 case QED_ROCE_QP_STATE_RESET: 2032 switch (new_state) { 2033 case QED_ROCE_QP_STATE_INIT: 2034 qp->prev_wqe_size = 0; 2035 qedr_reset_qp_hwq_info(&qp->sq); 2036 qedr_reset_qp_hwq_info(&qp->rq); 2037 break; 2038 default: 2039 status = -EINVAL; 2040 break; 2041 } 2042 break; 2043 case QED_ROCE_QP_STATE_INIT: 2044 switch (new_state) { 2045 case QED_ROCE_QP_STATE_RTR: 2046 /* Update doorbell (in case post_recv was 2047 * done before move to RTR) 2048 */ 2049 2050 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2051 writel(qp->rq.db_data.raw, qp->rq.db); 2052 } 2053 break; 2054 case QED_ROCE_QP_STATE_ERR: 2055 break; 2056 default: 2057 /* Invalid state change. */ 2058 status = -EINVAL; 2059 break; 2060 } 2061 break; 2062 case QED_ROCE_QP_STATE_RTR: 2063 /* RTR->XXX */ 2064 switch (new_state) { 2065 case QED_ROCE_QP_STATE_RTS: 2066 break; 2067 case QED_ROCE_QP_STATE_ERR: 2068 break; 2069 default: 2070 /* Invalid state change. */ 2071 status = -EINVAL; 2072 break; 2073 } 2074 break; 2075 case QED_ROCE_QP_STATE_RTS: 2076 /* RTS->XXX */ 2077 switch (new_state) { 2078 case QED_ROCE_QP_STATE_SQD: 2079 break; 2080 case QED_ROCE_QP_STATE_ERR: 2081 break; 2082 default: 2083 /* Invalid state change. */ 2084 status = -EINVAL; 2085 break; 2086 } 2087 break; 2088 case QED_ROCE_QP_STATE_SQD: 2089 /* SQD->XXX */ 2090 switch (new_state) { 2091 case QED_ROCE_QP_STATE_RTS: 2092 case QED_ROCE_QP_STATE_ERR: 2093 break; 2094 default: 2095 /* Invalid state change. */ 2096 status = -EINVAL; 2097 break; 2098 } 2099 break; 2100 case QED_ROCE_QP_STATE_ERR: 2101 /* ERR->XXX */ 2102 switch (new_state) { 2103 case QED_ROCE_QP_STATE_RESET: 2104 if ((qp->rq.prod != qp->rq.cons) || 2105 (qp->sq.prod != qp->sq.cons)) { 2106 DP_NOTICE(dev, 2107 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n", 2108 qp->rq.prod, qp->rq.cons, qp->sq.prod, 2109 qp->sq.cons); 2110 status = -EINVAL; 2111 } 2112 break; 2113 default: 2114 status = -EINVAL; 2115 break; 2116 } 2117 break; 2118 default: 2119 status = -EINVAL; 2120 break; 2121 } 2122 2123 return status; 2124 } 2125 2126 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2127 int attr_mask, struct ib_udata *udata) 2128 { 2129 struct qedr_qp *qp = get_qedr_qp(ibqp); 2130 struct qed_rdma_modify_qp_in_params qp_params = { 0 }; 2131 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); 2132 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 2133 enum ib_qp_state old_qp_state, new_qp_state; 2134 enum qed_roce_qp_state cur_state; 2135 int rc = 0; 2136 2137 DP_DEBUG(dev, QEDR_MSG_QP, 2138 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask, 2139 attr->qp_state); 2140 2141 old_qp_state = qedr_get_ibqp_state(qp->state); 2142 if (attr_mask & IB_QP_STATE) 2143 new_qp_state = attr->qp_state; 2144 else 2145 new_qp_state = old_qp_state; 2146 2147 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2148 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state, 2149 ibqp->qp_type, attr_mask)) { 2150 DP_ERR(dev, 2151 "modify qp: invalid attribute mask=0x%x specified for\n" 2152 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n", 2153 attr_mask, qp->qp_id, ibqp->qp_type, 2154 old_qp_state, new_qp_state); 2155 rc = -EINVAL; 2156 goto err; 2157 } 2158 } 2159 2160 /* Translate the masks... */ 2161 if (attr_mask & IB_QP_STATE) { 2162 SET_FIELD(qp_params.modify_flags, 2163 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 2164 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state); 2165 } 2166 2167 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 2168 qp_params.sqd_async = true; 2169 2170 if (attr_mask & IB_QP_PKEY_INDEX) { 2171 SET_FIELD(qp_params.modify_flags, 2172 QED_ROCE_MODIFY_QP_VALID_PKEY, 1); 2173 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) { 2174 rc = -EINVAL; 2175 goto err; 2176 } 2177 2178 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT; 2179 } 2180 2181 if (attr_mask & IB_QP_QKEY) 2182 qp->qkey = attr->qkey; 2183 2184 if (attr_mask & IB_QP_ACCESS_FLAGS) { 2185 SET_FIELD(qp_params.modify_flags, 2186 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 2187 qp_params.incoming_rdma_read_en = attr->qp_access_flags & 2188 IB_ACCESS_REMOTE_READ; 2189 qp_params.incoming_rdma_write_en = attr->qp_access_flags & 2190 IB_ACCESS_REMOTE_WRITE; 2191 qp_params.incoming_atomic_en = attr->qp_access_flags & 2192 IB_ACCESS_REMOTE_ATOMIC; 2193 } 2194 2195 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 2196 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2197 return -EINVAL; 2198 2199 if (attr_mask & IB_QP_PATH_MTU) { 2200 if (attr->path_mtu < IB_MTU_256 || 2201 attr->path_mtu > IB_MTU_4096) { 2202 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n"); 2203 rc = -EINVAL; 2204 goto err; 2205 } 2206 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 2207 ib_mtu_enum_to_int(iboe_get_mtu 2208 (dev->ndev->mtu))); 2209 } 2210 2211 if (!qp->mtu) { 2212 qp->mtu = 2213 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2214 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu); 2215 } 2216 2217 SET_FIELD(qp_params.modify_flags, 2218 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1); 2219 2220 qp_params.traffic_class_tos = grh->traffic_class; 2221 qp_params.flow_label = grh->flow_label; 2222 qp_params.hop_limit_ttl = grh->hop_limit; 2223 2224 qp->sgid_idx = grh->sgid_index; 2225 2226 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params); 2227 if (rc) { 2228 DP_ERR(dev, 2229 "modify qp: problems with GID index %d (rc=%d)\n", 2230 grh->sgid_index, rc); 2231 return rc; 2232 } 2233 2234 rc = qedr_get_dmac(dev, &attr->ah_attr, 2235 qp_params.remote_mac_addr); 2236 if (rc) 2237 return rc; 2238 2239 qp_params.use_local_mac = true; 2240 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr); 2241 2242 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n", 2243 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 2244 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 2245 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n", 2246 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 2247 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 2248 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n", 2249 qp_params.remote_mac_addr); 2250 2251 qp_params.mtu = qp->mtu; 2252 qp_params.lb_indication = false; 2253 } 2254 2255 if (!qp_params.mtu) { 2256 /* Stay with current MTU */ 2257 if (qp->mtu) 2258 qp_params.mtu = qp->mtu; 2259 else 2260 qp_params.mtu = 2261 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2262 } 2263 2264 if (attr_mask & IB_QP_TIMEOUT) { 2265 SET_FIELD(qp_params.modify_flags, 2266 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 2267 2268 /* The received timeout value is an exponent used like this: 2269 * "12.7.34 LOCAL ACK TIMEOUT 2270 * Value representing the transport (ACK) timeout for use by 2271 * the remote, expressed as: 4.096 * 2^timeout [usec]" 2272 * The FW expects timeout in msec so we need to divide the usec 2273 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2, 2274 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8). 2275 * The value of zero means infinite so we use a 'max_t' to make 2276 * sure that sub 1 msec values will be configured as 1 msec. 2277 */ 2278 if (attr->timeout) 2279 qp_params.ack_timeout = 2280 1 << max_t(int, attr->timeout - 8, 0); 2281 else 2282 qp_params.ack_timeout = 0; 2283 } 2284 2285 if (attr_mask & IB_QP_RETRY_CNT) { 2286 SET_FIELD(qp_params.modify_flags, 2287 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 2288 qp_params.retry_cnt = attr->retry_cnt; 2289 } 2290 2291 if (attr_mask & IB_QP_RNR_RETRY) { 2292 SET_FIELD(qp_params.modify_flags, 2293 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1); 2294 qp_params.rnr_retry_cnt = attr->rnr_retry; 2295 } 2296 2297 if (attr_mask & IB_QP_RQ_PSN) { 2298 SET_FIELD(qp_params.modify_flags, 2299 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1); 2300 qp_params.rq_psn = attr->rq_psn; 2301 qp->rq_psn = attr->rq_psn; 2302 } 2303 2304 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 2305 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) { 2306 rc = -EINVAL; 2307 DP_ERR(dev, 2308 "unsupported max_rd_atomic=%d, supported=%d\n", 2309 attr->max_rd_atomic, 2310 dev->attr.max_qp_req_rd_atomic_resc); 2311 goto err; 2312 } 2313 2314 SET_FIELD(qp_params.modify_flags, 2315 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1); 2316 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 2317 } 2318 2319 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 2320 SET_FIELD(qp_params.modify_flags, 2321 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1); 2322 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 2323 } 2324 2325 if (attr_mask & IB_QP_SQ_PSN) { 2326 SET_FIELD(qp_params.modify_flags, 2327 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1); 2328 qp_params.sq_psn = attr->sq_psn; 2329 qp->sq_psn = attr->sq_psn; 2330 } 2331 2332 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 2333 if (attr->max_dest_rd_atomic > 2334 dev->attr.max_qp_resp_rd_atomic_resc) { 2335 DP_ERR(dev, 2336 "unsupported max_dest_rd_atomic=%d, supported=%d\n", 2337 attr->max_dest_rd_atomic, 2338 dev->attr.max_qp_resp_rd_atomic_resc); 2339 2340 rc = -EINVAL; 2341 goto err; 2342 } 2343 2344 SET_FIELD(qp_params.modify_flags, 2345 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1); 2346 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 2347 } 2348 2349 if (attr_mask & IB_QP_DEST_QPN) { 2350 SET_FIELD(qp_params.modify_flags, 2351 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1); 2352 2353 qp_params.dest_qp = attr->dest_qp_num; 2354 qp->dest_qp_num = attr->dest_qp_num; 2355 } 2356 2357 cur_state = qp->state; 2358 2359 /* Update the QP state before the actual ramrod to prevent a race with 2360 * fast path. Modifying the QP state to error will cause the device to 2361 * flush the CQEs and while polling the flushed CQEs will considered as 2362 * a potential issue if the QP isn't in error state. 2363 */ 2364 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI && 2365 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR) 2366 qp->state = QED_ROCE_QP_STATE_ERR; 2367 2368 if (qp->qp_type != IB_QPT_GSI) 2369 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx, 2370 qp->qed_qp, &qp_params); 2371 2372 if (attr_mask & IB_QP_STATE) { 2373 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 2374 rc = qedr_update_qp_state(dev, qp, cur_state, 2375 qp_params.new_state); 2376 qp->state = qp_params.new_state; 2377 } 2378 2379 err: 2380 return rc; 2381 } 2382 2383 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params) 2384 { 2385 int ib_qp_acc_flags = 0; 2386 2387 if (params->incoming_rdma_write_en) 2388 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 2389 if (params->incoming_rdma_read_en) 2390 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 2391 if (params->incoming_atomic_en) 2392 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 2393 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 2394 return ib_qp_acc_flags; 2395 } 2396 2397 int qedr_query_qp(struct ib_qp *ibqp, 2398 struct ib_qp_attr *qp_attr, 2399 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 2400 { 2401 struct qed_rdma_query_qp_out_params params; 2402 struct qedr_qp *qp = get_qedr_qp(ibqp); 2403 struct qedr_dev *dev = qp->dev; 2404 int rc = 0; 2405 2406 memset(¶ms, 0, sizeof(params)); 2407 2408 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms); 2409 if (rc) 2410 goto err; 2411 2412 memset(qp_attr, 0, sizeof(*qp_attr)); 2413 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 2414 2415 qp_attr->qp_state = qedr_get_ibqp_state(params.state); 2416 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state); 2417 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu); 2418 qp_attr->path_mig_state = IB_MIG_MIGRATED; 2419 qp_attr->rq_psn = params.rq_psn; 2420 qp_attr->sq_psn = params.sq_psn; 2421 qp_attr->dest_qp_num = params.dest_qp; 2422 2423 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms); 2424 2425 qp_attr->cap.max_send_wr = qp->sq.max_wr; 2426 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 2427 qp_attr->cap.max_send_sge = qp->sq.max_sges; 2428 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 2429 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; 2430 qp_init_attr->cap = qp_attr->cap; 2431 2432 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2433 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 2434 params.flow_label, qp->sgid_idx, 2435 params.hop_limit_ttl, params.traffic_class_tos); 2436 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); 2437 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 2438 rdma_ah_set_sl(&qp_attr->ah_attr, 0); 2439 qp_attr->timeout = params.timeout; 2440 qp_attr->rnr_retry = params.rnr_retry; 2441 qp_attr->retry_cnt = params.retry_cnt; 2442 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 2443 qp_attr->pkey_index = params.pkey_index; 2444 qp_attr->port_num = 1; 2445 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 2446 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 2447 qp_attr->alt_pkey_index = 0; 2448 qp_attr->alt_port_num = 0; 2449 qp_attr->alt_timeout = 0; 2450 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 2451 2452 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0; 2453 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 2454 qp_attr->max_rd_atomic = params.max_rd_atomic; 2455 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0; 2456 2457 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n", 2458 qp_attr->cap.max_inline_data); 2459 2460 err: 2461 return rc; 2462 } 2463 2464 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp, 2465 struct ib_udata *udata) 2466 { 2467 int rc = 0; 2468 2469 if (qp->qp_type != IB_QPT_GSI) { 2470 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2471 if (rc) 2472 return rc; 2473 } 2474 2475 if (udata) 2476 qedr_cleanup_user(dev, qp); 2477 else 2478 qedr_cleanup_kernel(dev, qp); 2479 2480 return 0; 2481 } 2482 2483 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 2484 { 2485 struct qedr_qp *qp = get_qedr_qp(ibqp); 2486 struct qedr_dev *dev = qp->dev; 2487 struct ib_qp_attr attr; 2488 int attr_mask = 0; 2489 int rc = 0; 2490 2491 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n", 2492 qp, qp->qp_type); 2493 2494 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2495 if ((qp->state != QED_ROCE_QP_STATE_RESET) && 2496 (qp->state != QED_ROCE_QP_STATE_ERR) && 2497 (qp->state != QED_ROCE_QP_STATE_INIT)) { 2498 2499 attr.qp_state = IB_QPS_ERR; 2500 attr_mask |= IB_QP_STATE; 2501 2502 /* Change the QP state to ERROR */ 2503 qedr_modify_qp(ibqp, &attr, attr_mask, NULL); 2504 } 2505 } else { 2506 /* Wait for the connect/accept to complete */ 2507 if (qp->ep) { 2508 int wait_count = 1; 2509 2510 while (qp->ep->during_connect) { 2511 DP_DEBUG(dev, QEDR_MSG_QP, 2512 "Still in during connect/accept\n"); 2513 2514 msleep(100); 2515 if (wait_count++ > 200) { 2516 DP_NOTICE(dev, 2517 "during connect timeout\n"); 2518 break; 2519 } 2520 } 2521 } 2522 } 2523 2524 if (qp->qp_type == IB_QPT_GSI) 2525 qedr_destroy_gsi_qp(dev); 2526 2527 qedr_free_qp_resources(dev, qp, udata); 2528 2529 if (atomic_dec_and_test(&qp->refcnt) && 2530 rdma_protocol_iwarp(&dev->ibdev, 1)) { 2531 xa_erase_irq(&dev->qps, qp->qp_id); 2532 kfree(qp); 2533 } 2534 return rc; 2535 } 2536 2537 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags, 2538 struct ib_udata *udata) 2539 { 2540 struct qedr_ah *ah = get_qedr_ah(ibah); 2541 2542 rdma_copy_ah_attr(&ah->attr, attr); 2543 2544 return 0; 2545 } 2546 2547 void qedr_destroy_ah(struct ib_ah *ibah, u32 flags) 2548 { 2549 struct qedr_ah *ah = get_qedr_ah(ibah); 2550 2551 rdma_destroy_ah_attr(&ah->attr); 2552 } 2553 2554 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) 2555 { 2556 struct qedr_pbl *pbl, *tmp; 2557 2558 if (info->pbl_table) 2559 list_add_tail(&info->pbl_table->list_entry, 2560 &info->free_pbl_list); 2561 2562 if (!list_empty(&info->inuse_pbl_list)) 2563 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 2564 2565 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 2566 list_del(&pbl->list_entry); 2567 qedr_free_pbl(dev, &info->pbl_info, pbl); 2568 } 2569 } 2570 2571 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info, 2572 size_t page_list_len, bool two_layered) 2573 { 2574 struct qedr_pbl *tmp; 2575 int rc; 2576 2577 INIT_LIST_HEAD(&info->free_pbl_list); 2578 INIT_LIST_HEAD(&info->inuse_pbl_list); 2579 2580 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info, 2581 page_list_len, two_layered); 2582 if (rc) 2583 goto done; 2584 2585 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2586 if (IS_ERR(info->pbl_table)) { 2587 rc = PTR_ERR(info->pbl_table); 2588 goto done; 2589 } 2590 2591 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n", 2592 &info->pbl_table->pa); 2593 2594 /* in usual case we use 2 PBLs, so we add one to free 2595 * list and allocating another one 2596 */ 2597 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2598 if (IS_ERR(tmp)) { 2599 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n"); 2600 goto done; 2601 } 2602 2603 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 2604 2605 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa); 2606 2607 done: 2608 if (rc) 2609 free_mr_info(dev, info); 2610 2611 return rc; 2612 } 2613 2614 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 2615 u64 usr_addr, int acc, struct ib_udata *udata) 2616 { 2617 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2618 struct qedr_mr *mr; 2619 struct qedr_pd *pd; 2620 int rc = -ENOMEM; 2621 2622 pd = get_qedr_pd(ibpd); 2623 DP_DEBUG(dev, QEDR_MSG_MR, 2624 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 2625 pd->pd_id, start, len, usr_addr, acc); 2626 2627 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 2628 return ERR_PTR(-EINVAL); 2629 2630 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2631 if (!mr) 2632 return ERR_PTR(rc); 2633 2634 mr->type = QEDR_MR_USER; 2635 2636 mr->umem = ib_umem_get(udata, start, len, acc, 0); 2637 if (IS_ERR(mr->umem)) { 2638 rc = -EFAULT; 2639 goto err0; 2640 } 2641 2642 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); 2643 if (rc) 2644 goto err1; 2645 2646 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 2647 &mr->info.pbl_info, PAGE_SHIFT); 2648 2649 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2650 if (rc) { 2651 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2652 goto err1; 2653 } 2654 2655 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2656 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2657 mr->hw_mr.key = 0; 2658 mr->hw_mr.pd = pd->pd_id; 2659 mr->hw_mr.local_read = 1; 2660 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2661 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2662 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2663 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2664 mr->hw_mr.mw_bind = false; 2665 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 2666 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2667 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2668 mr->hw_mr.page_size_log = PAGE_SHIFT; 2669 mr->hw_mr.fbo = ib_umem_offset(mr->umem); 2670 mr->hw_mr.length = len; 2671 mr->hw_mr.vaddr = usr_addr; 2672 mr->hw_mr.zbva = false; 2673 mr->hw_mr.phy_mr = false; 2674 mr->hw_mr.dma_mr = false; 2675 2676 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2677 if (rc) { 2678 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2679 goto err2; 2680 } 2681 2682 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2683 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2684 mr->hw_mr.remote_atomic) 2685 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2686 2687 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n", 2688 mr->ibmr.lkey); 2689 return &mr->ibmr; 2690 2691 err2: 2692 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2693 err1: 2694 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2695 err0: 2696 kfree(mr); 2697 return ERR_PTR(rc); 2698 } 2699 2700 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 2701 { 2702 struct qedr_mr *mr = get_qedr_mr(ib_mr); 2703 struct qedr_dev *dev = get_qedr_dev(ib_mr->device); 2704 int rc = 0; 2705 2706 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid); 2707 if (rc) 2708 return rc; 2709 2710 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2711 2712 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR)) 2713 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2714 2715 /* it could be user registered memory. */ 2716 if (mr->umem) 2717 ib_umem_release(mr->umem); 2718 2719 kfree(mr); 2720 2721 return rc; 2722 } 2723 2724 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, 2725 int max_page_list_len) 2726 { 2727 struct qedr_pd *pd = get_qedr_pd(ibpd); 2728 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2729 struct qedr_mr *mr; 2730 int rc = -ENOMEM; 2731 2732 DP_DEBUG(dev, QEDR_MSG_MR, 2733 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id, 2734 max_page_list_len); 2735 2736 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2737 if (!mr) 2738 return ERR_PTR(rc); 2739 2740 mr->dev = dev; 2741 mr->type = QEDR_MR_FRMR; 2742 2743 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1); 2744 if (rc) 2745 goto err0; 2746 2747 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2748 if (rc) { 2749 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2750 goto err0; 2751 } 2752 2753 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2754 mr->hw_mr.tid_type = QED_RDMA_TID_FMR; 2755 mr->hw_mr.key = 0; 2756 mr->hw_mr.pd = pd->pd_id; 2757 mr->hw_mr.local_read = 1; 2758 mr->hw_mr.local_write = 0; 2759 mr->hw_mr.remote_read = 0; 2760 mr->hw_mr.remote_write = 0; 2761 mr->hw_mr.remote_atomic = 0; 2762 mr->hw_mr.mw_bind = false; 2763 mr->hw_mr.pbl_ptr = 0; 2764 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2765 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2766 mr->hw_mr.fbo = 0; 2767 mr->hw_mr.length = 0; 2768 mr->hw_mr.vaddr = 0; 2769 mr->hw_mr.zbva = false; 2770 mr->hw_mr.phy_mr = true; 2771 mr->hw_mr.dma_mr = false; 2772 2773 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2774 if (rc) { 2775 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2776 goto err1; 2777 } 2778 2779 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2780 mr->ibmr.rkey = mr->ibmr.lkey; 2781 2782 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey); 2783 return mr; 2784 2785 err1: 2786 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2787 err0: 2788 kfree(mr); 2789 return ERR_PTR(rc); 2790 } 2791 2792 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 2793 u32 max_num_sg, struct ib_udata *udata) 2794 { 2795 struct qedr_mr *mr; 2796 2797 if (mr_type != IB_MR_TYPE_MEM_REG) 2798 return ERR_PTR(-EINVAL); 2799 2800 mr = __qedr_alloc_mr(ibpd, max_num_sg); 2801 2802 if (IS_ERR(mr)) 2803 return ERR_PTR(-EINVAL); 2804 2805 return &mr->ibmr; 2806 } 2807 2808 static int qedr_set_page(struct ib_mr *ibmr, u64 addr) 2809 { 2810 struct qedr_mr *mr = get_qedr_mr(ibmr); 2811 struct qedr_pbl *pbl_table; 2812 struct regpair *pbe; 2813 u32 pbes_in_page; 2814 2815 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 2816 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages); 2817 return -ENOMEM; 2818 } 2819 2820 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n", 2821 mr->npages, addr); 2822 2823 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 2824 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 2825 pbe = (struct regpair *)pbl_table->va; 2826 pbe += mr->npages % pbes_in_page; 2827 pbe->lo = cpu_to_le32((u32)addr); 2828 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 2829 2830 mr->npages++; 2831 2832 return 0; 2833 } 2834 2835 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info) 2836 { 2837 int work = info->completed - info->completed_handled - 1; 2838 2839 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work); 2840 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 2841 struct qedr_pbl *pbl; 2842 2843 /* Free all the page list that are possible to be freed 2844 * (all the ones that were invalidated), under the assumption 2845 * that if an FMR was completed successfully that means that 2846 * if there was an invalidate operation before it also ended 2847 */ 2848 pbl = list_first_entry(&info->inuse_pbl_list, 2849 struct qedr_pbl, list_entry); 2850 list_move_tail(&pbl->list_entry, &info->free_pbl_list); 2851 info->completed_handled++; 2852 } 2853 } 2854 2855 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 2856 int sg_nents, unsigned int *sg_offset) 2857 { 2858 struct qedr_mr *mr = get_qedr_mr(ibmr); 2859 2860 mr->npages = 0; 2861 2862 handle_completed_mrs(mr->dev, &mr->info); 2863 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page); 2864 } 2865 2866 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc) 2867 { 2868 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2869 struct qedr_pd *pd = get_qedr_pd(ibpd); 2870 struct qedr_mr *mr; 2871 int rc; 2872 2873 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2874 if (!mr) 2875 return ERR_PTR(-ENOMEM); 2876 2877 mr->type = QEDR_MR_DMA; 2878 2879 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2880 if (rc) { 2881 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2882 goto err1; 2883 } 2884 2885 /* index only, 18 bit long, lkey = itid << 8 | key */ 2886 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2887 mr->hw_mr.pd = pd->pd_id; 2888 mr->hw_mr.local_read = 1; 2889 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2890 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2891 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2892 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2893 mr->hw_mr.dma_mr = true; 2894 2895 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2896 if (rc) { 2897 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2898 goto err2; 2899 } 2900 2901 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2902 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2903 mr->hw_mr.remote_atomic) 2904 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2905 2906 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey); 2907 return &mr->ibmr; 2908 2909 err2: 2910 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2911 err1: 2912 kfree(mr); 2913 return ERR_PTR(rc); 2914 } 2915 2916 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq) 2917 { 2918 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 2919 } 2920 2921 static int sge_data_len(struct ib_sge *sg_list, int num_sge) 2922 { 2923 int i, len = 0; 2924 2925 for (i = 0; i < num_sge; i++) 2926 len += sg_list[i].length; 2927 2928 return len; 2929 } 2930 2931 static void swap_wqe_data64(u64 *p) 2932 { 2933 int i; 2934 2935 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 2936 *p = cpu_to_be64(cpu_to_le64(*p)); 2937 } 2938 2939 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev, 2940 struct qedr_qp *qp, u8 *wqe_size, 2941 const struct ib_send_wr *wr, 2942 const struct ib_send_wr **bad_wr, 2943 u8 *bits, u8 bit) 2944 { 2945 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 2946 char *seg_prt, *wqe; 2947 int i, seg_siz; 2948 2949 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 2950 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size); 2951 *bad_wr = wr; 2952 return 0; 2953 } 2954 2955 if (!data_size) 2956 return data_size; 2957 2958 *bits |= bit; 2959 2960 seg_prt = NULL; 2961 wqe = NULL; 2962 seg_siz = 0; 2963 2964 /* Copy data inline */ 2965 for (i = 0; i < wr->num_sge; i++) { 2966 u32 len = wr->sg_list[i].length; 2967 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 2968 2969 while (len > 0) { 2970 u32 cur; 2971 2972 /* New segment required */ 2973 if (!seg_siz) { 2974 wqe = (char *)qed_chain_produce(&qp->sq.pbl); 2975 seg_prt = wqe; 2976 seg_siz = sizeof(struct rdma_sq_common_wqe); 2977 (*wqe_size)++; 2978 } 2979 2980 /* Calculate currently allowed length */ 2981 cur = min_t(u32, len, seg_siz); 2982 memcpy(seg_prt, src, cur); 2983 2984 /* Update segment variables */ 2985 seg_prt += cur; 2986 seg_siz -= cur; 2987 2988 /* Update sge variables */ 2989 src += cur; 2990 len -= cur; 2991 2992 /* Swap fully-completed segments */ 2993 if (!seg_siz) 2994 swap_wqe_data64((u64 *)wqe); 2995 } 2996 } 2997 2998 /* swap last not completed segment */ 2999 if (seg_siz) 3000 swap_wqe_data64((u64 *)wqe); 3001 3002 return data_size; 3003 } 3004 3005 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 3006 do { \ 3007 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3008 (sge)->length = cpu_to_le32(vlength); \ 3009 (sge)->flags = cpu_to_le32(vflags); \ 3010 } while (0) 3011 3012 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 3013 do { \ 3014 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \ 3015 (hdr)->num_sges = num_sge; \ 3016 } while (0) 3017 3018 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 3019 do { \ 3020 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3021 (sge)->length = cpu_to_le32(vlength); \ 3022 (sge)->l_key = cpu_to_le32(vlkey); \ 3023 } while (0) 3024 3025 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, 3026 const struct ib_send_wr *wr) 3027 { 3028 u32 data_size = 0; 3029 int i; 3030 3031 for (i = 0; i < wr->num_sge; i++) { 3032 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl); 3033 3034 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr); 3035 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 3036 sge->length = cpu_to_le32(wr->sg_list[i].length); 3037 data_size += wr->sg_list[i].length; 3038 } 3039 3040 if (wqe_size) 3041 *wqe_size += wr->num_sge; 3042 3043 return data_size; 3044 } 3045 3046 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev, 3047 struct qedr_qp *qp, 3048 struct rdma_sq_rdma_wqe_1st *rwqe, 3049 struct rdma_sq_rdma_wqe_2nd *rwqe2, 3050 const struct ib_send_wr *wr, 3051 const struct ib_send_wr **bad_wr) 3052 { 3053 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 3054 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr); 3055 3056 if (wr->send_flags & IB_SEND_INLINE && 3057 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 3058 wr->opcode == IB_WR_RDMA_WRITE)) { 3059 u8 flags = 0; 3060 3061 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 3062 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, 3063 bad_wr, &rwqe->flags, flags); 3064 } 3065 3066 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr); 3067 } 3068 3069 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev, 3070 struct qedr_qp *qp, 3071 struct rdma_sq_send_wqe_1st *swqe, 3072 struct rdma_sq_send_wqe_2st *swqe2, 3073 const struct ib_send_wr *wr, 3074 const struct ib_send_wr **bad_wr) 3075 { 3076 memset(swqe2, 0, sizeof(*swqe2)); 3077 if (wr->send_flags & IB_SEND_INLINE) { 3078 u8 flags = 0; 3079 3080 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 3081 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, 3082 bad_wr, &swqe->flags, flags); 3083 } 3084 3085 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr); 3086 } 3087 3088 static int qedr_prepare_reg(struct qedr_qp *qp, 3089 struct rdma_sq_fmr_wqe_1st *fwqe1, 3090 const struct ib_reg_wr *wr) 3091 { 3092 struct qedr_mr *mr = get_qedr_mr(wr->mr); 3093 struct rdma_sq_fmr_wqe_2nd *fwqe2; 3094 3095 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl); 3096 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 3097 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 3098 fwqe1->l_key = wr->key; 3099 3100 fwqe2->access_ctrl = 0; 3101 3102 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 3103 !!(wr->access & IB_ACCESS_REMOTE_READ)); 3104 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 3105 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 3106 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 3107 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 3108 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 3109 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 3110 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 3111 fwqe2->fmr_ctrl = 0; 3112 3113 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 3114 ilog2(mr->ibmr.page_size) - 12); 3115 3116 fwqe2->length_hi = 0; 3117 fwqe2->length_lo = mr->ibmr.length; 3118 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 3119 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 3120 3121 qp->wqe_wr_id[qp->sq.prod].mr = mr; 3122 3123 return 0; 3124 } 3125 3126 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 3127 { 3128 switch (opcode) { 3129 case IB_WR_RDMA_WRITE: 3130 case IB_WR_RDMA_WRITE_WITH_IMM: 3131 return IB_WC_RDMA_WRITE; 3132 case IB_WR_SEND_WITH_IMM: 3133 case IB_WR_SEND: 3134 case IB_WR_SEND_WITH_INV: 3135 return IB_WC_SEND; 3136 case IB_WR_RDMA_READ: 3137 case IB_WR_RDMA_READ_WITH_INV: 3138 return IB_WC_RDMA_READ; 3139 case IB_WR_ATOMIC_CMP_AND_SWP: 3140 return IB_WC_COMP_SWAP; 3141 case IB_WR_ATOMIC_FETCH_AND_ADD: 3142 return IB_WC_FETCH_ADD; 3143 case IB_WR_REG_MR: 3144 return IB_WC_REG_MR; 3145 case IB_WR_LOCAL_INV: 3146 return IB_WC_LOCAL_INV; 3147 default: 3148 return IB_WC_SEND; 3149 } 3150 } 3151 3152 static inline bool qedr_can_post_send(struct qedr_qp *qp, 3153 const struct ib_send_wr *wr) 3154 { 3155 int wq_is_full, err_wr, pbl_is_full; 3156 struct qedr_dev *dev = qp->dev; 3157 3158 /* prevent SQ overflow and/or processing of a bad WR */ 3159 err_wr = wr->num_sge > qp->sq.max_sges; 3160 wq_is_full = qedr_wq_is_full(&qp->sq); 3161 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) < 3162 QEDR_MAX_SQE_ELEMENTS_PER_SQE; 3163 if (wq_is_full || err_wr || pbl_is_full) { 3164 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) { 3165 DP_ERR(dev, 3166 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n", 3167 qp); 3168 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL; 3169 } 3170 3171 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) { 3172 DP_ERR(dev, 3173 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n", 3174 qp); 3175 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR; 3176 } 3177 3178 if (pbl_is_full && 3179 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) { 3180 DP_ERR(dev, 3181 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n", 3182 qp); 3183 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL; 3184 } 3185 return false; 3186 } 3187 return true; 3188 } 3189 3190 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3191 const struct ib_send_wr **bad_wr) 3192 { 3193 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3194 struct qedr_qp *qp = get_qedr_qp(ibqp); 3195 struct rdma_sq_atomic_wqe_1st *awqe1; 3196 struct rdma_sq_atomic_wqe_2nd *awqe2; 3197 struct rdma_sq_atomic_wqe_3rd *awqe3; 3198 struct rdma_sq_send_wqe_2st *swqe2; 3199 struct rdma_sq_local_inv_wqe *iwqe; 3200 struct rdma_sq_rdma_wqe_2nd *rwqe2; 3201 struct rdma_sq_send_wqe_1st *swqe; 3202 struct rdma_sq_rdma_wqe_1st *rwqe; 3203 struct rdma_sq_fmr_wqe_1st *fwqe1; 3204 struct rdma_sq_common_wqe *wqe; 3205 u32 length; 3206 int rc = 0; 3207 bool comp; 3208 3209 if (!qedr_can_post_send(qp, wr)) { 3210 *bad_wr = wr; 3211 return -ENOMEM; 3212 } 3213 3214 wqe = qed_chain_produce(&qp->sq.pbl); 3215 qp->wqe_wr_id[qp->sq.prod].signaled = 3216 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 3217 3218 wqe->flags = 0; 3219 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, 3220 !!(wr->send_flags & IB_SEND_SOLICITED)); 3221 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled; 3222 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 3223 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, 3224 !!(wr->send_flags & IB_SEND_FENCE)); 3225 wqe->prev_wqe_size = qp->prev_wqe_size; 3226 3227 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode); 3228 3229 switch (wr->opcode) { 3230 case IB_WR_SEND_WITH_IMM: 3231 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3232 rc = -EINVAL; 3233 *bad_wr = wr; 3234 break; 3235 } 3236 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 3237 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3238 swqe->wqe_size = 2; 3239 swqe2 = qed_chain_produce(&qp->sq.pbl); 3240 3241 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 3242 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3243 wr, bad_wr); 3244 swqe->length = cpu_to_le32(length); 3245 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3246 qp->prev_wqe_size = swqe->wqe_size; 3247 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3248 break; 3249 case IB_WR_SEND: 3250 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 3251 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3252 3253 swqe->wqe_size = 2; 3254 swqe2 = qed_chain_produce(&qp->sq.pbl); 3255 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3256 wr, bad_wr); 3257 swqe->length = cpu_to_le32(length); 3258 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3259 qp->prev_wqe_size = swqe->wqe_size; 3260 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3261 break; 3262 case IB_WR_SEND_WITH_INV: 3263 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 3264 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3265 swqe2 = qed_chain_produce(&qp->sq.pbl); 3266 swqe->wqe_size = 2; 3267 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey); 3268 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3269 wr, bad_wr); 3270 swqe->length = cpu_to_le32(length); 3271 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3272 qp->prev_wqe_size = swqe->wqe_size; 3273 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3274 break; 3275 3276 case IB_WR_RDMA_WRITE_WITH_IMM: 3277 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3278 rc = -EINVAL; 3279 *bad_wr = wr; 3280 break; 3281 } 3282 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 3283 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3284 3285 rwqe->wqe_size = 2; 3286 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 3287 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3288 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3289 wr, bad_wr); 3290 rwqe->length = cpu_to_le32(length); 3291 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3292 qp->prev_wqe_size = rwqe->wqe_size; 3293 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3294 break; 3295 case IB_WR_RDMA_WRITE: 3296 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 3297 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3298 3299 rwqe->wqe_size = 2; 3300 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3301 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3302 wr, bad_wr); 3303 rwqe->length = cpu_to_le32(length); 3304 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3305 qp->prev_wqe_size = rwqe->wqe_size; 3306 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3307 break; 3308 case IB_WR_RDMA_READ_WITH_INV: 3309 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1); 3310 /* fallthrough -- same is identical to RDMA READ */ 3311 3312 case IB_WR_RDMA_READ: 3313 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 3314 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3315 3316 rwqe->wqe_size = 2; 3317 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3318 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3319 wr, bad_wr); 3320 rwqe->length = cpu_to_le32(length); 3321 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3322 qp->prev_wqe_size = rwqe->wqe_size; 3323 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3324 break; 3325 3326 case IB_WR_ATOMIC_CMP_AND_SWP: 3327 case IB_WR_ATOMIC_FETCH_AND_ADD: 3328 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe; 3329 awqe1->wqe_size = 4; 3330 3331 awqe2 = qed_chain_produce(&qp->sq.pbl); 3332 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr); 3333 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 3334 3335 awqe3 = qed_chain_produce(&qp->sq.pbl); 3336 3337 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 3338 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 3339 DMA_REGPAIR_LE(awqe3->swap_data, 3340 atomic_wr(wr)->compare_add); 3341 } else { 3342 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 3343 DMA_REGPAIR_LE(awqe3->swap_data, 3344 atomic_wr(wr)->swap); 3345 DMA_REGPAIR_LE(awqe3->cmp_data, 3346 atomic_wr(wr)->compare_add); 3347 } 3348 3349 qedr_prepare_sq_sges(qp, NULL, wr); 3350 3351 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size; 3352 qp->prev_wqe_size = awqe1->wqe_size; 3353 break; 3354 3355 case IB_WR_LOCAL_INV: 3356 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 3357 iwqe->wqe_size = 1; 3358 3359 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 3360 iwqe->inv_l_key = wr->ex.invalidate_rkey; 3361 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size; 3362 qp->prev_wqe_size = iwqe->wqe_size; 3363 break; 3364 case IB_WR_REG_MR: 3365 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n"); 3366 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 3367 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 3368 fwqe1->wqe_size = 2; 3369 3370 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr)); 3371 if (rc) { 3372 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc); 3373 *bad_wr = wr; 3374 break; 3375 } 3376 3377 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 3378 qp->prev_wqe_size = fwqe1->wqe_size; 3379 break; 3380 default: 3381 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode); 3382 rc = -EINVAL; 3383 *bad_wr = wr; 3384 break; 3385 } 3386 3387 if (*bad_wr) { 3388 u16 value; 3389 3390 /* Restore prod to its position before 3391 * this WR was processed 3392 */ 3393 value = le16_to_cpu(qp->sq.db_data.data.value); 3394 qed_chain_set_prod(&qp->sq.pbl, value, wqe); 3395 3396 /* Restore prev_wqe_size */ 3397 qp->prev_wqe_size = wqe->prev_wqe_size; 3398 rc = -EINVAL; 3399 DP_ERR(dev, "POST SEND FAILED\n"); 3400 } 3401 3402 return rc; 3403 } 3404 3405 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3406 const struct ib_send_wr **bad_wr) 3407 { 3408 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3409 struct qedr_qp *qp = get_qedr_qp(ibqp); 3410 unsigned long flags; 3411 int rc = 0; 3412 3413 *bad_wr = NULL; 3414 3415 if (qp->qp_type == IB_QPT_GSI) 3416 return qedr_gsi_post_send(ibqp, wr, bad_wr); 3417 3418 spin_lock_irqsave(&qp->q_lock, flags); 3419 3420 if (rdma_protocol_roce(&dev->ibdev, 1)) { 3421 if ((qp->state != QED_ROCE_QP_STATE_RTS) && 3422 (qp->state != QED_ROCE_QP_STATE_ERR) && 3423 (qp->state != QED_ROCE_QP_STATE_SQD)) { 3424 spin_unlock_irqrestore(&qp->q_lock, flags); 3425 *bad_wr = wr; 3426 DP_DEBUG(dev, QEDR_MSG_CQ, 3427 "QP in wrong state! QP icid=0x%x state %d\n", 3428 qp->icid, qp->state); 3429 return -EINVAL; 3430 } 3431 } 3432 3433 while (wr) { 3434 rc = __qedr_post_send(ibqp, wr, bad_wr); 3435 if (rc) 3436 break; 3437 3438 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 3439 3440 qedr_inc_sw_prod(&qp->sq); 3441 3442 qp->sq.db_data.data.value++; 3443 3444 wr = wr->next; 3445 } 3446 3447 /* Trigger doorbell 3448 * If there was a failure in the first WR then it will be triggered in 3449 * vane. However this is not harmful (as long as the producer value is 3450 * unchanged). For performance reasons we avoid checking for this 3451 * redundant doorbell. 3452 * 3453 * qp->wqe_wr_id is accessed during qedr_poll_cq, as 3454 * soon as we give the doorbell, we could get a completion 3455 * for this wr, therefore we need to make sure that the 3456 * memory is updated before giving the doorbell. 3457 * During qedr_poll_cq, rmb is called before accessing the 3458 * cqe. This covers for the smp_rmb as well. 3459 */ 3460 smp_wmb(); 3461 writel(qp->sq.db_data.raw, qp->sq.db); 3462 3463 spin_unlock_irqrestore(&qp->q_lock, flags); 3464 3465 return rc; 3466 } 3467 3468 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq) 3469 { 3470 u32 used; 3471 3472 /* Calculate number of elements used based on producer 3473 * count and consumer count and subtract it from max 3474 * work request supported so that we get elements left. 3475 */ 3476 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt; 3477 3478 return hw_srq->max_wr - used; 3479 } 3480 3481 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 3482 const struct ib_recv_wr **bad_wr) 3483 { 3484 struct qedr_srq *srq = get_qedr_srq(ibsrq); 3485 struct qedr_srq_hwq_info *hw_srq; 3486 struct qedr_dev *dev = srq->dev; 3487 struct qed_chain *pbl; 3488 unsigned long flags; 3489 int status = 0; 3490 u32 num_sge; 3491 u32 offset; 3492 3493 spin_lock_irqsave(&srq->lock, flags); 3494 3495 hw_srq = &srq->hw_srq; 3496 pbl = &srq->hw_srq.pbl; 3497 while (wr) { 3498 struct rdma_srq_wqe_header *hdr; 3499 int i; 3500 3501 if (!qedr_srq_elem_left(hw_srq) || 3502 wr->num_sge > srq->hw_srq.max_sges) { 3503 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n", 3504 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt, 3505 wr->num_sge, srq->hw_srq.max_sges); 3506 status = -ENOMEM; 3507 *bad_wr = wr; 3508 break; 3509 } 3510 3511 hdr = qed_chain_produce(pbl); 3512 num_sge = wr->num_sge; 3513 /* Set number of sge and work request id in header */ 3514 SRQ_HDR_SET(hdr, wr->wr_id, num_sge); 3515 3516 srq->hw_srq.wr_prod_cnt++; 3517 hw_srq->wqe_prod++; 3518 hw_srq->sge_prod++; 3519 3520 DP_DEBUG(dev, QEDR_MSG_SRQ, 3521 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n", 3522 wr->num_sge, hw_srq->wqe_prod, wr->wr_id); 3523 3524 for (i = 0; i < wr->num_sge; i++) { 3525 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl); 3526 3527 /* Set SGE length, lkey and address */ 3528 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, 3529 wr->sg_list[i].length, wr->sg_list[i].lkey); 3530 3531 DP_DEBUG(dev, QEDR_MSG_SRQ, 3532 "[%d]: len %d key %x addr %x:%x\n", 3533 i, srq_sge->length, srq_sge->l_key, 3534 srq_sge->addr.hi, srq_sge->addr.lo); 3535 hw_srq->sge_prod++; 3536 } 3537 3538 /* Flush WQE and SGE information before 3539 * updating producer. 3540 */ 3541 wmb(); 3542 3543 /* SRQ producer is 8 bytes. Need to update SGE producer index 3544 * in first 4 bytes and need to update WQE producer in 3545 * next 4 bytes. 3546 */ 3547 *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod; 3548 offset = offsetof(struct rdma_srq_producers, wqe_prod); 3549 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) = 3550 hw_srq->wqe_prod; 3551 3552 /* Flush producer after updating it. */ 3553 wmb(); 3554 wr = wr->next; 3555 } 3556 3557 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n", 3558 qed_chain_get_elem_left(pbl)); 3559 spin_unlock_irqrestore(&srq->lock, flags); 3560 3561 return status; 3562 } 3563 3564 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 3565 const struct ib_recv_wr **bad_wr) 3566 { 3567 struct qedr_qp *qp = get_qedr_qp(ibqp); 3568 struct qedr_dev *dev = qp->dev; 3569 unsigned long flags; 3570 int status = 0; 3571 3572 if (qp->qp_type == IB_QPT_GSI) 3573 return qedr_gsi_post_recv(ibqp, wr, bad_wr); 3574 3575 spin_lock_irqsave(&qp->q_lock, flags); 3576 3577 if (qp->state == QED_ROCE_QP_STATE_RESET) { 3578 spin_unlock_irqrestore(&qp->q_lock, flags); 3579 *bad_wr = wr; 3580 return -EINVAL; 3581 } 3582 3583 while (wr) { 3584 int i; 3585 3586 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) < 3587 QEDR_MAX_RQE_ELEMENTS_PER_RQE || 3588 wr->num_sge > qp->rq.max_sges) { 3589 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n", 3590 qed_chain_get_elem_left_u32(&qp->rq.pbl), 3591 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge, 3592 qp->rq.max_sges); 3593 status = -ENOMEM; 3594 *bad_wr = wr; 3595 break; 3596 } 3597 for (i = 0; i < wr->num_sge; i++) { 3598 u32 flags = 0; 3599 struct rdma_rq_sge *rqe = 3600 qed_chain_produce(&qp->rq.pbl); 3601 3602 /* First one must include the number 3603 * of SGE in the list 3604 */ 3605 if (!i) 3606 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 3607 wr->num_sge); 3608 3609 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 3610 wr->sg_list[i].lkey); 3611 3612 RQ_SGE_SET(rqe, wr->sg_list[i].addr, 3613 wr->sg_list[i].length, flags); 3614 } 3615 3616 /* Special case of no sges. FW requires between 1-4 sges... 3617 * in this case we need to post 1 sge with length zero. this is 3618 * because rdma write with immediate consumes an RQ. 3619 */ 3620 if (!wr->num_sge) { 3621 u32 flags = 0; 3622 struct rdma_rq_sge *rqe = 3623 qed_chain_produce(&qp->rq.pbl); 3624 3625 /* First one must include the number 3626 * of SGE in the list 3627 */ 3628 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0); 3629 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 3630 3631 RQ_SGE_SET(rqe, 0, 0, flags); 3632 i = 1; 3633 } 3634 3635 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 3636 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 3637 3638 qedr_inc_sw_prod(&qp->rq); 3639 3640 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as 3641 * soon as we give the doorbell, we could get a completion 3642 * for this wr, therefore we need to make sure that the 3643 * memory is update before giving the doorbell. 3644 * During qedr_poll_cq, rmb is called before accessing the 3645 * cqe. This covers for the smp_rmb as well. 3646 */ 3647 smp_wmb(); 3648 3649 qp->rq.db_data.data.value++; 3650 3651 writel(qp->rq.db_data.raw, qp->rq.db); 3652 3653 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 3654 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 3655 } 3656 3657 wr = wr->next; 3658 } 3659 3660 spin_unlock_irqrestore(&qp->q_lock, flags); 3661 3662 return status; 3663 } 3664 3665 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe) 3666 { 3667 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3668 3669 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) == 3670 cq->pbl_toggle; 3671 } 3672 3673 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe) 3674 { 3675 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3676 struct qedr_qp *qp; 3677 3678 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi, 3679 resp_cqe->qp_handle.lo, 3680 u64); 3681 return qp; 3682 } 3683 3684 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe) 3685 { 3686 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3687 3688 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 3689 } 3690 3691 /* Return latest CQE (needs processing) */ 3692 static union rdma_cqe *get_cqe(struct qedr_cq *cq) 3693 { 3694 return cq->latest_cqe; 3695 } 3696 3697 /* In fmr we need to increase the number of fmr completed counter for the fmr 3698 * algorithm determining whether we can free a pbl or not. 3699 * we need to perform this whether the work request was signaled or not. for 3700 * this purpose we call this function from the condition that checks if a wr 3701 * should be skipped, to make sure we don't miss it ( possibly this fmr 3702 * operation was not signalted) 3703 */ 3704 static inline void qedr_chk_if_fmr(struct qedr_qp *qp) 3705 { 3706 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 3707 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3708 } 3709 3710 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, 3711 struct qedr_cq *cq, int num_entries, 3712 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, 3713 int force) 3714 { 3715 u16 cnt = 0; 3716 3717 while (num_entries && qp->sq.wqe_cons != hw_cons) { 3718 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 3719 qedr_chk_if_fmr(qp); 3720 /* skip WC */ 3721 goto next_cqe; 3722 } 3723 3724 /* fill WC */ 3725 wc->status = status; 3726 wc->vendor_err = 0; 3727 wc->wc_flags = 0; 3728 wc->src_qp = qp->id; 3729 wc->qp = &qp->ibqp; 3730 3731 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 3732 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 3733 3734 switch (wc->opcode) { 3735 case IB_WC_RDMA_WRITE: 3736 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3737 break; 3738 case IB_WC_COMP_SWAP: 3739 case IB_WC_FETCH_ADD: 3740 wc->byte_len = 8; 3741 break; 3742 case IB_WC_REG_MR: 3743 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3744 break; 3745 case IB_WC_RDMA_READ: 3746 case IB_WC_SEND: 3747 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3748 break; 3749 default: 3750 break; 3751 } 3752 3753 num_entries--; 3754 wc++; 3755 cnt++; 3756 next_cqe: 3757 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 3758 qed_chain_consume(&qp->sq.pbl); 3759 qedr_inc_sw_cons(&qp->sq); 3760 } 3761 3762 return cnt; 3763 } 3764 3765 static int qedr_poll_cq_req(struct qedr_dev *dev, 3766 struct qedr_qp *qp, struct qedr_cq *cq, 3767 int num_entries, struct ib_wc *wc, 3768 struct rdma_cqe_requester *req) 3769 { 3770 int cnt = 0; 3771 3772 switch (req->status) { 3773 case RDMA_CQE_REQ_STS_OK: 3774 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3775 IB_WC_SUCCESS, 0); 3776 break; 3777 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 3778 if (qp->state != QED_ROCE_QP_STATE_ERR) 3779 DP_DEBUG(dev, QEDR_MSG_CQ, 3780 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3781 cq->icid, qp->icid); 3782 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3783 IB_WC_WR_FLUSH_ERR, 1); 3784 break; 3785 default: 3786 /* process all WQE before the cosumer */ 3787 qp->state = QED_ROCE_QP_STATE_ERR; 3788 cnt = process_req(dev, qp, cq, num_entries, wc, 3789 req->sq_cons - 1, IB_WC_SUCCESS, 0); 3790 wc += cnt; 3791 /* if we have extra WC fill it with actual error info */ 3792 if (cnt < num_entries) { 3793 enum ib_wc_status wc_status; 3794 3795 switch (req->status) { 3796 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 3797 DP_ERR(dev, 3798 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3799 cq->icid, qp->icid); 3800 wc_status = IB_WC_BAD_RESP_ERR; 3801 break; 3802 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 3803 DP_ERR(dev, 3804 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3805 cq->icid, qp->icid); 3806 wc_status = IB_WC_LOC_LEN_ERR; 3807 break; 3808 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 3809 DP_ERR(dev, 3810 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3811 cq->icid, qp->icid); 3812 wc_status = IB_WC_LOC_QP_OP_ERR; 3813 break; 3814 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 3815 DP_ERR(dev, 3816 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3817 cq->icid, qp->icid); 3818 wc_status = IB_WC_LOC_PROT_ERR; 3819 break; 3820 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 3821 DP_ERR(dev, 3822 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3823 cq->icid, qp->icid); 3824 wc_status = IB_WC_MW_BIND_ERR; 3825 break; 3826 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 3827 DP_ERR(dev, 3828 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3829 cq->icid, qp->icid); 3830 wc_status = IB_WC_REM_INV_REQ_ERR; 3831 break; 3832 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 3833 DP_ERR(dev, 3834 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3835 cq->icid, qp->icid); 3836 wc_status = IB_WC_REM_ACCESS_ERR; 3837 break; 3838 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 3839 DP_ERR(dev, 3840 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3841 cq->icid, qp->icid); 3842 wc_status = IB_WC_REM_OP_ERR; 3843 break; 3844 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 3845 DP_ERR(dev, 3846 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3847 cq->icid, qp->icid); 3848 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 3849 break; 3850 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 3851 DP_ERR(dev, 3852 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3853 cq->icid, qp->icid); 3854 wc_status = IB_WC_RETRY_EXC_ERR; 3855 break; 3856 default: 3857 DP_ERR(dev, 3858 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3859 cq->icid, qp->icid); 3860 wc_status = IB_WC_GENERAL_ERR; 3861 } 3862 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 3863 wc_status, 1); 3864 } 3865 } 3866 3867 return cnt; 3868 } 3869 3870 static inline int qedr_cqe_resp_status_to_ib(u8 status) 3871 { 3872 switch (status) { 3873 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 3874 return IB_WC_LOC_ACCESS_ERR; 3875 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 3876 return IB_WC_LOC_LEN_ERR; 3877 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 3878 return IB_WC_LOC_QP_OP_ERR; 3879 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 3880 return IB_WC_LOC_PROT_ERR; 3881 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 3882 return IB_WC_MW_BIND_ERR; 3883 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 3884 return IB_WC_REM_INV_RD_REQ_ERR; 3885 case RDMA_CQE_RESP_STS_OK: 3886 return IB_WC_SUCCESS; 3887 default: 3888 return IB_WC_GENERAL_ERR; 3889 } 3890 } 3891 3892 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp, 3893 struct ib_wc *wc) 3894 { 3895 wc->status = IB_WC_SUCCESS; 3896 wc->byte_len = le32_to_cpu(resp->length); 3897 3898 if (resp->flags & QEDR_RESP_IMM) { 3899 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key)); 3900 wc->wc_flags |= IB_WC_WITH_IMM; 3901 3902 if (resp->flags & QEDR_RESP_RDMA) 3903 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 3904 3905 if (resp->flags & QEDR_RESP_INV) 3906 return -EINVAL; 3907 3908 } else if (resp->flags & QEDR_RESP_INV) { 3909 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 3910 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3911 3912 if (resp->flags & QEDR_RESP_RDMA) 3913 return -EINVAL; 3914 3915 } else if (resp->flags & QEDR_RESP_RDMA) { 3916 return -EINVAL; 3917 } 3918 3919 return 0; 3920 } 3921 3922 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 3923 struct qedr_cq *cq, struct ib_wc *wc, 3924 struct rdma_cqe_responder *resp, u64 wr_id) 3925 { 3926 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */ 3927 wc->opcode = IB_WC_RECV; 3928 wc->wc_flags = 0; 3929 3930 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) { 3931 if (qedr_set_ok_cqe_resp_wc(resp, wc)) 3932 DP_ERR(dev, 3933 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n", 3934 cq, cq->icid, resp->flags); 3935 3936 } else { 3937 wc->status = qedr_cqe_resp_status_to_ib(resp->status); 3938 if (wc->status == IB_WC_GENERAL_ERR) 3939 DP_ERR(dev, 3940 "CQ %p (icid=%d) contains an invalid CQE status %d\n", 3941 cq, cq->icid, resp->status); 3942 } 3943 3944 /* Fill the rest of the WC */ 3945 wc->vendor_err = 0; 3946 wc->src_qp = qp->id; 3947 wc->qp = &qp->ibqp; 3948 wc->wr_id = wr_id; 3949 } 3950 3951 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp, 3952 struct qedr_cq *cq, struct ib_wc *wc, 3953 struct rdma_cqe_responder *resp) 3954 { 3955 struct qedr_srq *srq = qp->srq; 3956 u64 wr_id; 3957 3958 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi), 3959 le32_to_cpu(resp->srq_wr_id.lo), u64); 3960 3961 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 3962 wc->status = IB_WC_WR_FLUSH_ERR; 3963 wc->vendor_err = 0; 3964 wc->wr_id = wr_id; 3965 wc->byte_len = 0; 3966 wc->src_qp = qp->id; 3967 wc->qp = &qp->ibqp; 3968 wc->wr_id = wr_id; 3969 } else { 3970 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 3971 } 3972 srq->hw_srq.wr_cons_cnt++; 3973 3974 return 1; 3975 } 3976 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 3977 struct qedr_cq *cq, struct ib_wc *wc, 3978 struct rdma_cqe_responder *resp) 3979 { 3980 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 3981 3982 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 3983 3984 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 3985 qed_chain_consume(&qp->rq.pbl); 3986 qedr_inc_sw_cons(&qp->rq); 3987 3988 return 1; 3989 } 3990 3991 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, 3992 int num_entries, struct ib_wc *wc, u16 hw_cons) 3993 { 3994 u16 cnt = 0; 3995 3996 while (num_entries && qp->rq.wqe_cons != hw_cons) { 3997 /* fill WC */ 3998 wc->status = IB_WC_WR_FLUSH_ERR; 3999 wc->vendor_err = 0; 4000 wc->wc_flags = 0; 4001 wc->src_qp = qp->id; 4002 wc->byte_len = 0; 4003 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4004 wc->qp = &qp->ibqp; 4005 num_entries--; 4006 wc++; 4007 cnt++; 4008 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4009 qed_chain_consume(&qp->rq.pbl); 4010 qedr_inc_sw_cons(&qp->rq); 4011 } 4012 4013 return cnt; 4014 } 4015 4016 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4017 struct rdma_cqe_responder *resp, int *update) 4018 { 4019 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) { 4020 consume_cqe(cq); 4021 *update |= 1; 4022 } 4023 } 4024 4025 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4026 struct qedr_cq *cq, int num_entries, 4027 struct ib_wc *wc, 4028 struct rdma_cqe_responder *resp) 4029 { 4030 int cnt; 4031 4032 cnt = process_resp_one_srq(dev, qp, cq, wc, resp); 4033 consume_cqe(cq); 4034 4035 return cnt; 4036 } 4037 4038 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, 4039 struct qedr_cq *cq, int num_entries, 4040 struct ib_wc *wc, struct rdma_cqe_responder *resp, 4041 int *update) 4042 { 4043 int cnt; 4044 4045 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4046 cnt = process_resp_flush(qp, cq, num_entries, wc, 4047 resp->rq_cons_or_srq_id); 4048 try_consume_resp_cqe(cq, qp, resp, update); 4049 } else { 4050 cnt = process_resp_one(dev, qp, cq, wc, resp); 4051 consume_cqe(cq); 4052 *update |= 1; 4053 } 4054 4055 return cnt; 4056 } 4057 4058 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4059 struct rdma_cqe_requester *req, int *update) 4060 { 4061 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 4062 consume_cqe(cq); 4063 *update |= 1; 4064 } 4065 } 4066 4067 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 4068 { 4069 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 4070 struct qedr_cq *cq = get_qedr_cq(ibcq); 4071 union rdma_cqe *cqe; 4072 u32 old_cons, new_cons; 4073 unsigned long flags; 4074 int update = 0; 4075 int done = 0; 4076 4077 if (cq->destroyed) { 4078 DP_ERR(dev, 4079 "warning: poll was invoked after destroy for cq %p (icid=%d)\n", 4080 cq, cq->icid); 4081 return 0; 4082 } 4083 4084 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 4085 return qedr_gsi_poll_cq(ibcq, num_entries, wc); 4086 4087 spin_lock_irqsave(&cq->cq_lock, flags); 4088 cqe = cq->latest_cqe; 4089 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4090 while (num_entries && is_valid_cqe(cq, cqe)) { 4091 struct qedr_qp *qp; 4092 int cnt = 0; 4093 4094 /* prevent speculative reads of any field of CQE */ 4095 rmb(); 4096 4097 qp = cqe_get_qp(cqe); 4098 if (!qp) { 4099 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe); 4100 break; 4101 } 4102 4103 wc->qp = &qp->ibqp; 4104 4105 switch (cqe_get_type(cqe)) { 4106 case RDMA_CQE_TYPE_REQUESTER: 4107 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc, 4108 &cqe->req); 4109 try_consume_req_cqe(cq, qp, &cqe->req, &update); 4110 break; 4111 case RDMA_CQE_TYPE_RESPONDER_RQ: 4112 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc, 4113 &cqe->resp, &update); 4114 break; 4115 case RDMA_CQE_TYPE_RESPONDER_SRQ: 4116 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries, 4117 wc, &cqe->resp); 4118 update = 1; 4119 break; 4120 case RDMA_CQE_TYPE_INVALID: 4121 default: 4122 DP_ERR(dev, "Error: invalid CQE type = %d\n", 4123 cqe_get_type(cqe)); 4124 } 4125 num_entries -= cnt; 4126 wc += cnt; 4127 done += cnt; 4128 4129 cqe = get_cqe(cq); 4130 } 4131 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4132 4133 cq->cq_cons += new_cons - old_cons; 4134 4135 if (update) 4136 /* doorbell notifies abount latest VALID entry, 4137 * but chain already point to the next INVALID one 4138 */ 4139 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 4140 4141 spin_unlock_irqrestore(&cq->cq_lock, flags); 4142 return done; 4143 } 4144 4145 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, 4146 u8 port_num, 4147 const struct ib_wc *in_wc, 4148 const struct ib_grh *in_grh, 4149 const struct ib_mad_hdr *mad_hdr, 4150 size_t in_mad_size, struct ib_mad_hdr *out_mad, 4151 size_t *out_mad_size, u16 *out_mad_pkey_index) 4152 { 4153 struct qedr_dev *dev = get_qedr_dev(ibdev); 4154 4155 DP_DEBUG(dev, QEDR_MSG_GSI, 4156 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n", 4157 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod, 4158 mad_hdr->class_specific, mad_hdr->class_version, 4159 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status); 4160 return IB_MAD_RESULT_SUCCESS; 4161 } 4162