1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/dma-mapping.h> 33 #include <linux/crc32.h> 34 #include <net/ip.h> 35 #include <net/ipv6.h> 36 #include <net/udp.h> 37 #include <linux/iommu.h> 38 39 #include <rdma/ib_verbs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/iw_cm.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/ib_addr.h> 44 #include <rdma/ib_cache.h> 45 #include <rdma/uverbs_ioctl.h> 46 47 #include <linux/qed/common_hsi.h> 48 #include "qedr_hsi_rdma.h" 49 #include <linux/qed/qed_if.h> 50 #include "qedr.h" 51 #include "verbs.h" 52 #include <rdma/qedr-abi.h> 53 #include "qedr_roce_cm.h" 54 #include "qedr_iw_cm.h" 55 56 #define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm) 57 #define RDMA_MAX_SGE_PER_SRQ (4) 58 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) 59 60 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 61 62 enum { 63 QEDR_USER_MMAP_IO_WC = 0, 64 QEDR_USER_MMAP_PHYS_PAGE, 65 }; 66 67 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, 68 size_t len) 69 { 70 size_t min_len = min_t(size_t, len, udata->outlen); 71 72 return ib_copy_to_udata(udata, src, min_len); 73 } 74 75 int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) 76 { 77 if (index >= QEDR_ROCE_PKEY_TABLE_LEN) 78 return -EINVAL; 79 80 *pkey = QEDR_ROCE_PKEY_DEFAULT; 81 return 0; 82 } 83 84 int qedr_iw_query_gid(struct ib_device *ibdev, u32 port, 85 int index, union ib_gid *sgid) 86 { 87 struct qedr_dev *dev = get_qedr_dev(ibdev); 88 89 memset(sgid->raw, 0, sizeof(sgid->raw)); 90 ether_addr_copy(sgid->raw, dev->ndev->dev_addr); 91 92 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index, 93 sgid->global.interface_id, sgid->global.subnet_prefix); 94 95 return 0; 96 } 97 98 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 99 { 100 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 101 struct qedr_device_attr *qattr = &dev->attr; 102 struct qedr_srq *srq = get_qedr_srq(ibsrq); 103 104 srq_attr->srq_limit = srq->srq_limit; 105 srq_attr->max_wr = qattr->max_srq_wr; 106 srq_attr->max_sge = qattr->max_sge; 107 108 return 0; 109 } 110 111 int qedr_query_device(struct ib_device *ibdev, 112 struct ib_device_attr *attr, struct ib_udata *udata) 113 { 114 struct qedr_dev *dev = get_qedr_dev(ibdev); 115 struct qedr_device_attr *qattr = &dev->attr; 116 117 if (!dev->rdma_ctx) { 118 DP_ERR(dev, 119 "qedr_query_device called with invalid params rdma_ctx=%p\n", 120 dev->rdma_ctx); 121 return -EINVAL; 122 } 123 124 memset(attr, 0, sizeof(*attr)); 125 126 attr->fw_ver = qattr->fw_ver; 127 attr->sys_image_guid = qattr->sys_image_guid; 128 attr->max_mr_size = qattr->max_mr_size; 129 attr->page_size_cap = qattr->page_size_caps; 130 attr->vendor_id = qattr->vendor_id; 131 attr->vendor_part_id = qattr->vendor_part_id; 132 attr->hw_ver = qattr->hw_ver; 133 attr->max_qp = qattr->max_qp; 134 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe); 135 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 136 IB_DEVICE_RC_RNR_NAK_GEN | 137 IB_DEVICE_MEM_MGT_EXTENSIONS; 138 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 139 140 if (!rdma_protocol_iwarp(&dev->ibdev, 1)) 141 attr->device_cap_flags |= IB_DEVICE_XRC; 142 attr->max_send_sge = qattr->max_sge; 143 attr->max_recv_sge = qattr->max_sge; 144 attr->max_sge_rd = qattr->max_sge; 145 attr->max_cq = qattr->max_cq; 146 attr->max_cqe = qattr->max_cqe; 147 attr->max_mr = qattr->max_mr; 148 attr->max_mw = qattr->max_mw; 149 attr->max_pd = qattr->max_pd; 150 attr->atomic_cap = dev->atomic_cap; 151 attr->max_qp_init_rd_atom = 152 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 153 attr->max_qp_rd_atom = 154 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 155 attr->max_qp_init_rd_atom); 156 157 attr->max_srq = qattr->max_srq; 158 attr->max_srq_sge = qattr->max_srq_sge; 159 attr->max_srq_wr = qattr->max_srq_wr; 160 161 attr->local_ca_ack_delay = qattr->dev_ack_delay; 162 attr->max_fast_reg_page_list_len = qattr->max_mr / 8; 163 attr->max_pkeys = qattr->max_pkey; 164 attr->max_ah = qattr->max_ah; 165 166 return 0; 167 } 168 169 static inline void get_link_speed_and_width(int speed, u16 *ib_speed, 170 u8 *ib_width) 171 { 172 switch (speed) { 173 case 1000: 174 *ib_speed = IB_SPEED_SDR; 175 *ib_width = IB_WIDTH_1X; 176 break; 177 case 10000: 178 *ib_speed = IB_SPEED_QDR; 179 *ib_width = IB_WIDTH_1X; 180 break; 181 182 case 20000: 183 *ib_speed = IB_SPEED_DDR; 184 *ib_width = IB_WIDTH_4X; 185 break; 186 187 case 25000: 188 *ib_speed = IB_SPEED_EDR; 189 *ib_width = IB_WIDTH_1X; 190 break; 191 192 case 40000: 193 *ib_speed = IB_SPEED_QDR; 194 *ib_width = IB_WIDTH_4X; 195 break; 196 197 case 50000: 198 *ib_speed = IB_SPEED_HDR; 199 *ib_width = IB_WIDTH_1X; 200 break; 201 202 case 100000: 203 *ib_speed = IB_SPEED_EDR; 204 *ib_width = IB_WIDTH_4X; 205 break; 206 207 default: 208 /* Unsupported */ 209 *ib_speed = IB_SPEED_SDR; 210 *ib_width = IB_WIDTH_1X; 211 } 212 } 213 214 int qedr_query_port(struct ib_device *ibdev, u32 port, 215 struct ib_port_attr *attr) 216 { 217 struct qedr_dev *dev; 218 struct qed_rdma_port *rdma_port; 219 220 dev = get_qedr_dev(ibdev); 221 222 if (!dev->rdma_ctx) { 223 DP_ERR(dev, "rdma_ctx is NULL\n"); 224 return -EINVAL; 225 } 226 227 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx); 228 229 /* *attr being zeroed by the caller, avoid zeroing it here */ 230 if (rdma_port->port_state == QED_RDMA_PORT_UP) { 231 attr->state = IB_PORT_ACTIVE; 232 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 233 } else { 234 attr->state = IB_PORT_DOWN; 235 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 236 } 237 attr->max_mtu = IB_MTU_4096; 238 attr->lid = 0; 239 attr->lmc = 0; 240 attr->sm_lid = 0; 241 attr->sm_sl = 0; 242 attr->ip_gids = true; 243 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 244 attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu); 245 attr->gid_tbl_len = 1; 246 } else { 247 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); 248 attr->gid_tbl_len = QEDR_MAX_SGID; 249 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; 250 } 251 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 252 attr->qkey_viol_cntr = 0; 253 get_link_speed_and_width(rdma_port->link_speed, 254 &attr->active_speed, &attr->active_width); 255 attr->max_msg_sz = rdma_port->max_msg_size; 256 attr->max_vl_num = 4; 257 258 return 0; 259 } 260 261 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 262 { 263 struct ib_device *ibdev = uctx->device; 264 int rc; 265 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx); 266 struct qedr_alloc_ucontext_resp uresp = {}; 267 struct qedr_alloc_ucontext_req ureq = {}; 268 struct qedr_dev *dev = get_qedr_dev(ibdev); 269 struct qed_rdma_add_user_out_params oparams; 270 struct qedr_user_mmap_entry *entry; 271 272 if (!udata) 273 return -EFAULT; 274 275 if (udata->inlen) { 276 rc = ib_copy_from_udata(&ureq, udata, 277 min(sizeof(ureq), udata->inlen)); 278 if (rc) { 279 DP_ERR(dev, "Problem copying data from user space\n"); 280 return -EFAULT; 281 } 282 ctx->edpm_mode = !!(ureq.context_flags & 283 QEDR_ALLOC_UCTX_EDPM_MODE); 284 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC); 285 } 286 287 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams); 288 if (rc) { 289 DP_ERR(dev, 290 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n", 291 rc); 292 return rc; 293 } 294 295 ctx->dpi = oparams.dpi; 296 ctx->dpi_addr = oparams.dpi_addr; 297 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 298 ctx->dpi_size = oparams.dpi_size; 299 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 300 if (!entry) { 301 rc = -ENOMEM; 302 goto err; 303 } 304 305 entry->io_address = ctx->dpi_phys_addr; 306 entry->length = ctx->dpi_size; 307 entry->mmap_flag = QEDR_USER_MMAP_IO_WC; 308 entry->dpi = ctx->dpi; 309 entry->dev = dev; 310 rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry, 311 ctx->dpi_size); 312 if (rc) { 313 kfree(entry); 314 goto err; 315 } 316 ctx->db_mmap_entry = &entry->rdma_entry; 317 318 if (!dev->user_dpm_enabled) 319 uresp.dpm_flags = 0; 320 else if (rdma_protocol_iwarp(&dev->ibdev, 1)) 321 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY; 322 else 323 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED | 324 QEDR_DPM_TYPE_ROCE_LEGACY | 325 QEDR_DPM_TYPE_ROCE_EDPM_MODE; 326 327 if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) { 328 uresp.dpm_flags |= QEDR_DPM_SIZES_SET; 329 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE; 330 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE; 331 uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE; 332 } 333 334 uresp.wids_enabled = 1; 335 uresp.wid_count = oparams.wid_count; 336 uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry); 337 uresp.db_size = ctx->dpi_size; 338 uresp.max_send_wr = dev->attr.max_sqe; 339 uresp.max_recv_wr = dev->attr.max_rqe; 340 uresp.max_srq_wr = dev->attr.max_srq_wr; 341 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE; 342 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE; 343 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 344 uresp.max_cqes = QEDR_MAX_CQES; 345 346 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 347 if (rc) 348 goto err; 349 350 ctx->dev = dev; 351 352 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n", 353 &ctx->ibucontext); 354 return 0; 355 356 err: 357 if (!ctx->db_mmap_entry) 358 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi); 359 else 360 rdma_user_mmap_entry_remove(ctx->db_mmap_entry); 361 362 return rc; 363 } 364 365 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx) 366 { 367 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx); 368 369 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n", 370 uctx); 371 372 rdma_user_mmap_entry_remove(uctx->db_mmap_entry); 373 } 374 375 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 376 { 377 struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry); 378 struct qedr_dev *dev = entry->dev; 379 380 if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE) 381 free_page((unsigned long)entry->address); 382 else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC) 383 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi); 384 385 kfree(entry); 386 } 387 388 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma) 389 { 390 struct ib_device *dev = ucontext->device; 391 size_t length = vma->vm_end - vma->vm_start; 392 struct rdma_user_mmap_entry *rdma_entry; 393 struct qedr_user_mmap_entry *entry; 394 int rc = 0; 395 u64 pfn; 396 397 ibdev_dbg(dev, 398 "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n", 399 vma->vm_start, vma->vm_end, length, vma->vm_pgoff); 400 401 rdma_entry = rdma_user_mmap_entry_get(ucontext, vma); 402 if (!rdma_entry) { 403 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n", 404 vma->vm_pgoff); 405 return -EINVAL; 406 } 407 entry = get_qedr_mmap_entry(rdma_entry); 408 ibdev_dbg(dev, 409 "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n", 410 entry->io_address, length, entry->mmap_flag); 411 412 switch (entry->mmap_flag) { 413 case QEDR_USER_MMAP_IO_WC: 414 pfn = entry->io_address >> PAGE_SHIFT; 415 rc = rdma_user_mmap_io(ucontext, vma, pfn, length, 416 pgprot_writecombine(vma->vm_page_prot), 417 rdma_entry); 418 break; 419 case QEDR_USER_MMAP_PHYS_PAGE: 420 rc = vm_insert_page(vma, vma->vm_start, 421 virt_to_page(entry->address)); 422 break; 423 default: 424 rc = -EINVAL; 425 } 426 427 if (rc) 428 ibdev_dbg(dev, 429 "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n", 430 entry->io_address, length, entry->mmap_flag, rc); 431 432 rdma_user_mmap_entry_put(rdma_entry); 433 return rc; 434 } 435 436 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 437 { 438 struct ib_device *ibdev = ibpd->device; 439 struct qedr_dev *dev = get_qedr_dev(ibdev); 440 struct qedr_pd *pd = get_qedr_pd(ibpd); 441 u16 pd_id; 442 int rc; 443 444 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n", 445 udata ? "User Lib" : "Kernel"); 446 447 if (!dev->rdma_ctx) { 448 DP_ERR(dev, "invalid RDMA context\n"); 449 return -EINVAL; 450 } 451 452 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id); 453 if (rc) 454 return rc; 455 456 pd->pd_id = pd_id; 457 458 if (udata) { 459 struct qedr_alloc_pd_uresp uresp = { 460 .pd_id = pd_id, 461 }; 462 struct qedr_ucontext *context = rdma_udata_to_drv_context( 463 udata, struct qedr_ucontext, ibucontext); 464 465 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 466 if (rc) { 467 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); 468 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); 469 return rc; 470 } 471 472 pd->uctx = context; 473 pd->uctx->pd = pd; 474 } 475 476 return 0; 477 } 478 479 int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 480 { 481 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 482 struct qedr_pd *pd = get_qedr_pd(ibpd); 483 484 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id); 485 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id); 486 return 0; 487 } 488 489 490 int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) 491 { 492 struct qedr_dev *dev = get_qedr_dev(ibxrcd->device); 493 struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd); 494 495 return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id); 496 } 497 498 int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) 499 { 500 struct qedr_dev *dev = get_qedr_dev(ibxrcd->device); 501 u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id; 502 503 dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id); 504 return 0; 505 } 506 static void qedr_free_pbl(struct qedr_dev *dev, 507 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) 508 { 509 struct pci_dev *pdev = dev->pdev; 510 int i; 511 512 for (i = 0; i < pbl_info->num_pbls; i++) { 513 if (!pbl[i].va) 514 continue; 515 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 516 pbl[i].va, pbl[i].pa); 517 } 518 519 kfree(pbl); 520 } 521 522 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024) 523 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024) 524 525 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 526 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 527 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE) 528 529 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev, 530 struct qedr_pbl_info *pbl_info, 531 gfp_t flags) 532 { 533 struct pci_dev *pdev = dev->pdev; 534 struct qedr_pbl *pbl_table; 535 dma_addr_t *pbl_main_tbl; 536 dma_addr_t pa; 537 void *va; 538 int i; 539 540 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags); 541 if (!pbl_table) 542 return ERR_PTR(-ENOMEM); 543 544 for (i = 0; i < pbl_info->num_pbls; i++) { 545 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa, 546 flags); 547 if (!va) 548 goto err; 549 550 pbl_table[i].va = va; 551 pbl_table[i].pa = pa; 552 } 553 554 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 555 * the first one with physical pointers to all of the rest 556 */ 557 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 558 for (i = 0; i < pbl_info->num_pbls - 1; i++) 559 pbl_main_tbl[i] = pbl_table[i + 1].pa; 560 561 return pbl_table; 562 563 err: 564 for (i--; i >= 0; i--) 565 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 566 pbl_table[i].va, pbl_table[i].pa); 567 568 qedr_free_pbl(dev, pbl_info, pbl_table); 569 570 return ERR_PTR(-ENOMEM); 571 } 572 573 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev, 574 struct qedr_pbl_info *pbl_info, 575 u32 num_pbes, int two_layer_capable) 576 { 577 u32 pbl_capacity; 578 u32 pbl_size; 579 u32 num_pbls; 580 581 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 582 if (num_pbes > MAX_PBES_TWO_LAYER) { 583 DP_ERR(dev, "prepare pbl table: too many pages %d\n", 584 num_pbes); 585 return -EINVAL; 586 } 587 588 /* calculate required pbl page size */ 589 pbl_size = MIN_FW_PBL_PAGE_SIZE; 590 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 591 NUM_PBES_ON_PAGE(pbl_size); 592 593 while (pbl_capacity < num_pbes) { 594 pbl_size *= 2; 595 pbl_capacity = pbl_size / sizeof(u64); 596 pbl_capacity = pbl_capacity * pbl_capacity; 597 } 598 599 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 600 num_pbls++; /* One for the layer0 ( points to the pbls) */ 601 pbl_info->two_layered = true; 602 } else { 603 /* One layered PBL */ 604 num_pbls = 1; 605 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, 606 roundup_pow_of_two((num_pbes * sizeof(u64)))); 607 pbl_info->two_layered = false; 608 } 609 610 pbl_info->num_pbls = num_pbls; 611 pbl_info->pbl_size = pbl_size; 612 pbl_info->num_pbes = num_pbes; 613 614 DP_DEBUG(dev, QEDR_MSG_MR, 615 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n", 616 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 617 618 return 0; 619 } 620 621 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, 622 struct qedr_pbl *pbl, 623 struct qedr_pbl_info *pbl_info, u32 pg_shift) 624 { 625 int pbe_cnt, total_num_pbes = 0; 626 struct qedr_pbl *pbl_tbl; 627 struct ib_block_iter biter; 628 struct regpair *pbe; 629 630 if (!pbl_info->num_pbes) 631 return; 632 633 /* If we have a two layered pbl, the first pbl points to the rest 634 * of the pbls and the first entry lays on the second pbl in the table 635 */ 636 if (pbl_info->two_layered) 637 pbl_tbl = &pbl[1]; 638 else 639 pbl_tbl = pbl; 640 641 pbe = (struct regpair *)pbl_tbl->va; 642 if (!pbe) { 643 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n"); 644 return; 645 } 646 647 pbe_cnt = 0; 648 649 rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) { 650 u64 pg_addr = rdma_block_iter_dma_address(&biter); 651 652 pbe->lo = cpu_to_le32(pg_addr); 653 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); 654 655 pbe_cnt++; 656 total_num_pbes++; 657 pbe++; 658 659 if (total_num_pbes == pbl_info->num_pbes) 660 return; 661 662 /* If the given pbl is full storing the pbes, move to next pbl. 663 */ 664 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { 665 pbl_tbl++; 666 pbe = (struct regpair *)pbl_tbl->va; 667 pbe_cnt = 0; 668 } 669 } 670 } 671 672 static int qedr_db_recovery_add(struct qedr_dev *dev, 673 void __iomem *db_addr, 674 void *db_data, 675 enum qed_db_rec_width db_width, 676 enum qed_db_rec_space db_space) 677 { 678 if (!db_data) { 679 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n"); 680 return 0; 681 } 682 683 return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data, 684 db_width, db_space); 685 } 686 687 static void qedr_db_recovery_del(struct qedr_dev *dev, 688 void __iomem *db_addr, 689 void *db_data) 690 { 691 if (!db_data) { 692 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n"); 693 return; 694 } 695 696 /* Ignore return code as there is not much we can do about it. Error 697 * log will be printed inside. 698 */ 699 dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data); 700 } 701 702 static int qedr_copy_cq_uresp(struct qedr_dev *dev, 703 struct qedr_cq *cq, struct ib_udata *udata, 704 u32 db_offset) 705 { 706 struct qedr_create_cq_uresp uresp; 707 int rc; 708 709 memset(&uresp, 0, sizeof(uresp)); 710 711 uresp.db_offset = db_offset; 712 uresp.icid = cq->icid; 713 if (cq->q.db_mmap_entry) 714 uresp.db_rec_addr = 715 rdma_user_mmap_get_offset(cq->q.db_mmap_entry); 716 717 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 718 if (rc) 719 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); 720 721 return rc; 722 } 723 724 static void consume_cqe(struct qedr_cq *cq) 725 { 726 if (cq->latest_cqe == cq->toggle_cqe) 727 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 728 729 cq->latest_cqe = qed_chain_consume(&cq->pbl); 730 } 731 732 static inline int qedr_align_cq_entries(int entries) 733 { 734 u64 size, aligned_size; 735 736 /* We allocate an extra entry that we don't report to the FW. */ 737 size = (entries + 1) * QEDR_CQE_SIZE; 738 aligned_size = ALIGN(size, PAGE_SIZE); 739 740 return aligned_size / QEDR_CQE_SIZE; 741 } 742 743 static int qedr_init_user_db_rec(struct ib_udata *udata, 744 struct qedr_dev *dev, struct qedr_userq *q, 745 bool requires_db_rec) 746 { 747 struct qedr_ucontext *uctx = 748 rdma_udata_to_drv_context(udata, struct qedr_ucontext, 749 ibucontext); 750 struct qedr_user_mmap_entry *entry; 751 int rc; 752 753 /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */ 754 if (requires_db_rec == 0 || !uctx->db_rec) 755 return 0; 756 757 /* Allocate a page for doorbell recovery, add to mmap */ 758 q->db_rec_data = (void *)get_zeroed_page(GFP_USER); 759 if (!q->db_rec_data) { 760 DP_ERR(dev, "get_zeroed_page failed\n"); 761 return -ENOMEM; 762 } 763 764 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 765 if (!entry) 766 goto err_free_db_data; 767 768 entry->address = q->db_rec_data; 769 entry->length = PAGE_SIZE; 770 entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE; 771 rc = rdma_user_mmap_entry_insert(&uctx->ibucontext, 772 &entry->rdma_entry, 773 PAGE_SIZE); 774 if (rc) 775 goto err_free_entry; 776 777 q->db_mmap_entry = &entry->rdma_entry; 778 779 return 0; 780 781 err_free_entry: 782 kfree(entry); 783 784 err_free_db_data: 785 free_page((unsigned long)q->db_rec_data); 786 q->db_rec_data = NULL; 787 return -ENOMEM; 788 } 789 790 static inline int qedr_init_user_queue(struct ib_udata *udata, 791 struct qedr_dev *dev, 792 struct qedr_userq *q, u64 buf_addr, 793 size_t buf_len, bool requires_db_rec, 794 int access, 795 int alloc_and_init) 796 { 797 u32 fw_pages; 798 int rc; 799 800 q->buf_addr = buf_addr; 801 q->buf_len = buf_len; 802 q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access); 803 if (IS_ERR(q->umem)) { 804 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", 805 PTR_ERR(q->umem)); 806 return PTR_ERR(q->umem); 807 } 808 809 fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT); 810 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); 811 if (rc) 812 goto err0; 813 814 if (alloc_and_init) { 815 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 816 if (IS_ERR(q->pbl_tbl)) { 817 rc = PTR_ERR(q->pbl_tbl); 818 goto err0; 819 } 820 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info, 821 FW_PAGE_SHIFT); 822 } else { 823 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); 824 if (!q->pbl_tbl) { 825 rc = -ENOMEM; 826 goto err0; 827 } 828 } 829 830 /* mmap the user address used to store doorbell data for recovery */ 831 return qedr_init_user_db_rec(udata, dev, q, requires_db_rec); 832 833 err0: 834 ib_umem_release(q->umem); 835 q->umem = NULL; 836 837 return rc; 838 } 839 840 static inline void qedr_init_cq_params(struct qedr_cq *cq, 841 struct qedr_ucontext *ctx, 842 struct qedr_dev *dev, int vector, 843 int chain_entries, int page_cnt, 844 u64 pbl_ptr, 845 struct qed_rdma_create_cq_in_params 846 *params) 847 { 848 memset(params, 0, sizeof(*params)); 849 params->cq_handle_hi = upper_32_bits((uintptr_t)cq); 850 params->cq_handle_lo = lower_32_bits((uintptr_t)cq); 851 params->cnq_id = vector; 852 params->cq_size = chain_entries - 1; 853 params->dpi = (ctx) ? ctx->dpi : dev->dpi; 854 params->pbl_num_pages = page_cnt; 855 params->pbl_ptr = pbl_ptr; 856 params->pbl_two_level = 0; 857 } 858 859 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags) 860 { 861 cq->db.data.agg_flags = flags; 862 cq->db.data.value = cpu_to_le32(cons); 863 writeq(cq->db.raw, cq->db_addr); 864 } 865 866 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 867 { 868 struct qedr_cq *cq = get_qedr_cq(ibcq); 869 unsigned long sflags; 870 struct qedr_dev *dev; 871 872 dev = get_qedr_dev(ibcq->device); 873 874 if (cq->destroyed) { 875 DP_ERR(dev, 876 "warning: arm was invoked after destroy for cq %p (icid=%d)\n", 877 cq, cq->icid); 878 return -EINVAL; 879 } 880 881 882 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 883 return 0; 884 885 spin_lock_irqsave(&cq->cq_lock, sflags); 886 887 cq->arm_flags = 0; 888 889 if (flags & IB_CQ_SOLICITED) 890 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 891 892 if (flags & IB_CQ_NEXT_COMP) 893 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 894 895 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 896 897 spin_unlock_irqrestore(&cq->cq_lock, sflags); 898 899 return 0; 900 } 901 902 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 903 struct ib_udata *udata) 904 { 905 struct ib_device *ibdev = ibcq->device; 906 struct qedr_ucontext *ctx = rdma_udata_to_drv_context( 907 udata, struct qedr_ucontext, ibucontext); 908 struct qed_rdma_destroy_cq_out_params destroy_oparams; 909 struct qed_rdma_destroy_cq_in_params destroy_iparams; 910 struct qed_chain_init_params chain_params = { 911 .mode = QED_CHAIN_MODE_PBL, 912 .intended_use = QED_CHAIN_USE_TO_CONSUME, 913 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 914 .elem_size = sizeof(union rdma_cqe), 915 }; 916 struct qedr_dev *dev = get_qedr_dev(ibdev); 917 struct qed_rdma_create_cq_in_params params; 918 struct qedr_create_cq_ureq ureq = {}; 919 int vector = attr->comp_vector; 920 int entries = attr->cqe; 921 struct qedr_cq *cq = get_qedr_cq(ibcq); 922 int chain_entries; 923 u32 db_offset; 924 int page_cnt; 925 u64 pbl_ptr; 926 u16 icid; 927 int rc; 928 929 DP_DEBUG(dev, QEDR_MSG_INIT, 930 "create_cq: called from %s. entries=%d, vector=%d\n", 931 udata ? "User Lib" : "Kernel", entries, vector); 932 933 if (attr->flags) 934 return -EOPNOTSUPP; 935 936 if (entries > QEDR_MAX_CQES) { 937 DP_ERR(dev, 938 "create cq: the number of entries %d is too high. Must be equal or below %d.\n", 939 entries, QEDR_MAX_CQES); 940 return -EINVAL; 941 } 942 943 chain_entries = qedr_align_cq_entries(entries); 944 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); 945 chain_params.num_elems = chain_entries; 946 947 /* calc db offset. user will add DPI base, kernel will add db addr */ 948 db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 949 950 if (udata) { 951 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 952 udata->inlen))) { 953 DP_ERR(dev, 954 "create cq: problem copying data from user space\n"); 955 goto err0; 956 } 957 958 if (!ureq.len) { 959 DP_ERR(dev, 960 "create cq: cannot create a cq with 0 entries\n"); 961 goto err0; 962 } 963 964 cq->cq_type = QEDR_CQ_TYPE_USER; 965 966 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr, 967 ureq.len, true, IB_ACCESS_LOCAL_WRITE, 968 1); 969 if (rc) 970 goto err0; 971 972 pbl_ptr = cq->q.pbl_tbl->pa; 973 page_cnt = cq->q.pbl_info.num_pbes; 974 975 cq->ibcq.cqe = chain_entries; 976 cq->q.db_addr = ctx->dpi_addr + db_offset; 977 } else { 978 cq->cq_type = QEDR_CQ_TYPE_KERNEL; 979 980 rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl, 981 &chain_params); 982 if (rc) 983 goto err0; 984 985 page_cnt = qed_chain_get_page_cnt(&cq->pbl); 986 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); 987 cq->ibcq.cqe = cq->pbl.capacity; 988 } 989 990 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, 991 pbl_ptr, ¶ms); 992 993 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 994 if (rc) 995 goto err1; 996 997 cq->icid = icid; 998 cq->sig = QEDR_CQ_MAGIC_NUMBER; 999 spin_lock_init(&cq->cq_lock); 1000 1001 if (udata) { 1002 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset); 1003 if (rc) 1004 goto err2; 1005 1006 rc = qedr_db_recovery_add(dev, cq->q.db_addr, 1007 &cq->q.db_rec_data->db_data, 1008 DB_REC_WIDTH_64B, 1009 DB_REC_USER); 1010 if (rc) 1011 goto err2; 1012 1013 } else { 1014 /* Generate doorbell address. */ 1015 cq->db.data.icid = cq->icid; 1016 cq->db_addr = dev->db_addr + db_offset; 1017 cq->db.data.params = DB_AGG_CMD_MAX << 1018 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 1019 1020 /* point to the very last element, passing it we will toggle */ 1021 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); 1022 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 1023 cq->latest_cqe = NULL; 1024 consume_cqe(cq); 1025 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 1026 1027 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data, 1028 DB_REC_WIDTH_64B, DB_REC_KERNEL); 1029 if (rc) 1030 goto err2; 1031 } 1032 1033 DP_DEBUG(dev, QEDR_MSG_CQ, 1034 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", 1035 cq->icid, cq, params.cq_size); 1036 1037 return 0; 1038 1039 err2: 1040 destroy_iparams.icid = cq->icid; 1041 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, 1042 &destroy_oparams); 1043 err1: 1044 if (udata) { 1045 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1046 ib_umem_release(cq->q.umem); 1047 if (cq->q.db_mmap_entry) 1048 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry); 1049 } else { 1050 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1051 } 1052 err0: 1053 return -EINVAL; 1054 } 1055 1056 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) 1057 #define QEDR_DESTROY_CQ_ITER_DURATION (10) 1058 1059 int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1060 { 1061 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1062 struct qed_rdma_destroy_cq_out_params oparams; 1063 struct qed_rdma_destroy_cq_in_params iparams; 1064 struct qedr_cq *cq = get_qedr_cq(ibcq); 1065 int iter; 1066 1067 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); 1068 1069 cq->destroyed = 1; 1070 1071 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 1072 if (cq->cq_type == QEDR_CQ_TYPE_GSI) { 1073 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data); 1074 return 0; 1075 } 1076 1077 iparams.icid = cq->icid; 1078 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 1079 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1080 1081 if (udata) { 1082 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1083 ib_umem_release(cq->q.umem); 1084 1085 if (cq->q.db_rec_data) { 1086 qedr_db_recovery_del(dev, cq->q.db_addr, 1087 &cq->q.db_rec_data->db_data); 1088 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry); 1089 } 1090 } else { 1091 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data); 1092 } 1093 1094 /* We don't want the IRQ handler to handle a non-existing CQ so we 1095 * wait until all CNQ interrupts, if any, are received. This will always 1096 * happen and will always happen very fast. If not, then a serious error 1097 * has occured. That is why we can use a long delay. 1098 * We spin for a short time so we don’t lose time on context switching 1099 * in case all the completions are handled in that span. Otherwise 1100 * we sleep for a while and check again. Since the CNQ may be 1101 * associated with (only) the current CPU we use msleep to allow the 1102 * current CPU to be freed. 1103 * The CNQ notification is increased in qedr_irq_handler(). 1104 */ 1105 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1106 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1107 udelay(QEDR_DESTROY_CQ_ITER_DURATION); 1108 iter--; 1109 } 1110 1111 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1112 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1113 msleep(QEDR_DESTROY_CQ_ITER_DURATION); 1114 iter--; 1115 } 1116 1117 /* Note that we don't need to have explicit code to wait for the 1118 * completion of the event handler because it is invoked from the EQ. 1119 * Since the destroy CQ ramrod has also been received on the EQ we can 1120 * be certain that there's no event handler in process. 1121 */ 1122 return 0; 1123 } 1124 1125 static inline int get_gid_info_from_table(struct ib_qp *ibqp, 1126 struct ib_qp_attr *attr, 1127 int attr_mask, 1128 struct qed_rdma_modify_qp_in_params 1129 *qp_params) 1130 { 1131 const struct ib_gid_attr *gid_attr; 1132 enum rdma_network_type nw_type; 1133 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1134 u32 ipv4_addr; 1135 int ret; 1136 int i; 1137 1138 gid_attr = grh->sgid_attr; 1139 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL); 1140 if (ret) 1141 return ret; 1142 1143 nw_type = rdma_gid_attr_network_type(gid_attr); 1144 switch (nw_type) { 1145 case RDMA_NETWORK_IPV6: 1146 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1147 sizeof(qp_params->sgid)); 1148 memcpy(&qp_params->dgid.bytes[0], 1149 &grh->dgid, 1150 sizeof(qp_params->dgid)); 1151 qp_params->roce_mode = ROCE_V2_IPV6; 1152 SET_FIELD(qp_params->modify_flags, 1153 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1154 break; 1155 case RDMA_NETWORK_ROCE_V1: 1156 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1157 sizeof(qp_params->sgid)); 1158 memcpy(&qp_params->dgid.bytes[0], 1159 &grh->dgid, 1160 sizeof(qp_params->dgid)); 1161 qp_params->roce_mode = ROCE_V1; 1162 break; 1163 case RDMA_NETWORK_IPV4: 1164 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid)); 1165 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid)); 1166 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw); 1167 qp_params->sgid.ipv4_addr = ipv4_addr; 1168 ipv4_addr = 1169 qedr_get_ipv4_from_gid(grh->dgid.raw); 1170 qp_params->dgid.ipv4_addr = ipv4_addr; 1171 SET_FIELD(qp_params->modify_flags, 1172 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1173 qp_params->roce_mode = ROCE_V2_IPV4; 1174 break; 1175 default: 1176 return -EINVAL; 1177 } 1178 1179 for (i = 0; i < 4; i++) { 1180 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 1181 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 1182 } 1183 1184 if (qp_params->vlan_id >= VLAN_CFI_MASK) 1185 qp_params->vlan_id = 0; 1186 1187 return 0; 1188 } 1189 1190 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, 1191 struct ib_qp_init_attr *attrs, 1192 struct ib_udata *udata) 1193 { 1194 struct qedr_device_attr *qattr = &dev->attr; 1195 1196 /* QP0... attrs->qp_type == IB_QPT_GSI */ 1197 if (attrs->qp_type != IB_QPT_RC && 1198 attrs->qp_type != IB_QPT_GSI && 1199 attrs->qp_type != IB_QPT_XRC_INI && 1200 attrs->qp_type != IB_QPT_XRC_TGT) { 1201 DP_DEBUG(dev, QEDR_MSG_QP, 1202 "create qp: unsupported qp type=0x%x requested\n", 1203 attrs->qp_type); 1204 return -EOPNOTSUPP; 1205 } 1206 1207 if (attrs->cap.max_send_wr > qattr->max_sqe) { 1208 DP_ERR(dev, 1209 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n", 1210 attrs->cap.max_send_wr, qattr->max_sqe); 1211 return -EINVAL; 1212 } 1213 1214 if (attrs->cap.max_inline_data > qattr->max_inline) { 1215 DP_ERR(dev, 1216 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n", 1217 attrs->cap.max_inline_data, qattr->max_inline); 1218 return -EINVAL; 1219 } 1220 1221 if (attrs->cap.max_send_sge > qattr->max_sge) { 1222 DP_ERR(dev, 1223 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n", 1224 attrs->cap.max_send_sge, qattr->max_sge); 1225 return -EINVAL; 1226 } 1227 1228 if (attrs->cap.max_recv_sge > qattr->max_sge) { 1229 DP_ERR(dev, 1230 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n", 1231 attrs->cap.max_recv_sge, qattr->max_sge); 1232 return -EINVAL; 1233 } 1234 1235 /* verify consumer QPs are not trying to use GSI QP's CQ. 1236 * TGT QP isn't associated with RQ/SQ 1237 */ 1238 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) && 1239 (attrs->qp_type != IB_QPT_XRC_TGT) && 1240 (attrs->qp_type != IB_QPT_XRC_INI)) { 1241 struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq); 1242 struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq); 1243 1244 if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) || 1245 (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) { 1246 DP_ERR(dev, 1247 "create qp: consumer QP cannot use GSI CQs.\n"); 1248 return -EINVAL; 1249 } 1250 } 1251 1252 return 0; 1253 } 1254 1255 static int qedr_copy_srq_uresp(struct qedr_dev *dev, 1256 struct qedr_srq *srq, struct ib_udata *udata) 1257 { 1258 struct qedr_create_srq_uresp uresp = {}; 1259 int rc; 1260 1261 uresp.srq_id = srq->srq_id; 1262 1263 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1264 if (rc) 1265 DP_ERR(dev, "create srq: problem copying data to user space\n"); 1266 1267 return rc; 1268 } 1269 1270 static void qedr_copy_rq_uresp(struct qedr_dev *dev, 1271 struct qedr_create_qp_uresp *uresp, 1272 struct qedr_qp *qp) 1273 { 1274 /* iWARP requires two doorbells per RQ. */ 1275 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1276 uresp->rq_db_offset = 1277 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1278 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1279 } else { 1280 uresp->rq_db_offset = 1281 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1282 } 1283 1284 uresp->rq_icid = qp->icid; 1285 if (qp->urq.db_mmap_entry) 1286 uresp->rq_db_rec_addr = 1287 rdma_user_mmap_get_offset(qp->urq.db_mmap_entry); 1288 } 1289 1290 static void qedr_copy_sq_uresp(struct qedr_dev *dev, 1291 struct qedr_create_qp_uresp *uresp, 1292 struct qedr_qp *qp) 1293 { 1294 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1295 1296 /* iWARP uses the same cid for rq and sq */ 1297 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1298 uresp->sq_icid = qp->icid; 1299 else 1300 uresp->sq_icid = qp->icid + 1; 1301 1302 if (qp->usq.db_mmap_entry) 1303 uresp->sq_db_rec_addr = 1304 rdma_user_mmap_get_offset(qp->usq.db_mmap_entry); 1305 } 1306 1307 static int qedr_copy_qp_uresp(struct qedr_dev *dev, 1308 struct qedr_qp *qp, struct ib_udata *udata, 1309 struct qedr_create_qp_uresp *uresp) 1310 { 1311 int rc; 1312 1313 memset(uresp, 0, sizeof(*uresp)); 1314 1315 if (qedr_qp_has_sq(qp)) 1316 qedr_copy_sq_uresp(dev, uresp, qp); 1317 1318 if (qedr_qp_has_rq(qp)) 1319 qedr_copy_rq_uresp(dev, uresp, qp); 1320 1321 uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 1322 uresp->qp_id = qp->qp_id; 1323 1324 rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp)); 1325 if (rc) 1326 DP_ERR(dev, 1327 "create qp: failed a copy to user space with qp icid=0x%x.\n", 1328 qp->icid); 1329 1330 return rc; 1331 } 1332 1333 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) 1334 { 1335 qed_chain_reset(&qph->pbl); 1336 qph->prod = 0; 1337 qph->cons = 0; 1338 qph->wqe_cons = 0; 1339 qph->db_data.data.value = cpu_to_le16(0); 1340 } 1341 1342 static void qedr_set_common_qp_params(struct qedr_dev *dev, 1343 struct qedr_qp *qp, 1344 struct qedr_pd *pd, 1345 struct ib_qp_init_attr *attrs) 1346 { 1347 spin_lock_init(&qp->q_lock); 1348 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1349 kref_init(&qp->refcnt); 1350 init_completion(&qp->iwarp_cm_comp); 1351 init_completion(&qp->qp_rel_comp); 1352 } 1353 1354 qp->pd = pd; 1355 qp->qp_type = attrs->qp_type; 1356 qp->max_inline_data = attrs->cap.max_inline_data; 1357 qp->state = QED_ROCE_QP_STATE_RESET; 1358 1359 qp->prev_wqe_size = 0; 1360 1361 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1362 qp->dev = dev; 1363 if (qedr_qp_has_sq(qp)) { 1364 qedr_reset_qp_hwq_info(&qp->sq); 1365 qp->sq.max_sges = attrs->cap.max_send_sge; 1366 qp->sq_cq = get_qedr_cq(attrs->send_cq); 1367 DP_DEBUG(dev, QEDR_MSG_QP, 1368 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1369 qp->sq.max_sges, qp->sq_cq->icid); 1370 } 1371 1372 if (attrs->srq) 1373 qp->srq = get_qedr_srq(attrs->srq); 1374 1375 if (qedr_qp_has_rq(qp)) { 1376 qedr_reset_qp_hwq_info(&qp->rq); 1377 qp->rq_cq = get_qedr_cq(attrs->recv_cq); 1378 qp->rq.max_sges = attrs->cap.max_recv_sge; 1379 DP_DEBUG(dev, QEDR_MSG_QP, 1380 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 1381 qp->rq.max_sges, qp->rq_cq->icid); 1382 } 1383 1384 DP_DEBUG(dev, QEDR_MSG_QP, 1385 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n", 1386 pd->pd_id, qp->qp_type, qp->max_inline_data, 1387 qp->state, qp->signaled, (attrs->srq) ? 1 : 0); 1388 DP_DEBUG(dev, QEDR_MSG_QP, 1389 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1390 qp->sq.max_sges, qp->sq_cq->icid); 1391 } 1392 1393 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1394 { 1395 int rc = 0; 1396 1397 if (qedr_qp_has_sq(qp)) { 1398 qp->sq.db = dev->db_addr + 1399 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1400 qp->sq.db_data.data.icid = qp->icid + 1; 1401 rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data, 1402 DB_REC_WIDTH_32B, DB_REC_KERNEL); 1403 if (rc) 1404 return rc; 1405 } 1406 1407 if (qedr_qp_has_rq(qp)) { 1408 qp->rq.db = dev->db_addr + 1409 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1410 qp->rq.db_data.data.icid = qp->icid; 1411 rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data, 1412 DB_REC_WIDTH_32B, DB_REC_KERNEL); 1413 if (rc && qedr_qp_has_sq(qp)) 1414 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data); 1415 } 1416 1417 return rc; 1418 } 1419 1420 static int qedr_check_srq_params(struct qedr_dev *dev, 1421 struct ib_srq_init_attr *attrs, 1422 struct ib_udata *udata) 1423 { 1424 struct qedr_device_attr *qattr = &dev->attr; 1425 1426 if (attrs->attr.max_wr > qattr->max_srq_wr) { 1427 DP_ERR(dev, 1428 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n", 1429 attrs->attr.max_wr, qattr->max_srq_wr); 1430 return -EINVAL; 1431 } 1432 1433 if (attrs->attr.max_sge > qattr->max_sge) { 1434 DP_ERR(dev, 1435 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", 1436 attrs->attr.max_sge, qattr->max_sge); 1437 } 1438 1439 if (!udata && attrs->srq_type == IB_SRQT_XRC) { 1440 DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n"); 1441 return -EINVAL; 1442 } 1443 1444 return 0; 1445 } 1446 1447 static void qedr_free_srq_user_params(struct qedr_srq *srq) 1448 { 1449 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1450 ib_umem_release(srq->usrq.umem); 1451 ib_umem_release(srq->prod_umem); 1452 } 1453 1454 static void qedr_free_srq_kernel_params(struct qedr_srq *srq) 1455 { 1456 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1457 struct qedr_dev *dev = srq->dev; 1458 1459 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl); 1460 1461 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1462 hw_srq->virt_prod_pair_addr, 1463 hw_srq->phy_prod_pair_addr); 1464 } 1465 1466 static int qedr_init_srq_user_params(struct ib_udata *udata, 1467 struct qedr_srq *srq, 1468 struct qedr_create_srq_ureq *ureq, 1469 int access) 1470 { 1471 struct scatterlist *sg; 1472 int rc; 1473 1474 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr, 1475 ureq->srq_len, false, access, 1); 1476 if (rc) 1477 return rc; 1478 1479 srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr, 1480 sizeof(struct rdma_srq_producers), access); 1481 if (IS_ERR(srq->prod_umem)) { 1482 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1483 ib_umem_release(srq->usrq.umem); 1484 DP_ERR(srq->dev, 1485 "create srq: failed ib_umem_get for producer, got %ld\n", 1486 PTR_ERR(srq->prod_umem)); 1487 return PTR_ERR(srq->prod_umem); 1488 } 1489 1490 sg = srq->prod_umem->sgt_append.sgt.sgl; 1491 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); 1492 1493 return 0; 1494 } 1495 1496 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq, 1497 struct qedr_dev *dev, 1498 struct ib_srq_init_attr *init_attr) 1499 { 1500 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1501 struct qed_chain_init_params params = { 1502 .mode = QED_CHAIN_MODE_PBL, 1503 .intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1504 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 1505 .elem_size = QEDR_SRQ_WQE_ELEM_SIZE, 1506 }; 1507 dma_addr_t phy_prod_pair_addr; 1508 u32 num_elems; 1509 void *va; 1510 int rc; 1511 1512 va = dma_alloc_coherent(&dev->pdev->dev, 1513 sizeof(struct rdma_srq_producers), 1514 &phy_prod_pair_addr, GFP_KERNEL); 1515 if (!va) { 1516 DP_ERR(dev, 1517 "create srq: failed to allocate dma memory for producer\n"); 1518 return -ENOMEM; 1519 } 1520 1521 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; 1522 hw_srq->virt_prod_pair_addr = va; 1523 1524 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE; 1525 params.num_elems = num_elems; 1526 1527 rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, ¶ms); 1528 if (rc) 1529 goto err0; 1530 1531 hw_srq->num_elems = num_elems; 1532 1533 return 0; 1534 1535 err0: 1536 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1537 va, phy_prod_pair_addr); 1538 return rc; 1539 } 1540 1541 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, 1542 struct ib_udata *udata) 1543 { 1544 struct qed_rdma_destroy_srq_in_params destroy_in_params; 1545 struct qed_rdma_create_srq_in_params in_params = {}; 1546 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1547 struct qed_rdma_create_srq_out_params out_params; 1548 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd); 1549 struct qedr_create_srq_ureq ureq = {}; 1550 u64 pbl_base_addr, phy_prod_pair_addr; 1551 struct qedr_srq_hwq_info *hw_srq; 1552 u32 page_cnt, page_size; 1553 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1554 int rc = 0; 1555 1556 DP_DEBUG(dev, QEDR_MSG_QP, 1557 "create SRQ called from %s (pd %p)\n", 1558 (udata) ? "User lib" : "kernel", pd); 1559 1560 if (init_attr->srq_type != IB_SRQT_BASIC && 1561 init_attr->srq_type != IB_SRQT_XRC) 1562 return -EOPNOTSUPP; 1563 1564 rc = qedr_check_srq_params(dev, init_attr, udata); 1565 if (rc) 1566 return -EINVAL; 1567 1568 srq->dev = dev; 1569 srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC); 1570 hw_srq = &srq->hw_srq; 1571 spin_lock_init(&srq->lock); 1572 1573 hw_srq->max_wr = init_attr->attr.max_wr; 1574 hw_srq->max_sges = init_attr->attr.max_sge; 1575 1576 if (udata) { 1577 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 1578 udata->inlen))) { 1579 DP_ERR(dev, 1580 "create srq: problem copying data from user space\n"); 1581 goto err0; 1582 } 1583 1584 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0); 1585 if (rc) 1586 goto err0; 1587 1588 page_cnt = srq->usrq.pbl_info.num_pbes; 1589 pbl_base_addr = srq->usrq.pbl_tbl->pa; 1590 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1591 page_size = PAGE_SIZE; 1592 } else { 1593 struct qed_chain *pbl; 1594 1595 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr); 1596 if (rc) 1597 goto err0; 1598 1599 pbl = &hw_srq->pbl; 1600 page_cnt = qed_chain_get_page_cnt(pbl); 1601 pbl_base_addr = qed_chain_get_pbl_phys(pbl); 1602 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1603 page_size = QED_CHAIN_PAGE_SIZE; 1604 } 1605 1606 in_params.pd_id = pd->pd_id; 1607 in_params.pbl_base_addr = pbl_base_addr; 1608 in_params.prod_pair_addr = phy_prod_pair_addr; 1609 in_params.num_pages = page_cnt; 1610 in_params.page_size = page_size; 1611 if (srq->is_xrc) { 1612 struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd); 1613 struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq); 1614 1615 in_params.is_xrc = 1; 1616 in_params.xrcd_id = xrcd->xrcd_id; 1617 in_params.cq_cid = cq->icid; 1618 } 1619 1620 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); 1621 if (rc) 1622 goto err1; 1623 1624 srq->srq_id = out_params.srq_id; 1625 1626 if (udata) { 1627 rc = qedr_copy_srq_uresp(dev, srq, udata); 1628 if (rc) 1629 goto err2; 1630 } 1631 1632 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL); 1633 if (rc) 1634 goto err2; 1635 1636 DP_DEBUG(dev, QEDR_MSG_SRQ, 1637 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id); 1638 return 0; 1639 1640 err2: 1641 destroy_in_params.srq_id = srq->srq_id; 1642 1643 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); 1644 err1: 1645 if (udata) 1646 qedr_free_srq_user_params(srq); 1647 else 1648 qedr_free_srq_kernel_params(srq); 1649 err0: 1650 return -EFAULT; 1651 } 1652 1653 int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 1654 { 1655 struct qed_rdma_destroy_srq_in_params in_params = {}; 1656 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1657 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1658 1659 xa_erase_irq(&dev->srqs, srq->srq_id); 1660 in_params.srq_id = srq->srq_id; 1661 in_params.is_xrc = srq->is_xrc; 1662 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params); 1663 1664 if (ibsrq->uobject) 1665 qedr_free_srq_user_params(srq); 1666 else 1667 qedr_free_srq_kernel_params(srq); 1668 1669 DP_DEBUG(dev, QEDR_MSG_SRQ, 1670 "destroy srq: destroyed srq with srq_id=0x%0x\n", 1671 srq->srq_id); 1672 return 0; 1673 } 1674 1675 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 1676 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 1677 { 1678 struct qed_rdma_modify_srq_in_params in_params = {}; 1679 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1680 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1681 int rc; 1682 1683 if (attr_mask & IB_SRQ_MAX_WR) { 1684 DP_ERR(dev, 1685 "modify srq: invalid attribute mask=0x%x specified for %p\n", 1686 attr_mask, srq); 1687 return -EINVAL; 1688 } 1689 1690 if (attr_mask & IB_SRQ_LIMIT) { 1691 if (attr->srq_limit >= srq->hw_srq.max_wr) { 1692 DP_ERR(dev, 1693 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n", 1694 attr->srq_limit, srq->hw_srq.max_wr); 1695 return -EINVAL; 1696 } 1697 1698 in_params.srq_id = srq->srq_id; 1699 in_params.wqe_limit = attr->srq_limit; 1700 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params); 1701 if (rc) 1702 return rc; 1703 } 1704 1705 srq->srq_limit = attr->srq_limit; 1706 1707 DP_DEBUG(dev, QEDR_MSG_SRQ, 1708 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id); 1709 1710 return 0; 1711 } 1712 1713 static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type) 1714 { 1715 switch (ib_qp_type) { 1716 case IB_QPT_RC: 1717 return QED_RDMA_QP_TYPE_RC; 1718 case IB_QPT_XRC_INI: 1719 return QED_RDMA_QP_TYPE_XRC_INI; 1720 case IB_QPT_XRC_TGT: 1721 return QED_RDMA_QP_TYPE_XRC_TGT; 1722 default: 1723 return QED_RDMA_QP_TYPE_INVAL; 1724 } 1725 } 1726 1727 static inline void 1728 qedr_init_common_qp_in_params(struct qedr_dev *dev, 1729 struct qedr_pd *pd, 1730 struct qedr_qp *qp, 1731 struct ib_qp_init_attr *attrs, 1732 bool fmr_and_reserved_lkey, 1733 struct qed_rdma_create_qp_in_params *params) 1734 { 1735 /* QP handle to be written in an async event */ 1736 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp); 1737 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp); 1738 1739 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 1740 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 1741 params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type); 1742 params->stats_queue = 0; 1743 1744 if (pd) { 1745 params->pd = pd->pd_id; 1746 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 1747 } 1748 1749 if (qedr_qp_has_sq(qp)) 1750 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; 1751 1752 if (qedr_qp_has_rq(qp)) 1753 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1754 1755 if (qedr_qp_has_srq(qp)) { 1756 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1757 params->srq_id = qp->srq->srq_id; 1758 params->use_srq = true; 1759 } else { 1760 params->srq_id = 0; 1761 params->use_srq = false; 1762 } 1763 } 1764 1765 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) 1766 { 1767 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. " 1768 "qp=%p. " 1769 "sq_addr=0x%llx, " 1770 "sq_len=%zd, " 1771 "rq_addr=0x%llx, " 1772 "rq_len=%zd" 1773 "\n", 1774 qp, 1775 qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0, 1776 qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0, 1777 qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0, 1778 qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0); 1779 } 1780 1781 static inline void 1782 qedr_iwarp_populate_user_qp(struct qedr_dev *dev, 1783 struct qedr_qp *qp, 1784 struct qed_rdma_create_qp_out_params *out_params) 1785 { 1786 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 1787 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 1788 1789 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 1790 &qp->usq.pbl_info, FW_PAGE_SHIFT); 1791 if (!qp->srq) { 1792 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 1793 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 1794 } 1795 1796 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 1797 &qp->urq.pbl_info, FW_PAGE_SHIFT); 1798 } 1799 1800 static void qedr_cleanup_user(struct qedr_dev *dev, 1801 struct qedr_ucontext *ctx, 1802 struct qedr_qp *qp) 1803 { 1804 if (qedr_qp_has_sq(qp)) { 1805 ib_umem_release(qp->usq.umem); 1806 qp->usq.umem = NULL; 1807 } 1808 1809 if (qedr_qp_has_rq(qp)) { 1810 ib_umem_release(qp->urq.umem); 1811 qp->urq.umem = NULL; 1812 } 1813 1814 if (rdma_protocol_roce(&dev->ibdev, 1)) { 1815 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl); 1816 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl); 1817 } else { 1818 kfree(qp->usq.pbl_tbl); 1819 kfree(qp->urq.pbl_tbl); 1820 } 1821 1822 if (qp->usq.db_rec_data) { 1823 qedr_db_recovery_del(dev, qp->usq.db_addr, 1824 &qp->usq.db_rec_data->db_data); 1825 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry); 1826 } 1827 1828 if (qp->urq.db_rec_data) { 1829 qedr_db_recovery_del(dev, qp->urq.db_addr, 1830 &qp->urq.db_rec_data->db_data); 1831 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry); 1832 } 1833 1834 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1835 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr, 1836 &qp->urq.db_rec_db2_data); 1837 } 1838 1839 static int qedr_create_user_qp(struct qedr_dev *dev, 1840 struct qedr_qp *qp, 1841 struct ib_pd *ibpd, 1842 struct ib_udata *udata, 1843 struct ib_qp_init_attr *attrs) 1844 { 1845 struct qed_rdma_create_qp_in_params in_params; 1846 struct qed_rdma_create_qp_out_params out_params; 1847 struct qedr_create_qp_uresp uresp = {}; 1848 struct qedr_create_qp_ureq ureq = {}; 1849 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1); 1850 struct qedr_ucontext *ctx = NULL; 1851 struct qedr_pd *pd = NULL; 1852 int rc = 0; 1853 1854 qp->create_type = QEDR_QP_CREATE_USER; 1855 1856 if (ibpd) { 1857 pd = get_qedr_pd(ibpd); 1858 ctx = pd->uctx; 1859 } 1860 1861 if (udata) { 1862 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 1863 udata->inlen)); 1864 if (rc) { 1865 DP_ERR(dev, "Problem copying data from user space\n"); 1866 return rc; 1867 } 1868 } 1869 1870 if (qedr_qp_has_sq(qp)) { 1871 /* SQ - read access only (0) */ 1872 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr, 1873 ureq.sq_len, true, 0, alloc_and_init); 1874 if (rc) 1875 return rc; 1876 } 1877 1878 if (qedr_qp_has_rq(qp)) { 1879 /* RQ - read access only (0) */ 1880 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr, 1881 ureq.rq_len, true, 0, alloc_and_init); 1882 if (rc) 1883 return rc; 1884 } 1885 1886 memset(&in_params, 0, sizeof(in_params)); 1887 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 1888 in_params.qp_handle_lo = ureq.qp_handle_lo; 1889 in_params.qp_handle_hi = ureq.qp_handle_hi; 1890 1891 if (qp->qp_type == IB_QPT_XRC_TGT) { 1892 struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd); 1893 1894 in_params.xrcd_id = xrcd->xrcd_id; 1895 in_params.qp_handle_lo = qp->qp_id; 1896 in_params.use_srq = 1; 1897 } 1898 1899 if (qedr_qp_has_sq(qp)) { 1900 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 1901 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 1902 } 1903 1904 if (qedr_qp_has_rq(qp)) { 1905 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 1906 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 1907 } 1908 1909 if (ctx) 1910 SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode); 1911 1912 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1913 &in_params, &out_params); 1914 1915 if (!qp->qed_qp) { 1916 rc = -ENOMEM; 1917 goto err1; 1918 } 1919 1920 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1921 qedr_iwarp_populate_user_qp(dev, qp, &out_params); 1922 1923 qp->qp_id = out_params.qp_id; 1924 qp->icid = out_params.icid; 1925 1926 if (udata) { 1927 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp); 1928 if (rc) 1929 goto err; 1930 } 1931 1932 /* db offset was calculated in copy_qp_uresp, now set in the user q */ 1933 if (qedr_qp_has_sq(qp)) { 1934 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset; 1935 qp->sq.max_wr = attrs->cap.max_send_wr; 1936 rc = qedr_db_recovery_add(dev, qp->usq.db_addr, 1937 &qp->usq.db_rec_data->db_data, 1938 DB_REC_WIDTH_32B, 1939 DB_REC_USER); 1940 if (rc) 1941 goto err; 1942 } 1943 1944 if (qedr_qp_has_rq(qp)) { 1945 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset; 1946 qp->rq.max_wr = attrs->cap.max_recv_wr; 1947 rc = qedr_db_recovery_add(dev, qp->urq.db_addr, 1948 &qp->urq.db_rec_data->db_data, 1949 DB_REC_WIDTH_32B, 1950 DB_REC_USER); 1951 if (rc) 1952 goto err; 1953 } 1954 1955 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1956 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset; 1957 1958 /* calculate the db_rec_db2 data since it is constant so no 1959 * need to reflect from user 1960 */ 1961 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid); 1962 qp->urq.db_rec_db2_data.data.value = 1963 cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD); 1964 1965 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr, 1966 &qp->urq.db_rec_db2_data, 1967 DB_REC_WIDTH_32B, 1968 DB_REC_USER); 1969 if (rc) 1970 goto err; 1971 } 1972 qedr_qp_user_print(dev, qp); 1973 return rc; 1974 err: 1975 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1976 if (rc) 1977 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc); 1978 1979 err1: 1980 qedr_cleanup_user(dev, ctx, qp); 1981 return rc; 1982 } 1983 1984 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1985 { 1986 int rc; 1987 1988 qp->sq.db = dev->db_addr + 1989 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1990 qp->sq.db_data.data.icid = qp->icid; 1991 1992 rc = qedr_db_recovery_add(dev, qp->sq.db, 1993 &qp->sq.db_data, 1994 DB_REC_WIDTH_32B, 1995 DB_REC_KERNEL); 1996 if (rc) 1997 return rc; 1998 1999 qp->rq.db = dev->db_addr + 2000 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 2001 qp->rq.db_data.data.icid = qp->icid; 2002 qp->rq.iwarp_db2 = dev->db_addr + 2003 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 2004 qp->rq.iwarp_db2_data.data.icid = qp->icid; 2005 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 2006 2007 rc = qedr_db_recovery_add(dev, qp->rq.db, 2008 &qp->rq.db_data, 2009 DB_REC_WIDTH_32B, 2010 DB_REC_KERNEL); 2011 if (rc) 2012 return rc; 2013 2014 rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2, 2015 &qp->rq.iwarp_db2_data, 2016 DB_REC_WIDTH_32B, 2017 DB_REC_KERNEL); 2018 return rc; 2019 } 2020 2021 static int 2022 qedr_roce_create_kernel_qp(struct qedr_dev *dev, 2023 struct qedr_qp *qp, 2024 struct qed_rdma_create_qp_in_params *in_params, 2025 u32 n_sq_elems, u32 n_rq_elems) 2026 { 2027 struct qed_rdma_create_qp_out_params out_params; 2028 struct qed_chain_init_params params = { 2029 .mode = QED_CHAIN_MODE_PBL, 2030 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 2031 }; 2032 int rc; 2033 2034 params.intended_use = QED_CHAIN_USE_TO_PRODUCE; 2035 params.num_elems = n_sq_elems; 2036 params.elem_size = QEDR_SQE_ELEMENT_SIZE; 2037 2038 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, ¶ms); 2039 if (rc) 2040 return rc; 2041 2042 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl); 2043 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl); 2044 2045 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE; 2046 params.num_elems = n_rq_elems; 2047 params.elem_size = QEDR_RQE_ELEMENT_SIZE; 2048 2049 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, ¶ms); 2050 if (rc) 2051 return rc; 2052 2053 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl); 2054 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl); 2055 2056 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 2057 in_params, &out_params); 2058 2059 if (!qp->qed_qp) 2060 return -EINVAL; 2061 2062 qp->qp_id = out_params.qp_id; 2063 qp->icid = out_params.icid; 2064 2065 return qedr_set_roce_db_info(dev, qp); 2066 } 2067 2068 static int 2069 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev, 2070 struct qedr_qp *qp, 2071 struct qed_rdma_create_qp_in_params *in_params, 2072 u32 n_sq_elems, u32 n_rq_elems) 2073 { 2074 struct qed_rdma_create_qp_out_params out_params; 2075 struct qed_chain_init_params params = { 2076 .mode = QED_CHAIN_MODE_PBL, 2077 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 2078 }; 2079 int rc; 2080 2081 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems, 2082 QEDR_SQE_ELEMENT_SIZE, 2083 QED_CHAIN_PAGE_SIZE, 2084 QED_CHAIN_MODE_PBL); 2085 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems, 2086 QEDR_RQE_ELEMENT_SIZE, 2087 QED_CHAIN_PAGE_SIZE, 2088 QED_CHAIN_MODE_PBL); 2089 2090 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 2091 in_params, &out_params); 2092 2093 if (!qp->qed_qp) 2094 return -EINVAL; 2095 2096 /* Now we allocate the chain */ 2097 2098 params.intended_use = QED_CHAIN_USE_TO_PRODUCE; 2099 params.num_elems = n_sq_elems; 2100 params.elem_size = QEDR_SQE_ELEMENT_SIZE; 2101 params.ext_pbl_virt = out_params.sq_pbl_virt; 2102 params.ext_pbl_phys = out_params.sq_pbl_phys; 2103 2104 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, ¶ms); 2105 if (rc) 2106 goto err; 2107 2108 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE; 2109 params.num_elems = n_rq_elems; 2110 params.elem_size = QEDR_RQE_ELEMENT_SIZE; 2111 params.ext_pbl_virt = out_params.rq_pbl_virt; 2112 params.ext_pbl_phys = out_params.rq_pbl_phys; 2113 2114 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, ¶ms); 2115 if (rc) 2116 goto err; 2117 2118 qp->qp_id = out_params.qp_id; 2119 qp->icid = out_params.icid; 2120 2121 return qedr_set_iwarp_db_info(dev, qp); 2122 2123 err: 2124 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2125 2126 return rc; 2127 } 2128 2129 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) 2130 { 2131 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl); 2132 kfree(qp->wqe_wr_id); 2133 2134 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl); 2135 kfree(qp->rqe_wr_id); 2136 2137 /* GSI qp is not registered to db mechanism so no need to delete */ 2138 if (qp->qp_type == IB_QPT_GSI) 2139 return; 2140 2141 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data); 2142 2143 if (!qp->srq) { 2144 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data); 2145 2146 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2147 qedr_db_recovery_del(dev, qp->rq.iwarp_db2, 2148 &qp->rq.iwarp_db2_data); 2149 } 2150 } 2151 2152 static int qedr_create_kernel_qp(struct qedr_dev *dev, 2153 struct qedr_qp *qp, 2154 struct ib_pd *ibpd, 2155 struct ib_qp_init_attr *attrs) 2156 { 2157 struct qed_rdma_create_qp_in_params in_params; 2158 struct qedr_pd *pd = get_qedr_pd(ibpd); 2159 int rc = -EINVAL; 2160 u32 n_rq_elems; 2161 u32 n_sq_elems; 2162 u32 n_sq_entries; 2163 2164 memset(&in_params, 0, sizeof(in_params)); 2165 qp->create_type = QEDR_QP_CREATE_KERNEL; 2166 2167 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in 2168 * the ring. The ring should allow at least a single WR, even if the 2169 * user requested none, due to allocation issues. 2170 * We should add an extra WR since the prod and cons indices of 2171 * wqe_wr_id are managed in such a way that the WQ is considered full 2172 * when (prod+1)%max_wr==cons. We currently don't do that because we 2173 * double the number of entries due an iSER issue that pushes far more 2174 * WRs than indicated. If we decline its ib_post_send() then we get 2175 * error prints in the dmesg we'd like to avoid. 2176 */ 2177 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 2178 dev->attr.max_sqe); 2179 2180 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id), 2181 GFP_KERNEL); 2182 if (!qp->wqe_wr_id) { 2183 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n"); 2184 return -ENOMEM; 2185 } 2186 2187 /* QP handle to be written in CQE */ 2188 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp); 2189 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp); 2190 2191 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in 2192 * the ring. There ring should allow at least a single WR, even if the 2193 * user requested none, due to allocation issues. 2194 */ 2195 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1); 2196 2197 /* Allocate driver internal RQ array */ 2198 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id), 2199 GFP_KERNEL); 2200 if (!qp->rqe_wr_id) { 2201 DP_ERR(dev, 2202 "create qp: failed RQ shadow memory allocation\n"); 2203 kfree(qp->wqe_wr_id); 2204 return -ENOMEM; 2205 } 2206 2207 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 2208 2209 n_sq_entries = attrs->cap.max_send_wr; 2210 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe); 2211 n_sq_entries = max_t(u32, n_sq_entries, 1); 2212 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE; 2213 2214 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE; 2215 2216 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2217 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params, 2218 n_sq_elems, n_rq_elems); 2219 else 2220 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params, 2221 n_sq_elems, n_rq_elems); 2222 if (rc) 2223 qedr_cleanup_kernel(dev, qp); 2224 2225 return rc; 2226 } 2227 2228 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp, 2229 struct ib_udata *udata) 2230 { 2231 struct qedr_ucontext *ctx = 2232 rdma_udata_to_drv_context(udata, struct qedr_ucontext, 2233 ibucontext); 2234 int rc; 2235 2236 if (qp->qp_type != IB_QPT_GSI) { 2237 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2238 if (rc) 2239 return rc; 2240 } 2241 2242 if (qp->create_type == QEDR_QP_CREATE_USER) 2243 qedr_cleanup_user(dev, ctx, qp); 2244 else 2245 qedr_cleanup_kernel(dev, qp); 2246 2247 return 0; 2248 } 2249 2250 int qedr_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, 2251 struct ib_udata *udata) 2252 { 2253 struct qedr_xrcd *xrcd = NULL; 2254 struct ib_pd *ibpd = ibqp->pd; 2255 struct qedr_pd *pd = get_qedr_pd(ibpd); 2256 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 2257 struct qedr_qp *qp = get_qedr_qp(ibqp); 2258 int rc = 0; 2259 2260 if (attrs->create_flags) 2261 return -EOPNOTSUPP; 2262 2263 if (attrs->qp_type == IB_QPT_XRC_TGT) 2264 xrcd = get_qedr_xrcd(attrs->xrcd); 2265 else 2266 pd = get_qedr_pd(ibpd); 2267 2268 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", 2269 udata ? "user library" : "kernel", pd); 2270 2271 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata); 2272 if (rc) 2273 return rc; 2274 2275 DP_DEBUG(dev, QEDR_MSG_QP, 2276 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 2277 udata ? "user library" : "kernel", attrs->event_handler, pd, 2278 get_qedr_cq(attrs->send_cq), 2279 get_qedr_cq(attrs->send_cq)->icid, 2280 get_qedr_cq(attrs->recv_cq), 2281 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0); 2282 2283 qedr_set_common_qp_params(dev, qp, pd, attrs); 2284 2285 if (attrs->qp_type == IB_QPT_GSI) 2286 return qedr_create_gsi_qp(dev, attrs, qp); 2287 2288 if (udata || xrcd) 2289 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); 2290 else 2291 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); 2292 2293 if (rc) 2294 return rc; 2295 2296 qp->ibqp.qp_num = qp->qp_id; 2297 2298 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 2299 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL); 2300 if (rc) 2301 goto out_free_qp_resources; 2302 } 2303 2304 return 0; 2305 2306 out_free_qp_resources: 2307 qedr_free_qp_resources(dev, qp, udata); 2308 return -EFAULT; 2309 } 2310 2311 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) 2312 { 2313 switch (qp_state) { 2314 case QED_ROCE_QP_STATE_RESET: 2315 return IB_QPS_RESET; 2316 case QED_ROCE_QP_STATE_INIT: 2317 return IB_QPS_INIT; 2318 case QED_ROCE_QP_STATE_RTR: 2319 return IB_QPS_RTR; 2320 case QED_ROCE_QP_STATE_RTS: 2321 return IB_QPS_RTS; 2322 case QED_ROCE_QP_STATE_SQD: 2323 return IB_QPS_SQD; 2324 case QED_ROCE_QP_STATE_ERR: 2325 return IB_QPS_ERR; 2326 case QED_ROCE_QP_STATE_SQE: 2327 return IB_QPS_SQE; 2328 } 2329 return IB_QPS_ERR; 2330 } 2331 2332 static enum qed_roce_qp_state qedr_get_state_from_ibqp( 2333 enum ib_qp_state qp_state) 2334 { 2335 switch (qp_state) { 2336 case IB_QPS_RESET: 2337 return QED_ROCE_QP_STATE_RESET; 2338 case IB_QPS_INIT: 2339 return QED_ROCE_QP_STATE_INIT; 2340 case IB_QPS_RTR: 2341 return QED_ROCE_QP_STATE_RTR; 2342 case IB_QPS_RTS: 2343 return QED_ROCE_QP_STATE_RTS; 2344 case IB_QPS_SQD: 2345 return QED_ROCE_QP_STATE_SQD; 2346 case IB_QPS_ERR: 2347 return QED_ROCE_QP_STATE_ERR; 2348 default: 2349 return QED_ROCE_QP_STATE_ERR; 2350 } 2351 } 2352 2353 static int qedr_update_qp_state(struct qedr_dev *dev, 2354 struct qedr_qp *qp, 2355 enum qed_roce_qp_state cur_state, 2356 enum qed_roce_qp_state new_state) 2357 { 2358 int status = 0; 2359 2360 if (new_state == cur_state) 2361 return 0; 2362 2363 switch (cur_state) { 2364 case QED_ROCE_QP_STATE_RESET: 2365 switch (new_state) { 2366 case QED_ROCE_QP_STATE_INIT: 2367 break; 2368 default: 2369 status = -EINVAL; 2370 break; 2371 } 2372 break; 2373 case QED_ROCE_QP_STATE_INIT: 2374 switch (new_state) { 2375 case QED_ROCE_QP_STATE_RTR: 2376 /* Update doorbell (in case post_recv was 2377 * done before move to RTR) 2378 */ 2379 2380 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2381 writel(qp->rq.db_data.raw, qp->rq.db); 2382 } 2383 break; 2384 case QED_ROCE_QP_STATE_ERR: 2385 break; 2386 default: 2387 /* Invalid state change. */ 2388 status = -EINVAL; 2389 break; 2390 } 2391 break; 2392 case QED_ROCE_QP_STATE_RTR: 2393 /* RTR->XXX */ 2394 switch (new_state) { 2395 case QED_ROCE_QP_STATE_RTS: 2396 break; 2397 case QED_ROCE_QP_STATE_ERR: 2398 break; 2399 default: 2400 /* Invalid state change. */ 2401 status = -EINVAL; 2402 break; 2403 } 2404 break; 2405 case QED_ROCE_QP_STATE_RTS: 2406 /* RTS->XXX */ 2407 switch (new_state) { 2408 case QED_ROCE_QP_STATE_SQD: 2409 break; 2410 case QED_ROCE_QP_STATE_ERR: 2411 break; 2412 default: 2413 /* Invalid state change. */ 2414 status = -EINVAL; 2415 break; 2416 } 2417 break; 2418 case QED_ROCE_QP_STATE_SQD: 2419 /* SQD->XXX */ 2420 switch (new_state) { 2421 case QED_ROCE_QP_STATE_RTS: 2422 case QED_ROCE_QP_STATE_ERR: 2423 break; 2424 default: 2425 /* Invalid state change. */ 2426 status = -EINVAL; 2427 break; 2428 } 2429 break; 2430 case QED_ROCE_QP_STATE_ERR: 2431 /* ERR->XXX */ 2432 switch (new_state) { 2433 case QED_ROCE_QP_STATE_RESET: 2434 if ((qp->rq.prod != qp->rq.cons) || 2435 (qp->sq.prod != qp->sq.cons)) { 2436 DP_NOTICE(dev, 2437 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n", 2438 qp->rq.prod, qp->rq.cons, qp->sq.prod, 2439 qp->sq.cons); 2440 status = -EINVAL; 2441 } 2442 break; 2443 default: 2444 status = -EINVAL; 2445 break; 2446 } 2447 break; 2448 default: 2449 status = -EINVAL; 2450 break; 2451 } 2452 2453 return status; 2454 } 2455 2456 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2457 int attr_mask, struct ib_udata *udata) 2458 { 2459 struct qedr_qp *qp = get_qedr_qp(ibqp); 2460 struct qed_rdma_modify_qp_in_params qp_params = { 0 }; 2461 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); 2462 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 2463 enum ib_qp_state old_qp_state, new_qp_state; 2464 enum qed_roce_qp_state cur_state; 2465 int rc = 0; 2466 2467 DP_DEBUG(dev, QEDR_MSG_QP, 2468 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask, 2469 attr->qp_state); 2470 2471 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 2472 return -EOPNOTSUPP; 2473 2474 old_qp_state = qedr_get_ibqp_state(qp->state); 2475 if (attr_mask & IB_QP_STATE) 2476 new_qp_state = attr->qp_state; 2477 else 2478 new_qp_state = old_qp_state; 2479 2480 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2481 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state, 2482 ibqp->qp_type, attr_mask)) { 2483 DP_ERR(dev, 2484 "modify qp: invalid attribute mask=0x%x specified for\n" 2485 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n", 2486 attr_mask, qp->qp_id, ibqp->qp_type, 2487 old_qp_state, new_qp_state); 2488 rc = -EINVAL; 2489 goto err; 2490 } 2491 } 2492 2493 /* Translate the masks... */ 2494 if (attr_mask & IB_QP_STATE) { 2495 SET_FIELD(qp_params.modify_flags, 2496 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 2497 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state); 2498 } 2499 2500 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 2501 qp_params.sqd_async = true; 2502 2503 if (attr_mask & IB_QP_PKEY_INDEX) { 2504 SET_FIELD(qp_params.modify_flags, 2505 QED_ROCE_MODIFY_QP_VALID_PKEY, 1); 2506 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) { 2507 rc = -EINVAL; 2508 goto err; 2509 } 2510 2511 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT; 2512 } 2513 2514 if (attr_mask & IB_QP_QKEY) 2515 qp->qkey = attr->qkey; 2516 2517 if (attr_mask & IB_QP_ACCESS_FLAGS) { 2518 SET_FIELD(qp_params.modify_flags, 2519 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 2520 qp_params.incoming_rdma_read_en = attr->qp_access_flags & 2521 IB_ACCESS_REMOTE_READ; 2522 qp_params.incoming_rdma_write_en = attr->qp_access_flags & 2523 IB_ACCESS_REMOTE_WRITE; 2524 qp_params.incoming_atomic_en = attr->qp_access_flags & 2525 IB_ACCESS_REMOTE_ATOMIC; 2526 } 2527 2528 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 2529 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2530 return -EINVAL; 2531 2532 if (attr_mask & IB_QP_PATH_MTU) { 2533 if (attr->path_mtu < IB_MTU_256 || 2534 attr->path_mtu > IB_MTU_4096) { 2535 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n"); 2536 rc = -EINVAL; 2537 goto err; 2538 } 2539 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 2540 ib_mtu_enum_to_int(iboe_get_mtu 2541 (dev->ndev->mtu))); 2542 } 2543 2544 if (!qp->mtu) { 2545 qp->mtu = 2546 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2547 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu); 2548 } 2549 2550 SET_FIELD(qp_params.modify_flags, 2551 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1); 2552 2553 qp_params.traffic_class_tos = grh->traffic_class; 2554 qp_params.flow_label = grh->flow_label; 2555 qp_params.hop_limit_ttl = grh->hop_limit; 2556 2557 qp->sgid_idx = grh->sgid_index; 2558 2559 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params); 2560 if (rc) { 2561 DP_ERR(dev, 2562 "modify qp: problems with GID index %d (rc=%d)\n", 2563 grh->sgid_index, rc); 2564 return rc; 2565 } 2566 2567 rc = qedr_get_dmac(dev, &attr->ah_attr, 2568 qp_params.remote_mac_addr); 2569 if (rc) 2570 return rc; 2571 2572 qp_params.use_local_mac = true; 2573 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr); 2574 2575 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n", 2576 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 2577 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 2578 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n", 2579 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 2580 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 2581 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n", 2582 qp_params.remote_mac_addr); 2583 2584 qp_params.mtu = qp->mtu; 2585 qp_params.lb_indication = false; 2586 } 2587 2588 if (!qp_params.mtu) { 2589 /* Stay with current MTU */ 2590 if (qp->mtu) 2591 qp_params.mtu = qp->mtu; 2592 else 2593 qp_params.mtu = 2594 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2595 } 2596 2597 if (attr_mask & IB_QP_TIMEOUT) { 2598 SET_FIELD(qp_params.modify_flags, 2599 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 2600 2601 /* The received timeout value is an exponent used like this: 2602 * "12.7.34 LOCAL ACK TIMEOUT 2603 * Value representing the transport (ACK) timeout for use by 2604 * the remote, expressed as: 4.096 * 2^timeout [usec]" 2605 * The FW expects timeout in msec so we need to divide the usec 2606 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2, 2607 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8). 2608 * The value of zero means infinite so we use a 'max_t' to make 2609 * sure that sub 1 msec values will be configured as 1 msec. 2610 */ 2611 if (attr->timeout) 2612 qp_params.ack_timeout = 2613 1 << max_t(int, attr->timeout - 8, 0); 2614 else 2615 qp_params.ack_timeout = 0; 2616 2617 qp->timeout = attr->timeout; 2618 } 2619 2620 if (attr_mask & IB_QP_RETRY_CNT) { 2621 SET_FIELD(qp_params.modify_flags, 2622 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 2623 qp_params.retry_cnt = attr->retry_cnt; 2624 } 2625 2626 if (attr_mask & IB_QP_RNR_RETRY) { 2627 SET_FIELD(qp_params.modify_flags, 2628 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1); 2629 qp_params.rnr_retry_cnt = attr->rnr_retry; 2630 } 2631 2632 if (attr_mask & IB_QP_RQ_PSN) { 2633 SET_FIELD(qp_params.modify_flags, 2634 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1); 2635 qp_params.rq_psn = attr->rq_psn; 2636 qp->rq_psn = attr->rq_psn; 2637 } 2638 2639 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 2640 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) { 2641 rc = -EINVAL; 2642 DP_ERR(dev, 2643 "unsupported max_rd_atomic=%d, supported=%d\n", 2644 attr->max_rd_atomic, 2645 dev->attr.max_qp_req_rd_atomic_resc); 2646 goto err; 2647 } 2648 2649 SET_FIELD(qp_params.modify_flags, 2650 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1); 2651 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 2652 } 2653 2654 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 2655 SET_FIELD(qp_params.modify_flags, 2656 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1); 2657 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 2658 } 2659 2660 if (attr_mask & IB_QP_SQ_PSN) { 2661 SET_FIELD(qp_params.modify_flags, 2662 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1); 2663 qp_params.sq_psn = attr->sq_psn; 2664 qp->sq_psn = attr->sq_psn; 2665 } 2666 2667 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 2668 if (attr->max_dest_rd_atomic > 2669 dev->attr.max_qp_resp_rd_atomic_resc) { 2670 DP_ERR(dev, 2671 "unsupported max_dest_rd_atomic=%d, supported=%d\n", 2672 attr->max_dest_rd_atomic, 2673 dev->attr.max_qp_resp_rd_atomic_resc); 2674 2675 rc = -EINVAL; 2676 goto err; 2677 } 2678 2679 SET_FIELD(qp_params.modify_flags, 2680 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1); 2681 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 2682 } 2683 2684 if (attr_mask & IB_QP_DEST_QPN) { 2685 SET_FIELD(qp_params.modify_flags, 2686 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1); 2687 2688 qp_params.dest_qp = attr->dest_qp_num; 2689 qp->dest_qp_num = attr->dest_qp_num; 2690 } 2691 2692 cur_state = qp->state; 2693 2694 /* Update the QP state before the actual ramrod to prevent a race with 2695 * fast path. Modifying the QP state to error will cause the device to 2696 * flush the CQEs and while polling the flushed CQEs will considered as 2697 * a potential issue if the QP isn't in error state. 2698 */ 2699 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI && 2700 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR) 2701 qp->state = QED_ROCE_QP_STATE_ERR; 2702 2703 if (qp->qp_type != IB_QPT_GSI) 2704 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx, 2705 qp->qed_qp, &qp_params); 2706 2707 if (attr_mask & IB_QP_STATE) { 2708 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 2709 rc = qedr_update_qp_state(dev, qp, cur_state, 2710 qp_params.new_state); 2711 qp->state = qp_params.new_state; 2712 } 2713 2714 err: 2715 return rc; 2716 } 2717 2718 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params) 2719 { 2720 int ib_qp_acc_flags = 0; 2721 2722 if (params->incoming_rdma_write_en) 2723 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 2724 if (params->incoming_rdma_read_en) 2725 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 2726 if (params->incoming_atomic_en) 2727 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 2728 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 2729 return ib_qp_acc_flags; 2730 } 2731 2732 int qedr_query_qp(struct ib_qp *ibqp, 2733 struct ib_qp_attr *qp_attr, 2734 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 2735 { 2736 struct qed_rdma_query_qp_out_params params; 2737 struct qedr_qp *qp = get_qedr_qp(ibqp); 2738 struct qedr_dev *dev = qp->dev; 2739 int rc = 0; 2740 2741 memset(¶ms, 0, sizeof(params)); 2742 memset(qp_attr, 0, sizeof(*qp_attr)); 2743 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 2744 2745 if (qp->qp_type != IB_QPT_GSI) { 2746 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms); 2747 if (rc) 2748 goto err; 2749 qp_attr->qp_state = qedr_get_ibqp_state(params.state); 2750 } else { 2751 qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS); 2752 } 2753 2754 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state); 2755 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu); 2756 qp_attr->path_mig_state = IB_MIG_MIGRATED; 2757 qp_attr->rq_psn = params.rq_psn; 2758 qp_attr->sq_psn = params.sq_psn; 2759 qp_attr->dest_qp_num = params.dest_qp; 2760 2761 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms); 2762 2763 qp_attr->cap.max_send_wr = qp->sq.max_wr; 2764 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 2765 qp_attr->cap.max_send_sge = qp->sq.max_sges; 2766 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 2767 qp_attr->cap.max_inline_data = dev->attr.max_inline; 2768 qp_init_attr->cap = qp_attr->cap; 2769 2770 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2771 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 2772 params.flow_label, qp->sgid_idx, 2773 params.hop_limit_ttl, params.traffic_class_tos); 2774 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); 2775 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 2776 rdma_ah_set_sl(&qp_attr->ah_attr, 0); 2777 qp_attr->timeout = qp->timeout; 2778 qp_attr->rnr_retry = params.rnr_retry; 2779 qp_attr->retry_cnt = params.retry_cnt; 2780 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 2781 qp_attr->pkey_index = params.pkey_index; 2782 qp_attr->port_num = 1; 2783 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 2784 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 2785 qp_attr->alt_pkey_index = 0; 2786 qp_attr->alt_port_num = 0; 2787 qp_attr->alt_timeout = 0; 2788 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 2789 2790 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0; 2791 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 2792 qp_attr->max_rd_atomic = params.max_rd_atomic; 2793 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0; 2794 2795 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n", 2796 qp_attr->cap.max_inline_data); 2797 2798 err: 2799 return rc; 2800 } 2801 2802 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 2803 { 2804 struct qedr_qp *qp = get_qedr_qp(ibqp); 2805 struct qedr_dev *dev = qp->dev; 2806 struct ib_qp_attr attr; 2807 int attr_mask = 0; 2808 2809 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n", 2810 qp, qp->qp_type); 2811 2812 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2813 if ((qp->state != QED_ROCE_QP_STATE_RESET) && 2814 (qp->state != QED_ROCE_QP_STATE_ERR) && 2815 (qp->state != QED_ROCE_QP_STATE_INIT)) { 2816 2817 attr.qp_state = IB_QPS_ERR; 2818 attr_mask |= IB_QP_STATE; 2819 2820 /* Change the QP state to ERROR */ 2821 qedr_modify_qp(ibqp, &attr, attr_mask, NULL); 2822 } 2823 } else { 2824 /* If connection establishment started the WAIT_FOR_CONNECT 2825 * bit will be on and we need to Wait for the establishment 2826 * to complete before destroying the qp. 2827 */ 2828 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT, 2829 &qp->iwarp_cm_flags)) 2830 wait_for_completion(&qp->iwarp_cm_comp); 2831 2832 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT 2833 * bit will be on, and we need to wait for the disconnect to 2834 * complete before continuing. We can use the same completion, 2835 * iwarp_cm_comp, since this is the only place that waits for 2836 * this completion and it is sequential. In addition, 2837 * disconnect can't occur before the connection is fully 2838 * established, therefore if WAIT_FOR_DISCONNECT is on it 2839 * means WAIT_FOR_CONNECT is also on and the completion for 2840 * CONNECT already occurred. 2841 */ 2842 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT, 2843 &qp->iwarp_cm_flags)) 2844 wait_for_completion(&qp->iwarp_cm_comp); 2845 } 2846 2847 if (qp->qp_type == IB_QPT_GSI) 2848 qedr_destroy_gsi_qp(dev); 2849 2850 /* We need to remove the entry from the xarray before we release the 2851 * qp_id to avoid a race of the qp_id being reallocated and failing 2852 * on xa_insert 2853 */ 2854 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2855 xa_erase(&dev->qps, qp->qp_id); 2856 2857 qedr_free_qp_resources(dev, qp, udata); 2858 2859 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 2860 qedr_iw_qp_rem_ref(&qp->ibqp); 2861 wait_for_completion(&qp->qp_rel_comp); 2862 } 2863 2864 return 0; 2865 } 2866 2867 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, 2868 struct ib_udata *udata) 2869 { 2870 struct qedr_ah *ah = get_qedr_ah(ibah); 2871 2872 rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr); 2873 2874 return 0; 2875 } 2876 2877 int qedr_destroy_ah(struct ib_ah *ibah, u32 flags) 2878 { 2879 struct qedr_ah *ah = get_qedr_ah(ibah); 2880 2881 rdma_destroy_ah_attr(&ah->attr); 2882 return 0; 2883 } 2884 2885 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) 2886 { 2887 struct qedr_pbl *pbl, *tmp; 2888 2889 if (info->pbl_table) 2890 list_add_tail(&info->pbl_table->list_entry, 2891 &info->free_pbl_list); 2892 2893 if (!list_empty(&info->inuse_pbl_list)) 2894 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 2895 2896 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 2897 list_del(&pbl->list_entry); 2898 qedr_free_pbl(dev, &info->pbl_info, pbl); 2899 } 2900 } 2901 2902 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info, 2903 size_t page_list_len, bool two_layered) 2904 { 2905 struct qedr_pbl *tmp; 2906 int rc; 2907 2908 INIT_LIST_HEAD(&info->free_pbl_list); 2909 INIT_LIST_HEAD(&info->inuse_pbl_list); 2910 2911 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info, 2912 page_list_len, two_layered); 2913 if (rc) 2914 goto done; 2915 2916 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2917 if (IS_ERR(info->pbl_table)) { 2918 rc = PTR_ERR(info->pbl_table); 2919 goto done; 2920 } 2921 2922 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n", 2923 &info->pbl_table->pa); 2924 2925 /* in usual case we use 2 PBLs, so we add one to free 2926 * list and allocating another one 2927 */ 2928 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2929 if (IS_ERR(tmp)) { 2930 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n"); 2931 goto done; 2932 } 2933 2934 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 2935 2936 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa); 2937 2938 done: 2939 if (rc) 2940 free_mr_info(dev, info); 2941 2942 return rc; 2943 } 2944 2945 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 2946 u64 usr_addr, int acc, struct ib_udata *udata) 2947 { 2948 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2949 struct qedr_mr *mr; 2950 struct qedr_pd *pd; 2951 int rc = -ENOMEM; 2952 2953 pd = get_qedr_pd(ibpd); 2954 DP_DEBUG(dev, QEDR_MSG_MR, 2955 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 2956 pd->pd_id, start, len, usr_addr, acc); 2957 2958 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 2959 return ERR_PTR(-EINVAL); 2960 2961 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2962 if (!mr) 2963 return ERR_PTR(rc); 2964 2965 mr->type = QEDR_MR_USER; 2966 2967 mr->umem = ib_umem_get(ibpd->device, start, len, acc); 2968 if (IS_ERR(mr->umem)) { 2969 rc = -EFAULT; 2970 goto err0; 2971 } 2972 2973 rc = init_mr_info(dev, &mr->info, 2974 ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1); 2975 if (rc) 2976 goto err1; 2977 2978 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 2979 &mr->info.pbl_info, PAGE_SHIFT); 2980 2981 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2982 if (rc) { 2983 if (rc == -EINVAL) 2984 DP_ERR(dev, "Out of MR resources\n"); 2985 else 2986 DP_ERR(dev, "roce alloc tid returned error %d\n", rc); 2987 2988 goto err1; 2989 } 2990 2991 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2992 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2993 mr->hw_mr.key = 0; 2994 mr->hw_mr.pd = pd->pd_id; 2995 mr->hw_mr.local_read = 1; 2996 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2997 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2998 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2999 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 3000 mr->hw_mr.mw_bind = false; 3001 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 3002 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 3003 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 3004 mr->hw_mr.page_size_log = PAGE_SHIFT; 3005 mr->hw_mr.length = len; 3006 mr->hw_mr.vaddr = usr_addr; 3007 mr->hw_mr.phy_mr = false; 3008 mr->hw_mr.dma_mr = false; 3009 3010 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 3011 if (rc) { 3012 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 3013 goto err2; 3014 } 3015 3016 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3017 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 3018 mr->hw_mr.remote_atomic) 3019 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3020 3021 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n", 3022 mr->ibmr.lkey); 3023 return &mr->ibmr; 3024 3025 err2: 3026 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3027 err1: 3028 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 3029 err0: 3030 kfree(mr); 3031 return ERR_PTR(rc); 3032 } 3033 3034 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 3035 { 3036 struct qedr_mr *mr = get_qedr_mr(ib_mr); 3037 struct qedr_dev *dev = get_qedr_dev(ib_mr->device); 3038 int rc = 0; 3039 3040 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid); 3041 if (rc) 3042 return rc; 3043 3044 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3045 3046 if (mr->type != QEDR_MR_DMA) 3047 free_mr_info(dev, &mr->info); 3048 3049 /* it could be user registered memory. */ 3050 ib_umem_release(mr->umem); 3051 3052 kfree(mr); 3053 3054 return rc; 3055 } 3056 3057 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, 3058 int max_page_list_len) 3059 { 3060 struct qedr_pd *pd = get_qedr_pd(ibpd); 3061 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 3062 struct qedr_mr *mr; 3063 int rc = -ENOMEM; 3064 3065 DP_DEBUG(dev, QEDR_MSG_MR, 3066 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id, 3067 max_page_list_len); 3068 3069 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3070 if (!mr) 3071 return ERR_PTR(rc); 3072 3073 mr->dev = dev; 3074 mr->type = QEDR_MR_FRMR; 3075 3076 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1); 3077 if (rc) 3078 goto err0; 3079 3080 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 3081 if (rc) { 3082 if (rc == -EINVAL) 3083 DP_ERR(dev, "Out of MR resources\n"); 3084 else 3085 DP_ERR(dev, "roce alloc tid returned error %d\n", rc); 3086 3087 goto err1; 3088 } 3089 3090 /* Index only, 18 bit long, lkey = itid << 8 | key */ 3091 mr->hw_mr.tid_type = QED_RDMA_TID_FMR; 3092 mr->hw_mr.key = 0; 3093 mr->hw_mr.pd = pd->pd_id; 3094 mr->hw_mr.local_read = 1; 3095 mr->hw_mr.local_write = 0; 3096 mr->hw_mr.remote_read = 0; 3097 mr->hw_mr.remote_write = 0; 3098 mr->hw_mr.remote_atomic = 0; 3099 mr->hw_mr.mw_bind = false; 3100 mr->hw_mr.pbl_ptr = 0; 3101 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 3102 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 3103 mr->hw_mr.length = 0; 3104 mr->hw_mr.vaddr = 0; 3105 mr->hw_mr.phy_mr = true; 3106 mr->hw_mr.dma_mr = false; 3107 3108 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 3109 if (rc) { 3110 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 3111 goto err2; 3112 } 3113 3114 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3115 mr->ibmr.rkey = mr->ibmr.lkey; 3116 3117 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey); 3118 return mr; 3119 3120 err2: 3121 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3122 err1: 3123 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 3124 err0: 3125 kfree(mr); 3126 return ERR_PTR(rc); 3127 } 3128 3129 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 3130 u32 max_num_sg) 3131 { 3132 struct qedr_mr *mr; 3133 3134 if (mr_type != IB_MR_TYPE_MEM_REG) 3135 return ERR_PTR(-EINVAL); 3136 3137 mr = __qedr_alloc_mr(ibpd, max_num_sg); 3138 3139 if (IS_ERR(mr)) 3140 return ERR_PTR(-EINVAL); 3141 3142 return &mr->ibmr; 3143 } 3144 3145 static int qedr_set_page(struct ib_mr *ibmr, u64 addr) 3146 { 3147 struct qedr_mr *mr = get_qedr_mr(ibmr); 3148 struct qedr_pbl *pbl_table; 3149 struct regpair *pbe; 3150 u32 pbes_in_page; 3151 3152 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 3153 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages); 3154 return -ENOMEM; 3155 } 3156 3157 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n", 3158 mr->npages, addr); 3159 3160 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 3161 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 3162 pbe = (struct regpair *)pbl_table->va; 3163 pbe += mr->npages % pbes_in_page; 3164 pbe->lo = cpu_to_le32((u32)addr); 3165 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 3166 3167 mr->npages++; 3168 3169 return 0; 3170 } 3171 3172 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info) 3173 { 3174 int work = info->completed - info->completed_handled - 1; 3175 3176 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work); 3177 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 3178 struct qedr_pbl *pbl; 3179 3180 /* Free all the page list that are possible to be freed 3181 * (all the ones that were invalidated), under the assumption 3182 * that if an FMR was completed successfully that means that 3183 * if there was an invalidate operation before it also ended 3184 */ 3185 pbl = list_first_entry(&info->inuse_pbl_list, 3186 struct qedr_pbl, list_entry); 3187 list_move_tail(&pbl->list_entry, &info->free_pbl_list); 3188 info->completed_handled++; 3189 } 3190 } 3191 3192 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 3193 int sg_nents, unsigned int *sg_offset) 3194 { 3195 struct qedr_mr *mr = get_qedr_mr(ibmr); 3196 3197 mr->npages = 0; 3198 3199 handle_completed_mrs(mr->dev, &mr->info); 3200 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page); 3201 } 3202 3203 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc) 3204 { 3205 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 3206 struct qedr_pd *pd = get_qedr_pd(ibpd); 3207 struct qedr_mr *mr; 3208 int rc; 3209 3210 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3211 if (!mr) 3212 return ERR_PTR(-ENOMEM); 3213 3214 mr->type = QEDR_MR_DMA; 3215 3216 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 3217 if (rc) { 3218 if (rc == -EINVAL) 3219 DP_ERR(dev, "Out of MR resources\n"); 3220 else 3221 DP_ERR(dev, "roce alloc tid returned error %d\n", rc); 3222 3223 goto err1; 3224 } 3225 3226 /* index only, 18 bit long, lkey = itid << 8 | key */ 3227 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 3228 mr->hw_mr.pd = pd->pd_id; 3229 mr->hw_mr.local_read = 1; 3230 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 3231 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 3232 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 3233 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 3234 mr->hw_mr.dma_mr = true; 3235 3236 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 3237 if (rc) { 3238 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 3239 goto err2; 3240 } 3241 3242 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3243 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 3244 mr->hw_mr.remote_atomic) 3245 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3246 3247 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey); 3248 return &mr->ibmr; 3249 3250 err2: 3251 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3252 err1: 3253 kfree(mr); 3254 return ERR_PTR(rc); 3255 } 3256 3257 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq) 3258 { 3259 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 3260 } 3261 3262 static int sge_data_len(struct ib_sge *sg_list, int num_sge) 3263 { 3264 int i, len = 0; 3265 3266 for (i = 0; i < num_sge; i++) 3267 len += sg_list[i].length; 3268 3269 return len; 3270 } 3271 3272 static void swap_wqe_data64(u64 *p) 3273 { 3274 int i; 3275 3276 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 3277 *p = cpu_to_be64(cpu_to_le64(*p)); 3278 } 3279 3280 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev, 3281 struct qedr_qp *qp, u8 *wqe_size, 3282 const struct ib_send_wr *wr, 3283 const struct ib_send_wr **bad_wr, 3284 u8 *bits, u8 bit) 3285 { 3286 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 3287 char *seg_prt, *wqe; 3288 int i, seg_siz; 3289 3290 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 3291 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size); 3292 *bad_wr = wr; 3293 return 0; 3294 } 3295 3296 if (!data_size) 3297 return data_size; 3298 3299 *bits |= bit; 3300 3301 seg_prt = NULL; 3302 wqe = NULL; 3303 seg_siz = 0; 3304 3305 /* Copy data inline */ 3306 for (i = 0; i < wr->num_sge; i++) { 3307 u32 len = wr->sg_list[i].length; 3308 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 3309 3310 while (len > 0) { 3311 u32 cur; 3312 3313 /* New segment required */ 3314 if (!seg_siz) { 3315 wqe = (char *)qed_chain_produce(&qp->sq.pbl); 3316 seg_prt = wqe; 3317 seg_siz = sizeof(struct rdma_sq_common_wqe); 3318 (*wqe_size)++; 3319 } 3320 3321 /* Calculate currently allowed length */ 3322 cur = min_t(u32, len, seg_siz); 3323 memcpy(seg_prt, src, cur); 3324 3325 /* Update segment variables */ 3326 seg_prt += cur; 3327 seg_siz -= cur; 3328 3329 /* Update sge variables */ 3330 src += cur; 3331 len -= cur; 3332 3333 /* Swap fully-completed segments */ 3334 if (!seg_siz) 3335 swap_wqe_data64((u64 *)wqe); 3336 } 3337 } 3338 3339 /* swap last not completed segment */ 3340 if (seg_siz) 3341 swap_wqe_data64((u64 *)wqe); 3342 3343 return data_size; 3344 } 3345 3346 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 3347 do { \ 3348 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3349 (sge)->length = cpu_to_le32(vlength); \ 3350 (sge)->flags = cpu_to_le32(vflags); \ 3351 } while (0) 3352 3353 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 3354 do { \ 3355 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \ 3356 (hdr)->num_sges = num_sge; \ 3357 } while (0) 3358 3359 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 3360 do { \ 3361 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3362 (sge)->length = cpu_to_le32(vlength); \ 3363 (sge)->l_key = cpu_to_le32(vlkey); \ 3364 } while (0) 3365 3366 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, 3367 const struct ib_send_wr *wr) 3368 { 3369 u32 data_size = 0; 3370 int i; 3371 3372 for (i = 0; i < wr->num_sge; i++) { 3373 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl); 3374 3375 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr); 3376 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 3377 sge->length = cpu_to_le32(wr->sg_list[i].length); 3378 data_size += wr->sg_list[i].length; 3379 } 3380 3381 if (wqe_size) 3382 *wqe_size += wr->num_sge; 3383 3384 return data_size; 3385 } 3386 3387 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev, 3388 struct qedr_qp *qp, 3389 struct rdma_sq_rdma_wqe_1st *rwqe, 3390 struct rdma_sq_rdma_wqe_2nd *rwqe2, 3391 const struct ib_send_wr *wr, 3392 const struct ib_send_wr **bad_wr) 3393 { 3394 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 3395 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr); 3396 3397 if (wr->send_flags & IB_SEND_INLINE && 3398 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 3399 wr->opcode == IB_WR_RDMA_WRITE)) { 3400 u8 flags = 0; 3401 3402 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 3403 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, 3404 bad_wr, &rwqe->flags, flags); 3405 } 3406 3407 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr); 3408 } 3409 3410 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev, 3411 struct qedr_qp *qp, 3412 struct rdma_sq_send_wqe_1st *swqe, 3413 struct rdma_sq_send_wqe_2st *swqe2, 3414 const struct ib_send_wr *wr, 3415 const struct ib_send_wr **bad_wr) 3416 { 3417 memset(swqe2, 0, sizeof(*swqe2)); 3418 if (wr->send_flags & IB_SEND_INLINE) { 3419 u8 flags = 0; 3420 3421 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 3422 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, 3423 bad_wr, &swqe->flags, flags); 3424 } 3425 3426 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr); 3427 } 3428 3429 static int qedr_prepare_reg(struct qedr_qp *qp, 3430 struct rdma_sq_fmr_wqe_1st *fwqe1, 3431 const struct ib_reg_wr *wr) 3432 { 3433 struct qedr_mr *mr = get_qedr_mr(wr->mr); 3434 struct rdma_sq_fmr_wqe_2nd *fwqe2; 3435 3436 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl); 3437 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 3438 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 3439 fwqe1->l_key = wr->key; 3440 3441 fwqe2->access_ctrl = 0; 3442 3443 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 3444 !!(wr->access & IB_ACCESS_REMOTE_READ)); 3445 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 3446 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 3447 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 3448 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 3449 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 3450 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 3451 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 3452 fwqe2->fmr_ctrl = 0; 3453 3454 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 3455 ilog2(mr->ibmr.page_size) - 12); 3456 3457 fwqe2->length_hi = 0; 3458 fwqe2->length_lo = mr->ibmr.length; 3459 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 3460 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 3461 3462 qp->wqe_wr_id[qp->sq.prod].mr = mr; 3463 3464 return 0; 3465 } 3466 3467 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 3468 { 3469 switch (opcode) { 3470 case IB_WR_RDMA_WRITE: 3471 case IB_WR_RDMA_WRITE_WITH_IMM: 3472 return IB_WC_RDMA_WRITE; 3473 case IB_WR_SEND_WITH_IMM: 3474 case IB_WR_SEND: 3475 case IB_WR_SEND_WITH_INV: 3476 return IB_WC_SEND; 3477 case IB_WR_RDMA_READ: 3478 case IB_WR_RDMA_READ_WITH_INV: 3479 return IB_WC_RDMA_READ; 3480 case IB_WR_ATOMIC_CMP_AND_SWP: 3481 return IB_WC_COMP_SWAP; 3482 case IB_WR_ATOMIC_FETCH_AND_ADD: 3483 return IB_WC_FETCH_ADD; 3484 case IB_WR_REG_MR: 3485 return IB_WC_REG_MR; 3486 case IB_WR_LOCAL_INV: 3487 return IB_WC_LOCAL_INV; 3488 default: 3489 return IB_WC_SEND; 3490 } 3491 } 3492 3493 static inline bool qedr_can_post_send(struct qedr_qp *qp, 3494 const struct ib_send_wr *wr) 3495 { 3496 int wq_is_full, err_wr, pbl_is_full; 3497 struct qedr_dev *dev = qp->dev; 3498 3499 /* prevent SQ overflow and/or processing of a bad WR */ 3500 err_wr = wr->num_sge > qp->sq.max_sges; 3501 wq_is_full = qedr_wq_is_full(&qp->sq); 3502 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) < 3503 QEDR_MAX_SQE_ELEMENTS_PER_SQE; 3504 if (wq_is_full || err_wr || pbl_is_full) { 3505 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) { 3506 DP_ERR(dev, 3507 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n", 3508 qp); 3509 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL; 3510 } 3511 3512 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) { 3513 DP_ERR(dev, 3514 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n", 3515 qp); 3516 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR; 3517 } 3518 3519 if (pbl_is_full && 3520 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) { 3521 DP_ERR(dev, 3522 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n", 3523 qp); 3524 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL; 3525 } 3526 return false; 3527 } 3528 return true; 3529 } 3530 3531 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3532 const struct ib_send_wr **bad_wr) 3533 { 3534 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3535 struct qedr_qp *qp = get_qedr_qp(ibqp); 3536 struct rdma_sq_atomic_wqe_1st *awqe1; 3537 struct rdma_sq_atomic_wqe_2nd *awqe2; 3538 struct rdma_sq_atomic_wqe_3rd *awqe3; 3539 struct rdma_sq_send_wqe_2st *swqe2; 3540 struct rdma_sq_local_inv_wqe *iwqe; 3541 struct rdma_sq_rdma_wqe_2nd *rwqe2; 3542 struct rdma_sq_send_wqe_1st *swqe; 3543 struct rdma_sq_rdma_wqe_1st *rwqe; 3544 struct rdma_sq_fmr_wqe_1st *fwqe1; 3545 struct rdma_sq_common_wqe *wqe; 3546 u32 length; 3547 int rc = 0; 3548 bool comp; 3549 3550 if (!qedr_can_post_send(qp, wr)) { 3551 *bad_wr = wr; 3552 return -ENOMEM; 3553 } 3554 3555 wqe = qed_chain_produce(&qp->sq.pbl); 3556 qp->wqe_wr_id[qp->sq.prod].signaled = 3557 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 3558 3559 wqe->flags = 0; 3560 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, 3561 !!(wr->send_flags & IB_SEND_SOLICITED)); 3562 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled; 3563 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 3564 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, 3565 !!(wr->send_flags & IB_SEND_FENCE)); 3566 wqe->prev_wqe_size = qp->prev_wqe_size; 3567 3568 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode); 3569 3570 switch (wr->opcode) { 3571 case IB_WR_SEND_WITH_IMM: 3572 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3573 rc = -EINVAL; 3574 *bad_wr = wr; 3575 break; 3576 } 3577 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 3578 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3579 swqe->wqe_size = 2; 3580 swqe2 = qed_chain_produce(&qp->sq.pbl); 3581 3582 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 3583 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3584 wr, bad_wr); 3585 swqe->length = cpu_to_le32(length); 3586 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3587 qp->prev_wqe_size = swqe->wqe_size; 3588 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3589 break; 3590 case IB_WR_SEND: 3591 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 3592 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3593 3594 swqe->wqe_size = 2; 3595 swqe2 = qed_chain_produce(&qp->sq.pbl); 3596 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3597 wr, bad_wr); 3598 swqe->length = cpu_to_le32(length); 3599 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3600 qp->prev_wqe_size = swqe->wqe_size; 3601 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3602 break; 3603 case IB_WR_SEND_WITH_INV: 3604 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 3605 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3606 swqe2 = qed_chain_produce(&qp->sq.pbl); 3607 swqe->wqe_size = 2; 3608 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey); 3609 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3610 wr, bad_wr); 3611 swqe->length = cpu_to_le32(length); 3612 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3613 qp->prev_wqe_size = swqe->wqe_size; 3614 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3615 break; 3616 3617 case IB_WR_RDMA_WRITE_WITH_IMM: 3618 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3619 rc = -EINVAL; 3620 *bad_wr = wr; 3621 break; 3622 } 3623 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 3624 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3625 3626 rwqe->wqe_size = 2; 3627 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 3628 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3629 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3630 wr, bad_wr); 3631 rwqe->length = cpu_to_le32(length); 3632 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3633 qp->prev_wqe_size = rwqe->wqe_size; 3634 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3635 break; 3636 case IB_WR_RDMA_WRITE: 3637 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 3638 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3639 3640 rwqe->wqe_size = 2; 3641 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3642 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3643 wr, bad_wr); 3644 rwqe->length = cpu_to_le32(length); 3645 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3646 qp->prev_wqe_size = rwqe->wqe_size; 3647 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3648 break; 3649 case IB_WR_RDMA_READ_WITH_INV: 3650 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1); 3651 fallthrough; /* same is identical to RDMA READ */ 3652 3653 case IB_WR_RDMA_READ: 3654 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 3655 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3656 3657 rwqe->wqe_size = 2; 3658 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3659 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3660 wr, bad_wr); 3661 rwqe->length = cpu_to_le32(length); 3662 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3663 qp->prev_wqe_size = rwqe->wqe_size; 3664 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3665 break; 3666 3667 case IB_WR_ATOMIC_CMP_AND_SWP: 3668 case IB_WR_ATOMIC_FETCH_AND_ADD: 3669 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe; 3670 awqe1->wqe_size = 4; 3671 3672 awqe2 = qed_chain_produce(&qp->sq.pbl); 3673 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr); 3674 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 3675 3676 awqe3 = qed_chain_produce(&qp->sq.pbl); 3677 3678 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 3679 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 3680 DMA_REGPAIR_LE(awqe3->swap_data, 3681 atomic_wr(wr)->compare_add); 3682 } else { 3683 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 3684 DMA_REGPAIR_LE(awqe3->swap_data, 3685 atomic_wr(wr)->swap); 3686 DMA_REGPAIR_LE(awqe3->cmp_data, 3687 atomic_wr(wr)->compare_add); 3688 } 3689 3690 qedr_prepare_sq_sges(qp, NULL, wr); 3691 3692 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size; 3693 qp->prev_wqe_size = awqe1->wqe_size; 3694 break; 3695 3696 case IB_WR_LOCAL_INV: 3697 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 3698 iwqe->wqe_size = 1; 3699 3700 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 3701 iwqe->inv_l_key = wr->ex.invalidate_rkey; 3702 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size; 3703 qp->prev_wqe_size = iwqe->wqe_size; 3704 break; 3705 case IB_WR_REG_MR: 3706 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n"); 3707 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 3708 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 3709 fwqe1->wqe_size = 2; 3710 3711 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr)); 3712 if (rc) { 3713 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc); 3714 *bad_wr = wr; 3715 break; 3716 } 3717 3718 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 3719 qp->prev_wqe_size = fwqe1->wqe_size; 3720 break; 3721 default: 3722 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode); 3723 rc = -EINVAL; 3724 *bad_wr = wr; 3725 break; 3726 } 3727 3728 if (*bad_wr) { 3729 u16 value; 3730 3731 /* Restore prod to its position before 3732 * this WR was processed 3733 */ 3734 value = le16_to_cpu(qp->sq.db_data.data.value); 3735 qed_chain_set_prod(&qp->sq.pbl, value, wqe); 3736 3737 /* Restore prev_wqe_size */ 3738 qp->prev_wqe_size = wqe->prev_wqe_size; 3739 rc = -EINVAL; 3740 DP_ERR(dev, "POST SEND FAILED\n"); 3741 } 3742 3743 return rc; 3744 } 3745 3746 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3747 const struct ib_send_wr **bad_wr) 3748 { 3749 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3750 struct qedr_qp *qp = get_qedr_qp(ibqp); 3751 unsigned long flags; 3752 int rc = 0; 3753 3754 *bad_wr = NULL; 3755 3756 if (qp->qp_type == IB_QPT_GSI) 3757 return qedr_gsi_post_send(ibqp, wr, bad_wr); 3758 3759 spin_lock_irqsave(&qp->q_lock, flags); 3760 3761 if (rdma_protocol_roce(&dev->ibdev, 1)) { 3762 if ((qp->state != QED_ROCE_QP_STATE_RTS) && 3763 (qp->state != QED_ROCE_QP_STATE_ERR) && 3764 (qp->state != QED_ROCE_QP_STATE_SQD)) { 3765 spin_unlock_irqrestore(&qp->q_lock, flags); 3766 *bad_wr = wr; 3767 DP_DEBUG(dev, QEDR_MSG_CQ, 3768 "QP in wrong state! QP icid=0x%x state %d\n", 3769 qp->icid, qp->state); 3770 return -EINVAL; 3771 } 3772 } 3773 3774 while (wr) { 3775 rc = __qedr_post_send(ibqp, wr, bad_wr); 3776 if (rc) 3777 break; 3778 3779 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 3780 3781 qedr_inc_sw_prod(&qp->sq); 3782 3783 qp->sq.db_data.data.value++; 3784 3785 wr = wr->next; 3786 } 3787 3788 /* Trigger doorbell 3789 * If there was a failure in the first WR then it will be triggered in 3790 * vane. However this is not harmful (as long as the producer value is 3791 * unchanged). For performance reasons we avoid checking for this 3792 * redundant doorbell. 3793 * 3794 * qp->wqe_wr_id is accessed during qedr_poll_cq, as 3795 * soon as we give the doorbell, we could get a completion 3796 * for this wr, therefore we need to make sure that the 3797 * memory is updated before giving the doorbell. 3798 * During qedr_poll_cq, rmb is called before accessing the 3799 * cqe. This covers for the smp_rmb as well. 3800 */ 3801 smp_wmb(); 3802 writel(qp->sq.db_data.raw, qp->sq.db); 3803 3804 spin_unlock_irqrestore(&qp->q_lock, flags); 3805 3806 return rc; 3807 } 3808 3809 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq) 3810 { 3811 u32 used; 3812 3813 /* Calculate number of elements used based on producer 3814 * count and consumer count and subtract it from max 3815 * work request supported so that we get elements left. 3816 */ 3817 used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt); 3818 3819 return hw_srq->max_wr - used; 3820 } 3821 3822 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 3823 const struct ib_recv_wr **bad_wr) 3824 { 3825 struct qedr_srq *srq = get_qedr_srq(ibsrq); 3826 struct qedr_srq_hwq_info *hw_srq; 3827 struct qedr_dev *dev = srq->dev; 3828 struct qed_chain *pbl; 3829 unsigned long flags; 3830 int status = 0; 3831 u32 num_sge; 3832 3833 spin_lock_irqsave(&srq->lock, flags); 3834 3835 hw_srq = &srq->hw_srq; 3836 pbl = &srq->hw_srq.pbl; 3837 while (wr) { 3838 struct rdma_srq_wqe_header *hdr; 3839 int i; 3840 3841 if (!qedr_srq_elem_left(hw_srq) || 3842 wr->num_sge > srq->hw_srq.max_sges) { 3843 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n", 3844 hw_srq->wr_prod_cnt, 3845 atomic_read(&hw_srq->wr_cons_cnt), 3846 wr->num_sge, srq->hw_srq.max_sges); 3847 status = -ENOMEM; 3848 *bad_wr = wr; 3849 break; 3850 } 3851 3852 hdr = qed_chain_produce(pbl); 3853 num_sge = wr->num_sge; 3854 /* Set number of sge and work request id in header */ 3855 SRQ_HDR_SET(hdr, wr->wr_id, num_sge); 3856 3857 srq->hw_srq.wr_prod_cnt++; 3858 hw_srq->wqe_prod++; 3859 hw_srq->sge_prod++; 3860 3861 DP_DEBUG(dev, QEDR_MSG_SRQ, 3862 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n", 3863 wr->num_sge, hw_srq->wqe_prod, wr->wr_id); 3864 3865 for (i = 0; i < wr->num_sge; i++) { 3866 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl); 3867 3868 /* Set SGE length, lkey and address */ 3869 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, 3870 wr->sg_list[i].length, wr->sg_list[i].lkey); 3871 3872 DP_DEBUG(dev, QEDR_MSG_SRQ, 3873 "[%d]: len %d key %x addr %x:%x\n", 3874 i, srq_sge->length, srq_sge->l_key, 3875 srq_sge->addr.hi, srq_sge->addr.lo); 3876 hw_srq->sge_prod++; 3877 } 3878 3879 /* Update WQE and SGE information before 3880 * updating producer. 3881 */ 3882 dma_wmb(); 3883 3884 /* SRQ producer is 8 bytes. Need to update SGE producer index 3885 * in first 4 bytes and need to update WQE producer in 3886 * next 4 bytes. 3887 */ 3888 srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod); 3889 /* Make sure sge producer is updated first */ 3890 dma_wmb(); 3891 srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod); 3892 3893 wr = wr->next; 3894 } 3895 3896 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n", 3897 qed_chain_get_elem_left(pbl)); 3898 spin_unlock_irqrestore(&srq->lock, flags); 3899 3900 return status; 3901 } 3902 3903 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 3904 const struct ib_recv_wr **bad_wr) 3905 { 3906 struct qedr_qp *qp = get_qedr_qp(ibqp); 3907 struct qedr_dev *dev = qp->dev; 3908 unsigned long flags; 3909 int status = 0; 3910 3911 if (qp->qp_type == IB_QPT_GSI) 3912 return qedr_gsi_post_recv(ibqp, wr, bad_wr); 3913 3914 spin_lock_irqsave(&qp->q_lock, flags); 3915 3916 while (wr) { 3917 int i; 3918 3919 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) < 3920 QEDR_MAX_RQE_ELEMENTS_PER_RQE || 3921 wr->num_sge > qp->rq.max_sges) { 3922 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n", 3923 qed_chain_get_elem_left_u32(&qp->rq.pbl), 3924 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge, 3925 qp->rq.max_sges); 3926 status = -ENOMEM; 3927 *bad_wr = wr; 3928 break; 3929 } 3930 for (i = 0; i < wr->num_sge; i++) { 3931 u32 flags = 0; 3932 struct rdma_rq_sge *rqe = 3933 qed_chain_produce(&qp->rq.pbl); 3934 3935 /* First one must include the number 3936 * of SGE in the list 3937 */ 3938 if (!i) 3939 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 3940 wr->num_sge); 3941 3942 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 3943 wr->sg_list[i].lkey); 3944 3945 RQ_SGE_SET(rqe, wr->sg_list[i].addr, 3946 wr->sg_list[i].length, flags); 3947 } 3948 3949 /* Special case of no sges. FW requires between 1-4 sges... 3950 * in this case we need to post 1 sge with length zero. this is 3951 * because rdma write with immediate consumes an RQ. 3952 */ 3953 if (!wr->num_sge) { 3954 u32 flags = 0; 3955 struct rdma_rq_sge *rqe = 3956 qed_chain_produce(&qp->rq.pbl); 3957 3958 /* First one must include the number 3959 * of SGE in the list 3960 */ 3961 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0); 3962 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 3963 3964 RQ_SGE_SET(rqe, 0, 0, flags); 3965 i = 1; 3966 } 3967 3968 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 3969 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 3970 3971 qedr_inc_sw_prod(&qp->rq); 3972 3973 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as 3974 * soon as we give the doorbell, we could get a completion 3975 * for this wr, therefore we need to make sure that the 3976 * memory is update before giving the doorbell. 3977 * During qedr_poll_cq, rmb is called before accessing the 3978 * cqe. This covers for the smp_rmb as well. 3979 */ 3980 smp_wmb(); 3981 3982 qp->rq.db_data.data.value++; 3983 3984 writel(qp->rq.db_data.raw, qp->rq.db); 3985 3986 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 3987 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 3988 } 3989 3990 wr = wr->next; 3991 } 3992 3993 spin_unlock_irqrestore(&qp->q_lock, flags); 3994 3995 return status; 3996 } 3997 3998 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe) 3999 { 4000 struct rdma_cqe_requester *resp_cqe = &cqe->req; 4001 4002 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) == 4003 cq->pbl_toggle; 4004 } 4005 4006 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe) 4007 { 4008 struct rdma_cqe_requester *resp_cqe = &cqe->req; 4009 struct qedr_qp *qp; 4010 4011 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi, 4012 resp_cqe->qp_handle.lo, 4013 u64); 4014 return qp; 4015 } 4016 4017 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe) 4018 { 4019 struct rdma_cqe_requester *resp_cqe = &cqe->req; 4020 4021 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 4022 } 4023 4024 /* Return latest CQE (needs processing) */ 4025 static union rdma_cqe *get_cqe(struct qedr_cq *cq) 4026 { 4027 return cq->latest_cqe; 4028 } 4029 4030 /* In fmr we need to increase the number of fmr completed counter for the fmr 4031 * algorithm determining whether we can free a pbl or not. 4032 * we need to perform this whether the work request was signaled or not. for 4033 * this purpose we call this function from the condition that checks if a wr 4034 * should be skipped, to make sure we don't miss it ( possibly this fmr 4035 * operation was not signalted) 4036 */ 4037 static inline void qedr_chk_if_fmr(struct qedr_qp *qp) 4038 { 4039 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 4040 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 4041 } 4042 4043 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, 4044 struct qedr_cq *cq, int num_entries, 4045 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, 4046 int force) 4047 { 4048 u16 cnt = 0; 4049 4050 while (num_entries && qp->sq.wqe_cons != hw_cons) { 4051 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 4052 qedr_chk_if_fmr(qp); 4053 /* skip WC */ 4054 goto next_cqe; 4055 } 4056 4057 /* fill WC */ 4058 wc->status = status; 4059 wc->vendor_err = 0; 4060 wc->wc_flags = 0; 4061 wc->src_qp = qp->id; 4062 wc->qp = &qp->ibqp; 4063 4064 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 4065 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 4066 4067 switch (wc->opcode) { 4068 case IB_WC_RDMA_WRITE: 4069 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 4070 break; 4071 case IB_WC_COMP_SWAP: 4072 case IB_WC_FETCH_ADD: 4073 wc->byte_len = 8; 4074 break; 4075 case IB_WC_REG_MR: 4076 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 4077 break; 4078 case IB_WC_RDMA_READ: 4079 case IB_WC_SEND: 4080 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 4081 break; 4082 default: 4083 break; 4084 } 4085 4086 num_entries--; 4087 wc++; 4088 cnt++; 4089 next_cqe: 4090 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 4091 qed_chain_consume(&qp->sq.pbl); 4092 qedr_inc_sw_cons(&qp->sq); 4093 } 4094 4095 return cnt; 4096 } 4097 4098 static int qedr_poll_cq_req(struct qedr_dev *dev, 4099 struct qedr_qp *qp, struct qedr_cq *cq, 4100 int num_entries, struct ib_wc *wc, 4101 struct rdma_cqe_requester *req) 4102 { 4103 int cnt = 0; 4104 4105 switch (req->status) { 4106 case RDMA_CQE_REQ_STS_OK: 4107 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 4108 IB_WC_SUCCESS, 0); 4109 break; 4110 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 4111 if (qp->state != QED_ROCE_QP_STATE_ERR) 4112 DP_DEBUG(dev, QEDR_MSG_CQ, 4113 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4114 cq->icid, qp->icid); 4115 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 4116 IB_WC_WR_FLUSH_ERR, 1); 4117 break; 4118 default: 4119 /* process all WQE before the cosumer */ 4120 qp->state = QED_ROCE_QP_STATE_ERR; 4121 cnt = process_req(dev, qp, cq, num_entries, wc, 4122 req->sq_cons - 1, IB_WC_SUCCESS, 0); 4123 wc += cnt; 4124 /* if we have extra WC fill it with actual error info */ 4125 if (cnt < num_entries) { 4126 enum ib_wc_status wc_status; 4127 4128 switch (req->status) { 4129 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 4130 DP_ERR(dev, 4131 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4132 cq->icid, qp->icid); 4133 wc_status = IB_WC_BAD_RESP_ERR; 4134 break; 4135 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 4136 DP_ERR(dev, 4137 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4138 cq->icid, qp->icid); 4139 wc_status = IB_WC_LOC_LEN_ERR; 4140 break; 4141 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 4142 DP_ERR(dev, 4143 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4144 cq->icid, qp->icid); 4145 wc_status = IB_WC_LOC_QP_OP_ERR; 4146 break; 4147 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 4148 DP_ERR(dev, 4149 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4150 cq->icid, qp->icid); 4151 wc_status = IB_WC_LOC_PROT_ERR; 4152 break; 4153 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 4154 DP_ERR(dev, 4155 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4156 cq->icid, qp->icid); 4157 wc_status = IB_WC_MW_BIND_ERR; 4158 break; 4159 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 4160 DP_ERR(dev, 4161 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4162 cq->icid, qp->icid); 4163 wc_status = IB_WC_REM_INV_REQ_ERR; 4164 break; 4165 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 4166 DP_ERR(dev, 4167 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4168 cq->icid, qp->icid); 4169 wc_status = IB_WC_REM_ACCESS_ERR; 4170 break; 4171 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 4172 DP_ERR(dev, 4173 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4174 cq->icid, qp->icid); 4175 wc_status = IB_WC_REM_OP_ERR; 4176 break; 4177 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 4178 DP_ERR(dev, 4179 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4180 cq->icid, qp->icid); 4181 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 4182 break; 4183 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 4184 DP_ERR(dev, 4185 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4186 cq->icid, qp->icid); 4187 wc_status = IB_WC_RETRY_EXC_ERR; 4188 break; 4189 default: 4190 DP_ERR(dev, 4191 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4192 cq->icid, qp->icid); 4193 wc_status = IB_WC_GENERAL_ERR; 4194 } 4195 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 4196 wc_status, 1); 4197 } 4198 } 4199 4200 return cnt; 4201 } 4202 4203 static inline int qedr_cqe_resp_status_to_ib(u8 status) 4204 { 4205 switch (status) { 4206 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 4207 return IB_WC_LOC_ACCESS_ERR; 4208 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 4209 return IB_WC_LOC_LEN_ERR; 4210 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 4211 return IB_WC_LOC_QP_OP_ERR; 4212 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 4213 return IB_WC_LOC_PROT_ERR; 4214 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 4215 return IB_WC_MW_BIND_ERR; 4216 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 4217 return IB_WC_REM_INV_RD_REQ_ERR; 4218 case RDMA_CQE_RESP_STS_OK: 4219 return IB_WC_SUCCESS; 4220 default: 4221 return IB_WC_GENERAL_ERR; 4222 } 4223 } 4224 4225 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp, 4226 struct ib_wc *wc) 4227 { 4228 wc->status = IB_WC_SUCCESS; 4229 wc->byte_len = le32_to_cpu(resp->length); 4230 4231 if (resp->flags & QEDR_RESP_IMM) { 4232 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key)); 4233 wc->wc_flags |= IB_WC_WITH_IMM; 4234 4235 if (resp->flags & QEDR_RESP_RDMA) 4236 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 4237 4238 if (resp->flags & QEDR_RESP_INV) 4239 return -EINVAL; 4240 4241 } else if (resp->flags & QEDR_RESP_INV) { 4242 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 4243 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4244 4245 if (resp->flags & QEDR_RESP_RDMA) 4246 return -EINVAL; 4247 4248 } else if (resp->flags & QEDR_RESP_RDMA) { 4249 return -EINVAL; 4250 } 4251 4252 return 0; 4253 } 4254 4255 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4256 struct qedr_cq *cq, struct ib_wc *wc, 4257 struct rdma_cqe_responder *resp, u64 wr_id) 4258 { 4259 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */ 4260 wc->opcode = IB_WC_RECV; 4261 wc->wc_flags = 0; 4262 4263 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) { 4264 if (qedr_set_ok_cqe_resp_wc(resp, wc)) 4265 DP_ERR(dev, 4266 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n", 4267 cq, cq->icid, resp->flags); 4268 4269 } else { 4270 wc->status = qedr_cqe_resp_status_to_ib(resp->status); 4271 if (wc->status == IB_WC_GENERAL_ERR) 4272 DP_ERR(dev, 4273 "CQ %p (icid=%d) contains an invalid CQE status %d\n", 4274 cq, cq->icid, resp->status); 4275 } 4276 4277 /* Fill the rest of the WC */ 4278 wc->vendor_err = 0; 4279 wc->src_qp = qp->id; 4280 wc->qp = &qp->ibqp; 4281 wc->wr_id = wr_id; 4282 } 4283 4284 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4285 struct qedr_cq *cq, struct ib_wc *wc, 4286 struct rdma_cqe_responder *resp) 4287 { 4288 struct qedr_srq *srq = qp->srq; 4289 u64 wr_id; 4290 4291 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi), 4292 le32_to_cpu(resp->srq_wr_id.lo), u64); 4293 4294 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4295 wc->status = IB_WC_WR_FLUSH_ERR; 4296 wc->vendor_err = 0; 4297 wc->wr_id = wr_id; 4298 wc->byte_len = 0; 4299 wc->src_qp = qp->id; 4300 wc->qp = &qp->ibqp; 4301 wc->wr_id = wr_id; 4302 } else { 4303 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4304 } 4305 atomic_inc(&srq->hw_srq.wr_cons_cnt); 4306 4307 return 1; 4308 } 4309 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4310 struct qedr_cq *cq, struct ib_wc *wc, 4311 struct rdma_cqe_responder *resp) 4312 { 4313 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4314 4315 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4316 4317 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4318 qed_chain_consume(&qp->rq.pbl); 4319 qedr_inc_sw_cons(&qp->rq); 4320 4321 return 1; 4322 } 4323 4324 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, 4325 int num_entries, struct ib_wc *wc, u16 hw_cons) 4326 { 4327 u16 cnt = 0; 4328 4329 while (num_entries && qp->rq.wqe_cons != hw_cons) { 4330 /* fill WC */ 4331 wc->status = IB_WC_WR_FLUSH_ERR; 4332 wc->vendor_err = 0; 4333 wc->wc_flags = 0; 4334 wc->src_qp = qp->id; 4335 wc->byte_len = 0; 4336 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4337 wc->qp = &qp->ibqp; 4338 num_entries--; 4339 wc++; 4340 cnt++; 4341 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4342 qed_chain_consume(&qp->rq.pbl); 4343 qedr_inc_sw_cons(&qp->rq); 4344 } 4345 4346 return cnt; 4347 } 4348 4349 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4350 struct rdma_cqe_responder *resp, int *update) 4351 { 4352 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) { 4353 consume_cqe(cq); 4354 *update |= 1; 4355 } 4356 } 4357 4358 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4359 struct qedr_cq *cq, int num_entries, 4360 struct ib_wc *wc, 4361 struct rdma_cqe_responder *resp) 4362 { 4363 int cnt; 4364 4365 cnt = process_resp_one_srq(dev, qp, cq, wc, resp); 4366 consume_cqe(cq); 4367 4368 return cnt; 4369 } 4370 4371 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, 4372 struct qedr_cq *cq, int num_entries, 4373 struct ib_wc *wc, struct rdma_cqe_responder *resp, 4374 int *update) 4375 { 4376 int cnt; 4377 4378 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4379 cnt = process_resp_flush(qp, cq, num_entries, wc, 4380 resp->rq_cons_or_srq_id); 4381 try_consume_resp_cqe(cq, qp, resp, update); 4382 } else { 4383 cnt = process_resp_one(dev, qp, cq, wc, resp); 4384 consume_cqe(cq); 4385 *update |= 1; 4386 } 4387 4388 return cnt; 4389 } 4390 4391 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4392 struct rdma_cqe_requester *req, int *update) 4393 { 4394 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 4395 consume_cqe(cq); 4396 *update |= 1; 4397 } 4398 } 4399 4400 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 4401 { 4402 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 4403 struct qedr_cq *cq = get_qedr_cq(ibcq); 4404 union rdma_cqe *cqe; 4405 u32 old_cons, new_cons; 4406 unsigned long flags; 4407 int update = 0; 4408 int done = 0; 4409 4410 if (cq->destroyed) { 4411 DP_ERR(dev, 4412 "warning: poll was invoked after destroy for cq %p (icid=%d)\n", 4413 cq, cq->icid); 4414 return 0; 4415 } 4416 4417 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 4418 return qedr_gsi_poll_cq(ibcq, num_entries, wc); 4419 4420 spin_lock_irqsave(&cq->cq_lock, flags); 4421 cqe = cq->latest_cqe; 4422 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4423 while (num_entries && is_valid_cqe(cq, cqe)) { 4424 struct qedr_qp *qp; 4425 int cnt = 0; 4426 4427 /* prevent speculative reads of any field of CQE */ 4428 rmb(); 4429 4430 qp = cqe_get_qp(cqe); 4431 if (!qp) { 4432 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe); 4433 break; 4434 } 4435 4436 wc->qp = &qp->ibqp; 4437 4438 switch (cqe_get_type(cqe)) { 4439 case RDMA_CQE_TYPE_REQUESTER: 4440 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc, 4441 &cqe->req); 4442 try_consume_req_cqe(cq, qp, &cqe->req, &update); 4443 break; 4444 case RDMA_CQE_TYPE_RESPONDER_RQ: 4445 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc, 4446 &cqe->resp, &update); 4447 break; 4448 case RDMA_CQE_TYPE_RESPONDER_SRQ: 4449 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries, 4450 wc, &cqe->resp); 4451 update = 1; 4452 break; 4453 case RDMA_CQE_TYPE_INVALID: 4454 default: 4455 DP_ERR(dev, "Error: invalid CQE type = %d\n", 4456 cqe_get_type(cqe)); 4457 } 4458 num_entries -= cnt; 4459 wc += cnt; 4460 done += cnt; 4461 4462 cqe = get_cqe(cq); 4463 } 4464 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4465 4466 cq->cq_cons += new_cons - old_cons; 4467 4468 if (update) 4469 /* doorbell notifies abount latest VALID entry, 4470 * but chain already point to the next INVALID one 4471 */ 4472 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 4473 4474 spin_unlock_irqrestore(&cq->cq_lock, flags); 4475 return done; 4476 } 4477 4478 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, 4479 u32 port_num, const struct ib_wc *in_wc, 4480 const struct ib_grh *in_grh, const struct ib_mad *in, 4481 struct ib_mad *out_mad, size_t *out_mad_size, 4482 u16 *out_mad_pkey_index) 4483 { 4484 return IB_MAD_RESULT_SUCCESS; 4485 } 4486