1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/dma-mapping.h> 33 #include <linux/crc32.h> 34 #include <net/ip.h> 35 #include <net/ipv6.h> 36 #include <net/udp.h> 37 #include <linux/iommu.h> 38 39 #include <rdma/ib_verbs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/iw_cm.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/ib_addr.h> 44 #include <rdma/ib_cache.h> 45 #include <rdma/uverbs_ioctl.h> 46 47 #include <linux/qed/common_hsi.h> 48 #include "qedr_hsi_rdma.h" 49 #include <linux/qed/qed_if.h> 50 #include "qedr.h" 51 #include "verbs.h" 52 #include <rdma/qedr-abi.h> 53 #include "qedr_roce_cm.h" 54 #include "qedr_iw_cm.h" 55 56 #define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm) 57 #define RDMA_MAX_SGE_PER_SRQ (4) 58 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) 59 60 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 61 62 enum { 63 QEDR_USER_MMAP_IO_WC = 0, 64 QEDR_USER_MMAP_PHYS_PAGE, 65 }; 66 67 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, 68 size_t len) 69 { 70 size_t min_len = min_t(size_t, len, udata->outlen); 71 72 return ib_copy_to_udata(udata, src, min_len); 73 } 74 75 int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) 76 { 77 if (index >= QEDR_ROCE_PKEY_TABLE_LEN) 78 return -EINVAL; 79 80 *pkey = QEDR_ROCE_PKEY_DEFAULT; 81 return 0; 82 } 83 84 int qedr_iw_query_gid(struct ib_device *ibdev, u32 port, 85 int index, union ib_gid *sgid) 86 { 87 struct qedr_dev *dev = get_qedr_dev(ibdev); 88 89 memset(sgid->raw, 0, sizeof(sgid->raw)); 90 ether_addr_copy(sgid->raw, dev->ndev->dev_addr); 91 92 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index, 93 sgid->global.interface_id, sgid->global.subnet_prefix); 94 95 return 0; 96 } 97 98 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 99 { 100 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 101 struct qedr_device_attr *qattr = &dev->attr; 102 struct qedr_srq *srq = get_qedr_srq(ibsrq); 103 104 srq_attr->srq_limit = srq->srq_limit; 105 srq_attr->max_wr = qattr->max_srq_wr; 106 srq_attr->max_sge = qattr->max_sge; 107 108 return 0; 109 } 110 111 int qedr_query_device(struct ib_device *ibdev, 112 struct ib_device_attr *attr, struct ib_udata *udata) 113 { 114 struct qedr_dev *dev = get_qedr_dev(ibdev); 115 struct qedr_device_attr *qattr = &dev->attr; 116 117 if (!dev->rdma_ctx) { 118 DP_ERR(dev, 119 "qedr_query_device called with invalid params rdma_ctx=%p\n", 120 dev->rdma_ctx); 121 return -EINVAL; 122 } 123 124 memset(attr, 0, sizeof(*attr)); 125 126 attr->fw_ver = qattr->fw_ver; 127 attr->sys_image_guid = qattr->sys_image_guid; 128 attr->max_mr_size = qattr->max_mr_size; 129 attr->page_size_cap = qattr->page_size_caps; 130 attr->vendor_id = qattr->vendor_id; 131 attr->vendor_part_id = qattr->vendor_part_id; 132 attr->hw_ver = qattr->hw_ver; 133 attr->max_qp = qattr->max_qp; 134 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe); 135 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 136 IB_DEVICE_RC_RNR_NAK_GEN | 137 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS; 138 139 if (!rdma_protocol_iwarp(&dev->ibdev, 1)) 140 attr->device_cap_flags |= IB_DEVICE_XRC; 141 attr->max_send_sge = qattr->max_sge; 142 attr->max_recv_sge = qattr->max_sge; 143 attr->max_sge_rd = qattr->max_sge; 144 attr->max_cq = qattr->max_cq; 145 attr->max_cqe = qattr->max_cqe; 146 attr->max_mr = qattr->max_mr; 147 attr->max_mw = qattr->max_mw; 148 attr->max_pd = qattr->max_pd; 149 attr->atomic_cap = dev->atomic_cap; 150 attr->max_qp_init_rd_atom = 151 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 152 attr->max_qp_rd_atom = 153 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 154 attr->max_qp_init_rd_atom); 155 156 attr->max_srq = qattr->max_srq; 157 attr->max_srq_sge = qattr->max_srq_sge; 158 attr->max_srq_wr = qattr->max_srq_wr; 159 160 attr->local_ca_ack_delay = qattr->dev_ack_delay; 161 attr->max_fast_reg_page_list_len = qattr->max_mr / 8; 162 attr->max_pkeys = qattr->max_pkey; 163 attr->max_ah = qattr->max_ah; 164 165 return 0; 166 } 167 168 static inline void get_link_speed_and_width(int speed, u16 *ib_speed, 169 u8 *ib_width) 170 { 171 switch (speed) { 172 case 1000: 173 *ib_speed = IB_SPEED_SDR; 174 *ib_width = IB_WIDTH_1X; 175 break; 176 case 10000: 177 *ib_speed = IB_SPEED_QDR; 178 *ib_width = IB_WIDTH_1X; 179 break; 180 181 case 20000: 182 *ib_speed = IB_SPEED_DDR; 183 *ib_width = IB_WIDTH_4X; 184 break; 185 186 case 25000: 187 *ib_speed = IB_SPEED_EDR; 188 *ib_width = IB_WIDTH_1X; 189 break; 190 191 case 40000: 192 *ib_speed = IB_SPEED_QDR; 193 *ib_width = IB_WIDTH_4X; 194 break; 195 196 case 50000: 197 *ib_speed = IB_SPEED_HDR; 198 *ib_width = IB_WIDTH_1X; 199 break; 200 201 case 100000: 202 *ib_speed = IB_SPEED_EDR; 203 *ib_width = IB_WIDTH_4X; 204 break; 205 206 default: 207 /* Unsupported */ 208 *ib_speed = IB_SPEED_SDR; 209 *ib_width = IB_WIDTH_1X; 210 } 211 } 212 213 int qedr_query_port(struct ib_device *ibdev, u32 port, 214 struct ib_port_attr *attr) 215 { 216 struct qedr_dev *dev; 217 struct qed_rdma_port *rdma_port; 218 219 dev = get_qedr_dev(ibdev); 220 221 if (!dev->rdma_ctx) { 222 DP_ERR(dev, "rdma_ctx is NULL\n"); 223 return -EINVAL; 224 } 225 226 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx); 227 228 /* *attr being zeroed by the caller, avoid zeroing it here */ 229 if (rdma_port->port_state == QED_RDMA_PORT_UP) { 230 attr->state = IB_PORT_ACTIVE; 231 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 232 } else { 233 attr->state = IB_PORT_DOWN; 234 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 235 } 236 attr->max_mtu = IB_MTU_4096; 237 attr->lid = 0; 238 attr->lmc = 0; 239 attr->sm_lid = 0; 240 attr->sm_sl = 0; 241 attr->ip_gids = true; 242 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 243 attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu); 244 attr->gid_tbl_len = 1; 245 } else { 246 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); 247 attr->gid_tbl_len = QEDR_MAX_SGID; 248 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; 249 } 250 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 251 attr->qkey_viol_cntr = 0; 252 get_link_speed_and_width(rdma_port->link_speed, 253 &attr->active_speed, &attr->active_width); 254 attr->max_msg_sz = rdma_port->max_msg_size; 255 attr->max_vl_num = 4; 256 257 return 0; 258 } 259 260 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 261 { 262 struct ib_device *ibdev = uctx->device; 263 int rc; 264 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx); 265 struct qedr_alloc_ucontext_resp uresp = {}; 266 struct qedr_alloc_ucontext_req ureq = {}; 267 struct qedr_dev *dev = get_qedr_dev(ibdev); 268 struct qed_rdma_add_user_out_params oparams; 269 struct qedr_user_mmap_entry *entry; 270 271 if (!udata) 272 return -EFAULT; 273 274 if (udata->inlen) { 275 rc = ib_copy_from_udata(&ureq, udata, 276 min(sizeof(ureq), udata->inlen)); 277 if (rc) { 278 DP_ERR(dev, "Problem copying data from user space\n"); 279 return -EFAULT; 280 } 281 ctx->edpm_mode = !!(ureq.context_flags & 282 QEDR_ALLOC_UCTX_EDPM_MODE); 283 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC); 284 } 285 286 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams); 287 if (rc) { 288 DP_ERR(dev, 289 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n", 290 rc); 291 return rc; 292 } 293 294 ctx->dpi = oparams.dpi; 295 ctx->dpi_addr = oparams.dpi_addr; 296 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 297 ctx->dpi_size = oparams.dpi_size; 298 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 299 if (!entry) { 300 rc = -ENOMEM; 301 goto err; 302 } 303 304 entry->io_address = ctx->dpi_phys_addr; 305 entry->length = ctx->dpi_size; 306 entry->mmap_flag = QEDR_USER_MMAP_IO_WC; 307 entry->dpi = ctx->dpi; 308 entry->dev = dev; 309 rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry, 310 ctx->dpi_size); 311 if (rc) { 312 kfree(entry); 313 goto err; 314 } 315 ctx->db_mmap_entry = &entry->rdma_entry; 316 317 if (!dev->user_dpm_enabled) 318 uresp.dpm_flags = 0; 319 else if (rdma_protocol_iwarp(&dev->ibdev, 1)) 320 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY; 321 else 322 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED | 323 QEDR_DPM_TYPE_ROCE_LEGACY | 324 QEDR_DPM_TYPE_ROCE_EDPM_MODE; 325 326 if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) { 327 uresp.dpm_flags |= QEDR_DPM_SIZES_SET; 328 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE; 329 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE; 330 uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE; 331 } 332 333 uresp.wids_enabled = 1; 334 uresp.wid_count = oparams.wid_count; 335 uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry); 336 uresp.db_size = ctx->dpi_size; 337 uresp.max_send_wr = dev->attr.max_sqe; 338 uresp.max_recv_wr = dev->attr.max_rqe; 339 uresp.max_srq_wr = dev->attr.max_srq_wr; 340 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE; 341 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE; 342 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 343 uresp.max_cqes = QEDR_MAX_CQES; 344 345 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 346 if (rc) 347 goto err; 348 349 ctx->dev = dev; 350 351 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n", 352 &ctx->ibucontext); 353 return 0; 354 355 err: 356 if (!ctx->db_mmap_entry) 357 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi); 358 else 359 rdma_user_mmap_entry_remove(ctx->db_mmap_entry); 360 361 return rc; 362 } 363 364 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx) 365 { 366 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx); 367 368 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n", 369 uctx); 370 371 rdma_user_mmap_entry_remove(uctx->db_mmap_entry); 372 } 373 374 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 375 { 376 struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry); 377 struct qedr_dev *dev = entry->dev; 378 379 if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE) 380 free_page((unsigned long)entry->address); 381 else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC) 382 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi); 383 384 kfree(entry); 385 } 386 387 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma) 388 { 389 struct ib_device *dev = ucontext->device; 390 size_t length = vma->vm_end - vma->vm_start; 391 struct rdma_user_mmap_entry *rdma_entry; 392 struct qedr_user_mmap_entry *entry; 393 int rc = 0; 394 u64 pfn; 395 396 ibdev_dbg(dev, 397 "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n", 398 vma->vm_start, vma->vm_end, length, vma->vm_pgoff); 399 400 rdma_entry = rdma_user_mmap_entry_get(ucontext, vma); 401 if (!rdma_entry) { 402 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n", 403 vma->vm_pgoff); 404 return -EINVAL; 405 } 406 entry = get_qedr_mmap_entry(rdma_entry); 407 ibdev_dbg(dev, 408 "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n", 409 entry->io_address, length, entry->mmap_flag); 410 411 switch (entry->mmap_flag) { 412 case QEDR_USER_MMAP_IO_WC: 413 pfn = entry->io_address >> PAGE_SHIFT; 414 rc = rdma_user_mmap_io(ucontext, vma, pfn, length, 415 pgprot_writecombine(vma->vm_page_prot), 416 rdma_entry); 417 break; 418 case QEDR_USER_MMAP_PHYS_PAGE: 419 rc = vm_insert_page(vma, vma->vm_start, 420 virt_to_page(entry->address)); 421 break; 422 default: 423 rc = -EINVAL; 424 } 425 426 if (rc) 427 ibdev_dbg(dev, 428 "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n", 429 entry->io_address, length, entry->mmap_flag, rc); 430 431 rdma_user_mmap_entry_put(rdma_entry); 432 return rc; 433 } 434 435 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 436 { 437 struct ib_device *ibdev = ibpd->device; 438 struct qedr_dev *dev = get_qedr_dev(ibdev); 439 struct qedr_pd *pd = get_qedr_pd(ibpd); 440 u16 pd_id; 441 int rc; 442 443 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n", 444 udata ? "User Lib" : "Kernel"); 445 446 if (!dev->rdma_ctx) { 447 DP_ERR(dev, "invalid RDMA context\n"); 448 return -EINVAL; 449 } 450 451 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id); 452 if (rc) 453 return rc; 454 455 pd->pd_id = pd_id; 456 457 if (udata) { 458 struct qedr_alloc_pd_uresp uresp = { 459 .pd_id = pd_id, 460 }; 461 struct qedr_ucontext *context = rdma_udata_to_drv_context( 462 udata, struct qedr_ucontext, ibucontext); 463 464 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 465 if (rc) { 466 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); 467 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); 468 return rc; 469 } 470 471 pd->uctx = context; 472 pd->uctx->pd = pd; 473 } 474 475 return 0; 476 } 477 478 int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 479 { 480 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 481 struct qedr_pd *pd = get_qedr_pd(ibpd); 482 483 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id); 484 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id); 485 return 0; 486 } 487 488 489 int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) 490 { 491 struct qedr_dev *dev = get_qedr_dev(ibxrcd->device); 492 struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd); 493 494 return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id); 495 } 496 497 int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) 498 { 499 struct qedr_dev *dev = get_qedr_dev(ibxrcd->device); 500 u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id; 501 502 dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id); 503 return 0; 504 } 505 static void qedr_free_pbl(struct qedr_dev *dev, 506 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) 507 { 508 struct pci_dev *pdev = dev->pdev; 509 int i; 510 511 for (i = 0; i < pbl_info->num_pbls; i++) { 512 if (!pbl[i].va) 513 continue; 514 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 515 pbl[i].va, pbl[i].pa); 516 } 517 518 kfree(pbl); 519 } 520 521 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024) 522 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024) 523 524 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 525 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 526 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE) 527 528 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev, 529 struct qedr_pbl_info *pbl_info, 530 gfp_t flags) 531 { 532 struct pci_dev *pdev = dev->pdev; 533 struct qedr_pbl *pbl_table; 534 dma_addr_t *pbl_main_tbl; 535 dma_addr_t pa; 536 void *va; 537 int i; 538 539 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags); 540 if (!pbl_table) 541 return ERR_PTR(-ENOMEM); 542 543 for (i = 0; i < pbl_info->num_pbls; i++) { 544 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa, 545 flags); 546 if (!va) 547 goto err; 548 549 pbl_table[i].va = va; 550 pbl_table[i].pa = pa; 551 } 552 553 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 554 * the first one with physical pointers to all of the rest 555 */ 556 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 557 for (i = 0; i < pbl_info->num_pbls - 1; i++) 558 pbl_main_tbl[i] = pbl_table[i + 1].pa; 559 560 return pbl_table; 561 562 err: 563 for (i--; i >= 0; i--) 564 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 565 pbl_table[i].va, pbl_table[i].pa); 566 567 qedr_free_pbl(dev, pbl_info, pbl_table); 568 569 return ERR_PTR(-ENOMEM); 570 } 571 572 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev, 573 struct qedr_pbl_info *pbl_info, 574 u32 num_pbes, int two_layer_capable) 575 { 576 u32 pbl_capacity; 577 u32 pbl_size; 578 u32 num_pbls; 579 580 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 581 if (num_pbes > MAX_PBES_TWO_LAYER) { 582 DP_ERR(dev, "prepare pbl table: too many pages %d\n", 583 num_pbes); 584 return -EINVAL; 585 } 586 587 /* calculate required pbl page size */ 588 pbl_size = MIN_FW_PBL_PAGE_SIZE; 589 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 590 NUM_PBES_ON_PAGE(pbl_size); 591 592 while (pbl_capacity < num_pbes) { 593 pbl_size *= 2; 594 pbl_capacity = pbl_size / sizeof(u64); 595 pbl_capacity = pbl_capacity * pbl_capacity; 596 } 597 598 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 599 num_pbls++; /* One for the layer0 ( points to the pbls) */ 600 pbl_info->two_layered = true; 601 } else { 602 /* One layered PBL */ 603 num_pbls = 1; 604 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, 605 roundup_pow_of_two((num_pbes * sizeof(u64)))); 606 pbl_info->two_layered = false; 607 } 608 609 pbl_info->num_pbls = num_pbls; 610 pbl_info->pbl_size = pbl_size; 611 pbl_info->num_pbes = num_pbes; 612 613 DP_DEBUG(dev, QEDR_MSG_MR, 614 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n", 615 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 616 617 return 0; 618 } 619 620 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, 621 struct qedr_pbl *pbl, 622 struct qedr_pbl_info *pbl_info, u32 pg_shift) 623 { 624 int pbe_cnt, total_num_pbes = 0; 625 struct qedr_pbl *pbl_tbl; 626 struct ib_block_iter biter; 627 struct regpair *pbe; 628 629 if (!pbl_info->num_pbes) 630 return; 631 632 /* If we have a two layered pbl, the first pbl points to the rest 633 * of the pbls and the first entry lays on the second pbl in the table 634 */ 635 if (pbl_info->two_layered) 636 pbl_tbl = &pbl[1]; 637 else 638 pbl_tbl = pbl; 639 640 pbe = (struct regpair *)pbl_tbl->va; 641 if (!pbe) { 642 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n"); 643 return; 644 } 645 646 pbe_cnt = 0; 647 648 rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) { 649 u64 pg_addr = rdma_block_iter_dma_address(&biter); 650 651 pbe->lo = cpu_to_le32(pg_addr); 652 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); 653 654 pbe_cnt++; 655 total_num_pbes++; 656 pbe++; 657 658 if (total_num_pbes == pbl_info->num_pbes) 659 return; 660 661 /* If the given pbl is full storing the pbes, move to next pbl. 662 */ 663 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { 664 pbl_tbl++; 665 pbe = (struct regpair *)pbl_tbl->va; 666 pbe_cnt = 0; 667 } 668 } 669 } 670 671 static int qedr_db_recovery_add(struct qedr_dev *dev, 672 void __iomem *db_addr, 673 void *db_data, 674 enum qed_db_rec_width db_width, 675 enum qed_db_rec_space db_space) 676 { 677 if (!db_data) { 678 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n"); 679 return 0; 680 } 681 682 return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data, 683 db_width, db_space); 684 } 685 686 static void qedr_db_recovery_del(struct qedr_dev *dev, 687 void __iomem *db_addr, 688 void *db_data) 689 { 690 if (!db_data) { 691 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n"); 692 return; 693 } 694 695 /* Ignore return code as there is not much we can do about it. Error 696 * log will be printed inside. 697 */ 698 dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data); 699 } 700 701 static int qedr_copy_cq_uresp(struct qedr_dev *dev, 702 struct qedr_cq *cq, struct ib_udata *udata, 703 u32 db_offset) 704 { 705 struct qedr_create_cq_uresp uresp; 706 int rc; 707 708 memset(&uresp, 0, sizeof(uresp)); 709 710 uresp.db_offset = db_offset; 711 uresp.icid = cq->icid; 712 if (cq->q.db_mmap_entry) 713 uresp.db_rec_addr = 714 rdma_user_mmap_get_offset(cq->q.db_mmap_entry); 715 716 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 717 if (rc) 718 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); 719 720 return rc; 721 } 722 723 static void consume_cqe(struct qedr_cq *cq) 724 { 725 if (cq->latest_cqe == cq->toggle_cqe) 726 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 727 728 cq->latest_cqe = qed_chain_consume(&cq->pbl); 729 } 730 731 static inline int qedr_align_cq_entries(int entries) 732 { 733 u64 size, aligned_size; 734 735 /* We allocate an extra entry that we don't report to the FW. */ 736 size = (entries + 1) * QEDR_CQE_SIZE; 737 aligned_size = ALIGN(size, PAGE_SIZE); 738 739 return aligned_size / QEDR_CQE_SIZE; 740 } 741 742 static int qedr_init_user_db_rec(struct ib_udata *udata, 743 struct qedr_dev *dev, struct qedr_userq *q, 744 bool requires_db_rec) 745 { 746 struct qedr_ucontext *uctx = 747 rdma_udata_to_drv_context(udata, struct qedr_ucontext, 748 ibucontext); 749 struct qedr_user_mmap_entry *entry; 750 int rc; 751 752 /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */ 753 if (requires_db_rec == 0 || !uctx->db_rec) 754 return 0; 755 756 /* Allocate a page for doorbell recovery, add to mmap */ 757 q->db_rec_data = (void *)get_zeroed_page(GFP_USER); 758 if (!q->db_rec_data) { 759 DP_ERR(dev, "get_zeroed_page failed\n"); 760 return -ENOMEM; 761 } 762 763 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 764 if (!entry) 765 goto err_free_db_data; 766 767 entry->address = q->db_rec_data; 768 entry->length = PAGE_SIZE; 769 entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE; 770 rc = rdma_user_mmap_entry_insert(&uctx->ibucontext, 771 &entry->rdma_entry, 772 PAGE_SIZE); 773 if (rc) 774 goto err_free_entry; 775 776 q->db_mmap_entry = &entry->rdma_entry; 777 778 return 0; 779 780 err_free_entry: 781 kfree(entry); 782 783 err_free_db_data: 784 free_page((unsigned long)q->db_rec_data); 785 q->db_rec_data = NULL; 786 return -ENOMEM; 787 } 788 789 static inline int qedr_init_user_queue(struct ib_udata *udata, 790 struct qedr_dev *dev, 791 struct qedr_userq *q, u64 buf_addr, 792 size_t buf_len, bool requires_db_rec, 793 int access, 794 int alloc_and_init) 795 { 796 u32 fw_pages; 797 int rc; 798 799 q->buf_addr = buf_addr; 800 q->buf_len = buf_len; 801 q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access); 802 if (IS_ERR(q->umem)) { 803 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", 804 PTR_ERR(q->umem)); 805 return PTR_ERR(q->umem); 806 } 807 808 fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT); 809 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); 810 if (rc) 811 goto err0; 812 813 if (alloc_and_init) { 814 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 815 if (IS_ERR(q->pbl_tbl)) { 816 rc = PTR_ERR(q->pbl_tbl); 817 goto err0; 818 } 819 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info, 820 FW_PAGE_SHIFT); 821 } else { 822 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); 823 if (!q->pbl_tbl) { 824 rc = -ENOMEM; 825 goto err0; 826 } 827 } 828 829 /* mmap the user address used to store doorbell data for recovery */ 830 return qedr_init_user_db_rec(udata, dev, q, requires_db_rec); 831 832 err0: 833 ib_umem_release(q->umem); 834 q->umem = NULL; 835 836 return rc; 837 } 838 839 static inline void qedr_init_cq_params(struct qedr_cq *cq, 840 struct qedr_ucontext *ctx, 841 struct qedr_dev *dev, int vector, 842 int chain_entries, int page_cnt, 843 u64 pbl_ptr, 844 struct qed_rdma_create_cq_in_params 845 *params) 846 { 847 memset(params, 0, sizeof(*params)); 848 params->cq_handle_hi = upper_32_bits((uintptr_t)cq); 849 params->cq_handle_lo = lower_32_bits((uintptr_t)cq); 850 params->cnq_id = vector; 851 params->cq_size = chain_entries - 1; 852 params->dpi = (ctx) ? ctx->dpi : dev->dpi; 853 params->pbl_num_pages = page_cnt; 854 params->pbl_ptr = pbl_ptr; 855 params->pbl_two_level = 0; 856 } 857 858 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags) 859 { 860 cq->db.data.agg_flags = flags; 861 cq->db.data.value = cpu_to_le32(cons); 862 writeq(cq->db.raw, cq->db_addr); 863 } 864 865 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 866 { 867 struct qedr_cq *cq = get_qedr_cq(ibcq); 868 unsigned long sflags; 869 struct qedr_dev *dev; 870 871 dev = get_qedr_dev(ibcq->device); 872 873 if (cq->destroyed) { 874 DP_ERR(dev, 875 "warning: arm was invoked after destroy for cq %p (icid=%d)\n", 876 cq, cq->icid); 877 return -EINVAL; 878 } 879 880 881 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 882 return 0; 883 884 spin_lock_irqsave(&cq->cq_lock, sflags); 885 886 cq->arm_flags = 0; 887 888 if (flags & IB_CQ_SOLICITED) 889 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 890 891 if (flags & IB_CQ_NEXT_COMP) 892 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 893 894 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 895 896 spin_unlock_irqrestore(&cq->cq_lock, sflags); 897 898 return 0; 899 } 900 901 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 902 struct ib_udata *udata) 903 { 904 struct ib_device *ibdev = ibcq->device; 905 struct qedr_ucontext *ctx = rdma_udata_to_drv_context( 906 udata, struct qedr_ucontext, ibucontext); 907 struct qed_rdma_destroy_cq_out_params destroy_oparams; 908 struct qed_rdma_destroy_cq_in_params destroy_iparams; 909 struct qed_chain_init_params chain_params = { 910 .mode = QED_CHAIN_MODE_PBL, 911 .intended_use = QED_CHAIN_USE_TO_CONSUME, 912 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 913 .elem_size = sizeof(union rdma_cqe), 914 }; 915 struct qedr_dev *dev = get_qedr_dev(ibdev); 916 struct qed_rdma_create_cq_in_params params; 917 struct qedr_create_cq_ureq ureq = {}; 918 int vector = attr->comp_vector; 919 int entries = attr->cqe; 920 struct qedr_cq *cq = get_qedr_cq(ibcq); 921 int chain_entries; 922 u32 db_offset; 923 int page_cnt; 924 u64 pbl_ptr; 925 u16 icid; 926 int rc; 927 928 DP_DEBUG(dev, QEDR_MSG_INIT, 929 "create_cq: called from %s. entries=%d, vector=%d\n", 930 udata ? "User Lib" : "Kernel", entries, vector); 931 932 if (attr->flags) 933 return -EOPNOTSUPP; 934 935 if (entries > QEDR_MAX_CQES) { 936 DP_ERR(dev, 937 "create cq: the number of entries %d is too high. Must be equal or below %d.\n", 938 entries, QEDR_MAX_CQES); 939 return -EINVAL; 940 } 941 942 chain_entries = qedr_align_cq_entries(entries); 943 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); 944 chain_params.num_elems = chain_entries; 945 946 /* calc db offset. user will add DPI base, kernel will add db addr */ 947 db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 948 949 if (udata) { 950 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 951 udata->inlen))) { 952 DP_ERR(dev, 953 "create cq: problem copying data from user space\n"); 954 goto err0; 955 } 956 957 if (!ureq.len) { 958 DP_ERR(dev, 959 "create cq: cannot create a cq with 0 entries\n"); 960 goto err0; 961 } 962 963 cq->cq_type = QEDR_CQ_TYPE_USER; 964 965 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr, 966 ureq.len, true, IB_ACCESS_LOCAL_WRITE, 967 1); 968 if (rc) 969 goto err0; 970 971 pbl_ptr = cq->q.pbl_tbl->pa; 972 page_cnt = cq->q.pbl_info.num_pbes; 973 974 cq->ibcq.cqe = chain_entries; 975 cq->q.db_addr = ctx->dpi_addr + db_offset; 976 } else { 977 cq->cq_type = QEDR_CQ_TYPE_KERNEL; 978 979 rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl, 980 &chain_params); 981 if (rc) 982 goto err0; 983 984 page_cnt = qed_chain_get_page_cnt(&cq->pbl); 985 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); 986 cq->ibcq.cqe = cq->pbl.capacity; 987 } 988 989 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, 990 pbl_ptr, ¶ms); 991 992 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 993 if (rc) 994 goto err1; 995 996 cq->icid = icid; 997 cq->sig = QEDR_CQ_MAGIC_NUMBER; 998 spin_lock_init(&cq->cq_lock); 999 1000 if (udata) { 1001 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset); 1002 if (rc) 1003 goto err2; 1004 1005 rc = qedr_db_recovery_add(dev, cq->q.db_addr, 1006 &cq->q.db_rec_data->db_data, 1007 DB_REC_WIDTH_64B, 1008 DB_REC_USER); 1009 if (rc) 1010 goto err2; 1011 1012 } else { 1013 /* Generate doorbell address. */ 1014 cq->db.data.icid = cq->icid; 1015 cq->db_addr = dev->db_addr + db_offset; 1016 cq->db.data.params = DB_AGG_CMD_MAX << 1017 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 1018 1019 /* point to the very last element, passing it we will toggle */ 1020 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); 1021 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 1022 cq->latest_cqe = NULL; 1023 consume_cqe(cq); 1024 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 1025 1026 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data, 1027 DB_REC_WIDTH_64B, DB_REC_KERNEL); 1028 if (rc) 1029 goto err2; 1030 } 1031 1032 DP_DEBUG(dev, QEDR_MSG_CQ, 1033 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", 1034 cq->icid, cq, params.cq_size); 1035 1036 return 0; 1037 1038 err2: 1039 destroy_iparams.icid = cq->icid; 1040 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, 1041 &destroy_oparams); 1042 err1: 1043 if (udata) { 1044 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1045 ib_umem_release(cq->q.umem); 1046 if (cq->q.db_mmap_entry) 1047 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry); 1048 } else { 1049 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1050 } 1051 err0: 1052 return -EINVAL; 1053 } 1054 1055 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 1056 { 1057 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1058 struct qedr_cq *cq = get_qedr_cq(ibcq); 1059 1060 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq); 1061 1062 return 0; 1063 } 1064 1065 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) 1066 #define QEDR_DESTROY_CQ_ITER_DURATION (10) 1067 1068 int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1069 { 1070 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1071 struct qed_rdma_destroy_cq_out_params oparams; 1072 struct qed_rdma_destroy_cq_in_params iparams; 1073 struct qedr_cq *cq = get_qedr_cq(ibcq); 1074 int iter; 1075 1076 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); 1077 1078 cq->destroyed = 1; 1079 1080 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 1081 if (cq->cq_type == QEDR_CQ_TYPE_GSI) { 1082 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data); 1083 return 0; 1084 } 1085 1086 iparams.icid = cq->icid; 1087 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 1088 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1089 1090 if (udata) { 1091 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1092 ib_umem_release(cq->q.umem); 1093 1094 if (cq->q.db_rec_data) { 1095 qedr_db_recovery_del(dev, cq->q.db_addr, 1096 &cq->q.db_rec_data->db_data); 1097 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry); 1098 } 1099 } else { 1100 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data); 1101 } 1102 1103 /* We don't want the IRQ handler to handle a non-existing CQ so we 1104 * wait until all CNQ interrupts, if any, are received. This will always 1105 * happen and will always happen very fast. If not, then a serious error 1106 * has occured. That is why we can use a long delay. 1107 * We spin for a short time so we don’t lose time on context switching 1108 * in case all the completions are handled in that span. Otherwise 1109 * we sleep for a while and check again. Since the CNQ may be 1110 * associated with (only) the current CPU we use msleep to allow the 1111 * current CPU to be freed. 1112 * The CNQ notification is increased in qedr_irq_handler(). 1113 */ 1114 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1115 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1116 udelay(QEDR_DESTROY_CQ_ITER_DURATION); 1117 iter--; 1118 } 1119 1120 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1121 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1122 msleep(QEDR_DESTROY_CQ_ITER_DURATION); 1123 iter--; 1124 } 1125 1126 /* Note that we don't need to have explicit code to wait for the 1127 * completion of the event handler because it is invoked from the EQ. 1128 * Since the destroy CQ ramrod has also been received on the EQ we can 1129 * be certain that there's no event handler in process. 1130 */ 1131 return 0; 1132 } 1133 1134 static inline int get_gid_info_from_table(struct ib_qp *ibqp, 1135 struct ib_qp_attr *attr, 1136 int attr_mask, 1137 struct qed_rdma_modify_qp_in_params 1138 *qp_params) 1139 { 1140 const struct ib_gid_attr *gid_attr; 1141 enum rdma_network_type nw_type; 1142 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1143 u32 ipv4_addr; 1144 int ret; 1145 int i; 1146 1147 gid_attr = grh->sgid_attr; 1148 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL); 1149 if (ret) 1150 return ret; 1151 1152 nw_type = rdma_gid_attr_network_type(gid_attr); 1153 switch (nw_type) { 1154 case RDMA_NETWORK_IPV6: 1155 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1156 sizeof(qp_params->sgid)); 1157 memcpy(&qp_params->dgid.bytes[0], 1158 &grh->dgid, 1159 sizeof(qp_params->dgid)); 1160 qp_params->roce_mode = ROCE_V2_IPV6; 1161 SET_FIELD(qp_params->modify_flags, 1162 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1163 break; 1164 case RDMA_NETWORK_ROCE_V1: 1165 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1166 sizeof(qp_params->sgid)); 1167 memcpy(&qp_params->dgid.bytes[0], 1168 &grh->dgid, 1169 sizeof(qp_params->dgid)); 1170 qp_params->roce_mode = ROCE_V1; 1171 break; 1172 case RDMA_NETWORK_IPV4: 1173 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid)); 1174 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid)); 1175 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw); 1176 qp_params->sgid.ipv4_addr = ipv4_addr; 1177 ipv4_addr = 1178 qedr_get_ipv4_from_gid(grh->dgid.raw); 1179 qp_params->dgid.ipv4_addr = ipv4_addr; 1180 SET_FIELD(qp_params->modify_flags, 1181 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1182 qp_params->roce_mode = ROCE_V2_IPV4; 1183 break; 1184 default: 1185 return -EINVAL; 1186 } 1187 1188 for (i = 0; i < 4; i++) { 1189 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 1190 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 1191 } 1192 1193 if (qp_params->vlan_id >= VLAN_CFI_MASK) 1194 qp_params->vlan_id = 0; 1195 1196 return 0; 1197 } 1198 1199 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, 1200 struct ib_qp_init_attr *attrs, 1201 struct ib_udata *udata) 1202 { 1203 struct qedr_device_attr *qattr = &dev->attr; 1204 1205 /* QP0... attrs->qp_type == IB_QPT_GSI */ 1206 if (attrs->qp_type != IB_QPT_RC && 1207 attrs->qp_type != IB_QPT_GSI && 1208 attrs->qp_type != IB_QPT_XRC_INI && 1209 attrs->qp_type != IB_QPT_XRC_TGT) { 1210 DP_DEBUG(dev, QEDR_MSG_QP, 1211 "create qp: unsupported qp type=0x%x requested\n", 1212 attrs->qp_type); 1213 return -EOPNOTSUPP; 1214 } 1215 1216 if (attrs->cap.max_send_wr > qattr->max_sqe) { 1217 DP_ERR(dev, 1218 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n", 1219 attrs->cap.max_send_wr, qattr->max_sqe); 1220 return -EINVAL; 1221 } 1222 1223 if (attrs->cap.max_inline_data > qattr->max_inline) { 1224 DP_ERR(dev, 1225 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n", 1226 attrs->cap.max_inline_data, qattr->max_inline); 1227 return -EINVAL; 1228 } 1229 1230 if (attrs->cap.max_send_sge > qattr->max_sge) { 1231 DP_ERR(dev, 1232 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n", 1233 attrs->cap.max_send_sge, qattr->max_sge); 1234 return -EINVAL; 1235 } 1236 1237 if (attrs->cap.max_recv_sge > qattr->max_sge) { 1238 DP_ERR(dev, 1239 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n", 1240 attrs->cap.max_recv_sge, qattr->max_sge); 1241 return -EINVAL; 1242 } 1243 1244 /* verify consumer QPs are not trying to use GSI QP's CQ. 1245 * TGT QP isn't associated with RQ/SQ 1246 */ 1247 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) && 1248 (attrs->qp_type != IB_QPT_XRC_TGT) && 1249 (attrs->qp_type != IB_QPT_XRC_INI)) { 1250 struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq); 1251 struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq); 1252 1253 if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) || 1254 (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) { 1255 DP_ERR(dev, 1256 "create qp: consumer QP cannot use GSI CQs.\n"); 1257 return -EINVAL; 1258 } 1259 } 1260 1261 return 0; 1262 } 1263 1264 static int qedr_copy_srq_uresp(struct qedr_dev *dev, 1265 struct qedr_srq *srq, struct ib_udata *udata) 1266 { 1267 struct qedr_create_srq_uresp uresp = {}; 1268 int rc; 1269 1270 uresp.srq_id = srq->srq_id; 1271 1272 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1273 if (rc) 1274 DP_ERR(dev, "create srq: problem copying data to user space\n"); 1275 1276 return rc; 1277 } 1278 1279 static void qedr_copy_rq_uresp(struct qedr_dev *dev, 1280 struct qedr_create_qp_uresp *uresp, 1281 struct qedr_qp *qp) 1282 { 1283 /* iWARP requires two doorbells per RQ. */ 1284 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1285 uresp->rq_db_offset = 1286 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1287 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1288 } else { 1289 uresp->rq_db_offset = 1290 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1291 } 1292 1293 uresp->rq_icid = qp->icid; 1294 if (qp->urq.db_mmap_entry) 1295 uresp->rq_db_rec_addr = 1296 rdma_user_mmap_get_offset(qp->urq.db_mmap_entry); 1297 } 1298 1299 static void qedr_copy_sq_uresp(struct qedr_dev *dev, 1300 struct qedr_create_qp_uresp *uresp, 1301 struct qedr_qp *qp) 1302 { 1303 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1304 1305 /* iWARP uses the same cid for rq and sq */ 1306 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1307 uresp->sq_icid = qp->icid; 1308 else 1309 uresp->sq_icid = qp->icid + 1; 1310 1311 if (qp->usq.db_mmap_entry) 1312 uresp->sq_db_rec_addr = 1313 rdma_user_mmap_get_offset(qp->usq.db_mmap_entry); 1314 } 1315 1316 static int qedr_copy_qp_uresp(struct qedr_dev *dev, 1317 struct qedr_qp *qp, struct ib_udata *udata, 1318 struct qedr_create_qp_uresp *uresp) 1319 { 1320 int rc; 1321 1322 memset(uresp, 0, sizeof(*uresp)); 1323 1324 if (qedr_qp_has_sq(qp)) 1325 qedr_copy_sq_uresp(dev, uresp, qp); 1326 1327 if (qedr_qp_has_rq(qp)) 1328 qedr_copy_rq_uresp(dev, uresp, qp); 1329 1330 uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 1331 uresp->qp_id = qp->qp_id; 1332 1333 rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp)); 1334 if (rc) 1335 DP_ERR(dev, 1336 "create qp: failed a copy to user space with qp icid=0x%x.\n", 1337 qp->icid); 1338 1339 return rc; 1340 } 1341 1342 static void qedr_set_common_qp_params(struct qedr_dev *dev, 1343 struct qedr_qp *qp, 1344 struct qedr_pd *pd, 1345 struct ib_qp_init_attr *attrs) 1346 { 1347 spin_lock_init(&qp->q_lock); 1348 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1349 kref_init(&qp->refcnt); 1350 init_completion(&qp->iwarp_cm_comp); 1351 } 1352 1353 qp->pd = pd; 1354 qp->qp_type = attrs->qp_type; 1355 qp->max_inline_data = attrs->cap.max_inline_data; 1356 qp->state = QED_ROCE_QP_STATE_RESET; 1357 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1358 qp->dev = dev; 1359 if (qedr_qp_has_sq(qp)) { 1360 qp->sq.max_sges = attrs->cap.max_send_sge; 1361 qp->sq_cq = get_qedr_cq(attrs->send_cq); 1362 DP_DEBUG(dev, QEDR_MSG_QP, 1363 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1364 qp->sq.max_sges, qp->sq_cq->icid); 1365 } 1366 1367 if (attrs->srq) 1368 qp->srq = get_qedr_srq(attrs->srq); 1369 1370 if (qedr_qp_has_rq(qp)) { 1371 qp->rq_cq = get_qedr_cq(attrs->recv_cq); 1372 qp->rq.max_sges = attrs->cap.max_recv_sge; 1373 DP_DEBUG(dev, QEDR_MSG_QP, 1374 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 1375 qp->rq.max_sges, qp->rq_cq->icid); 1376 } 1377 1378 DP_DEBUG(dev, QEDR_MSG_QP, 1379 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n", 1380 pd->pd_id, qp->qp_type, qp->max_inline_data, 1381 qp->state, qp->signaled, (attrs->srq) ? 1 : 0); 1382 DP_DEBUG(dev, QEDR_MSG_QP, 1383 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1384 qp->sq.max_sges, qp->sq_cq->icid); 1385 } 1386 1387 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1388 { 1389 int rc = 0; 1390 1391 if (qedr_qp_has_sq(qp)) { 1392 qp->sq.db = dev->db_addr + 1393 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1394 qp->sq.db_data.data.icid = qp->icid + 1; 1395 rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data, 1396 DB_REC_WIDTH_32B, DB_REC_KERNEL); 1397 if (rc) 1398 return rc; 1399 } 1400 1401 if (qedr_qp_has_rq(qp)) { 1402 qp->rq.db = dev->db_addr + 1403 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1404 qp->rq.db_data.data.icid = qp->icid; 1405 rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data, 1406 DB_REC_WIDTH_32B, DB_REC_KERNEL); 1407 if (rc && qedr_qp_has_sq(qp)) 1408 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data); 1409 } 1410 1411 return rc; 1412 } 1413 1414 static int qedr_check_srq_params(struct qedr_dev *dev, 1415 struct ib_srq_init_attr *attrs, 1416 struct ib_udata *udata) 1417 { 1418 struct qedr_device_attr *qattr = &dev->attr; 1419 1420 if (attrs->attr.max_wr > qattr->max_srq_wr) { 1421 DP_ERR(dev, 1422 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n", 1423 attrs->attr.max_wr, qattr->max_srq_wr); 1424 return -EINVAL; 1425 } 1426 1427 if (attrs->attr.max_sge > qattr->max_sge) { 1428 DP_ERR(dev, 1429 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", 1430 attrs->attr.max_sge, qattr->max_sge); 1431 } 1432 1433 if (!udata && attrs->srq_type == IB_SRQT_XRC) { 1434 DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n"); 1435 return -EINVAL; 1436 } 1437 1438 return 0; 1439 } 1440 1441 static void qedr_free_srq_user_params(struct qedr_srq *srq) 1442 { 1443 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1444 ib_umem_release(srq->usrq.umem); 1445 ib_umem_release(srq->prod_umem); 1446 } 1447 1448 static void qedr_free_srq_kernel_params(struct qedr_srq *srq) 1449 { 1450 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1451 struct qedr_dev *dev = srq->dev; 1452 1453 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl); 1454 1455 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1456 hw_srq->virt_prod_pair_addr, 1457 hw_srq->phy_prod_pair_addr); 1458 } 1459 1460 static int qedr_init_srq_user_params(struct ib_udata *udata, 1461 struct qedr_srq *srq, 1462 struct qedr_create_srq_ureq *ureq, 1463 int access) 1464 { 1465 struct scatterlist *sg; 1466 int rc; 1467 1468 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr, 1469 ureq->srq_len, false, access, 1); 1470 if (rc) 1471 return rc; 1472 1473 srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr, 1474 sizeof(struct rdma_srq_producers), access); 1475 if (IS_ERR(srq->prod_umem)) { 1476 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1477 ib_umem_release(srq->usrq.umem); 1478 DP_ERR(srq->dev, 1479 "create srq: failed ib_umem_get for producer, got %ld\n", 1480 PTR_ERR(srq->prod_umem)); 1481 return PTR_ERR(srq->prod_umem); 1482 } 1483 1484 sg = srq->prod_umem->sg_head.sgl; 1485 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); 1486 1487 return 0; 1488 } 1489 1490 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq, 1491 struct qedr_dev *dev, 1492 struct ib_srq_init_attr *init_attr) 1493 { 1494 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1495 struct qed_chain_init_params params = { 1496 .mode = QED_CHAIN_MODE_PBL, 1497 .intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1498 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 1499 .elem_size = QEDR_SRQ_WQE_ELEM_SIZE, 1500 }; 1501 dma_addr_t phy_prod_pair_addr; 1502 u32 num_elems; 1503 void *va; 1504 int rc; 1505 1506 va = dma_alloc_coherent(&dev->pdev->dev, 1507 sizeof(struct rdma_srq_producers), 1508 &phy_prod_pair_addr, GFP_KERNEL); 1509 if (!va) { 1510 DP_ERR(dev, 1511 "create srq: failed to allocate dma memory for producer\n"); 1512 return -ENOMEM; 1513 } 1514 1515 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; 1516 hw_srq->virt_prod_pair_addr = va; 1517 1518 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE; 1519 params.num_elems = num_elems; 1520 1521 rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, ¶ms); 1522 if (rc) 1523 goto err0; 1524 1525 hw_srq->num_elems = num_elems; 1526 1527 return 0; 1528 1529 err0: 1530 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1531 va, phy_prod_pair_addr); 1532 return rc; 1533 } 1534 1535 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, 1536 struct ib_udata *udata) 1537 { 1538 struct qed_rdma_destroy_srq_in_params destroy_in_params; 1539 struct qed_rdma_create_srq_in_params in_params = {}; 1540 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1541 struct qed_rdma_create_srq_out_params out_params; 1542 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd); 1543 struct qedr_create_srq_ureq ureq = {}; 1544 u64 pbl_base_addr, phy_prod_pair_addr; 1545 struct qedr_srq_hwq_info *hw_srq; 1546 u32 page_cnt, page_size; 1547 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1548 int rc = 0; 1549 1550 DP_DEBUG(dev, QEDR_MSG_QP, 1551 "create SRQ called from %s (pd %p)\n", 1552 (udata) ? "User lib" : "kernel", pd); 1553 1554 if (init_attr->srq_type != IB_SRQT_BASIC && 1555 init_attr->srq_type != IB_SRQT_XRC) 1556 return -EOPNOTSUPP; 1557 1558 rc = qedr_check_srq_params(dev, init_attr, udata); 1559 if (rc) 1560 return -EINVAL; 1561 1562 srq->dev = dev; 1563 srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC); 1564 hw_srq = &srq->hw_srq; 1565 spin_lock_init(&srq->lock); 1566 1567 hw_srq->max_wr = init_attr->attr.max_wr; 1568 hw_srq->max_sges = init_attr->attr.max_sge; 1569 1570 if (udata) { 1571 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 1572 udata->inlen))) { 1573 DP_ERR(dev, 1574 "create srq: problem copying data from user space\n"); 1575 goto err0; 1576 } 1577 1578 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0); 1579 if (rc) 1580 goto err0; 1581 1582 page_cnt = srq->usrq.pbl_info.num_pbes; 1583 pbl_base_addr = srq->usrq.pbl_tbl->pa; 1584 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1585 page_size = PAGE_SIZE; 1586 } else { 1587 struct qed_chain *pbl; 1588 1589 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr); 1590 if (rc) 1591 goto err0; 1592 1593 pbl = &hw_srq->pbl; 1594 page_cnt = qed_chain_get_page_cnt(pbl); 1595 pbl_base_addr = qed_chain_get_pbl_phys(pbl); 1596 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1597 page_size = QED_CHAIN_PAGE_SIZE; 1598 } 1599 1600 in_params.pd_id = pd->pd_id; 1601 in_params.pbl_base_addr = pbl_base_addr; 1602 in_params.prod_pair_addr = phy_prod_pair_addr; 1603 in_params.num_pages = page_cnt; 1604 in_params.page_size = page_size; 1605 if (srq->is_xrc) { 1606 struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd); 1607 struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq); 1608 1609 in_params.is_xrc = 1; 1610 in_params.xrcd_id = xrcd->xrcd_id; 1611 in_params.cq_cid = cq->icid; 1612 } 1613 1614 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); 1615 if (rc) 1616 goto err1; 1617 1618 srq->srq_id = out_params.srq_id; 1619 1620 if (udata) { 1621 rc = qedr_copy_srq_uresp(dev, srq, udata); 1622 if (rc) 1623 goto err2; 1624 } 1625 1626 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL); 1627 if (rc) 1628 goto err2; 1629 1630 DP_DEBUG(dev, QEDR_MSG_SRQ, 1631 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id); 1632 return 0; 1633 1634 err2: 1635 destroy_in_params.srq_id = srq->srq_id; 1636 1637 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); 1638 err1: 1639 if (udata) 1640 qedr_free_srq_user_params(srq); 1641 else 1642 qedr_free_srq_kernel_params(srq); 1643 err0: 1644 return -EFAULT; 1645 } 1646 1647 int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 1648 { 1649 struct qed_rdma_destroy_srq_in_params in_params = {}; 1650 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1651 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1652 1653 xa_erase_irq(&dev->srqs, srq->srq_id); 1654 in_params.srq_id = srq->srq_id; 1655 in_params.is_xrc = srq->is_xrc; 1656 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params); 1657 1658 if (ibsrq->uobject) 1659 qedr_free_srq_user_params(srq); 1660 else 1661 qedr_free_srq_kernel_params(srq); 1662 1663 DP_DEBUG(dev, QEDR_MSG_SRQ, 1664 "destroy srq: destroyed srq with srq_id=0x%0x\n", 1665 srq->srq_id); 1666 return 0; 1667 } 1668 1669 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 1670 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 1671 { 1672 struct qed_rdma_modify_srq_in_params in_params = {}; 1673 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1674 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1675 int rc; 1676 1677 if (attr_mask & IB_SRQ_MAX_WR) { 1678 DP_ERR(dev, 1679 "modify srq: invalid attribute mask=0x%x specified for %p\n", 1680 attr_mask, srq); 1681 return -EINVAL; 1682 } 1683 1684 if (attr_mask & IB_SRQ_LIMIT) { 1685 if (attr->srq_limit >= srq->hw_srq.max_wr) { 1686 DP_ERR(dev, 1687 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n", 1688 attr->srq_limit, srq->hw_srq.max_wr); 1689 return -EINVAL; 1690 } 1691 1692 in_params.srq_id = srq->srq_id; 1693 in_params.wqe_limit = attr->srq_limit; 1694 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params); 1695 if (rc) 1696 return rc; 1697 } 1698 1699 srq->srq_limit = attr->srq_limit; 1700 1701 DP_DEBUG(dev, QEDR_MSG_SRQ, 1702 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id); 1703 1704 return 0; 1705 } 1706 1707 static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type) 1708 { 1709 switch (ib_qp_type) { 1710 case IB_QPT_RC: 1711 return QED_RDMA_QP_TYPE_RC; 1712 case IB_QPT_XRC_INI: 1713 return QED_RDMA_QP_TYPE_XRC_INI; 1714 case IB_QPT_XRC_TGT: 1715 return QED_RDMA_QP_TYPE_XRC_TGT; 1716 default: 1717 return QED_RDMA_QP_TYPE_INVAL; 1718 } 1719 } 1720 1721 static inline void 1722 qedr_init_common_qp_in_params(struct qedr_dev *dev, 1723 struct qedr_pd *pd, 1724 struct qedr_qp *qp, 1725 struct ib_qp_init_attr *attrs, 1726 bool fmr_and_reserved_lkey, 1727 struct qed_rdma_create_qp_in_params *params) 1728 { 1729 /* QP handle to be written in an async event */ 1730 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp); 1731 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp); 1732 1733 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 1734 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 1735 params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type); 1736 params->stats_queue = 0; 1737 1738 if (pd) { 1739 params->pd = pd->pd_id; 1740 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 1741 } 1742 1743 if (qedr_qp_has_sq(qp)) 1744 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; 1745 1746 if (qedr_qp_has_rq(qp)) 1747 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1748 1749 if (qedr_qp_has_srq(qp)) { 1750 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1751 params->srq_id = qp->srq->srq_id; 1752 params->use_srq = true; 1753 } else { 1754 params->srq_id = 0; 1755 params->use_srq = false; 1756 } 1757 } 1758 1759 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) 1760 { 1761 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. " 1762 "qp=%p. " 1763 "sq_addr=0x%llx, " 1764 "sq_len=%zd, " 1765 "rq_addr=0x%llx, " 1766 "rq_len=%zd" 1767 "\n", 1768 qp, 1769 qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0, 1770 qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0, 1771 qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0, 1772 qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0); 1773 } 1774 1775 static inline void 1776 qedr_iwarp_populate_user_qp(struct qedr_dev *dev, 1777 struct qedr_qp *qp, 1778 struct qed_rdma_create_qp_out_params *out_params) 1779 { 1780 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 1781 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 1782 1783 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 1784 &qp->usq.pbl_info, FW_PAGE_SHIFT); 1785 if (!qp->srq) { 1786 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 1787 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 1788 } 1789 1790 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 1791 &qp->urq.pbl_info, FW_PAGE_SHIFT); 1792 } 1793 1794 static void qedr_cleanup_user(struct qedr_dev *dev, 1795 struct qedr_ucontext *ctx, 1796 struct qedr_qp *qp) 1797 { 1798 if (qedr_qp_has_sq(qp)) { 1799 ib_umem_release(qp->usq.umem); 1800 qp->usq.umem = NULL; 1801 } 1802 1803 if (qedr_qp_has_rq(qp)) { 1804 ib_umem_release(qp->urq.umem); 1805 qp->urq.umem = NULL; 1806 } 1807 1808 if (rdma_protocol_roce(&dev->ibdev, 1)) { 1809 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl); 1810 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl); 1811 } else { 1812 kfree(qp->usq.pbl_tbl); 1813 kfree(qp->urq.pbl_tbl); 1814 } 1815 1816 if (qp->usq.db_rec_data) { 1817 qedr_db_recovery_del(dev, qp->usq.db_addr, 1818 &qp->usq.db_rec_data->db_data); 1819 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry); 1820 } 1821 1822 if (qp->urq.db_rec_data) { 1823 qedr_db_recovery_del(dev, qp->urq.db_addr, 1824 &qp->urq.db_rec_data->db_data); 1825 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry); 1826 } 1827 1828 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1829 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr, 1830 &qp->urq.db_rec_db2_data); 1831 } 1832 1833 static int qedr_create_user_qp(struct qedr_dev *dev, 1834 struct qedr_qp *qp, 1835 struct ib_pd *ibpd, 1836 struct ib_udata *udata, 1837 struct ib_qp_init_attr *attrs) 1838 { 1839 struct qed_rdma_create_qp_in_params in_params; 1840 struct qed_rdma_create_qp_out_params out_params; 1841 struct qedr_create_qp_uresp uresp = {}; 1842 struct qedr_create_qp_ureq ureq = {}; 1843 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1); 1844 struct qedr_ucontext *ctx = NULL; 1845 struct qedr_pd *pd = NULL; 1846 int rc = 0; 1847 1848 qp->create_type = QEDR_QP_CREATE_USER; 1849 1850 if (ibpd) { 1851 pd = get_qedr_pd(ibpd); 1852 ctx = pd->uctx; 1853 } 1854 1855 if (udata) { 1856 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 1857 udata->inlen)); 1858 if (rc) { 1859 DP_ERR(dev, "Problem copying data from user space\n"); 1860 return rc; 1861 } 1862 } 1863 1864 if (qedr_qp_has_sq(qp)) { 1865 /* SQ - read access only (0) */ 1866 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr, 1867 ureq.sq_len, true, 0, alloc_and_init); 1868 if (rc) 1869 return rc; 1870 } 1871 1872 if (qedr_qp_has_rq(qp)) { 1873 /* RQ - read access only (0) */ 1874 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr, 1875 ureq.rq_len, true, 0, alloc_and_init); 1876 if (rc) 1877 return rc; 1878 } 1879 1880 memset(&in_params, 0, sizeof(in_params)); 1881 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 1882 in_params.qp_handle_lo = ureq.qp_handle_lo; 1883 in_params.qp_handle_hi = ureq.qp_handle_hi; 1884 1885 if (qp->qp_type == IB_QPT_XRC_TGT) { 1886 struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd); 1887 1888 in_params.xrcd_id = xrcd->xrcd_id; 1889 in_params.qp_handle_lo = qp->qp_id; 1890 in_params.use_srq = 1; 1891 } 1892 1893 if (qedr_qp_has_sq(qp)) { 1894 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 1895 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 1896 } 1897 1898 if (qedr_qp_has_rq(qp)) { 1899 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 1900 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 1901 } 1902 1903 if (ctx) 1904 SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode); 1905 1906 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1907 &in_params, &out_params); 1908 1909 if (!qp->qed_qp) { 1910 rc = -ENOMEM; 1911 goto err1; 1912 } 1913 1914 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1915 qedr_iwarp_populate_user_qp(dev, qp, &out_params); 1916 1917 qp->qp_id = out_params.qp_id; 1918 qp->icid = out_params.icid; 1919 1920 if (udata) { 1921 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp); 1922 if (rc) 1923 goto err; 1924 } 1925 1926 /* db offset was calculated in copy_qp_uresp, now set in the user q */ 1927 if (qedr_qp_has_sq(qp)) { 1928 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset; 1929 rc = qedr_db_recovery_add(dev, qp->usq.db_addr, 1930 &qp->usq.db_rec_data->db_data, 1931 DB_REC_WIDTH_32B, 1932 DB_REC_USER); 1933 if (rc) 1934 goto err; 1935 } 1936 1937 if (qedr_qp_has_rq(qp)) { 1938 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset; 1939 rc = qedr_db_recovery_add(dev, qp->urq.db_addr, 1940 &qp->urq.db_rec_data->db_data, 1941 DB_REC_WIDTH_32B, 1942 DB_REC_USER); 1943 if (rc) 1944 goto err; 1945 } 1946 1947 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1948 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset; 1949 1950 /* calculate the db_rec_db2 data since it is constant so no 1951 * need to reflect from user 1952 */ 1953 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid); 1954 qp->urq.db_rec_db2_data.data.value = 1955 cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD); 1956 1957 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr, 1958 &qp->urq.db_rec_db2_data, 1959 DB_REC_WIDTH_32B, 1960 DB_REC_USER); 1961 if (rc) 1962 goto err; 1963 } 1964 qedr_qp_user_print(dev, qp); 1965 return rc; 1966 err: 1967 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1968 if (rc) 1969 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc); 1970 1971 err1: 1972 qedr_cleanup_user(dev, ctx, qp); 1973 return rc; 1974 } 1975 1976 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1977 { 1978 int rc; 1979 1980 qp->sq.db = dev->db_addr + 1981 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1982 qp->sq.db_data.data.icid = qp->icid; 1983 1984 rc = qedr_db_recovery_add(dev, qp->sq.db, 1985 &qp->sq.db_data, 1986 DB_REC_WIDTH_32B, 1987 DB_REC_KERNEL); 1988 if (rc) 1989 return rc; 1990 1991 qp->rq.db = dev->db_addr + 1992 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1993 qp->rq.db_data.data.icid = qp->icid; 1994 qp->rq.iwarp_db2 = dev->db_addr + 1995 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1996 qp->rq.iwarp_db2_data.data.icid = qp->icid; 1997 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 1998 1999 rc = qedr_db_recovery_add(dev, qp->rq.db, 2000 &qp->rq.db_data, 2001 DB_REC_WIDTH_32B, 2002 DB_REC_KERNEL); 2003 if (rc) 2004 return rc; 2005 2006 rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2, 2007 &qp->rq.iwarp_db2_data, 2008 DB_REC_WIDTH_32B, 2009 DB_REC_KERNEL); 2010 return rc; 2011 } 2012 2013 static int 2014 qedr_roce_create_kernel_qp(struct qedr_dev *dev, 2015 struct qedr_qp *qp, 2016 struct qed_rdma_create_qp_in_params *in_params, 2017 u32 n_sq_elems, u32 n_rq_elems) 2018 { 2019 struct qed_rdma_create_qp_out_params out_params; 2020 struct qed_chain_init_params params = { 2021 .mode = QED_CHAIN_MODE_PBL, 2022 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 2023 }; 2024 int rc; 2025 2026 params.intended_use = QED_CHAIN_USE_TO_PRODUCE; 2027 params.num_elems = n_sq_elems; 2028 params.elem_size = QEDR_SQE_ELEMENT_SIZE; 2029 2030 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, ¶ms); 2031 if (rc) 2032 return rc; 2033 2034 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl); 2035 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl); 2036 2037 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE; 2038 params.num_elems = n_rq_elems; 2039 params.elem_size = QEDR_RQE_ELEMENT_SIZE; 2040 2041 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, ¶ms); 2042 if (rc) 2043 return rc; 2044 2045 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl); 2046 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl); 2047 2048 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 2049 in_params, &out_params); 2050 2051 if (!qp->qed_qp) 2052 return -EINVAL; 2053 2054 qp->qp_id = out_params.qp_id; 2055 qp->icid = out_params.icid; 2056 2057 return qedr_set_roce_db_info(dev, qp); 2058 } 2059 2060 static int 2061 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev, 2062 struct qedr_qp *qp, 2063 struct qed_rdma_create_qp_in_params *in_params, 2064 u32 n_sq_elems, u32 n_rq_elems) 2065 { 2066 struct qed_rdma_create_qp_out_params out_params; 2067 struct qed_chain_init_params params = { 2068 .mode = QED_CHAIN_MODE_PBL, 2069 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 2070 }; 2071 int rc; 2072 2073 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems, 2074 QEDR_SQE_ELEMENT_SIZE, 2075 QED_CHAIN_PAGE_SIZE, 2076 QED_CHAIN_MODE_PBL); 2077 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems, 2078 QEDR_RQE_ELEMENT_SIZE, 2079 QED_CHAIN_PAGE_SIZE, 2080 QED_CHAIN_MODE_PBL); 2081 2082 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 2083 in_params, &out_params); 2084 2085 if (!qp->qed_qp) 2086 return -EINVAL; 2087 2088 /* Now we allocate the chain */ 2089 2090 params.intended_use = QED_CHAIN_USE_TO_PRODUCE; 2091 params.num_elems = n_sq_elems; 2092 params.elem_size = QEDR_SQE_ELEMENT_SIZE; 2093 params.ext_pbl_virt = out_params.sq_pbl_virt; 2094 params.ext_pbl_phys = out_params.sq_pbl_phys; 2095 2096 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, ¶ms); 2097 if (rc) 2098 goto err; 2099 2100 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE; 2101 params.num_elems = n_rq_elems; 2102 params.elem_size = QEDR_RQE_ELEMENT_SIZE; 2103 params.ext_pbl_virt = out_params.rq_pbl_virt; 2104 params.ext_pbl_phys = out_params.rq_pbl_phys; 2105 2106 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, ¶ms); 2107 if (rc) 2108 goto err; 2109 2110 qp->qp_id = out_params.qp_id; 2111 qp->icid = out_params.icid; 2112 2113 return qedr_set_iwarp_db_info(dev, qp); 2114 2115 err: 2116 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2117 2118 return rc; 2119 } 2120 2121 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) 2122 { 2123 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl); 2124 kfree(qp->wqe_wr_id); 2125 2126 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl); 2127 kfree(qp->rqe_wr_id); 2128 2129 /* GSI qp is not registered to db mechanism so no need to delete */ 2130 if (qp->qp_type == IB_QPT_GSI) 2131 return; 2132 2133 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data); 2134 2135 if (!qp->srq) { 2136 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data); 2137 2138 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2139 qedr_db_recovery_del(dev, qp->rq.iwarp_db2, 2140 &qp->rq.iwarp_db2_data); 2141 } 2142 } 2143 2144 static int qedr_create_kernel_qp(struct qedr_dev *dev, 2145 struct qedr_qp *qp, 2146 struct ib_pd *ibpd, 2147 struct ib_qp_init_attr *attrs) 2148 { 2149 struct qed_rdma_create_qp_in_params in_params; 2150 struct qedr_pd *pd = get_qedr_pd(ibpd); 2151 int rc = -EINVAL; 2152 u32 n_rq_elems; 2153 u32 n_sq_elems; 2154 u32 n_sq_entries; 2155 2156 memset(&in_params, 0, sizeof(in_params)); 2157 qp->create_type = QEDR_QP_CREATE_KERNEL; 2158 2159 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in 2160 * the ring. The ring should allow at least a single WR, even if the 2161 * user requested none, due to allocation issues. 2162 * We should add an extra WR since the prod and cons indices of 2163 * wqe_wr_id are managed in such a way that the WQ is considered full 2164 * when (prod+1)%max_wr==cons. We currently don't do that because we 2165 * double the number of entries due an iSER issue that pushes far more 2166 * WRs than indicated. If we decline its ib_post_send() then we get 2167 * error prints in the dmesg we'd like to avoid. 2168 */ 2169 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 2170 dev->attr.max_sqe); 2171 2172 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id), 2173 GFP_KERNEL); 2174 if (!qp->wqe_wr_id) { 2175 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n"); 2176 return -ENOMEM; 2177 } 2178 2179 /* QP handle to be written in CQE */ 2180 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp); 2181 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp); 2182 2183 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in 2184 * the ring. There ring should allow at least a single WR, even if the 2185 * user requested none, due to allocation issues. 2186 */ 2187 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1); 2188 2189 /* Allocate driver internal RQ array */ 2190 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id), 2191 GFP_KERNEL); 2192 if (!qp->rqe_wr_id) { 2193 DP_ERR(dev, 2194 "create qp: failed RQ shadow memory allocation\n"); 2195 kfree(qp->wqe_wr_id); 2196 return -ENOMEM; 2197 } 2198 2199 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 2200 2201 n_sq_entries = attrs->cap.max_send_wr; 2202 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe); 2203 n_sq_entries = max_t(u32, n_sq_entries, 1); 2204 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE; 2205 2206 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE; 2207 2208 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2209 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params, 2210 n_sq_elems, n_rq_elems); 2211 else 2212 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params, 2213 n_sq_elems, n_rq_elems); 2214 if (rc) 2215 qedr_cleanup_kernel(dev, qp); 2216 2217 return rc; 2218 } 2219 2220 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp, 2221 struct ib_udata *udata) 2222 { 2223 struct qedr_ucontext *ctx = 2224 rdma_udata_to_drv_context(udata, struct qedr_ucontext, 2225 ibucontext); 2226 int rc; 2227 2228 if (qp->qp_type != IB_QPT_GSI) { 2229 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2230 if (rc) 2231 return rc; 2232 } 2233 2234 if (qp->create_type == QEDR_QP_CREATE_USER) 2235 qedr_cleanup_user(dev, ctx, qp); 2236 else 2237 qedr_cleanup_kernel(dev, qp); 2238 2239 return 0; 2240 } 2241 2242 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, 2243 struct ib_qp_init_attr *attrs, 2244 struct ib_udata *udata) 2245 { 2246 struct qedr_xrcd *xrcd = NULL; 2247 struct qedr_pd *pd = NULL; 2248 struct qedr_dev *dev; 2249 struct qedr_qp *qp; 2250 struct ib_qp *ibqp; 2251 int rc = 0; 2252 2253 if (attrs->create_flags) 2254 return ERR_PTR(-EOPNOTSUPP); 2255 2256 if (attrs->qp_type == IB_QPT_XRC_TGT) { 2257 xrcd = get_qedr_xrcd(attrs->xrcd); 2258 dev = get_qedr_dev(xrcd->ibxrcd.device); 2259 } else { 2260 pd = get_qedr_pd(ibpd); 2261 dev = get_qedr_dev(ibpd->device); 2262 } 2263 2264 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", 2265 udata ? "user library" : "kernel", pd); 2266 2267 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata); 2268 if (rc) 2269 return ERR_PTR(rc); 2270 2271 DP_DEBUG(dev, QEDR_MSG_QP, 2272 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 2273 udata ? "user library" : "kernel", attrs->event_handler, pd, 2274 get_qedr_cq(attrs->send_cq), 2275 get_qedr_cq(attrs->send_cq)->icid, 2276 get_qedr_cq(attrs->recv_cq), 2277 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0); 2278 2279 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 2280 if (!qp) { 2281 DP_ERR(dev, "create qp: failed allocating memory\n"); 2282 return ERR_PTR(-ENOMEM); 2283 } 2284 2285 qedr_set_common_qp_params(dev, qp, pd, attrs); 2286 2287 if (attrs->qp_type == IB_QPT_GSI) { 2288 ibqp = qedr_create_gsi_qp(dev, attrs, qp); 2289 if (IS_ERR(ibqp)) 2290 kfree(qp); 2291 return ibqp; 2292 } 2293 2294 if (udata || xrcd) 2295 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); 2296 else 2297 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); 2298 2299 if (rc) 2300 goto out_free_qp; 2301 2302 qp->ibqp.qp_num = qp->qp_id; 2303 2304 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 2305 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL); 2306 if (rc) 2307 goto out_free_qp_resources; 2308 } 2309 2310 return &qp->ibqp; 2311 2312 out_free_qp_resources: 2313 qedr_free_qp_resources(dev, qp, udata); 2314 out_free_qp: 2315 kfree(qp); 2316 2317 return ERR_PTR(-EFAULT); 2318 } 2319 2320 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) 2321 { 2322 switch (qp_state) { 2323 case QED_ROCE_QP_STATE_RESET: 2324 return IB_QPS_RESET; 2325 case QED_ROCE_QP_STATE_INIT: 2326 return IB_QPS_INIT; 2327 case QED_ROCE_QP_STATE_RTR: 2328 return IB_QPS_RTR; 2329 case QED_ROCE_QP_STATE_RTS: 2330 return IB_QPS_RTS; 2331 case QED_ROCE_QP_STATE_SQD: 2332 return IB_QPS_SQD; 2333 case QED_ROCE_QP_STATE_ERR: 2334 return IB_QPS_ERR; 2335 case QED_ROCE_QP_STATE_SQE: 2336 return IB_QPS_SQE; 2337 } 2338 return IB_QPS_ERR; 2339 } 2340 2341 static enum qed_roce_qp_state qedr_get_state_from_ibqp( 2342 enum ib_qp_state qp_state) 2343 { 2344 switch (qp_state) { 2345 case IB_QPS_RESET: 2346 return QED_ROCE_QP_STATE_RESET; 2347 case IB_QPS_INIT: 2348 return QED_ROCE_QP_STATE_INIT; 2349 case IB_QPS_RTR: 2350 return QED_ROCE_QP_STATE_RTR; 2351 case IB_QPS_RTS: 2352 return QED_ROCE_QP_STATE_RTS; 2353 case IB_QPS_SQD: 2354 return QED_ROCE_QP_STATE_SQD; 2355 case IB_QPS_ERR: 2356 return QED_ROCE_QP_STATE_ERR; 2357 default: 2358 return QED_ROCE_QP_STATE_ERR; 2359 } 2360 } 2361 2362 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) 2363 { 2364 qed_chain_reset(&qph->pbl); 2365 qph->prod = 0; 2366 qph->cons = 0; 2367 qph->wqe_cons = 0; 2368 qph->db_data.data.value = cpu_to_le16(0); 2369 } 2370 2371 static int qedr_update_qp_state(struct qedr_dev *dev, 2372 struct qedr_qp *qp, 2373 enum qed_roce_qp_state cur_state, 2374 enum qed_roce_qp_state new_state) 2375 { 2376 int status = 0; 2377 2378 if (new_state == cur_state) 2379 return 0; 2380 2381 switch (cur_state) { 2382 case QED_ROCE_QP_STATE_RESET: 2383 switch (new_state) { 2384 case QED_ROCE_QP_STATE_INIT: 2385 qp->prev_wqe_size = 0; 2386 qedr_reset_qp_hwq_info(&qp->sq); 2387 qedr_reset_qp_hwq_info(&qp->rq); 2388 break; 2389 default: 2390 status = -EINVAL; 2391 break; 2392 } 2393 break; 2394 case QED_ROCE_QP_STATE_INIT: 2395 switch (new_state) { 2396 case QED_ROCE_QP_STATE_RTR: 2397 /* Update doorbell (in case post_recv was 2398 * done before move to RTR) 2399 */ 2400 2401 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2402 writel(qp->rq.db_data.raw, qp->rq.db); 2403 } 2404 break; 2405 case QED_ROCE_QP_STATE_ERR: 2406 break; 2407 default: 2408 /* Invalid state change. */ 2409 status = -EINVAL; 2410 break; 2411 } 2412 break; 2413 case QED_ROCE_QP_STATE_RTR: 2414 /* RTR->XXX */ 2415 switch (new_state) { 2416 case QED_ROCE_QP_STATE_RTS: 2417 break; 2418 case QED_ROCE_QP_STATE_ERR: 2419 break; 2420 default: 2421 /* Invalid state change. */ 2422 status = -EINVAL; 2423 break; 2424 } 2425 break; 2426 case QED_ROCE_QP_STATE_RTS: 2427 /* RTS->XXX */ 2428 switch (new_state) { 2429 case QED_ROCE_QP_STATE_SQD: 2430 break; 2431 case QED_ROCE_QP_STATE_ERR: 2432 break; 2433 default: 2434 /* Invalid state change. */ 2435 status = -EINVAL; 2436 break; 2437 } 2438 break; 2439 case QED_ROCE_QP_STATE_SQD: 2440 /* SQD->XXX */ 2441 switch (new_state) { 2442 case QED_ROCE_QP_STATE_RTS: 2443 case QED_ROCE_QP_STATE_ERR: 2444 break; 2445 default: 2446 /* Invalid state change. */ 2447 status = -EINVAL; 2448 break; 2449 } 2450 break; 2451 case QED_ROCE_QP_STATE_ERR: 2452 /* ERR->XXX */ 2453 switch (new_state) { 2454 case QED_ROCE_QP_STATE_RESET: 2455 if ((qp->rq.prod != qp->rq.cons) || 2456 (qp->sq.prod != qp->sq.cons)) { 2457 DP_NOTICE(dev, 2458 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n", 2459 qp->rq.prod, qp->rq.cons, qp->sq.prod, 2460 qp->sq.cons); 2461 status = -EINVAL; 2462 } 2463 break; 2464 default: 2465 status = -EINVAL; 2466 break; 2467 } 2468 break; 2469 default: 2470 status = -EINVAL; 2471 break; 2472 } 2473 2474 return status; 2475 } 2476 2477 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2478 int attr_mask, struct ib_udata *udata) 2479 { 2480 struct qedr_qp *qp = get_qedr_qp(ibqp); 2481 struct qed_rdma_modify_qp_in_params qp_params = { 0 }; 2482 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); 2483 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 2484 enum ib_qp_state old_qp_state, new_qp_state; 2485 enum qed_roce_qp_state cur_state; 2486 int rc = 0; 2487 2488 DP_DEBUG(dev, QEDR_MSG_QP, 2489 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask, 2490 attr->qp_state); 2491 2492 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 2493 return -EOPNOTSUPP; 2494 2495 old_qp_state = qedr_get_ibqp_state(qp->state); 2496 if (attr_mask & IB_QP_STATE) 2497 new_qp_state = attr->qp_state; 2498 else 2499 new_qp_state = old_qp_state; 2500 2501 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2502 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state, 2503 ibqp->qp_type, attr_mask)) { 2504 DP_ERR(dev, 2505 "modify qp: invalid attribute mask=0x%x specified for\n" 2506 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n", 2507 attr_mask, qp->qp_id, ibqp->qp_type, 2508 old_qp_state, new_qp_state); 2509 rc = -EINVAL; 2510 goto err; 2511 } 2512 } 2513 2514 /* Translate the masks... */ 2515 if (attr_mask & IB_QP_STATE) { 2516 SET_FIELD(qp_params.modify_flags, 2517 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 2518 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state); 2519 } 2520 2521 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 2522 qp_params.sqd_async = true; 2523 2524 if (attr_mask & IB_QP_PKEY_INDEX) { 2525 SET_FIELD(qp_params.modify_flags, 2526 QED_ROCE_MODIFY_QP_VALID_PKEY, 1); 2527 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) { 2528 rc = -EINVAL; 2529 goto err; 2530 } 2531 2532 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT; 2533 } 2534 2535 if (attr_mask & IB_QP_QKEY) 2536 qp->qkey = attr->qkey; 2537 2538 if (attr_mask & IB_QP_ACCESS_FLAGS) { 2539 SET_FIELD(qp_params.modify_flags, 2540 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 2541 qp_params.incoming_rdma_read_en = attr->qp_access_flags & 2542 IB_ACCESS_REMOTE_READ; 2543 qp_params.incoming_rdma_write_en = attr->qp_access_flags & 2544 IB_ACCESS_REMOTE_WRITE; 2545 qp_params.incoming_atomic_en = attr->qp_access_flags & 2546 IB_ACCESS_REMOTE_ATOMIC; 2547 } 2548 2549 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 2550 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2551 return -EINVAL; 2552 2553 if (attr_mask & IB_QP_PATH_MTU) { 2554 if (attr->path_mtu < IB_MTU_256 || 2555 attr->path_mtu > IB_MTU_4096) { 2556 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n"); 2557 rc = -EINVAL; 2558 goto err; 2559 } 2560 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 2561 ib_mtu_enum_to_int(iboe_get_mtu 2562 (dev->ndev->mtu))); 2563 } 2564 2565 if (!qp->mtu) { 2566 qp->mtu = 2567 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2568 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu); 2569 } 2570 2571 SET_FIELD(qp_params.modify_flags, 2572 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1); 2573 2574 qp_params.traffic_class_tos = grh->traffic_class; 2575 qp_params.flow_label = grh->flow_label; 2576 qp_params.hop_limit_ttl = grh->hop_limit; 2577 2578 qp->sgid_idx = grh->sgid_index; 2579 2580 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params); 2581 if (rc) { 2582 DP_ERR(dev, 2583 "modify qp: problems with GID index %d (rc=%d)\n", 2584 grh->sgid_index, rc); 2585 return rc; 2586 } 2587 2588 rc = qedr_get_dmac(dev, &attr->ah_attr, 2589 qp_params.remote_mac_addr); 2590 if (rc) 2591 return rc; 2592 2593 qp_params.use_local_mac = true; 2594 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr); 2595 2596 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n", 2597 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 2598 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 2599 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n", 2600 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 2601 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 2602 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n", 2603 qp_params.remote_mac_addr); 2604 2605 qp_params.mtu = qp->mtu; 2606 qp_params.lb_indication = false; 2607 } 2608 2609 if (!qp_params.mtu) { 2610 /* Stay with current MTU */ 2611 if (qp->mtu) 2612 qp_params.mtu = qp->mtu; 2613 else 2614 qp_params.mtu = 2615 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2616 } 2617 2618 if (attr_mask & IB_QP_TIMEOUT) { 2619 SET_FIELD(qp_params.modify_flags, 2620 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 2621 2622 /* The received timeout value is an exponent used like this: 2623 * "12.7.34 LOCAL ACK TIMEOUT 2624 * Value representing the transport (ACK) timeout for use by 2625 * the remote, expressed as: 4.096 * 2^timeout [usec]" 2626 * The FW expects timeout in msec so we need to divide the usec 2627 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2, 2628 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8). 2629 * The value of zero means infinite so we use a 'max_t' to make 2630 * sure that sub 1 msec values will be configured as 1 msec. 2631 */ 2632 if (attr->timeout) 2633 qp_params.ack_timeout = 2634 1 << max_t(int, attr->timeout - 8, 0); 2635 else 2636 qp_params.ack_timeout = 0; 2637 } 2638 2639 if (attr_mask & IB_QP_RETRY_CNT) { 2640 SET_FIELD(qp_params.modify_flags, 2641 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 2642 qp_params.retry_cnt = attr->retry_cnt; 2643 } 2644 2645 if (attr_mask & IB_QP_RNR_RETRY) { 2646 SET_FIELD(qp_params.modify_flags, 2647 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1); 2648 qp_params.rnr_retry_cnt = attr->rnr_retry; 2649 } 2650 2651 if (attr_mask & IB_QP_RQ_PSN) { 2652 SET_FIELD(qp_params.modify_flags, 2653 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1); 2654 qp_params.rq_psn = attr->rq_psn; 2655 qp->rq_psn = attr->rq_psn; 2656 } 2657 2658 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 2659 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) { 2660 rc = -EINVAL; 2661 DP_ERR(dev, 2662 "unsupported max_rd_atomic=%d, supported=%d\n", 2663 attr->max_rd_atomic, 2664 dev->attr.max_qp_req_rd_atomic_resc); 2665 goto err; 2666 } 2667 2668 SET_FIELD(qp_params.modify_flags, 2669 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1); 2670 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 2671 } 2672 2673 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 2674 SET_FIELD(qp_params.modify_flags, 2675 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1); 2676 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 2677 } 2678 2679 if (attr_mask & IB_QP_SQ_PSN) { 2680 SET_FIELD(qp_params.modify_flags, 2681 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1); 2682 qp_params.sq_psn = attr->sq_psn; 2683 qp->sq_psn = attr->sq_psn; 2684 } 2685 2686 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 2687 if (attr->max_dest_rd_atomic > 2688 dev->attr.max_qp_resp_rd_atomic_resc) { 2689 DP_ERR(dev, 2690 "unsupported max_dest_rd_atomic=%d, supported=%d\n", 2691 attr->max_dest_rd_atomic, 2692 dev->attr.max_qp_resp_rd_atomic_resc); 2693 2694 rc = -EINVAL; 2695 goto err; 2696 } 2697 2698 SET_FIELD(qp_params.modify_flags, 2699 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1); 2700 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 2701 } 2702 2703 if (attr_mask & IB_QP_DEST_QPN) { 2704 SET_FIELD(qp_params.modify_flags, 2705 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1); 2706 2707 qp_params.dest_qp = attr->dest_qp_num; 2708 qp->dest_qp_num = attr->dest_qp_num; 2709 } 2710 2711 cur_state = qp->state; 2712 2713 /* Update the QP state before the actual ramrod to prevent a race with 2714 * fast path. Modifying the QP state to error will cause the device to 2715 * flush the CQEs and while polling the flushed CQEs will considered as 2716 * a potential issue if the QP isn't in error state. 2717 */ 2718 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI && 2719 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR) 2720 qp->state = QED_ROCE_QP_STATE_ERR; 2721 2722 if (qp->qp_type != IB_QPT_GSI) 2723 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx, 2724 qp->qed_qp, &qp_params); 2725 2726 if (attr_mask & IB_QP_STATE) { 2727 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 2728 rc = qedr_update_qp_state(dev, qp, cur_state, 2729 qp_params.new_state); 2730 qp->state = qp_params.new_state; 2731 } 2732 2733 err: 2734 return rc; 2735 } 2736 2737 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params) 2738 { 2739 int ib_qp_acc_flags = 0; 2740 2741 if (params->incoming_rdma_write_en) 2742 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 2743 if (params->incoming_rdma_read_en) 2744 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 2745 if (params->incoming_atomic_en) 2746 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 2747 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 2748 return ib_qp_acc_flags; 2749 } 2750 2751 int qedr_query_qp(struct ib_qp *ibqp, 2752 struct ib_qp_attr *qp_attr, 2753 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 2754 { 2755 struct qed_rdma_query_qp_out_params params; 2756 struct qedr_qp *qp = get_qedr_qp(ibqp); 2757 struct qedr_dev *dev = qp->dev; 2758 int rc = 0; 2759 2760 memset(¶ms, 0, sizeof(params)); 2761 2762 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms); 2763 if (rc) 2764 goto err; 2765 2766 memset(qp_attr, 0, sizeof(*qp_attr)); 2767 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 2768 2769 qp_attr->qp_state = qedr_get_ibqp_state(params.state); 2770 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state); 2771 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu); 2772 qp_attr->path_mig_state = IB_MIG_MIGRATED; 2773 qp_attr->rq_psn = params.rq_psn; 2774 qp_attr->sq_psn = params.sq_psn; 2775 qp_attr->dest_qp_num = params.dest_qp; 2776 2777 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms); 2778 2779 qp_attr->cap.max_send_wr = qp->sq.max_wr; 2780 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 2781 qp_attr->cap.max_send_sge = qp->sq.max_sges; 2782 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 2783 qp_attr->cap.max_inline_data = dev->attr.max_inline; 2784 qp_init_attr->cap = qp_attr->cap; 2785 2786 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2787 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 2788 params.flow_label, qp->sgid_idx, 2789 params.hop_limit_ttl, params.traffic_class_tos); 2790 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); 2791 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 2792 rdma_ah_set_sl(&qp_attr->ah_attr, 0); 2793 qp_attr->timeout = params.timeout; 2794 qp_attr->rnr_retry = params.rnr_retry; 2795 qp_attr->retry_cnt = params.retry_cnt; 2796 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 2797 qp_attr->pkey_index = params.pkey_index; 2798 qp_attr->port_num = 1; 2799 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 2800 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 2801 qp_attr->alt_pkey_index = 0; 2802 qp_attr->alt_port_num = 0; 2803 qp_attr->alt_timeout = 0; 2804 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 2805 2806 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0; 2807 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 2808 qp_attr->max_rd_atomic = params.max_rd_atomic; 2809 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0; 2810 2811 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n", 2812 qp_attr->cap.max_inline_data); 2813 2814 err: 2815 return rc; 2816 } 2817 2818 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 2819 { 2820 struct qedr_qp *qp = get_qedr_qp(ibqp); 2821 struct qedr_dev *dev = qp->dev; 2822 struct ib_qp_attr attr; 2823 int attr_mask = 0; 2824 2825 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n", 2826 qp, qp->qp_type); 2827 2828 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2829 if ((qp->state != QED_ROCE_QP_STATE_RESET) && 2830 (qp->state != QED_ROCE_QP_STATE_ERR) && 2831 (qp->state != QED_ROCE_QP_STATE_INIT)) { 2832 2833 attr.qp_state = IB_QPS_ERR; 2834 attr_mask |= IB_QP_STATE; 2835 2836 /* Change the QP state to ERROR */ 2837 qedr_modify_qp(ibqp, &attr, attr_mask, NULL); 2838 } 2839 } else { 2840 /* If connection establishment started the WAIT_FOR_CONNECT 2841 * bit will be on and we need to Wait for the establishment 2842 * to complete before destroying the qp. 2843 */ 2844 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT, 2845 &qp->iwarp_cm_flags)) 2846 wait_for_completion(&qp->iwarp_cm_comp); 2847 2848 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT 2849 * bit will be on, and we need to wait for the disconnect to 2850 * complete before continuing. We can use the same completion, 2851 * iwarp_cm_comp, since this is the only place that waits for 2852 * this completion and it is sequential. In addition, 2853 * disconnect can't occur before the connection is fully 2854 * established, therefore if WAIT_FOR_DISCONNECT is on it 2855 * means WAIT_FOR_CONNECT is also on and the completion for 2856 * CONNECT already occurred. 2857 */ 2858 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT, 2859 &qp->iwarp_cm_flags)) 2860 wait_for_completion(&qp->iwarp_cm_comp); 2861 } 2862 2863 if (qp->qp_type == IB_QPT_GSI) 2864 qedr_destroy_gsi_qp(dev); 2865 2866 /* We need to remove the entry from the xarray before we release the 2867 * qp_id to avoid a race of the qp_id being reallocated and failing 2868 * on xa_insert 2869 */ 2870 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2871 xa_erase(&dev->qps, qp->qp_id); 2872 2873 qedr_free_qp_resources(dev, qp, udata); 2874 2875 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2876 qedr_iw_qp_rem_ref(&qp->ibqp); 2877 else 2878 kfree(qp); 2879 2880 return 0; 2881 } 2882 2883 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, 2884 struct ib_udata *udata) 2885 { 2886 struct qedr_ah *ah = get_qedr_ah(ibah); 2887 2888 rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr); 2889 2890 return 0; 2891 } 2892 2893 int qedr_destroy_ah(struct ib_ah *ibah, u32 flags) 2894 { 2895 struct qedr_ah *ah = get_qedr_ah(ibah); 2896 2897 rdma_destroy_ah_attr(&ah->attr); 2898 return 0; 2899 } 2900 2901 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) 2902 { 2903 struct qedr_pbl *pbl, *tmp; 2904 2905 if (info->pbl_table) 2906 list_add_tail(&info->pbl_table->list_entry, 2907 &info->free_pbl_list); 2908 2909 if (!list_empty(&info->inuse_pbl_list)) 2910 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 2911 2912 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 2913 list_del(&pbl->list_entry); 2914 qedr_free_pbl(dev, &info->pbl_info, pbl); 2915 } 2916 } 2917 2918 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info, 2919 size_t page_list_len, bool two_layered) 2920 { 2921 struct qedr_pbl *tmp; 2922 int rc; 2923 2924 INIT_LIST_HEAD(&info->free_pbl_list); 2925 INIT_LIST_HEAD(&info->inuse_pbl_list); 2926 2927 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info, 2928 page_list_len, two_layered); 2929 if (rc) 2930 goto done; 2931 2932 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2933 if (IS_ERR(info->pbl_table)) { 2934 rc = PTR_ERR(info->pbl_table); 2935 goto done; 2936 } 2937 2938 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n", 2939 &info->pbl_table->pa); 2940 2941 /* in usual case we use 2 PBLs, so we add one to free 2942 * list and allocating another one 2943 */ 2944 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2945 if (IS_ERR(tmp)) { 2946 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n"); 2947 goto done; 2948 } 2949 2950 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 2951 2952 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa); 2953 2954 done: 2955 if (rc) 2956 free_mr_info(dev, info); 2957 2958 return rc; 2959 } 2960 2961 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 2962 u64 usr_addr, int acc, struct ib_udata *udata) 2963 { 2964 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2965 struct qedr_mr *mr; 2966 struct qedr_pd *pd; 2967 int rc = -ENOMEM; 2968 2969 pd = get_qedr_pd(ibpd); 2970 DP_DEBUG(dev, QEDR_MSG_MR, 2971 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 2972 pd->pd_id, start, len, usr_addr, acc); 2973 2974 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 2975 return ERR_PTR(-EINVAL); 2976 2977 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2978 if (!mr) 2979 return ERR_PTR(rc); 2980 2981 mr->type = QEDR_MR_USER; 2982 2983 mr->umem = ib_umem_get(ibpd->device, start, len, acc); 2984 if (IS_ERR(mr->umem)) { 2985 rc = -EFAULT; 2986 goto err0; 2987 } 2988 2989 rc = init_mr_info(dev, &mr->info, 2990 ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1); 2991 if (rc) 2992 goto err1; 2993 2994 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 2995 &mr->info.pbl_info, PAGE_SHIFT); 2996 2997 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2998 if (rc) { 2999 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 3000 goto err1; 3001 } 3002 3003 /* Index only, 18 bit long, lkey = itid << 8 | key */ 3004 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 3005 mr->hw_mr.key = 0; 3006 mr->hw_mr.pd = pd->pd_id; 3007 mr->hw_mr.local_read = 1; 3008 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 3009 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 3010 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 3011 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 3012 mr->hw_mr.mw_bind = false; 3013 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 3014 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 3015 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 3016 mr->hw_mr.page_size_log = PAGE_SHIFT; 3017 mr->hw_mr.length = len; 3018 mr->hw_mr.vaddr = usr_addr; 3019 mr->hw_mr.phy_mr = false; 3020 mr->hw_mr.dma_mr = false; 3021 3022 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 3023 if (rc) { 3024 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 3025 goto err2; 3026 } 3027 3028 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3029 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 3030 mr->hw_mr.remote_atomic) 3031 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3032 3033 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n", 3034 mr->ibmr.lkey); 3035 return &mr->ibmr; 3036 3037 err2: 3038 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3039 err1: 3040 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 3041 err0: 3042 kfree(mr); 3043 return ERR_PTR(rc); 3044 } 3045 3046 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 3047 { 3048 struct qedr_mr *mr = get_qedr_mr(ib_mr); 3049 struct qedr_dev *dev = get_qedr_dev(ib_mr->device); 3050 int rc = 0; 3051 3052 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid); 3053 if (rc) 3054 return rc; 3055 3056 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3057 3058 if (mr->type != QEDR_MR_DMA) 3059 free_mr_info(dev, &mr->info); 3060 3061 /* it could be user registered memory. */ 3062 ib_umem_release(mr->umem); 3063 3064 kfree(mr); 3065 3066 return rc; 3067 } 3068 3069 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, 3070 int max_page_list_len) 3071 { 3072 struct qedr_pd *pd = get_qedr_pd(ibpd); 3073 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 3074 struct qedr_mr *mr; 3075 int rc = -ENOMEM; 3076 3077 DP_DEBUG(dev, QEDR_MSG_MR, 3078 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id, 3079 max_page_list_len); 3080 3081 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3082 if (!mr) 3083 return ERR_PTR(rc); 3084 3085 mr->dev = dev; 3086 mr->type = QEDR_MR_FRMR; 3087 3088 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1); 3089 if (rc) 3090 goto err0; 3091 3092 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 3093 if (rc) { 3094 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 3095 goto err0; 3096 } 3097 3098 /* Index only, 18 bit long, lkey = itid << 8 | key */ 3099 mr->hw_mr.tid_type = QED_RDMA_TID_FMR; 3100 mr->hw_mr.key = 0; 3101 mr->hw_mr.pd = pd->pd_id; 3102 mr->hw_mr.local_read = 1; 3103 mr->hw_mr.local_write = 0; 3104 mr->hw_mr.remote_read = 0; 3105 mr->hw_mr.remote_write = 0; 3106 mr->hw_mr.remote_atomic = 0; 3107 mr->hw_mr.mw_bind = false; 3108 mr->hw_mr.pbl_ptr = 0; 3109 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 3110 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 3111 mr->hw_mr.length = 0; 3112 mr->hw_mr.vaddr = 0; 3113 mr->hw_mr.phy_mr = true; 3114 mr->hw_mr.dma_mr = false; 3115 3116 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 3117 if (rc) { 3118 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 3119 goto err1; 3120 } 3121 3122 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3123 mr->ibmr.rkey = mr->ibmr.lkey; 3124 3125 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey); 3126 return mr; 3127 3128 err1: 3129 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3130 err0: 3131 kfree(mr); 3132 return ERR_PTR(rc); 3133 } 3134 3135 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 3136 u32 max_num_sg) 3137 { 3138 struct qedr_mr *mr; 3139 3140 if (mr_type != IB_MR_TYPE_MEM_REG) 3141 return ERR_PTR(-EINVAL); 3142 3143 mr = __qedr_alloc_mr(ibpd, max_num_sg); 3144 3145 if (IS_ERR(mr)) 3146 return ERR_PTR(-EINVAL); 3147 3148 return &mr->ibmr; 3149 } 3150 3151 static int qedr_set_page(struct ib_mr *ibmr, u64 addr) 3152 { 3153 struct qedr_mr *mr = get_qedr_mr(ibmr); 3154 struct qedr_pbl *pbl_table; 3155 struct regpair *pbe; 3156 u32 pbes_in_page; 3157 3158 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 3159 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages); 3160 return -ENOMEM; 3161 } 3162 3163 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n", 3164 mr->npages, addr); 3165 3166 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 3167 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 3168 pbe = (struct regpair *)pbl_table->va; 3169 pbe += mr->npages % pbes_in_page; 3170 pbe->lo = cpu_to_le32((u32)addr); 3171 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 3172 3173 mr->npages++; 3174 3175 return 0; 3176 } 3177 3178 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info) 3179 { 3180 int work = info->completed - info->completed_handled - 1; 3181 3182 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work); 3183 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 3184 struct qedr_pbl *pbl; 3185 3186 /* Free all the page list that are possible to be freed 3187 * (all the ones that were invalidated), under the assumption 3188 * that if an FMR was completed successfully that means that 3189 * if there was an invalidate operation before it also ended 3190 */ 3191 pbl = list_first_entry(&info->inuse_pbl_list, 3192 struct qedr_pbl, list_entry); 3193 list_move_tail(&pbl->list_entry, &info->free_pbl_list); 3194 info->completed_handled++; 3195 } 3196 } 3197 3198 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 3199 int sg_nents, unsigned int *sg_offset) 3200 { 3201 struct qedr_mr *mr = get_qedr_mr(ibmr); 3202 3203 mr->npages = 0; 3204 3205 handle_completed_mrs(mr->dev, &mr->info); 3206 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page); 3207 } 3208 3209 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc) 3210 { 3211 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 3212 struct qedr_pd *pd = get_qedr_pd(ibpd); 3213 struct qedr_mr *mr; 3214 int rc; 3215 3216 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3217 if (!mr) 3218 return ERR_PTR(-ENOMEM); 3219 3220 mr->type = QEDR_MR_DMA; 3221 3222 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 3223 if (rc) { 3224 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 3225 goto err1; 3226 } 3227 3228 /* index only, 18 bit long, lkey = itid << 8 | key */ 3229 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 3230 mr->hw_mr.pd = pd->pd_id; 3231 mr->hw_mr.local_read = 1; 3232 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 3233 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 3234 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 3235 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 3236 mr->hw_mr.dma_mr = true; 3237 3238 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 3239 if (rc) { 3240 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 3241 goto err2; 3242 } 3243 3244 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3245 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 3246 mr->hw_mr.remote_atomic) 3247 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3248 3249 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey); 3250 return &mr->ibmr; 3251 3252 err2: 3253 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3254 err1: 3255 kfree(mr); 3256 return ERR_PTR(rc); 3257 } 3258 3259 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq) 3260 { 3261 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 3262 } 3263 3264 static int sge_data_len(struct ib_sge *sg_list, int num_sge) 3265 { 3266 int i, len = 0; 3267 3268 for (i = 0; i < num_sge; i++) 3269 len += sg_list[i].length; 3270 3271 return len; 3272 } 3273 3274 static void swap_wqe_data64(u64 *p) 3275 { 3276 int i; 3277 3278 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 3279 *p = cpu_to_be64(cpu_to_le64(*p)); 3280 } 3281 3282 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev, 3283 struct qedr_qp *qp, u8 *wqe_size, 3284 const struct ib_send_wr *wr, 3285 const struct ib_send_wr **bad_wr, 3286 u8 *bits, u8 bit) 3287 { 3288 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 3289 char *seg_prt, *wqe; 3290 int i, seg_siz; 3291 3292 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 3293 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size); 3294 *bad_wr = wr; 3295 return 0; 3296 } 3297 3298 if (!data_size) 3299 return data_size; 3300 3301 *bits |= bit; 3302 3303 seg_prt = NULL; 3304 wqe = NULL; 3305 seg_siz = 0; 3306 3307 /* Copy data inline */ 3308 for (i = 0; i < wr->num_sge; i++) { 3309 u32 len = wr->sg_list[i].length; 3310 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 3311 3312 while (len > 0) { 3313 u32 cur; 3314 3315 /* New segment required */ 3316 if (!seg_siz) { 3317 wqe = (char *)qed_chain_produce(&qp->sq.pbl); 3318 seg_prt = wqe; 3319 seg_siz = sizeof(struct rdma_sq_common_wqe); 3320 (*wqe_size)++; 3321 } 3322 3323 /* Calculate currently allowed length */ 3324 cur = min_t(u32, len, seg_siz); 3325 memcpy(seg_prt, src, cur); 3326 3327 /* Update segment variables */ 3328 seg_prt += cur; 3329 seg_siz -= cur; 3330 3331 /* Update sge variables */ 3332 src += cur; 3333 len -= cur; 3334 3335 /* Swap fully-completed segments */ 3336 if (!seg_siz) 3337 swap_wqe_data64((u64 *)wqe); 3338 } 3339 } 3340 3341 /* swap last not completed segment */ 3342 if (seg_siz) 3343 swap_wqe_data64((u64 *)wqe); 3344 3345 return data_size; 3346 } 3347 3348 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 3349 do { \ 3350 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3351 (sge)->length = cpu_to_le32(vlength); \ 3352 (sge)->flags = cpu_to_le32(vflags); \ 3353 } while (0) 3354 3355 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 3356 do { \ 3357 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \ 3358 (hdr)->num_sges = num_sge; \ 3359 } while (0) 3360 3361 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 3362 do { \ 3363 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3364 (sge)->length = cpu_to_le32(vlength); \ 3365 (sge)->l_key = cpu_to_le32(vlkey); \ 3366 } while (0) 3367 3368 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, 3369 const struct ib_send_wr *wr) 3370 { 3371 u32 data_size = 0; 3372 int i; 3373 3374 for (i = 0; i < wr->num_sge; i++) { 3375 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl); 3376 3377 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr); 3378 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 3379 sge->length = cpu_to_le32(wr->sg_list[i].length); 3380 data_size += wr->sg_list[i].length; 3381 } 3382 3383 if (wqe_size) 3384 *wqe_size += wr->num_sge; 3385 3386 return data_size; 3387 } 3388 3389 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev, 3390 struct qedr_qp *qp, 3391 struct rdma_sq_rdma_wqe_1st *rwqe, 3392 struct rdma_sq_rdma_wqe_2nd *rwqe2, 3393 const struct ib_send_wr *wr, 3394 const struct ib_send_wr **bad_wr) 3395 { 3396 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 3397 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr); 3398 3399 if (wr->send_flags & IB_SEND_INLINE && 3400 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 3401 wr->opcode == IB_WR_RDMA_WRITE)) { 3402 u8 flags = 0; 3403 3404 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 3405 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, 3406 bad_wr, &rwqe->flags, flags); 3407 } 3408 3409 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr); 3410 } 3411 3412 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev, 3413 struct qedr_qp *qp, 3414 struct rdma_sq_send_wqe_1st *swqe, 3415 struct rdma_sq_send_wqe_2st *swqe2, 3416 const struct ib_send_wr *wr, 3417 const struct ib_send_wr **bad_wr) 3418 { 3419 memset(swqe2, 0, sizeof(*swqe2)); 3420 if (wr->send_flags & IB_SEND_INLINE) { 3421 u8 flags = 0; 3422 3423 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 3424 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, 3425 bad_wr, &swqe->flags, flags); 3426 } 3427 3428 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr); 3429 } 3430 3431 static int qedr_prepare_reg(struct qedr_qp *qp, 3432 struct rdma_sq_fmr_wqe_1st *fwqe1, 3433 const struct ib_reg_wr *wr) 3434 { 3435 struct qedr_mr *mr = get_qedr_mr(wr->mr); 3436 struct rdma_sq_fmr_wqe_2nd *fwqe2; 3437 3438 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl); 3439 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 3440 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 3441 fwqe1->l_key = wr->key; 3442 3443 fwqe2->access_ctrl = 0; 3444 3445 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 3446 !!(wr->access & IB_ACCESS_REMOTE_READ)); 3447 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 3448 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 3449 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 3450 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 3451 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 3452 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 3453 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 3454 fwqe2->fmr_ctrl = 0; 3455 3456 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 3457 ilog2(mr->ibmr.page_size) - 12); 3458 3459 fwqe2->length_hi = 0; 3460 fwqe2->length_lo = mr->ibmr.length; 3461 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 3462 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 3463 3464 qp->wqe_wr_id[qp->sq.prod].mr = mr; 3465 3466 return 0; 3467 } 3468 3469 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 3470 { 3471 switch (opcode) { 3472 case IB_WR_RDMA_WRITE: 3473 case IB_WR_RDMA_WRITE_WITH_IMM: 3474 return IB_WC_RDMA_WRITE; 3475 case IB_WR_SEND_WITH_IMM: 3476 case IB_WR_SEND: 3477 case IB_WR_SEND_WITH_INV: 3478 return IB_WC_SEND; 3479 case IB_WR_RDMA_READ: 3480 case IB_WR_RDMA_READ_WITH_INV: 3481 return IB_WC_RDMA_READ; 3482 case IB_WR_ATOMIC_CMP_AND_SWP: 3483 return IB_WC_COMP_SWAP; 3484 case IB_WR_ATOMIC_FETCH_AND_ADD: 3485 return IB_WC_FETCH_ADD; 3486 case IB_WR_REG_MR: 3487 return IB_WC_REG_MR; 3488 case IB_WR_LOCAL_INV: 3489 return IB_WC_LOCAL_INV; 3490 default: 3491 return IB_WC_SEND; 3492 } 3493 } 3494 3495 static inline bool qedr_can_post_send(struct qedr_qp *qp, 3496 const struct ib_send_wr *wr) 3497 { 3498 int wq_is_full, err_wr, pbl_is_full; 3499 struct qedr_dev *dev = qp->dev; 3500 3501 /* prevent SQ overflow and/or processing of a bad WR */ 3502 err_wr = wr->num_sge > qp->sq.max_sges; 3503 wq_is_full = qedr_wq_is_full(&qp->sq); 3504 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) < 3505 QEDR_MAX_SQE_ELEMENTS_PER_SQE; 3506 if (wq_is_full || err_wr || pbl_is_full) { 3507 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) { 3508 DP_ERR(dev, 3509 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n", 3510 qp); 3511 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL; 3512 } 3513 3514 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) { 3515 DP_ERR(dev, 3516 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n", 3517 qp); 3518 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR; 3519 } 3520 3521 if (pbl_is_full && 3522 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) { 3523 DP_ERR(dev, 3524 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n", 3525 qp); 3526 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL; 3527 } 3528 return false; 3529 } 3530 return true; 3531 } 3532 3533 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3534 const struct ib_send_wr **bad_wr) 3535 { 3536 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3537 struct qedr_qp *qp = get_qedr_qp(ibqp); 3538 struct rdma_sq_atomic_wqe_1st *awqe1; 3539 struct rdma_sq_atomic_wqe_2nd *awqe2; 3540 struct rdma_sq_atomic_wqe_3rd *awqe3; 3541 struct rdma_sq_send_wqe_2st *swqe2; 3542 struct rdma_sq_local_inv_wqe *iwqe; 3543 struct rdma_sq_rdma_wqe_2nd *rwqe2; 3544 struct rdma_sq_send_wqe_1st *swqe; 3545 struct rdma_sq_rdma_wqe_1st *rwqe; 3546 struct rdma_sq_fmr_wqe_1st *fwqe1; 3547 struct rdma_sq_common_wqe *wqe; 3548 u32 length; 3549 int rc = 0; 3550 bool comp; 3551 3552 if (!qedr_can_post_send(qp, wr)) { 3553 *bad_wr = wr; 3554 return -ENOMEM; 3555 } 3556 3557 wqe = qed_chain_produce(&qp->sq.pbl); 3558 qp->wqe_wr_id[qp->sq.prod].signaled = 3559 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 3560 3561 wqe->flags = 0; 3562 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, 3563 !!(wr->send_flags & IB_SEND_SOLICITED)); 3564 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled; 3565 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 3566 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, 3567 !!(wr->send_flags & IB_SEND_FENCE)); 3568 wqe->prev_wqe_size = qp->prev_wqe_size; 3569 3570 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode); 3571 3572 switch (wr->opcode) { 3573 case IB_WR_SEND_WITH_IMM: 3574 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3575 rc = -EINVAL; 3576 *bad_wr = wr; 3577 break; 3578 } 3579 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 3580 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3581 swqe->wqe_size = 2; 3582 swqe2 = qed_chain_produce(&qp->sq.pbl); 3583 3584 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 3585 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3586 wr, bad_wr); 3587 swqe->length = cpu_to_le32(length); 3588 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3589 qp->prev_wqe_size = swqe->wqe_size; 3590 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3591 break; 3592 case IB_WR_SEND: 3593 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 3594 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3595 3596 swqe->wqe_size = 2; 3597 swqe2 = qed_chain_produce(&qp->sq.pbl); 3598 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3599 wr, bad_wr); 3600 swqe->length = cpu_to_le32(length); 3601 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3602 qp->prev_wqe_size = swqe->wqe_size; 3603 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3604 break; 3605 case IB_WR_SEND_WITH_INV: 3606 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 3607 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3608 swqe2 = qed_chain_produce(&qp->sq.pbl); 3609 swqe->wqe_size = 2; 3610 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey); 3611 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3612 wr, bad_wr); 3613 swqe->length = cpu_to_le32(length); 3614 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3615 qp->prev_wqe_size = swqe->wqe_size; 3616 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3617 break; 3618 3619 case IB_WR_RDMA_WRITE_WITH_IMM: 3620 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3621 rc = -EINVAL; 3622 *bad_wr = wr; 3623 break; 3624 } 3625 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 3626 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3627 3628 rwqe->wqe_size = 2; 3629 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 3630 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3631 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3632 wr, bad_wr); 3633 rwqe->length = cpu_to_le32(length); 3634 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3635 qp->prev_wqe_size = rwqe->wqe_size; 3636 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3637 break; 3638 case IB_WR_RDMA_WRITE: 3639 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 3640 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3641 3642 rwqe->wqe_size = 2; 3643 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3644 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3645 wr, bad_wr); 3646 rwqe->length = cpu_to_le32(length); 3647 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3648 qp->prev_wqe_size = rwqe->wqe_size; 3649 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3650 break; 3651 case IB_WR_RDMA_READ_WITH_INV: 3652 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1); 3653 fallthrough; /* same is identical to RDMA READ */ 3654 3655 case IB_WR_RDMA_READ: 3656 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 3657 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3658 3659 rwqe->wqe_size = 2; 3660 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3661 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3662 wr, bad_wr); 3663 rwqe->length = cpu_to_le32(length); 3664 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3665 qp->prev_wqe_size = rwqe->wqe_size; 3666 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3667 break; 3668 3669 case IB_WR_ATOMIC_CMP_AND_SWP: 3670 case IB_WR_ATOMIC_FETCH_AND_ADD: 3671 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe; 3672 awqe1->wqe_size = 4; 3673 3674 awqe2 = qed_chain_produce(&qp->sq.pbl); 3675 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr); 3676 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 3677 3678 awqe3 = qed_chain_produce(&qp->sq.pbl); 3679 3680 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 3681 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 3682 DMA_REGPAIR_LE(awqe3->swap_data, 3683 atomic_wr(wr)->compare_add); 3684 } else { 3685 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 3686 DMA_REGPAIR_LE(awqe3->swap_data, 3687 atomic_wr(wr)->swap); 3688 DMA_REGPAIR_LE(awqe3->cmp_data, 3689 atomic_wr(wr)->compare_add); 3690 } 3691 3692 qedr_prepare_sq_sges(qp, NULL, wr); 3693 3694 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size; 3695 qp->prev_wqe_size = awqe1->wqe_size; 3696 break; 3697 3698 case IB_WR_LOCAL_INV: 3699 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 3700 iwqe->wqe_size = 1; 3701 3702 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 3703 iwqe->inv_l_key = wr->ex.invalidate_rkey; 3704 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size; 3705 qp->prev_wqe_size = iwqe->wqe_size; 3706 break; 3707 case IB_WR_REG_MR: 3708 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n"); 3709 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 3710 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 3711 fwqe1->wqe_size = 2; 3712 3713 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr)); 3714 if (rc) { 3715 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc); 3716 *bad_wr = wr; 3717 break; 3718 } 3719 3720 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 3721 qp->prev_wqe_size = fwqe1->wqe_size; 3722 break; 3723 default: 3724 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode); 3725 rc = -EINVAL; 3726 *bad_wr = wr; 3727 break; 3728 } 3729 3730 if (*bad_wr) { 3731 u16 value; 3732 3733 /* Restore prod to its position before 3734 * this WR was processed 3735 */ 3736 value = le16_to_cpu(qp->sq.db_data.data.value); 3737 qed_chain_set_prod(&qp->sq.pbl, value, wqe); 3738 3739 /* Restore prev_wqe_size */ 3740 qp->prev_wqe_size = wqe->prev_wqe_size; 3741 rc = -EINVAL; 3742 DP_ERR(dev, "POST SEND FAILED\n"); 3743 } 3744 3745 return rc; 3746 } 3747 3748 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3749 const struct ib_send_wr **bad_wr) 3750 { 3751 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3752 struct qedr_qp *qp = get_qedr_qp(ibqp); 3753 unsigned long flags; 3754 int rc = 0; 3755 3756 *bad_wr = NULL; 3757 3758 if (qp->qp_type == IB_QPT_GSI) 3759 return qedr_gsi_post_send(ibqp, wr, bad_wr); 3760 3761 spin_lock_irqsave(&qp->q_lock, flags); 3762 3763 if (rdma_protocol_roce(&dev->ibdev, 1)) { 3764 if ((qp->state != QED_ROCE_QP_STATE_RTS) && 3765 (qp->state != QED_ROCE_QP_STATE_ERR) && 3766 (qp->state != QED_ROCE_QP_STATE_SQD)) { 3767 spin_unlock_irqrestore(&qp->q_lock, flags); 3768 *bad_wr = wr; 3769 DP_DEBUG(dev, QEDR_MSG_CQ, 3770 "QP in wrong state! QP icid=0x%x state %d\n", 3771 qp->icid, qp->state); 3772 return -EINVAL; 3773 } 3774 } 3775 3776 while (wr) { 3777 rc = __qedr_post_send(ibqp, wr, bad_wr); 3778 if (rc) 3779 break; 3780 3781 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 3782 3783 qedr_inc_sw_prod(&qp->sq); 3784 3785 qp->sq.db_data.data.value++; 3786 3787 wr = wr->next; 3788 } 3789 3790 /* Trigger doorbell 3791 * If there was a failure in the first WR then it will be triggered in 3792 * vane. However this is not harmful (as long as the producer value is 3793 * unchanged). For performance reasons we avoid checking for this 3794 * redundant doorbell. 3795 * 3796 * qp->wqe_wr_id is accessed during qedr_poll_cq, as 3797 * soon as we give the doorbell, we could get a completion 3798 * for this wr, therefore we need to make sure that the 3799 * memory is updated before giving the doorbell. 3800 * During qedr_poll_cq, rmb is called before accessing the 3801 * cqe. This covers for the smp_rmb as well. 3802 */ 3803 smp_wmb(); 3804 writel(qp->sq.db_data.raw, qp->sq.db); 3805 3806 spin_unlock_irqrestore(&qp->q_lock, flags); 3807 3808 return rc; 3809 } 3810 3811 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq) 3812 { 3813 u32 used; 3814 3815 /* Calculate number of elements used based on producer 3816 * count and consumer count and subtract it from max 3817 * work request supported so that we get elements left. 3818 */ 3819 used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt); 3820 3821 return hw_srq->max_wr - used; 3822 } 3823 3824 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 3825 const struct ib_recv_wr **bad_wr) 3826 { 3827 struct qedr_srq *srq = get_qedr_srq(ibsrq); 3828 struct qedr_srq_hwq_info *hw_srq; 3829 struct qedr_dev *dev = srq->dev; 3830 struct qed_chain *pbl; 3831 unsigned long flags; 3832 int status = 0; 3833 u32 num_sge; 3834 3835 spin_lock_irqsave(&srq->lock, flags); 3836 3837 hw_srq = &srq->hw_srq; 3838 pbl = &srq->hw_srq.pbl; 3839 while (wr) { 3840 struct rdma_srq_wqe_header *hdr; 3841 int i; 3842 3843 if (!qedr_srq_elem_left(hw_srq) || 3844 wr->num_sge > srq->hw_srq.max_sges) { 3845 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n", 3846 hw_srq->wr_prod_cnt, 3847 atomic_read(&hw_srq->wr_cons_cnt), 3848 wr->num_sge, srq->hw_srq.max_sges); 3849 status = -ENOMEM; 3850 *bad_wr = wr; 3851 break; 3852 } 3853 3854 hdr = qed_chain_produce(pbl); 3855 num_sge = wr->num_sge; 3856 /* Set number of sge and work request id in header */ 3857 SRQ_HDR_SET(hdr, wr->wr_id, num_sge); 3858 3859 srq->hw_srq.wr_prod_cnt++; 3860 hw_srq->wqe_prod++; 3861 hw_srq->sge_prod++; 3862 3863 DP_DEBUG(dev, QEDR_MSG_SRQ, 3864 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n", 3865 wr->num_sge, hw_srq->wqe_prod, wr->wr_id); 3866 3867 for (i = 0; i < wr->num_sge; i++) { 3868 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl); 3869 3870 /* Set SGE length, lkey and address */ 3871 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, 3872 wr->sg_list[i].length, wr->sg_list[i].lkey); 3873 3874 DP_DEBUG(dev, QEDR_MSG_SRQ, 3875 "[%d]: len %d key %x addr %x:%x\n", 3876 i, srq_sge->length, srq_sge->l_key, 3877 srq_sge->addr.hi, srq_sge->addr.lo); 3878 hw_srq->sge_prod++; 3879 } 3880 3881 /* Update WQE and SGE information before 3882 * updating producer. 3883 */ 3884 dma_wmb(); 3885 3886 /* SRQ producer is 8 bytes. Need to update SGE producer index 3887 * in first 4 bytes and need to update WQE producer in 3888 * next 4 bytes. 3889 */ 3890 srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod); 3891 /* Make sure sge producer is updated first */ 3892 dma_wmb(); 3893 srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod); 3894 3895 wr = wr->next; 3896 } 3897 3898 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n", 3899 qed_chain_get_elem_left(pbl)); 3900 spin_unlock_irqrestore(&srq->lock, flags); 3901 3902 return status; 3903 } 3904 3905 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 3906 const struct ib_recv_wr **bad_wr) 3907 { 3908 struct qedr_qp *qp = get_qedr_qp(ibqp); 3909 struct qedr_dev *dev = qp->dev; 3910 unsigned long flags; 3911 int status = 0; 3912 3913 if (qp->qp_type == IB_QPT_GSI) 3914 return qedr_gsi_post_recv(ibqp, wr, bad_wr); 3915 3916 spin_lock_irqsave(&qp->q_lock, flags); 3917 3918 if (qp->state == QED_ROCE_QP_STATE_RESET) { 3919 spin_unlock_irqrestore(&qp->q_lock, flags); 3920 *bad_wr = wr; 3921 return -EINVAL; 3922 } 3923 3924 while (wr) { 3925 int i; 3926 3927 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) < 3928 QEDR_MAX_RQE_ELEMENTS_PER_RQE || 3929 wr->num_sge > qp->rq.max_sges) { 3930 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n", 3931 qed_chain_get_elem_left_u32(&qp->rq.pbl), 3932 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge, 3933 qp->rq.max_sges); 3934 status = -ENOMEM; 3935 *bad_wr = wr; 3936 break; 3937 } 3938 for (i = 0; i < wr->num_sge; i++) { 3939 u32 flags = 0; 3940 struct rdma_rq_sge *rqe = 3941 qed_chain_produce(&qp->rq.pbl); 3942 3943 /* First one must include the number 3944 * of SGE in the list 3945 */ 3946 if (!i) 3947 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 3948 wr->num_sge); 3949 3950 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 3951 wr->sg_list[i].lkey); 3952 3953 RQ_SGE_SET(rqe, wr->sg_list[i].addr, 3954 wr->sg_list[i].length, flags); 3955 } 3956 3957 /* Special case of no sges. FW requires between 1-4 sges... 3958 * in this case we need to post 1 sge with length zero. this is 3959 * because rdma write with immediate consumes an RQ. 3960 */ 3961 if (!wr->num_sge) { 3962 u32 flags = 0; 3963 struct rdma_rq_sge *rqe = 3964 qed_chain_produce(&qp->rq.pbl); 3965 3966 /* First one must include the number 3967 * of SGE in the list 3968 */ 3969 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0); 3970 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 3971 3972 RQ_SGE_SET(rqe, 0, 0, flags); 3973 i = 1; 3974 } 3975 3976 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 3977 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 3978 3979 qedr_inc_sw_prod(&qp->rq); 3980 3981 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as 3982 * soon as we give the doorbell, we could get a completion 3983 * for this wr, therefore we need to make sure that the 3984 * memory is update before giving the doorbell. 3985 * During qedr_poll_cq, rmb is called before accessing the 3986 * cqe. This covers for the smp_rmb as well. 3987 */ 3988 smp_wmb(); 3989 3990 qp->rq.db_data.data.value++; 3991 3992 writel(qp->rq.db_data.raw, qp->rq.db); 3993 3994 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 3995 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 3996 } 3997 3998 wr = wr->next; 3999 } 4000 4001 spin_unlock_irqrestore(&qp->q_lock, flags); 4002 4003 return status; 4004 } 4005 4006 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe) 4007 { 4008 struct rdma_cqe_requester *resp_cqe = &cqe->req; 4009 4010 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) == 4011 cq->pbl_toggle; 4012 } 4013 4014 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe) 4015 { 4016 struct rdma_cqe_requester *resp_cqe = &cqe->req; 4017 struct qedr_qp *qp; 4018 4019 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi, 4020 resp_cqe->qp_handle.lo, 4021 u64); 4022 return qp; 4023 } 4024 4025 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe) 4026 { 4027 struct rdma_cqe_requester *resp_cqe = &cqe->req; 4028 4029 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 4030 } 4031 4032 /* Return latest CQE (needs processing) */ 4033 static union rdma_cqe *get_cqe(struct qedr_cq *cq) 4034 { 4035 return cq->latest_cqe; 4036 } 4037 4038 /* In fmr we need to increase the number of fmr completed counter for the fmr 4039 * algorithm determining whether we can free a pbl or not. 4040 * we need to perform this whether the work request was signaled or not. for 4041 * this purpose we call this function from the condition that checks if a wr 4042 * should be skipped, to make sure we don't miss it ( possibly this fmr 4043 * operation was not signalted) 4044 */ 4045 static inline void qedr_chk_if_fmr(struct qedr_qp *qp) 4046 { 4047 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 4048 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 4049 } 4050 4051 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, 4052 struct qedr_cq *cq, int num_entries, 4053 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, 4054 int force) 4055 { 4056 u16 cnt = 0; 4057 4058 while (num_entries && qp->sq.wqe_cons != hw_cons) { 4059 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 4060 qedr_chk_if_fmr(qp); 4061 /* skip WC */ 4062 goto next_cqe; 4063 } 4064 4065 /* fill WC */ 4066 wc->status = status; 4067 wc->vendor_err = 0; 4068 wc->wc_flags = 0; 4069 wc->src_qp = qp->id; 4070 wc->qp = &qp->ibqp; 4071 4072 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 4073 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 4074 4075 switch (wc->opcode) { 4076 case IB_WC_RDMA_WRITE: 4077 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 4078 break; 4079 case IB_WC_COMP_SWAP: 4080 case IB_WC_FETCH_ADD: 4081 wc->byte_len = 8; 4082 break; 4083 case IB_WC_REG_MR: 4084 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 4085 break; 4086 case IB_WC_RDMA_READ: 4087 case IB_WC_SEND: 4088 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 4089 break; 4090 default: 4091 break; 4092 } 4093 4094 num_entries--; 4095 wc++; 4096 cnt++; 4097 next_cqe: 4098 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 4099 qed_chain_consume(&qp->sq.pbl); 4100 qedr_inc_sw_cons(&qp->sq); 4101 } 4102 4103 return cnt; 4104 } 4105 4106 static int qedr_poll_cq_req(struct qedr_dev *dev, 4107 struct qedr_qp *qp, struct qedr_cq *cq, 4108 int num_entries, struct ib_wc *wc, 4109 struct rdma_cqe_requester *req) 4110 { 4111 int cnt = 0; 4112 4113 switch (req->status) { 4114 case RDMA_CQE_REQ_STS_OK: 4115 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 4116 IB_WC_SUCCESS, 0); 4117 break; 4118 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 4119 if (qp->state != QED_ROCE_QP_STATE_ERR) 4120 DP_DEBUG(dev, QEDR_MSG_CQ, 4121 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4122 cq->icid, qp->icid); 4123 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 4124 IB_WC_WR_FLUSH_ERR, 1); 4125 break; 4126 default: 4127 /* process all WQE before the cosumer */ 4128 qp->state = QED_ROCE_QP_STATE_ERR; 4129 cnt = process_req(dev, qp, cq, num_entries, wc, 4130 req->sq_cons - 1, IB_WC_SUCCESS, 0); 4131 wc += cnt; 4132 /* if we have extra WC fill it with actual error info */ 4133 if (cnt < num_entries) { 4134 enum ib_wc_status wc_status; 4135 4136 switch (req->status) { 4137 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 4138 DP_ERR(dev, 4139 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4140 cq->icid, qp->icid); 4141 wc_status = IB_WC_BAD_RESP_ERR; 4142 break; 4143 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 4144 DP_ERR(dev, 4145 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4146 cq->icid, qp->icid); 4147 wc_status = IB_WC_LOC_LEN_ERR; 4148 break; 4149 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 4150 DP_ERR(dev, 4151 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4152 cq->icid, qp->icid); 4153 wc_status = IB_WC_LOC_QP_OP_ERR; 4154 break; 4155 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 4156 DP_ERR(dev, 4157 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4158 cq->icid, qp->icid); 4159 wc_status = IB_WC_LOC_PROT_ERR; 4160 break; 4161 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 4162 DP_ERR(dev, 4163 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4164 cq->icid, qp->icid); 4165 wc_status = IB_WC_MW_BIND_ERR; 4166 break; 4167 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 4168 DP_ERR(dev, 4169 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4170 cq->icid, qp->icid); 4171 wc_status = IB_WC_REM_INV_REQ_ERR; 4172 break; 4173 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 4174 DP_ERR(dev, 4175 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4176 cq->icid, qp->icid); 4177 wc_status = IB_WC_REM_ACCESS_ERR; 4178 break; 4179 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 4180 DP_ERR(dev, 4181 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4182 cq->icid, qp->icid); 4183 wc_status = IB_WC_REM_OP_ERR; 4184 break; 4185 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 4186 DP_ERR(dev, 4187 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4188 cq->icid, qp->icid); 4189 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 4190 break; 4191 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 4192 DP_ERR(dev, 4193 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4194 cq->icid, qp->icid); 4195 wc_status = IB_WC_RETRY_EXC_ERR; 4196 break; 4197 default: 4198 DP_ERR(dev, 4199 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4200 cq->icid, qp->icid); 4201 wc_status = IB_WC_GENERAL_ERR; 4202 } 4203 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 4204 wc_status, 1); 4205 } 4206 } 4207 4208 return cnt; 4209 } 4210 4211 static inline int qedr_cqe_resp_status_to_ib(u8 status) 4212 { 4213 switch (status) { 4214 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 4215 return IB_WC_LOC_ACCESS_ERR; 4216 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 4217 return IB_WC_LOC_LEN_ERR; 4218 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 4219 return IB_WC_LOC_QP_OP_ERR; 4220 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 4221 return IB_WC_LOC_PROT_ERR; 4222 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 4223 return IB_WC_MW_BIND_ERR; 4224 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 4225 return IB_WC_REM_INV_RD_REQ_ERR; 4226 case RDMA_CQE_RESP_STS_OK: 4227 return IB_WC_SUCCESS; 4228 default: 4229 return IB_WC_GENERAL_ERR; 4230 } 4231 } 4232 4233 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp, 4234 struct ib_wc *wc) 4235 { 4236 wc->status = IB_WC_SUCCESS; 4237 wc->byte_len = le32_to_cpu(resp->length); 4238 4239 if (resp->flags & QEDR_RESP_IMM) { 4240 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key)); 4241 wc->wc_flags |= IB_WC_WITH_IMM; 4242 4243 if (resp->flags & QEDR_RESP_RDMA) 4244 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 4245 4246 if (resp->flags & QEDR_RESP_INV) 4247 return -EINVAL; 4248 4249 } else if (resp->flags & QEDR_RESP_INV) { 4250 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 4251 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4252 4253 if (resp->flags & QEDR_RESP_RDMA) 4254 return -EINVAL; 4255 4256 } else if (resp->flags & QEDR_RESP_RDMA) { 4257 return -EINVAL; 4258 } 4259 4260 return 0; 4261 } 4262 4263 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4264 struct qedr_cq *cq, struct ib_wc *wc, 4265 struct rdma_cqe_responder *resp, u64 wr_id) 4266 { 4267 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */ 4268 wc->opcode = IB_WC_RECV; 4269 wc->wc_flags = 0; 4270 4271 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) { 4272 if (qedr_set_ok_cqe_resp_wc(resp, wc)) 4273 DP_ERR(dev, 4274 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n", 4275 cq, cq->icid, resp->flags); 4276 4277 } else { 4278 wc->status = qedr_cqe_resp_status_to_ib(resp->status); 4279 if (wc->status == IB_WC_GENERAL_ERR) 4280 DP_ERR(dev, 4281 "CQ %p (icid=%d) contains an invalid CQE status %d\n", 4282 cq, cq->icid, resp->status); 4283 } 4284 4285 /* Fill the rest of the WC */ 4286 wc->vendor_err = 0; 4287 wc->src_qp = qp->id; 4288 wc->qp = &qp->ibqp; 4289 wc->wr_id = wr_id; 4290 } 4291 4292 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4293 struct qedr_cq *cq, struct ib_wc *wc, 4294 struct rdma_cqe_responder *resp) 4295 { 4296 struct qedr_srq *srq = qp->srq; 4297 u64 wr_id; 4298 4299 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi), 4300 le32_to_cpu(resp->srq_wr_id.lo), u64); 4301 4302 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4303 wc->status = IB_WC_WR_FLUSH_ERR; 4304 wc->vendor_err = 0; 4305 wc->wr_id = wr_id; 4306 wc->byte_len = 0; 4307 wc->src_qp = qp->id; 4308 wc->qp = &qp->ibqp; 4309 wc->wr_id = wr_id; 4310 } else { 4311 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4312 } 4313 atomic_inc(&srq->hw_srq.wr_cons_cnt); 4314 4315 return 1; 4316 } 4317 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4318 struct qedr_cq *cq, struct ib_wc *wc, 4319 struct rdma_cqe_responder *resp) 4320 { 4321 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4322 4323 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4324 4325 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4326 qed_chain_consume(&qp->rq.pbl); 4327 qedr_inc_sw_cons(&qp->rq); 4328 4329 return 1; 4330 } 4331 4332 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, 4333 int num_entries, struct ib_wc *wc, u16 hw_cons) 4334 { 4335 u16 cnt = 0; 4336 4337 while (num_entries && qp->rq.wqe_cons != hw_cons) { 4338 /* fill WC */ 4339 wc->status = IB_WC_WR_FLUSH_ERR; 4340 wc->vendor_err = 0; 4341 wc->wc_flags = 0; 4342 wc->src_qp = qp->id; 4343 wc->byte_len = 0; 4344 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4345 wc->qp = &qp->ibqp; 4346 num_entries--; 4347 wc++; 4348 cnt++; 4349 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4350 qed_chain_consume(&qp->rq.pbl); 4351 qedr_inc_sw_cons(&qp->rq); 4352 } 4353 4354 return cnt; 4355 } 4356 4357 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4358 struct rdma_cqe_responder *resp, int *update) 4359 { 4360 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) { 4361 consume_cqe(cq); 4362 *update |= 1; 4363 } 4364 } 4365 4366 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4367 struct qedr_cq *cq, int num_entries, 4368 struct ib_wc *wc, 4369 struct rdma_cqe_responder *resp) 4370 { 4371 int cnt; 4372 4373 cnt = process_resp_one_srq(dev, qp, cq, wc, resp); 4374 consume_cqe(cq); 4375 4376 return cnt; 4377 } 4378 4379 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, 4380 struct qedr_cq *cq, int num_entries, 4381 struct ib_wc *wc, struct rdma_cqe_responder *resp, 4382 int *update) 4383 { 4384 int cnt; 4385 4386 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4387 cnt = process_resp_flush(qp, cq, num_entries, wc, 4388 resp->rq_cons_or_srq_id); 4389 try_consume_resp_cqe(cq, qp, resp, update); 4390 } else { 4391 cnt = process_resp_one(dev, qp, cq, wc, resp); 4392 consume_cqe(cq); 4393 *update |= 1; 4394 } 4395 4396 return cnt; 4397 } 4398 4399 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4400 struct rdma_cqe_requester *req, int *update) 4401 { 4402 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 4403 consume_cqe(cq); 4404 *update |= 1; 4405 } 4406 } 4407 4408 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 4409 { 4410 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 4411 struct qedr_cq *cq = get_qedr_cq(ibcq); 4412 union rdma_cqe *cqe; 4413 u32 old_cons, new_cons; 4414 unsigned long flags; 4415 int update = 0; 4416 int done = 0; 4417 4418 if (cq->destroyed) { 4419 DP_ERR(dev, 4420 "warning: poll was invoked after destroy for cq %p (icid=%d)\n", 4421 cq, cq->icid); 4422 return 0; 4423 } 4424 4425 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 4426 return qedr_gsi_poll_cq(ibcq, num_entries, wc); 4427 4428 spin_lock_irqsave(&cq->cq_lock, flags); 4429 cqe = cq->latest_cqe; 4430 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4431 while (num_entries && is_valid_cqe(cq, cqe)) { 4432 struct qedr_qp *qp; 4433 int cnt = 0; 4434 4435 /* prevent speculative reads of any field of CQE */ 4436 rmb(); 4437 4438 qp = cqe_get_qp(cqe); 4439 if (!qp) { 4440 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe); 4441 break; 4442 } 4443 4444 wc->qp = &qp->ibqp; 4445 4446 switch (cqe_get_type(cqe)) { 4447 case RDMA_CQE_TYPE_REQUESTER: 4448 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc, 4449 &cqe->req); 4450 try_consume_req_cqe(cq, qp, &cqe->req, &update); 4451 break; 4452 case RDMA_CQE_TYPE_RESPONDER_RQ: 4453 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc, 4454 &cqe->resp, &update); 4455 break; 4456 case RDMA_CQE_TYPE_RESPONDER_SRQ: 4457 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries, 4458 wc, &cqe->resp); 4459 update = 1; 4460 break; 4461 case RDMA_CQE_TYPE_INVALID: 4462 default: 4463 DP_ERR(dev, "Error: invalid CQE type = %d\n", 4464 cqe_get_type(cqe)); 4465 } 4466 num_entries -= cnt; 4467 wc += cnt; 4468 done += cnt; 4469 4470 cqe = get_cqe(cq); 4471 } 4472 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4473 4474 cq->cq_cons += new_cons - old_cons; 4475 4476 if (update) 4477 /* doorbell notifies abount latest VALID entry, 4478 * but chain already point to the next INVALID one 4479 */ 4480 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 4481 4482 spin_unlock_irqrestore(&cq->cq_lock, flags); 4483 return done; 4484 } 4485 4486 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, 4487 u32 port_num, const struct ib_wc *in_wc, 4488 const struct ib_grh *in_grh, const struct ib_mad *in, 4489 struct ib_mad *out_mad, size_t *out_mad_size, 4490 u16 *out_mad_pkey_index) 4491 { 4492 return IB_MAD_RESULT_SUCCESS; 4493 } 4494