1 /* This file is part of the Emulex RoCE Device Driver for 2 * RoCE (RDMA over Converged Ethernet) adapters. 3 * Copyright (C) 2012-2015 Emulex. All rights reserved. 4 * EMULEX and SLI are trademarks of Emulex. 5 * www.emulex.com 6 * 7 * This software is available to you under a choice of one of two licenses. 8 * You may choose to be licensed under the terms of the GNU General Public 9 * License (GPL) Version 2, available from the file COPYING in the main 10 * directory of this source tree, or the BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 16 * - Redistributions of source code must retain the above copyright notice, 17 * this list of conditions and the following disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 30 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 31 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 32 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 33 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * Contact Information: 36 * linux-drivers@emulex.com 37 * 38 * Emulex 39 * 3333 Susan Street 40 * Costa Mesa, CA 92626 41 */ 42 43 #include <linux/dma-mapping.h> 44 #include <rdma/ib_verbs.h> 45 #include <rdma/ib_user_verbs.h> 46 #include <rdma/iw_cm.h> 47 #include <rdma/ib_umem.h> 48 #include <rdma/ib_addr.h> 49 #include <rdma/ib_cache.h> 50 51 #include "ocrdma.h" 52 #include "ocrdma_hw.h" 53 #include "ocrdma_verbs.h" 54 #include <rdma/ocrdma-abi.h> 55 56 int ocrdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 57 { 58 if (index > 1) 59 return -EINVAL; 60 61 *pkey = 0xffff; 62 return 0; 63 } 64 65 int ocrdma_query_gid(struct ib_device *ibdev, u8 port, 66 int index, union ib_gid *sgid) 67 { 68 int ret; 69 70 memset(sgid, 0, sizeof(*sgid)); 71 if (index >= OCRDMA_MAX_SGID) 72 return -EINVAL; 73 74 ret = ib_get_cached_gid(ibdev, port, index, sgid, NULL); 75 if (ret == -EAGAIN) { 76 memcpy(sgid, &zgid, sizeof(*sgid)); 77 return 0; 78 } 79 80 return ret; 81 } 82 83 int ocrdma_add_gid(struct ib_device *device, 84 u8 port_num, 85 unsigned int index, 86 const union ib_gid *gid, 87 const struct ib_gid_attr *attr, 88 void **context) { 89 return 0; 90 } 91 92 int ocrdma_del_gid(struct ib_device *device, 93 u8 port_num, 94 unsigned int index, 95 void **context) { 96 return 0; 97 } 98 99 int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 100 struct ib_udata *uhw) 101 { 102 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 103 104 if (uhw->inlen || uhw->outlen) 105 return -EINVAL; 106 107 memset(attr, 0, sizeof *attr); 108 memcpy(&attr->fw_ver, &dev->attr.fw_ver[0], 109 min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver))); 110 ocrdma_get_guid(dev, (u8 *)&attr->sys_image_guid); 111 attr->max_mr_size = dev->attr.max_mr_size; 112 attr->page_size_cap = 0xffff000; 113 attr->vendor_id = dev->nic_info.pdev->vendor; 114 attr->vendor_part_id = dev->nic_info.pdev->device; 115 attr->hw_ver = dev->asic_id; 116 attr->max_qp = dev->attr.max_qp; 117 attr->max_ah = OCRDMA_MAX_AH; 118 attr->max_qp_wr = dev->attr.max_wqe; 119 120 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 121 IB_DEVICE_RC_RNR_NAK_GEN | 122 IB_DEVICE_SHUTDOWN_PORT | 123 IB_DEVICE_SYS_IMAGE_GUID | 124 IB_DEVICE_LOCAL_DMA_LKEY | 125 IB_DEVICE_MEM_MGT_EXTENSIONS; 126 attr->max_sge = min(dev->attr.max_send_sge, dev->attr.max_recv_sge); 127 attr->max_sge_rd = dev->attr.max_rdma_sge; 128 attr->max_cq = dev->attr.max_cq; 129 attr->max_cqe = dev->attr.max_cqe; 130 attr->max_mr = dev->attr.max_mr; 131 attr->max_mw = dev->attr.max_mw; 132 attr->max_pd = dev->attr.max_pd; 133 attr->atomic_cap = 0; 134 attr->max_fmr = 0; 135 attr->max_map_per_fmr = 0; 136 attr->max_qp_rd_atom = 137 min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp); 138 attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp; 139 attr->max_srq = dev->attr.max_srq; 140 attr->max_srq_sge = dev->attr.max_srq_sge; 141 attr->max_srq_wr = dev->attr.max_rqe; 142 attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay; 143 attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr; 144 attr->max_pkeys = 1; 145 return 0; 146 } 147 148 struct net_device *ocrdma_get_netdev(struct ib_device *ibdev, u8 port_num) 149 { 150 struct ocrdma_dev *dev; 151 struct net_device *ndev = NULL; 152 153 rcu_read_lock(); 154 155 dev = get_ocrdma_dev(ibdev); 156 if (dev) 157 ndev = dev->nic_info.netdev; 158 if (ndev) 159 dev_hold(ndev); 160 161 rcu_read_unlock(); 162 163 return ndev; 164 } 165 166 static inline void get_link_speed_and_width(struct ocrdma_dev *dev, 167 u8 *ib_speed, u8 *ib_width) 168 { 169 int status; 170 u8 speed; 171 172 status = ocrdma_mbx_get_link_speed(dev, &speed, NULL); 173 if (status) 174 speed = OCRDMA_PHYS_LINK_SPEED_ZERO; 175 176 switch (speed) { 177 case OCRDMA_PHYS_LINK_SPEED_1GBPS: 178 *ib_speed = IB_SPEED_SDR; 179 *ib_width = IB_WIDTH_1X; 180 break; 181 182 case OCRDMA_PHYS_LINK_SPEED_10GBPS: 183 *ib_speed = IB_SPEED_QDR; 184 *ib_width = IB_WIDTH_1X; 185 break; 186 187 case OCRDMA_PHYS_LINK_SPEED_20GBPS: 188 *ib_speed = IB_SPEED_DDR; 189 *ib_width = IB_WIDTH_4X; 190 break; 191 192 case OCRDMA_PHYS_LINK_SPEED_40GBPS: 193 *ib_speed = IB_SPEED_QDR; 194 *ib_width = IB_WIDTH_4X; 195 break; 196 197 default: 198 /* Unsupported */ 199 *ib_speed = IB_SPEED_SDR; 200 *ib_width = IB_WIDTH_1X; 201 } 202 } 203 204 int ocrdma_query_port(struct ib_device *ibdev, 205 u8 port, struct ib_port_attr *props) 206 { 207 enum ib_port_state port_state; 208 struct ocrdma_dev *dev; 209 struct net_device *netdev; 210 211 /* props being zeroed by the caller, avoid zeroing it here */ 212 dev = get_ocrdma_dev(ibdev); 213 if (port > 1) { 214 pr_err("%s(%d) invalid_port=0x%x\n", __func__, 215 dev->id, port); 216 return -EINVAL; 217 } 218 netdev = dev->nic_info.netdev; 219 if (netif_running(netdev) && netif_oper_up(netdev)) { 220 port_state = IB_PORT_ACTIVE; 221 props->phys_state = 5; 222 } else { 223 port_state = IB_PORT_DOWN; 224 props->phys_state = 3; 225 } 226 props->max_mtu = IB_MTU_4096; 227 props->active_mtu = iboe_get_mtu(netdev->mtu); 228 props->lid = 0; 229 props->lmc = 0; 230 props->sm_lid = 0; 231 props->sm_sl = 0; 232 props->state = port_state; 233 props->port_cap_flags = 234 IB_PORT_CM_SUP | 235 IB_PORT_REINIT_SUP | 236 IB_PORT_DEVICE_MGMT_SUP | IB_PORT_VENDOR_CLASS_SUP | 237 IB_PORT_IP_BASED_GIDS; 238 props->gid_tbl_len = OCRDMA_MAX_SGID; 239 props->pkey_tbl_len = 1; 240 props->bad_pkey_cntr = 0; 241 props->qkey_viol_cntr = 0; 242 get_link_speed_and_width(dev, &props->active_speed, 243 &props->active_width); 244 props->max_msg_sz = 0x80000000; 245 props->max_vl_num = 4; 246 return 0; 247 } 248 249 int ocrdma_modify_port(struct ib_device *ibdev, u8 port, int mask, 250 struct ib_port_modify *props) 251 { 252 struct ocrdma_dev *dev; 253 254 dev = get_ocrdma_dev(ibdev); 255 if (port > 1) { 256 pr_err("%s(%d) invalid_port=0x%x\n", __func__, dev->id, port); 257 return -EINVAL; 258 } 259 return 0; 260 } 261 262 static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 263 unsigned long len) 264 { 265 struct ocrdma_mm *mm; 266 267 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 268 if (mm == NULL) 269 return -ENOMEM; 270 mm->key.phy_addr = phy_addr; 271 mm->key.len = len; 272 INIT_LIST_HEAD(&mm->entry); 273 274 mutex_lock(&uctx->mm_list_lock); 275 list_add_tail(&mm->entry, &uctx->mm_head); 276 mutex_unlock(&uctx->mm_list_lock); 277 return 0; 278 } 279 280 static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 281 unsigned long len) 282 { 283 struct ocrdma_mm *mm, *tmp; 284 285 mutex_lock(&uctx->mm_list_lock); 286 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 287 if (len != mm->key.len && phy_addr != mm->key.phy_addr) 288 continue; 289 290 list_del(&mm->entry); 291 kfree(mm); 292 break; 293 } 294 mutex_unlock(&uctx->mm_list_lock); 295 } 296 297 static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 298 unsigned long len) 299 { 300 bool found = false; 301 struct ocrdma_mm *mm; 302 303 mutex_lock(&uctx->mm_list_lock); 304 list_for_each_entry(mm, &uctx->mm_head, entry) { 305 if (len != mm->key.len && phy_addr != mm->key.phy_addr) 306 continue; 307 308 found = true; 309 break; 310 } 311 mutex_unlock(&uctx->mm_list_lock); 312 return found; 313 } 314 315 316 static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool) 317 { 318 u16 pd_bitmap_idx = 0; 319 const unsigned long *pd_bitmap; 320 321 if (dpp_pool) { 322 pd_bitmap = dev->pd_mgr->pd_dpp_bitmap; 323 pd_bitmap_idx = find_first_zero_bit(pd_bitmap, 324 dev->pd_mgr->max_dpp_pd); 325 __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_dpp_bitmap); 326 dev->pd_mgr->pd_dpp_count++; 327 if (dev->pd_mgr->pd_dpp_count > dev->pd_mgr->pd_dpp_thrsh) 328 dev->pd_mgr->pd_dpp_thrsh = dev->pd_mgr->pd_dpp_count; 329 } else { 330 pd_bitmap = dev->pd_mgr->pd_norm_bitmap; 331 pd_bitmap_idx = find_first_zero_bit(pd_bitmap, 332 dev->pd_mgr->max_normal_pd); 333 __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_norm_bitmap); 334 dev->pd_mgr->pd_norm_count++; 335 if (dev->pd_mgr->pd_norm_count > dev->pd_mgr->pd_norm_thrsh) 336 dev->pd_mgr->pd_norm_thrsh = dev->pd_mgr->pd_norm_count; 337 } 338 return pd_bitmap_idx; 339 } 340 341 static int _ocrdma_pd_mgr_put_bitmap(struct ocrdma_dev *dev, u16 pd_id, 342 bool dpp_pool) 343 { 344 u16 pd_count; 345 u16 pd_bit_index; 346 347 pd_count = dpp_pool ? dev->pd_mgr->pd_dpp_count : 348 dev->pd_mgr->pd_norm_count; 349 if (pd_count == 0) 350 return -EINVAL; 351 352 if (dpp_pool) { 353 pd_bit_index = pd_id - dev->pd_mgr->pd_dpp_start; 354 if (pd_bit_index >= dev->pd_mgr->max_dpp_pd) { 355 return -EINVAL; 356 } else { 357 __clear_bit(pd_bit_index, dev->pd_mgr->pd_dpp_bitmap); 358 dev->pd_mgr->pd_dpp_count--; 359 } 360 } else { 361 pd_bit_index = pd_id - dev->pd_mgr->pd_norm_start; 362 if (pd_bit_index >= dev->pd_mgr->max_normal_pd) { 363 return -EINVAL; 364 } else { 365 __clear_bit(pd_bit_index, dev->pd_mgr->pd_norm_bitmap); 366 dev->pd_mgr->pd_norm_count--; 367 } 368 } 369 370 return 0; 371 } 372 373 static int ocrdma_put_pd_num(struct ocrdma_dev *dev, u16 pd_id, 374 bool dpp_pool) 375 { 376 int status; 377 378 mutex_lock(&dev->dev_lock); 379 status = _ocrdma_pd_mgr_put_bitmap(dev, pd_id, dpp_pool); 380 mutex_unlock(&dev->dev_lock); 381 return status; 382 } 383 384 static int ocrdma_get_pd_num(struct ocrdma_dev *dev, struct ocrdma_pd *pd) 385 { 386 u16 pd_idx = 0; 387 int status = 0; 388 389 mutex_lock(&dev->dev_lock); 390 if (pd->dpp_enabled) { 391 /* try allocating DPP PD, if not available then normal PD */ 392 if (dev->pd_mgr->pd_dpp_count < dev->pd_mgr->max_dpp_pd) { 393 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, true); 394 pd->id = dev->pd_mgr->pd_dpp_start + pd_idx; 395 pd->dpp_page = dev->pd_mgr->dpp_page_index + pd_idx; 396 } else if (dev->pd_mgr->pd_norm_count < 397 dev->pd_mgr->max_normal_pd) { 398 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false); 399 pd->id = dev->pd_mgr->pd_norm_start + pd_idx; 400 pd->dpp_enabled = false; 401 } else { 402 status = -EINVAL; 403 } 404 } else { 405 if (dev->pd_mgr->pd_norm_count < dev->pd_mgr->max_normal_pd) { 406 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false); 407 pd->id = dev->pd_mgr->pd_norm_start + pd_idx; 408 } else { 409 status = -EINVAL; 410 } 411 } 412 mutex_unlock(&dev->dev_lock); 413 return status; 414 } 415 416 static struct ocrdma_pd *_ocrdma_alloc_pd(struct ocrdma_dev *dev, 417 struct ocrdma_ucontext *uctx, 418 struct ib_udata *udata) 419 { 420 struct ocrdma_pd *pd = NULL; 421 int status; 422 423 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 424 if (!pd) 425 return ERR_PTR(-ENOMEM); 426 427 if (udata && uctx && dev->attr.max_dpp_pds) { 428 pd->dpp_enabled = 429 ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R; 430 pd->num_dpp_qp = 431 pd->dpp_enabled ? (dev->nic_info.db_page_size / 432 dev->attr.wqe_size) : 0; 433 } 434 435 if (dev->pd_mgr->pd_prealloc_valid) { 436 status = ocrdma_get_pd_num(dev, pd); 437 if (status == 0) { 438 return pd; 439 } else { 440 kfree(pd); 441 return ERR_PTR(status); 442 } 443 } 444 445 retry: 446 status = ocrdma_mbx_alloc_pd(dev, pd); 447 if (status) { 448 if (pd->dpp_enabled) { 449 pd->dpp_enabled = false; 450 pd->num_dpp_qp = 0; 451 goto retry; 452 } else { 453 kfree(pd); 454 return ERR_PTR(status); 455 } 456 } 457 458 return pd; 459 } 460 461 static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx, 462 struct ocrdma_pd *pd) 463 { 464 return (uctx->cntxt_pd == pd ? true : false); 465 } 466 467 static int _ocrdma_dealloc_pd(struct ocrdma_dev *dev, 468 struct ocrdma_pd *pd) 469 { 470 int status; 471 472 if (dev->pd_mgr->pd_prealloc_valid) 473 status = ocrdma_put_pd_num(dev, pd->id, pd->dpp_enabled); 474 else 475 status = ocrdma_mbx_dealloc_pd(dev, pd); 476 477 kfree(pd); 478 return status; 479 } 480 481 static int ocrdma_alloc_ucontext_pd(struct ocrdma_dev *dev, 482 struct ocrdma_ucontext *uctx, 483 struct ib_udata *udata) 484 { 485 int status = 0; 486 487 uctx->cntxt_pd = _ocrdma_alloc_pd(dev, uctx, udata); 488 if (IS_ERR(uctx->cntxt_pd)) { 489 status = PTR_ERR(uctx->cntxt_pd); 490 uctx->cntxt_pd = NULL; 491 goto err; 492 } 493 494 uctx->cntxt_pd->uctx = uctx; 495 uctx->cntxt_pd->ibpd.device = &dev->ibdev; 496 err: 497 return status; 498 } 499 500 static int ocrdma_dealloc_ucontext_pd(struct ocrdma_ucontext *uctx) 501 { 502 struct ocrdma_pd *pd = uctx->cntxt_pd; 503 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device); 504 505 if (uctx->pd_in_use) { 506 pr_err("%s(%d) Freeing in use pdid=0x%x.\n", 507 __func__, dev->id, pd->id); 508 } 509 uctx->cntxt_pd = NULL; 510 (void)_ocrdma_dealloc_pd(dev, pd); 511 return 0; 512 } 513 514 static struct ocrdma_pd *ocrdma_get_ucontext_pd(struct ocrdma_ucontext *uctx) 515 { 516 struct ocrdma_pd *pd = NULL; 517 518 mutex_lock(&uctx->mm_list_lock); 519 if (!uctx->pd_in_use) { 520 uctx->pd_in_use = true; 521 pd = uctx->cntxt_pd; 522 } 523 mutex_unlock(&uctx->mm_list_lock); 524 525 return pd; 526 } 527 528 static void ocrdma_release_ucontext_pd(struct ocrdma_ucontext *uctx) 529 { 530 mutex_lock(&uctx->mm_list_lock); 531 uctx->pd_in_use = false; 532 mutex_unlock(&uctx->mm_list_lock); 533 } 534 535 struct ib_ucontext *ocrdma_alloc_ucontext(struct ib_device *ibdev, 536 struct ib_udata *udata) 537 { 538 int status; 539 struct ocrdma_ucontext *ctx; 540 struct ocrdma_alloc_ucontext_resp resp; 541 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 542 struct pci_dev *pdev = dev->nic_info.pdev; 543 u32 map_len = roundup(sizeof(u32) * 2048, PAGE_SIZE); 544 545 if (!udata) 546 return ERR_PTR(-EFAULT); 547 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 548 if (!ctx) 549 return ERR_PTR(-ENOMEM); 550 INIT_LIST_HEAD(&ctx->mm_head); 551 mutex_init(&ctx->mm_list_lock); 552 553 ctx->ah_tbl.va = dma_alloc_coherent(&pdev->dev, map_len, 554 &ctx->ah_tbl.pa, GFP_KERNEL); 555 if (!ctx->ah_tbl.va) { 556 kfree(ctx); 557 return ERR_PTR(-ENOMEM); 558 } 559 memset(ctx->ah_tbl.va, 0, map_len); 560 ctx->ah_tbl.len = map_len; 561 562 memset(&resp, 0, sizeof(resp)); 563 resp.ah_tbl_len = ctx->ah_tbl.len; 564 resp.ah_tbl_page = virt_to_phys(ctx->ah_tbl.va); 565 566 status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len); 567 if (status) 568 goto map_err; 569 570 status = ocrdma_alloc_ucontext_pd(dev, ctx, udata); 571 if (status) 572 goto pd_err; 573 574 resp.dev_id = dev->id; 575 resp.max_inline_data = dev->attr.max_inline_data; 576 resp.wqe_size = dev->attr.wqe_size; 577 resp.rqe_size = dev->attr.rqe_size; 578 resp.dpp_wqe_size = dev->attr.wqe_size; 579 580 memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver)); 581 status = ib_copy_to_udata(udata, &resp, sizeof(resp)); 582 if (status) 583 goto cpy_err; 584 return &ctx->ibucontext; 585 586 cpy_err: 587 pd_err: 588 ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len); 589 map_err: 590 dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va, 591 ctx->ah_tbl.pa); 592 kfree(ctx); 593 return ERR_PTR(status); 594 } 595 596 int ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx) 597 { 598 int status; 599 struct ocrdma_mm *mm, *tmp; 600 struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx); 601 struct ocrdma_dev *dev = get_ocrdma_dev(ibctx->device); 602 struct pci_dev *pdev = dev->nic_info.pdev; 603 604 status = ocrdma_dealloc_ucontext_pd(uctx); 605 606 ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len); 607 dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va, 608 uctx->ah_tbl.pa); 609 610 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 611 list_del(&mm->entry); 612 kfree(mm); 613 } 614 kfree(uctx); 615 return status; 616 } 617 618 int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 619 { 620 struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context); 621 struct ocrdma_dev *dev = get_ocrdma_dev(context->device); 622 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 623 u64 unmapped_db = (u64) dev->nic_info.unmapped_db; 624 unsigned long len = (vma->vm_end - vma->vm_start); 625 int status; 626 bool found; 627 628 if (vma->vm_start & (PAGE_SIZE - 1)) 629 return -EINVAL; 630 found = ocrdma_search_mmap(ucontext, vma->vm_pgoff << PAGE_SHIFT, len); 631 if (!found) 632 return -EINVAL; 633 634 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + 635 dev->nic_info.db_total_size)) && 636 (len <= dev->nic_info.db_page_size)) { 637 if (vma->vm_flags & VM_READ) 638 return -EPERM; 639 640 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 641 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 642 len, vma->vm_page_prot); 643 } else if (dev->nic_info.dpp_unmapped_len && 644 (vm_page >= (u64) dev->nic_info.dpp_unmapped_addr) && 645 (vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr + 646 dev->nic_info.dpp_unmapped_len)) && 647 (len <= dev->nic_info.dpp_unmapped_len)) { 648 if (vma->vm_flags & VM_READ) 649 return -EPERM; 650 651 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 652 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 653 len, vma->vm_page_prot); 654 } else { 655 status = remap_pfn_range(vma, vma->vm_start, 656 vma->vm_pgoff, len, vma->vm_page_prot); 657 } 658 return status; 659 } 660 661 static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd, 662 struct ib_ucontext *ib_ctx, 663 struct ib_udata *udata) 664 { 665 int status; 666 u64 db_page_addr; 667 u64 dpp_page_addr = 0; 668 u32 db_page_size; 669 struct ocrdma_alloc_pd_uresp rsp; 670 struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ib_ctx); 671 672 memset(&rsp, 0, sizeof(rsp)); 673 rsp.id = pd->id; 674 rsp.dpp_enabled = pd->dpp_enabled; 675 db_page_addr = ocrdma_get_db_addr(dev, pd->id); 676 db_page_size = dev->nic_info.db_page_size; 677 678 status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size); 679 if (status) 680 return status; 681 682 if (pd->dpp_enabled) { 683 dpp_page_addr = dev->nic_info.dpp_unmapped_addr + 684 (pd->id * PAGE_SIZE); 685 status = ocrdma_add_mmap(uctx, dpp_page_addr, 686 PAGE_SIZE); 687 if (status) 688 goto dpp_map_err; 689 rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr); 690 rsp.dpp_page_addr_lo = dpp_page_addr; 691 } 692 693 status = ib_copy_to_udata(udata, &rsp, sizeof(rsp)); 694 if (status) 695 goto ucopy_err; 696 697 pd->uctx = uctx; 698 return 0; 699 700 ucopy_err: 701 if (pd->dpp_enabled) 702 ocrdma_del_mmap(pd->uctx, dpp_page_addr, PAGE_SIZE); 703 dpp_map_err: 704 ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size); 705 return status; 706 } 707 708 struct ib_pd *ocrdma_alloc_pd(struct ib_device *ibdev, 709 struct ib_ucontext *context, 710 struct ib_udata *udata) 711 { 712 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 713 struct ocrdma_pd *pd; 714 struct ocrdma_ucontext *uctx = NULL; 715 int status; 716 u8 is_uctx_pd = false; 717 718 if (udata && context) { 719 uctx = get_ocrdma_ucontext(context); 720 pd = ocrdma_get_ucontext_pd(uctx); 721 if (pd) { 722 is_uctx_pd = true; 723 goto pd_mapping; 724 } 725 } 726 727 pd = _ocrdma_alloc_pd(dev, uctx, udata); 728 if (IS_ERR(pd)) { 729 status = PTR_ERR(pd); 730 goto exit; 731 } 732 733 pd_mapping: 734 if (udata && context) { 735 status = ocrdma_copy_pd_uresp(dev, pd, context, udata); 736 if (status) 737 goto err; 738 } 739 return &pd->ibpd; 740 741 err: 742 if (is_uctx_pd) { 743 ocrdma_release_ucontext_pd(uctx); 744 } else { 745 if (_ocrdma_dealloc_pd(dev, pd)) 746 pr_err("%s: _ocrdma_dealloc_pd() failed\n", __func__); 747 } 748 exit: 749 return ERR_PTR(status); 750 } 751 752 int ocrdma_dealloc_pd(struct ib_pd *ibpd) 753 { 754 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 755 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 756 struct ocrdma_ucontext *uctx = NULL; 757 int status = 0; 758 u64 usr_db; 759 760 uctx = pd->uctx; 761 if (uctx) { 762 u64 dpp_db = dev->nic_info.dpp_unmapped_addr + 763 (pd->id * PAGE_SIZE); 764 if (pd->dpp_enabled) 765 ocrdma_del_mmap(pd->uctx, dpp_db, PAGE_SIZE); 766 usr_db = ocrdma_get_db_addr(dev, pd->id); 767 ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size); 768 769 if (is_ucontext_pd(uctx, pd)) { 770 ocrdma_release_ucontext_pd(uctx); 771 return status; 772 } 773 } 774 status = _ocrdma_dealloc_pd(dev, pd); 775 return status; 776 } 777 778 static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 779 u32 pdid, int acc, u32 num_pbls, u32 addr_check) 780 { 781 int status; 782 783 mr->hwmr.fr_mr = 0; 784 mr->hwmr.local_rd = 1; 785 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 786 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 787 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 788 mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0; 789 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 790 mr->hwmr.num_pbls = num_pbls; 791 792 status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check); 793 if (status) 794 return status; 795 796 mr->ibmr.lkey = mr->hwmr.lkey; 797 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd) 798 mr->ibmr.rkey = mr->hwmr.lkey; 799 return 0; 800 } 801 802 struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc) 803 { 804 int status; 805 struct ocrdma_mr *mr; 806 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 807 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 808 809 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { 810 pr_err("%s err, invalid access rights\n", __func__); 811 return ERR_PTR(-EINVAL); 812 } 813 814 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 815 if (!mr) 816 return ERR_PTR(-ENOMEM); 817 818 status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0, 819 OCRDMA_ADDR_CHECK_DISABLE); 820 if (status) { 821 kfree(mr); 822 return ERR_PTR(status); 823 } 824 825 return &mr->ibmr; 826 } 827 828 static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev, 829 struct ocrdma_hw_mr *mr) 830 { 831 struct pci_dev *pdev = dev->nic_info.pdev; 832 int i = 0; 833 834 if (mr->pbl_table) { 835 for (i = 0; i < mr->num_pbls; i++) { 836 if (!mr->pbl_table[i].va) 837 continue; 838 dma_free_coherent(&pdev->dev, mr->pbl_size, 839 mr->pbl_table[i].va, 840 mr->pbl_table[i].pa); 841 } 842 kfree(mr->pbl_table); 843 mr->pbl_table = NULL; 844 } 845 } 846 847 static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 848 u32 num_pbes) 849 { 850 u32 num_pbls = 0; 851 u32 idx = 0; 852 int status = 0; 853 u32 pbl_size; 854 855 do { 856 pbl_size = OCRDMA_MIN_HPAGE_SIZE * (1 << idx); 857 if (pbl_size > MAX_OCRDMA_PBL_SIZE) { 858 status = -EFAULT; 859 break; 860 } 861 num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64))); 862 num_pbls = num_pbls / (pbl_size / sizeof(u64)); 863 idx++; 864 } while (num_pbls >= dev->attr.max_num_mr_pbl); 865 866 mr->hwmr.num_pbes = num_pbes; 867 mr->hwmr.num_pbls = num_pbls; 868 mr->hwmr.pbl_size = pbl_size; 869 return status; 870 } 871 872 static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr) 873 { 874 int status = 0; 875 int i; 876 u32 dma_len = mr->pbl_size; 877 struct pci_dev *pdev = dev->nic_info.pdev; 878 void *va; 879 dma_addr_t pa; 880 881 mr->pbl_table = kzalloc(sizeof(struct ocrdma_pbl) * 882 mr->num_pbls, GFP_KERNEL); 883 884 if (!mr->pbl_table) 885 return -ENOMEM; 886 887 for (i = 0; i < mr->num_pbls; i++) { 888 va = dma_alloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL); 889 if (!va) { 890 ocrdma_free_mr_pbl_tbl(dev, mr); 891 status = -ENOMEM; 892 break; 893 } 894 memset(va, 0, dma_len); 895 mr->pbl_table[i].va = va; 896 mr->pbl_table[i].pa = pa; 897 } 898 return status; 899 } 900 901 static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 902 u32 num_pbes) 903 { 904 struct ocrdma_pbe *pbe; 905 struct scatterlist *sg; 906 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; 907 struct ib_umem *umem = mr->umem; 908 int shift, pg_cnt, pages, pbe_cnt, entry, total_num_pbes = 0; 909 910 if (!mr->hwmr.num_pbes) 911 return; 912 913 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 914 pbe_cnt = 0; 915 916 shift = umem->page_shift; 917 918 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 919 pages = sg_dma_len(sg) >> shift; 920 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 921 /* store the page address in pbe */ 922 pbe->pa_lo = 923 cpu_to_le32(sg_dma_address(sg) + 924 (pg_cnt << shift)); 925 pbe->pa_hi = 926 cpu_to_le32(upper_32_bits(sg_dma_address(sg) + 927 (pg_cnt << shift))); 928 pbe_cnt += 1; 929 total_num_pbes += 1; 930 pbe++; 931 932 /* if done building pbes, issue the mbx cmd. */ 933 if (total_num_pbes == num_pbes) 934 return; 935 936 /* if the given pbl is full storing the pbes, 937 * move to next pbl. 938 */ 939 if (pbe_cnt == 940 (mr->hwmr.pbl_size / sizeof(u64))) { 941 pbl_tbl++; 942 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 943 pbe_cnt = 0; 944 } 945 946 } 947 } 948 } 949 950 struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 951 u64 usr_addr, int acc, struct ib_udata *udata) 952 { 953 int status = -ENOMEM; 954 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 955 struct ocrdma_mr *mr; 956 struct ocrdma_pd *pd; 957 u32 num_pbes; 958 959 pd = get_ocrdma_pd(ibpd); 960 961 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 962 return ERR_PTR(-EINVAL); 963 964 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 965 if (!mr) 966 return ERR_PTR(status); 967 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0); 968 if (IS_ERR(mr->umem)) { 969 status = -EFAULT; 970 goto umem_err; 971 } 972 num_pbes = ib_umem_page_count(mr->umem); 973 status = ocrdma_get_pbl_info(dev, mr, num_pbes); 974 if (status) 975 goto umem_err; 976 977 mr->hwmr.pbe_size = BIT(mr->umem->page_shift); 978 mr->hwmr.fbo = ib_umem_offset(mr->umem); 979 mr->hwmr.va = usr_addr; 980 mr->hwmr.len = len; 981 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 982 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 983 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 984 mr->hwmr.local_rd = 1; 985 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 986 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr); 987 if (status) 988 goto umem_err; 989 build_user_pbes(dev, mr, num_pbes); 990 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc); 991 if (status) 992 goto mbx_err; 993 mr->ibmr.lkey = mr->hwmr.lkey; 994 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd) 995 mr->ibmr.rkey = mr->hwmr.lkey; 996 997 return &mr->ibmr; 998 999 mbx_err: 1000 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 1001 umem_err: 1002 kfree(mr); 1003 return ERR_PTR(status); 1004 } 1005 1006 int ocrdma_dereg_mr(struct ib_mr *ib_mr) 1007 { 1008 struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr); 1009 struct ocrdma_dev *dev = get_ocrdma_dev(ib_mr->device); 1010 1011 (void) ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey); 1012 1013 kfree(mr->pages); 1014 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 1015 1016 /* it could be user registered memory. */ 1017 if (mr->umem) 1018 ib_umem_release(mr->umem); 1019 kfree(mr); 1020 1021 /* Don't stop cleanup, in case FW is unresponsive */ 1022 if (dev->mqe_ctx.fw_error_state) { 1023 pr_err("%s(%d) fw not responding.\n", 1024 __func__, dev->id); 1025 } 1026 return 0; 1027 } 1028 1029 static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq, 1030 struct ib_udata *udata, 1031 struct ib_ucontext *ib_ctx) 1032 { 1033 int status; 1034 struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ib_ctx); 1035 struct ocrdma_create_cq_uresp uresp; 1036 1037 memset(&uresp, 0, sizeof(uresp)); 1038 uresp.cq_id = cq->id; 1039 uresp.page_size = PAGE_ALIGN(cq->len); 1040 uresp.num_pages = 1; 1041 uresp.max_hw_cqe = cq->max_hw_cqe; 1042 uresp.page_addr[0] = virt_to_phys(cq->va); 1043 uresp.db_page_addr = ocrdma_get_db_addr(dev, uctx->cntxt_pd->id); 1044 uresp.db_page_size = dev->nic_info.db_page_size; 1045 uresp.phase_change = cq->phase_change ? 1 : 0; 1046 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1047 if (status) { 1048 pr_err("%s(%d) copy error cqid=0x%x.\n", 1049 __func__, dev->id, cq->id); 1050 goto err; 1051 } 1052 status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); 1053 if (status) 1054 goto err; 1055 status = ocrdma_add_mmap(uctx, uresp.page_addr[0], uresp.page_size); 1056 if (status) { 1057 ocrdma_del_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); 1058 goto err; 1059 } 1060 cq->ucontext = uctx; 1061 err: 1062 return status; 1063 } 1064 1065 struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, 1066 const struct ib_cq_init_attr *attr, 1067 struct ib_ucontext *ib_ctx, 1068 struct ib_udata *udata) 1069 { 1070 int entries = attr->cqe; 1071 struct ocrdma_cq *cq; 1072 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 1073 struct ocrdma_ucontext *uctx = NULL; 1074 u16 pd_id = 0; 1075 int status; 1076 struct ocrdma_create_cq_ureq ureq; 1077 1078 if (attr->flags) 1079 return ERR_PTR(-EINVAL); 1080 1081 if (udata) { 1082 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 1083 return ERR_PTR(-EFAULT); 1084 } else 1085 ureq.dpp_cq = 0; 1086 cq = kzalloc(sizeof(*cq), GFP_KERNEL); 1087 if (!cq) 1088 return ERR_PTR(-ENOMEM); 1089 1090 spin_lock_init(&cq->cq_lock); 1091 spin_lock_init(&cq->comp_handler_lock); 1092 INIT_LIST_HEAD(&cq->sq_head); 1093 INIT_LIST_HEAD(&cq->rq_head); 1094 1095 if (ib_ctx) { 1096 uctx = get_ocrdma_ucontext(ib_ctx); 1097 pd_id = uctx->cntxt_pd->id; 1098 } 1099 1100 status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id); 1101 if (status) { 1102 kfree(cq); 1103 return ERR_PTR(status); 1104 } 1105 if (ib_ctx) { 1106 status = ocrdma_copy_cq_uresp(dev, cq, udata, ib_ctx); 1107 if (status) 1108 goto ctx_err; 1109 } 1110 cq->phase = OCRDMA_CQE_VALID; 1111 dev->cq_tbl[cq->id] = cq; 1112 return &cq->ibcq; 1113 1114 ctx_err: 1115 ocrdma_mbx_destroy_cq(dev, cq); 1116 kfree(cq); 1117 return ERR_PTR(status); 1118 } 1119 1120 int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt, 1121 struct ib_udata *udata) 1122 { 1123 int status = 0; 1124 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 1125 1126 if (new_cnt < 1 || new_cnt > cq->max_hw_cqe) { 1127 status = -EINVAL; 1128 return status; 1129 } 1130 ibcq->cqe = new_cnt; 1131 return status; 1132 } 1133 1134 static void ocrdma_flush_cq(struct ocrdma_cq *cq) 1135 { 1136 int cqe_cnt; 1137 int valid_count = 0; 1138 unsigned long flags; 1139 1140 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device); 1141 struct ocrdma_cqe *cqe = NULL; 1142 1143 cqe = cq->va; 1144 cqe_cnt = cq->cqe_cnt; 1145 1146 /* Last irq might have scheduled a polling thread 1147 * sync-up with it before hard flushing. 1148 */ 1149 spin_lock_irqsave(&cq->cq_lock, flags); 1150 while (cqe_cnt) { 1151 if (is_cqe_valid(cq, cqe)) 1152 valid_count++; 1153 cqe++; 1154 cqe_cnt--; 1155 } 1156 ocrdma_ring_cq_db(dev, cq->id, false, false, valid_count); 1157 spin_unlock_irqrestore(&cq->cq_lock, flags); 1158 } 1159 1160 int ocrdma_destroy_cq(struct ib_cq *ibcq) 1161 { 1162 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 1163 struct ocrdma_eq *eq = NULL; 1164 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 1165 int pdid = 0; 1166 u32 irq, indx; 1167 1168 dev->cq_tbl[cq->id] = NULL; 1169 indx = ocrdma_get_eq_table_index(dev, cq->eqn); 1170 BUG_ON(indx == -EINVAL); 1171 1172 eq = &dev->eq_tbl[indx]; 1173 irq = ocrdma_get_irq(dev, eq); 1174 synchronize_irq(irq); 1175 ocrdma_flush_cq(cq); 1176 1177 (void)ocrdma_mbx_destroy_cq(dev, cq); 1178 if (cq->ucontext) { 1179 pdid = cq->ucontext->cntxt_pd->id; 1180 ocrdma_del_mmap(cq->ucontext, (u64) cq->pa, 1181 PAGE_ALIGN(cq->len)); 1182 ocrdma_del_mmap(cq->ucontext, 1183 ocrdma_get_db_addr(dev, pdid), 1184 dev->nic_info.db_page_size); 1185 } 1186 1187 kfree(cq); 1188 return 0; 1189 } 1190 1191 static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) 1192 { 1193 int status = -EINVAL; 1194 1195 if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) { 1196 dev->qp_tbl[qp->id] = qp; 1197 status = 0; 1198 } 1199 return status; 1200 } 1201 1202 static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) 1203 { 1204 dev->qp_tbl[qp->id] = NULL; 1205 } 1206 1207 static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev, 1208 struct ib_qp_init_attr *attrs) 1209 { 1210 if ((attrs->qp_type != IB_QPT_GSI) && 1211 (attrs->qp_type != IB_QPT_RC) && 1212 (attrs->qp_type != IB_QPT_UC) && 1213 (attrs->qp_type != IB_QPT_UD)) { 1214 pr_err("%s(%d) unsupported qp type=0x%x requested\n", 1215 __func__, dev->id, attrs->qp_type); 1216 return -EINVAL; 1217 } 1218 /* Skip the check for QP1 to support CM size of 128 */ 1219 if ((attrs->qp_type != IB_QPT_GSI) && 1220 (attrs->cap.max_send_wr > dev->attr.max_wqe)) { 1221 pr_err("%s(%d) unsupported send_wr=0x%x requested\n", 1222 __func__, dev->id, attrs->cap.max_send_wr); 1223 pr_err("%s(%d) supported send_wr=0x%x\n", 1224 __func__, dev->id, dev->attr.max_wqe); 1225 return -EINVAL; 1226 } 1227 if (!attrs->srq && (attrs->cap.max_recv_wr > dev->attr.max_rqe)) { 1228 pr_err("%s(%d) unsupported recv_wr=0x%x requested\n", 1229 __func__, dev->id, attrs->cap.max_recv_wr); 1230 pr_err("%s(%d) supported recv_wr=0x%x\n", 1231 __func__, dev->id, dev->attr.max_rqe); 1232 return -EINVAL; 1233 } 1234 if (attrs->cap.max_inline_data > dev->attr.max_inline_data) { 1235 pr_err("%s(%d) unsupported inline data size=0x%x requested\n", 1236 __func__, dev->id, attrs->cap.max_inline_data); 1237 pr_err("%s(%d) supported inline data size=0x%x\n", 1238 __func__, dev->id, dev->attr.max_inline_data); 1239 return -EINVAL; 1240 } 1241 if (attrs->cap.max_send_sge > dev->attr.max_send_sge) { 1242 pr_err("%s(%d) unsupported send_sge=0x%x requested\n", 1243 __func__, dev->id, attrs->cap.max_send_sge); 1244 pr_err("%s(%d) supported send_sge=0x%x\n", 1245 __func__, dev->id, dev->attr.max_send_sge); 1246 return -EINVAL; 1247 } 1248 if (attrs->cap.max_recv_sge > dev->attr.max_recv_sge) { 1249 pr_err("%s(%d) unsupported recv_sge=0x%x requested\n", 1250 __func__, dev->id, attrs->cap.max_recv_sge); 1251 pr_err("%s(%d) supported recv_sge=0x%x\n", 1252 __func__, dev->id, dev->attr.max_recv_sge); 1253 return -EINVAL; 1254 } 1255 /* unprivileged user space cannot create special QP */ 1256 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) { 1257 pr_err 1258 ("%s(%d) Userspace can't create special QPs of type=0x%x\n", 1259 __func__, dev->id, attrs->qp_type); 1260 return -EINVAL; 1261 } 1262 /* allow creating only one GSI type of QP */ 1263 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) { 1264 pr_err("%s(%d) GSI special QPs already created.\n", 1265 __func__, dev->id); 1266 return -EINVAL; 1267 } 1268 /* verify consumer QPs are not trying to use GSI QP's CQ */ 1269 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) { 1270 if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) || 1271 (dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) { 1272 pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n", 1273 __func__, dev->id); 1274 return -EINVAL; 1275 } 1276 } 1277 return 0; 1278 } 1279 1280 static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp, 1281 struct ib_udata *udata, int dpp_offset, 1282 int dpp_credit_lmt, int srq) 1283 { 1284 int status; 1285 u64 usr_db; 1286 struct ocrdma_create_qp_uresp uresp; 1287 struct ocrdma_pd *pd = qp->pd; 1288 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device); 1289 1290 memset(&uresp, 0, sizeof(uresp)); 1291 usr_db = dev->nic_info.unmapped_db + 1292 (pd->id * dev->nic_info.db_page_size); 1293 uresp.qp_id = qp->id; 1294 uresp.sq_dbid = qp->sq.dbid; 1295 uresp.num_sq_pages = 1; 1296 uresp.sq_page_size = PAGE_ALIGN(qp->sq.len); 1297 uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va); 1298 uresp.num_wqe_allocated = qp->sq.max_cnt; 1299 if (!srq) { 1300 uresp.rq_dbid = qp->rq.dbid; 1301 uresp.num_rq_pages = 1; 1302 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len); 1303 uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va); 1304 uresp.num_rqe_allocated = qp->rq.max_cnt; 1305 } 1306 uresp.db_page_addr = usr_db; 1307 uresp.db_page_size = dev->nic_info.db_page_size; 1308 uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET; 1309 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET; 1310 uresp.db_shift = OCRDMA_DB_RQ_SHIFT; 1311 1312 if (qp->dpp_enabled) { 1313 uresp.dpp_credit = dpp_credit_lmt; 1314 uresp.dpp_offset = dpp_offset; 1315 } 1316 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1317 if (status) { 1318 pr_err("%s(%d) user copy error.\n", __func__, dev->id); 1319 goto err; 1320 } 1321 status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0], 1322 uresp.sq_page_size); 1323 if (status) 1324 goto err; 1325 1326 if (!srq) { 1327 status = ocrdma_add_mmap(pd->uctx, uresp.rq_page_addr[0], 1328 uresp.rq_page_size); 1329 if (status) 1330 goto rq_map_err; 1331 } 1332 return status; 1333 rq_map_err: 1334 ocrdma_del_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size); 1335 err: 1336 return status; 1337 } 1338 1339 static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp, 1340 struct ocrdma_pd *pd) 1341 { 1342 if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) { 1343 qp->sq_db = dev->nic_info.db + 1344 (pd->id * dev->nic_info.db_page_size) + 1345 OCRDMA_DB_GEN2_SQ_OFFSET; 1346 qp->rq_db = dev->nic_info.db + 1347 (pd->id * dev->nic_info.db_page_size) + 1348 OCRDMA_DB_GEN2_RQ_OFFSET; 1349 } else { 1350 qp->sq_db = dev->nic_info.db + 1351 (pd->id * dev->nic_info.db_page_size) + 1352 OCRDMA_DB_SQ_OFFSET; 1353 qp->rq_db = dev->nic_info.db + 1354 (pd->id * dev->nic_info.db_page_size) + 1355 OCRDMA_DB_RQ_OFFSET; 1356 } 1357 } 1358 1359 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp) 1360 { 1361 qp->wqe_wr_id_tbl = 1362 kzalloc(sizeof(*(qp->wqe_wr_id_tbl)) * qp->sq.max_cnt, 1363 GFP_KERNEL); 1364 if (qp->wqe_wr_id_tbl == NULL) 1365 return -ENOMEM; 1366 qp->rqe_wr_id_tbl = 1367 kzalloc(sizeof(u64) * qp->rq.max_cnt, GFP_KERNEL); 1368 if (qp->rqe_wr_id_tbl == NULL) 1369 return -ENOMEM; 1370 1371 return 0; 1372 } 1373 1374 static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp, 1375 struct ocrdma_pd *pd, 1376 struct ib_qp_init_attr *attrs) 1377 { 1378 qp->pd = pd; 1379 spin_lock_init(&qp->q_lock); 1380 INIT_LIST_HEAD(&qp->sq_entry); 1381 INIT_LIST_HEAD(&qp->rq_entry); 1382 1383 qp->qp_type = attrs->qp_type; 1384 qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR; 1385 qp->max_inline_data = attrs->cap.max_inline_data; 1386 qp->sq.max_sges = attrs->cap.max_send_sge; 1387 qp->rq.max_sges = attrs->cap.max_recv_sge; 1388 qp->state = OCRDMA_QPS_RST; 1389 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1390 } 1391 1392 static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev, 1393 struct ib_qp_init_attr *attrs) 1394 { 1395 if (attrs->qp_type == IB_QPT_GSI) { 1396 dev->gsi_qp_created = 1; 1397 dev->gsi_sqcq = get_ocrdma_cq(attrs->send_cq); 1398 dev->gsi_rqcq = get_ocrdma_cq(attrs->recv_cq); 1399 } 1400 } 1401 1402 struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd, 1403 struct ib_qp_init_attr *attrs, 1404 struct ib_udata *udata) 1405 { 1406 int status; 1407 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 1408 struct ocrdma_qp *qp; 1409 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 1410 struct ocrdma_create_qp_ureq ureq; 1411 u16 dpp_credit_lmt, dpp_offset; 1412 1413 status = ocrdma_check_qp_params(ibpd, dev, attrs); 1414 if (status) 1415 goto gen_err; 1416 1417 memset(&ureq, 0, sizeof(ureq)); 1418 if (udata) { 1419 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 1420 return ERR_PTR(-EFAULT); 1421 } 1422 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1423 if (!qp) { 1424 status = -ENOMEM; 1425 goto gen_err; 1426 } 1427 ocrdma_set_qp_init_params(qp, pd, attrs); 1428 if (udata == NULL) 1429 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 | 1430 OCRDMA_QP_FAST_REG); 1431 1432 mutex_lock(&dev->dev_lock); 1433 status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq, 1434 ureq.dpp_cq_id, 1435 &dpp_offset, &dpp_credit_lmt); 1436 if (status) 1437 goto mbx_err; 1438 1439 /* user space QP's wr_id table are managed in library */ 1440 if (udata == NULL) { 1441 status = ocrdma_alloc_wr_id_tbl(qp); 1442 if (status) 1443 goto map_err; 1444 } 1445 1446 status = ocrdma_add_qpn_map(dev, qp); 1447 if (status) 1448 goto map_err; 1449 ocrdma_set_qp_db(dev, qp, pd); 1450 if (udata) { 1451 status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset, 1452 dpp_credit_lmt, 1453 (attrs->srq != NULL)); 1454 if (status) 1455 goto cpy_err; 1456 } 1457 ocrdma_store_gsi_qp_cq(dev, attrs); 1458 qp->ibqp.qp_num = qp->id; 1459 mutex_unlock(&dev->dev_lock); 1460 return &qp->ibqp; 1461 1462 cpy_err: 1463 ocrdma_del_qpn_map(dev, qp); 1464 map_err: 1465 ocrdma_mbx_destroy_qp(dev, qp); 1466 mbx_err: 1467 mutex_unlock(&dev->dev_lock); 1468 kfree(qp->wqe_wr_id_tbl); 1469 kfree(qp->rqe_wr_id_tbl); 1470 kfree(qp); 1471 pr_err("%s(%d) error=%d\n", __func__, dev->id, status); 1472 gen_err: 1473 return ERR_PTR(status); 1474 } 1475 1476 int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1477 int attr_mask) 1478 { 1479 int status = 0; 1480 struct ocrdma_qp *qp; 1481 struct ocrdma_dev *dev; 1482 enum ib_qp_state old_qps; 1483 1484 qp = get_ocrdma_qp(ibqp); 1485 dev = get_ocrdma_dev(ibqp->device); 1486 if (attr_mask & IB_QP_STATE) 1487 status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps); 1488 /* if new and previous states are same hw doesn't need to 1489 * know about it. 1490 */ 1491 if (status < 0) 1492 return status; 1493 return ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask); 1494 } 1495 1496 int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1497 int attr_mask, struct ib_udata *udata) 1498 { 1499 unsigned long flags; 1500 int status = -EINVAL; 1501 struct ocrdma_qp *qp; 1502 struct ocrdma_dev *dev; 1503 enum ib_qp_state old_qps, new_qps; 1504 1505 qp = get_ocrdma_qp(ibqp); 1506 dev = get_ocrdma_dev(ibqp->device); 1507 1508 /* syncronize with multiple context trying to change, retrive qps */ 1509 mutex_lock(&dev->dev_lock); 1510 /* syncronize with wqe, rqe posting and cqe processing contexts */ 1511 spin_lock_irqsave(&qp->q_lock, flags); 1512 old_qps = get_ibqp_state(qp->state); 1513 if (attr_mask & IB_QP_STATE) 1514 new_qps = attr->qp_state; 1515 else 1516 new_qps = old_qps; 1517 spin_unlock_irqrestore(&qp->q_lock, flags); 1518 1519 if (!ib_modify_qp_is_ok(old_qps, new_qps, ibqp->qp_type, attr_mask, 1520 IB_LINK_LAYER_ETHERNET)) { 1521 pr_err("%s(%d) invalid attribute mask=0x%x specified for\n" 1522 "qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n", 1523 __func__, dev->id, attr_mask, qp->id, ibqp->qp_type, 1524 old_qps, new_qps); 1525 goto param_err; 1526 } 1527 1528 status = _ocrdma_modify_qp(ibqp, attr, attr_mask); 1529 if (status > 0) 1530 status = 0; 1531 param_err: 1532 mutex_unlock(&dev->dev_lock); 1533 return status; 1534 } 1535 1536 static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu) 1537 { 1538 switch (mtu) { 1539 case 256: 1540 return IB_MTU_256; 1541 case 512: 1542 return IB_MTU_512; 1543 case 1024: 1544 return IB_MTU_1024; 1545 case 2048: 1546 return IB_MTU_2048; 1547 case 4096: 1548 return IB_MTU_4096; 1549 default: 1550 return IB_MTU_1024; 1551 } 1552 } 1553 1554 static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags) 1555 { 1556 int ib_qp_acc_flags = 0; 1557 1558 if (qp_cap_flags & OCRDMA_QP_INB_WR) 1559 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 1560 if (qp_cap_flags & OCRDMA_QP_INB_RD) 1561 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 1562 return ib_qp_acc_flags; 1563 } 1564 1565 int ocrdma_query_qp(struct ib_qp *ibqp, 1566 struct ib_qp_attr *qp_attr, 1567 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 1568 { 1569 int status; 1570 u32 qp_state; 1571 struct ocrdma_qp_params params; 1572 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 1573 struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device); 1574 1575 memset(¶ms, 0, sizeof(params)); 1576 mutex_lock(&dev->dev_lock); 1577 status = ocrdma_mbx_query_qp(dev, qp, ¶ms); 1578 mutex_unlock(&dev->dev_lock); 1579 if (status) 1580 goto mbx_err; 1581 if (qp->qp_type == IB_QPT_UD) 1582 qp_attr->qkey = params.qkey; 1583 qp_attr->path_mtu = 1584 ocrdma_mtu_int_to_enum(params.path_mtu_pkey_indx & 1585 OCRDMA_QP_PARAMS_PATH_MTU_MASK) >> 1586 OCRDMA_QP_PARAMS_PATH_MTU_SHIFT; 1587 qp_attr->path_mig_state = IB_MIG_MIGRATED; 1588 qp_attr->rq_psn = params.hop_lmt_rq_psn & OCRDMA_QP_PARAMS_RQ_PSN_MASK; 1589 qp_attr->sq_psn = params.tclass_sq_psn & OCRDMA_QP_PARAMS_SQ_PSN_MASK; 1590 qp_attr->dest_qp_num = 1591 params.ack_to_rnr_rtc_dest_qpn & OCRDMA_QP_PARAMS_DEST_QPN_MASK; 1592 1593 qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags); 1594 qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1; 1595 qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1; 1596 qp_attr->cap.max_send_sge = qp->sq.max_sges; 1597 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 1598 qp_attr->cap.max_inline_data = qp->max_inline_data; 1599 qp_init_attr->cap = qp_attr->cap; 1600 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 1601 1602 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 1603 params.rnt_rc_sl_fl & 1604 OCRDMA_QP_PARAMS_FLOW_LABEL_MASK, 1605 qp->sgid_idx, 1606 (params.hop_lmt_rq_psn & 1607 OCRDMA_QP_PARAMS_HOP_LMT_MASK) >> 1608 OCRDMA_QP_PARAMS_HOP_LMT_SHIFT, 1609 (params.tclass_sq_psn & 1610 OCRDMA_QP_PARAMS_TCLASS_MASK) >> 1611 OCRDMA_QP_PARAMS_TCLASS_SHIFT); 1612 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid[0]); 1613 1614 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 1615 rdma_ah_set_sl(&qp_attr->ah_attr, (params.rnt_rc_sl_fl & 1616 OCRDMA_QP_PARAMS_SL_MASK) >> 1617 OCRDMA_QP_PARAMS_SL_SHIFT); 1618 qp_attr->timeout = (params.ack_to_rnr_rtc_dest_qpn & 1619 OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK) >> 1620 OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT; 1621 qp_attr->rnr_retry = (params.ack_to_rnr_rtc_dest_qpn & 1622 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK) >> 1623 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT; 1624 qp_attr->retry_cnt = 1625 (params.rnt_rc_sl_fl & OCRDMA_QP_PARAMS_RETRY_CNT_MASK) >> 1626 OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT; 1627 qp_attr->min_rnr_timer = 0; 1628 qp_attr->pkey_index = 0; 1629 qp_attr->port_num = 1; 1630 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 1631 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 1632 qp_attr->alt_pkey_index = 0; 1633 qp_attr->alt_port_num = 0; 1634 qp_attr->alt_timeout = 0; 1635 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 1636 qp_state = (params.max_sge_recv_flags & OCRDMA_QP_PARAMS_STATE_MASK) >> 1637 OCRDMA_QP_PARAMS_STATE_SHIFT; 1638 qp_attr->qp_state = get_ibqp_state(qp_state); 1639 qp_attr->cur_qp_state = qp_attr->qp_state; 1640 qp_attr->sq_draining = (qp_state == OCRDMA_QPS_SQ_DRAINING) ? 1 : 0; 1641 qp_attr->max_dest_rd_atomic = 1642 params.max_ord_ird >> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT; 1643 qp_attr->max_rd_atomic = 1644 params.max_ord_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK; 1645 qp_attr->en_sqd_async_notify = (params.max_sge_recv_flags & 1646 OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC) ? 1 : 0; 1647 /* Sync driver QP state with FW */ 1648 ocrdma_qp_state_change(qp, qp_attr->qp_state, NULL); 1649 mbx_err: 1650 return status; 1651 } 1652 1653 static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, unsigned int idx) 1654 { 1655 unsigned int i = idx / 32; 1656 u32 mask = (1U << (idx % 32)); 1657 1658 srq->idx_bit_fields[i] ^= mask; 1659 } 1660 1661 static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q) 1662 { 1663 return ((q->max_wqe_idx - q->head) + q->tail) % q->max_cnt; 1664 } 1665 1666 static int is_hw_sq_empty(struct ocrdma_qp *qp) 1667 { 1668 return (qp->sq.tail == qp->sq.head); 1669 } 1670 1671 static int is_hw_rq_empty(struct ocrdma_qp *qp) 1672 { 1673 return (qp->rq.tail == qp->rq.head); 1674 } 1675 1676 static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q) 1677 { 1678 return q->va + (q->head * q->entry_size); 1679 } 1680 1681 static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q, 1682 u32 idx) 1683 { 1684 return q->va + (idx * q->entry_size); 1685 } 1686 1687 static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q) 1688 { 1689 q->head = (q->head + 1) & q->max_wqe_idx; 1690 } 1691 1692 static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q) 1693 { 1694 q->tail = (q->tail + 1) & q->max_wqe_idx; 1695 } 1696 1697 /* discard the cqe for a given QP */ 1698 static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq) 1699 { 1700 unsigned long cq_flags; 1701 unsigned long flags; 1702 int discard_cnt = 0; 1703 u32 cur_getp, stop_getp; 1704 struct ocrdma_cqe *cqe; 1705 u32 qpn = 0, wqe_idx = 0; 1706 1707 spin_lock_irqsave(&cq->cq_lock, cq_flags); 1708 1709 /* traverse through the CQEs in the hw CQ, 1710 * find the matching CQE for a given qp, 1711 * mark the matching one discarded by clearing qpn. 1712 * ring the doorbell in the poll_cq() as 1713 * we don't complete out of order cqe. 1714 */ 1715 1716 cur_getp = cq->getp; 1717 /* find upto when do we reap the cq. */ 1718 stop_getp = cur_getp; 1719 do { 1720 if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp))) 1721 break; 1722 1723 cqe = cq->va + cur_getp; 1724 /* if (a) done reaping whole hw cq, or 1725 * (b) qp_xq becomes empty. 1726 * then exit 1727 */ 1728 qpn = cqe->cmn.qpn & OCRDMA_CQE_QPN_MASK; 1729 /* if previously discarded cqe found, skip that too. */ 1730 /* check for matching qp */ 1731 if (qpn == 0 || qpn != qp->id) 1732 goto skip_cqe; 1733 1734 if (is_cqe_for_sq(cqe)) { 1735 ocrdma_hwq_inc_tail(&qp->sq); 1736 } else { 1737 if (qp->srq) { 1738 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >> 1739 OCRDMA_CQE_BUFTAG_SHIFT) & 1740 qp->srq->rq.max_wqe_idx; 1741 BUG_ON(wqe_idx < 1); 1742 spin_lock_irqsave(&qp->srq->q_lock, flags); 1743 ocrdma_hwq_inc_tail(&qp->srq->rq); 1744 ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1); 1745 spin_unlock_irqrestore(&qp->srq->q_lock, flags); 1746 1747 } else { 1748 ocrdma_hwq_inc_tail(&qp->rq); 1749 } 1750 } 1751 /* mark cqe discarded so that it is not picked up later 1752 * in the poll_cq(). 1753 */ 1754 discard_cnt += 1; 1755 cqe->cmn.qpn = 0; 1756 skip_cqe: 1757 cur_getp = (cur_getp + 1) % cq->max_hw_cqe; 1758 } while (cur_getp != stop_getp); 1759 spin_unlock_irqrestore(&cq->cq_lock, cq_flags); 1760 } 1761 1762 void ocrdma_del_flush_qp(struct ocrdma_qp *qp) 1763 { 1764 int found = false; 1765 unsigned long flags; 1766 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 1767 /* sync with any active CQ poll */ 1768 1769 spin_lock_irqsave(&dev->flush_q_lock, flags); 1770 found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp); 1771 if (found) 1772 list_del(&qp->sq_entry); 1773 if (!qp->srq) { 1774 found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp); 1775 if (found) 1776 list_del(&qp->rq_entry); 1777 } 1778 spin_unlock_irqrestore(&dev->flush_q_lock, flags); 1779 } 1780 1781 int ocrdma_destroy_qp(struct ib_qp *ibqp) 1782 { 1783 struct ocrdma_pd *pd; 1784 struct ocrdma_qp *qp; 1785 struct ocrdma_dev *dev; 1786 struct ib_qp_attr attrs; 1787 int attr_mask; 1788 unsigned long flags; 1789 1790 qp = get_ocrdma_qp(ibqp); 1791 dev = get_ocrdma_dev(ibqp->device); 1792 1793 pd = qp->pd; 1794 1795 /* change the QP state to ERROR */ 1796 if (qp->state != OCRDMA_QPS_RST) { 1797 attrs.qp_state = IB_QPS_ERR; 1798 attr_mask = IB_QP_STATE; 1799 _ocrdma_modify_qp(ibqp, &attrs, attr_mask); 1800 } 1801 /* ensure that CQEs for newly created QP (whose id may be same with 1802 * one which just getting destroyed are same), dont get 1803 * discarded until the old CQEs are discarded. 1804 */ 1805 mutex_lock(&dev->dev_lock); 1806 (void) ocrdma_mbx_destroy_qp(dev, qp); 1807 1808 /* 1809 * acquire CQ lock while destroy is in progress, in order to 1810 * protect against proessing in-flight CQEs for this QP. 1811 */ 1812 spin_lock_irqsave(&qp->sq_cq->cq_lock, flags); 1813 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) 1814 spin_lock(&qp->rq_cq->cq_lock); 1815 1816 ocrdma_del_qpn_map(dev, qp); 1817 1818 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) 1819 spin_unlock(&qp->rq_cq->cq_lock); 1820 spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags); 1821 1822 if (!pd->uctx) { 1823 ocrdma_discard_cqes(qp, qp->sq_cq); 1824 ocrdma_discard_cqes(qp, qp->rq_cq); 1825 } 1826 mutex_unlock(&dev->dev_lock); 1827 1828 if (pd->uctx) { 1829 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa, 1830 PAGE_ALIGN(qp->sq.len)); 1831 if (!qp->srq) 1832 ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa, 1833 PAGE_ALIGN(qp->rq.len)); 1834 } 1835 1836 ocrdma_del_flush_qp(qp); 1837 1838 kfree(qp->wqe_wr_id_tbl); 1839 kfree(qp->rqe_wr_id_tbl); 1840 kfree(qp); 1841 return 0; 1842 } 1843 1844 static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq, 1845 struct ib_udata *udata) 1846 { 1847 int status; 1848 struct ocrdma_create_srq_uresp uresp; 1849 1850 memset(&uresp, 0, sizeof(uresp)); 1851 uresp.rq_dbid = srq->rq.dbid; 1852 uresp.num_rq_pages = 1; 1853 uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va); 1854 uresp.rq_page_size = srq->rq.len; 1855 uresp.db_page_addr = dev->nic_info.unmapped_db + 1856 (srq->pd->id * dev->nic_info.db_page_size); 1857 uresp.db_page_size = dev->nic_info.db_page_size; 1858 uresp.num_rqe_allocated = srq->rq.max_cnt; 1859 if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) { 1860 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET; 1861 uresp.db_shift = 24; 1862 } else { 1863 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET; 1864 uresp.db_shift = 16; 1865 } 1866 1867 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1868 if (status) 1869 return status; 1870 status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0], 1871 uresp.rq_page_size); 1872 if (status) 1873 return status; 1874 return status; 1875 } 1876 1877 struct ib_srq *ocrdma_create_srq(struct ib_pd *ibpd, 1878 struct ib_srq_init_attr *init_attr, 1879 struct ib_udata *udata) 1880 { 1881 int status = -ENOMEM; 1882 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 1883 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 1884 struct ocrdma_srq *srq; 1885 1886 if (init_attr->attr.max_sge > dev->attr.max_recv_sge) 1887 return ERR_PTR(-EINVAL); 1888 if (init_attr->attr.max_wr > dev->attr.max_rqe) 1889 return ERR_PTR(-EINVAL); 1890 1891 srq = kzalloc(sizeof(*srq), GFP_KERNEL); 1892 if (!srq) 1893 return ERR_PTR(status); 1894 1895 spin_lock_init(&srq->q_lock); 1896 srq->pd = pd; 1897 srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size); 1898 status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd); 1899 if (status) 1900 goto err; 1901 1902 if (udata == NULL) { 1903 status = -ENOMEM; 1904 srq->rqe_wr_id_tbl = kzalloc(sizeof(u64) * srq->rq.max_cnt, 1905 GFP_KERNEL); 1906 if (srq->rqe_wr_id_tbl == NULL) 1907 goto arm_err; 1908 1909 srq->bit_fields_len = (srq->rq.max_cnt / 32) + 1910 (srq->rq.max_cnt % 32 ? 1 : 0); 1911 srq->idx_bit_fields = 1912 kmalloc(srq->bit_fields_len * sizeof(u32), GFP_KERNEL); 1913 if (srq->idx_bit_fields == NULL) 1914 goto arm_err; 1915 memset(srq->idx_bit_fields, 0xff, 1916 srq->bit_fields_len * sizeof(u32)); 1917 } 1918 1919 if (init_attr->attr.srq_limit) { 1920 status = ocrdma_mbx_modify_srq(srq, &init_attr->attr); 1921 if (status) 1922 goto arm_err; 1923 } 1924 1925 if (udata) { 1926 status = ocrdma_copy_srq_uresp(dev, srq, udata); 1927 if (status) 1928 goto arm_err; 1929 } 1930 1931 return &srq->ibsrq; 1932 1933 arm_err: 1934 ocrdma_mbx_destroy_srq(dev, srq); 1935 err: 1936 kfree(srq->rqe_wr_id_tbl); 1937 kfree(srq->idx_bit_fields); 1938 kfree(srq); 1939 return ERR_PTR(status); 1940 } 1941 1942 int ocrdma_modify_srq(struct ib_srq *ibsrq, 1943 struct ib_srq_attr *srq_attr, 1944 enum ib_srq_attr_mask srq_attr_mask, 1945 struct ib_udata *udata) 1946 { 1947 int status; 1948 struct ocrdma_srq *srq; 1949 1950 srq = get_ocrdma_srq(ibsrq); 1951 if (srq_attr_mask & IB_SRQ_MAX_WR) 1952 status = -EINVAL; 1953 else 1954 status = ocrdma_mbx_modify_srq(srq, srq_attr); 1955 return status; 1956 } 1957 1958 int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 1959 { 1960 int status; 1961 struct ocrdma_srq *srq; 1962 1963 srq = get_ocrdma_srq(ibsrq); 1964 status = ocrdma_mbx_query_srq(srq, srq_attr); 1965 return status; 1966 } 1967 1968 int ocrdma_destroy_srq(struct ib_srq *ibsrq) 1969 { 1970 int status; 1971 struct ocrdma_srq *srq; 1972 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device); 1973 1974 srq = get_ocrdma_srq(ibsrq); 1975 1976 status = ocrdma_mbx_destroy_srq(dev, srq); 1977 1978 if (srq->pd->uctx) 1979 ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa, 1980 PAGE_ALIGN(srq->rq.len)); 1981 1982 kfree(srq->idx_bit_fields); 1983 kfree(srq->rqe_wr_id_tbl); 1984 kfree(srq); 1985 return status; 1986 } 1987 1988 /* unprivileged verbs and their support functions. */ 1989 static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp, 1990 struct ocrdma_hdr_wqe *hdr, 1991 struct ib_send_wr *wr) 1992 { 1993 struct ocrdma_ewqe_ud_hdr *ud_hdr = 1994 (struct ocrdma_ewqe_ud_hdr *)(hdr + 1); 1995 struct ocrdma_ah *ah = get_ocrdma_ah(ud_wr(wr)->ah); 1996 1997 ud_hdr->rsvd_dest_qpn = ud_wr(wr)->remote_qpn; 1998 if (qp->qp_type == IB_QPT_GSI) 1999 ud_hdr->qkey = qp->qkey; 2000 else 2001 ud_hdr->qkey = ud_wr(wr)->remote_qkey; 2002 ud_hdr->rsvd_ahid = ah->id; 2003 ud_hdr->hdr_type = ah->hdr_type; 2004 if (ah->av->valid & OCRDMA_AV_VLAN_VALID) 2005 hdr->cw |= (OCRDMA_FLAG_AH_VLAN_PR << OCRDMA_WQE_FLAGS_SHIFT); 2006 } 2007 2008 static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr, 2009 struct ocrdma_sge *sge, int num_sge, 2010 struct ib_sge *sg_list) 2011 { 2012 int i; 2013 2014 for (i = 0; i < num_sge; i++) { 2015 sge[i].lrkey = sg_list[i].lkey; 2016 sge[i].addr_lo = sg_list[i].addr; 2017 sge[i].addr_hi = upper_32_bits(sg_list[i].addr); 2018 sge[i].len = sg_list[i].length; 2019 hdr->total_len += sg_list[i].length; 2020 } 2021 if (num_sge == 0) 2022 memset(sge, 0, sizeof(*sge)); 2023 } 2024 2025 static inline uint32_t ocrdma_sglist_len(struct ib_sge *sg_list, int num_sge) 2026 { 2027 uint32_t total_len = 0, i; 2028 2029 for (i = 0; i < num_sge; i++) 2030 total_len += sg_list[i].length; 2031 return total_len; 2032 } 2033 2034 2035 static int ocrdma_build_inline_sges(struct ocrdma_qp *qp, 2036 struct ocrdma_hdr_wqe *hdr, 2037 struct ocrdma_sge *sge, 2038 struct ib_send_wr *wr, u32 wqe_size) 2039 { 2040 int i; 2041 char *dpp_addr; 2042 2043 if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) { 2044 hdr->total_len = ocrdma_sglist_len(wr->sg_list, wr->num_sge); 2045 if (unlikely(hdr->total_len > qp->max_inline_data)) { 2046 pr_err("%s() supported_len=0x%x,\n" 2047 " unsupported len req=0x%x\n", __func__, 2048 qp->max_inline_data, hdr->total_len); 2049 return -EINVAL; 2050 } 2051 dpp_addr = (char *)sge; 2052 for (i = 0; i < wr->num_sge; i++) { 2053 memcpy(dpp_addr, 2054 (void *)(unsigned long)wr->sg_list[i].addr, 2055 wr->sg_list[i].length); 2056 dpp_addr += wr->sg_list[i].length; 2057 } 2058 2059 wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES); 2060 if (0 == hdr->total_len) 2061 wqe_size += sizeof(struct ocrdma_sge); 2062 hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT); 2063 } else { 2064 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list); 2065 if (wr->num_sge) 2066 wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge)); 2067 else 2068 wqe_size += sizeof(struct ocrdma_sge); 2069 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 2070 } 2071 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 2072 return 0; 2073 } 2074 2075 static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 2076 struct ib_send_wr *wr) 2077 { 2078 int status; 2079 struct ocrdma_sge *sge; 2080 u32 wqe_size = sizeof(*hdr); 2081 2082 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 2083 ocrdma_build_ud_hdr(qp, hdr, wr); 2084 sge = (struct ocrdma_sge *)(hdr + 2); 2085 wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr); 2086 } else { 2087 sge = (struct ocrdma_sge *)(hdr + 1); 2088 } 2089 2090 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); 2091 return status; 2092 } 2093 2094 static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 2095 struct ib_send_wr *wr) 2096 { 2097 int status; 2098 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1); 2099 struct ocrdma_sge *sge = ext_rw + 1; 2100 u32 wqe_size = sizeof(*hdr) + sizeof(*ext_rw); 2101 2102 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); 2103 if (status) 2104 return status; 2105 ext_rw->addr_lo = rdma_wr(wr)->remote_addr; 2106 ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr); 2107 ext_rw->lrkey = rdma_wr(wr)->rkey; 2108 ext_rw->len = hdr->total_len; 2109 return 0; 2110 } 2111 2112 static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 2113 struct ib_send_wr *wr) 2114 { 2115 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1); 2116 struct ocrdma_sge *sge = ext_rw + 1; 2117 u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) + 2118 sizeof(struct ocrdma_hdr_wqe); 2119 2120 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list); 2121 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 2122 hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT); 2123 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 2124 2125 ext_rw->addr_lo = rdma_wr(wr)->remote_addr; 2126 ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr); 2127 ext_rw->lrkey = rdma_wr(wr)->rkey; 2128 ext_rw->len = hdr->total_len; 2129 } 2130 2131 static int get_encoded_page_size(int pg_sz) 2132 { 2133 /* Max size is 256M 4096 << 16 */ 2134 int i = 0; 2135 for (; i < 17; i++) 2136 if (pg_sz == (4096 << i)) 2137 break; 2138 return i; 2139 } 2140 2141 static int ocrdma_build_reg(struct ocrdma_qp *qp, 2142 struct ocrdma_hdr_wqe *hdr, 2143 struct ib_reg_wr *wr) 2144 { 2145 u64 fbo; 2146 struct ocrdma_ewqe_fr *fast_reg = (struct ocrdma_ewqe_fr *)(hdr + 1); 2147 struct ocrdma_mr *mr = get_ocrdma_mr(wr->mr); 2148 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; 2149 struct ocrdma_pbe *pbe; 2150 u32 wqe_size = sizeof(*fast_reg) + sizeof(*hdr); 2151 int num_pbes = 0, i; 2152 2153 wqe_size = roundup(wqe_size, OCRDMA_WQE_ALIGN_BYTES); 2154 2155 hdr->cw |= (OCRDMA_FR_MR << OCRDMA_WQE_OPCODE_SHIFT); 2156 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 2157 2158 if (wr->access & IB_ACCESS_LOCAL_WRITE) 2159 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_LOCAL_WR; 2160 if (wr->access & IB_ACCESS_REMOTE_WRITE) 2161 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_WR; 2162 if (wr->access & IB_ACCESS_REMOTE_READ) 2163 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_RD; 2164 hdr->lkey = wr->key; 2165 hdr->total_len = mr->ibmr.length; 2166 2167 fbo = mr->ibmr.iova - mr->pages[0]; 2168 2169 fast_reg->va_hi = upper_32_bits(mr->ibmr.iova); 2170 fast_reg->va_lo = (u32) (mr->ibmr.iova & 0xffffffff); 2171 fast_reg->fbo_hi = upper_32_bits(fbo); 2172 fast_reg->fbo_lo = (u32) fbo & 0xffffffff; 2173 fast_reg->num_sges = mr->npages; 2174 fast_reg->size_sge = get_encoded_page_size(mr->ibmr.page_size); 2175 2176 pbe = pbl_tbl->va; 2177 for (i = 0; i < mr->npages; i++) { 2178 u64 buf_addr = mr->pages[i]; 2179 2180 pbe->pa_lo = cpu_to_le32((u32) (buf_addr & PAGE_MASK)); 2181 pbe->pa_hi = cpu_to_le32((u32) upper_32_bits(buf_addr)); 2182 num_pbes += 1; 2183 pbe++; 2184 2185 /* if the pbl is full storing the pbes, 2186 * move to next pbl. 2187 */ 2188 if (num_pbes == (mr->hwmr.pbl_size/sizeof(u64))) { 2189 pbl_tbl++; 2190 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 2191 } 2192 } 2193 2194 return 0; 2195 } 2196 2197 static void ocrdma_ring_sq_db(struct ocrdma_qp *qp) 2198 { 2199 u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT); 2200 2201 iowrite32(val, qp->sq_db); 2202 } 2203 2204 int ocrdma_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 2205 struct ib_send_wr **bad_wr) 2206 { 2207 int status = 0; 2208 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 2209 struct ocrdma_hdr_wqe *hdr; 2210 unsigned long flags; 2211 2212 spin_lock_irqsave(&qp->q_lock, flags); 2213 if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) { 2214 spin_unlock_irqrestore(&qp->q_lock, flags); 2215 *bad_wr = wr; 2216 return -EINVAL; 2217 } 2218 2219 while (wr) { 2220 if (qp->qp_type == IB_QPT_UD && 2221 (wr->opcode != IB_WR_SEND && 2222 wr->opcode != IB_WR_SEND_WITH_IMM)) { 2223 *bad_wr = wr; 2224 status = -EINVAL; 2225 break; 2226 } 2227 if (ocrdma_hwq_free_cnt(&qp->sq) == 0 || 2228 wr->num_sge > qp->sq.max_sges) { 2229 *bad_wr = wr; 2230 status = -ENOMEM; 2231 break; 2232 } 2233 hdr = ocrdma_hwq_head(&qp->sq); 2234 hdr->cw = 0; 2235 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) 2236 hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT); 2237 if (wr->send_flags & IB_SEND_FENCE) 2238 hdr->cw |= 2239 (OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT); 2240 if (wr->send_flags & IB_SEND_SOLICITED) 2241 hdr->cw |= 2242 (OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT); 2243 hdr->total_len = 0; 2244 switch (wr->opcode) { 2245 case IB_WR_SEND_WITH_IMM: 2246 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT); 2247 hdr->immdt = ntohl(wr->ex.imm_data); 2248 /* fall through */ 2249 case IB_WR_SEND: 2250 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT); 2251 ocrdma_build_send(qp, hdr, wr); 2252 break; 2253 case IB_WR_SEND_WITH_INV: 2254 hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT); 2255 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT); 2256 hdr->lkey = wr->ex.invalidate_rkey; 2257 status = ocrdma_build_send(qp, hdr, wr); 2258 break; 2259 case IB_WR_RDMA_WRITE_WITH_IMM: 2260 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT); 2261 hdr->immdt = ntohl(wr->ex.imm_data); 2262 /* fall through */ 2263 case IB_WR_RDMA_WRITE: 2264 hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT); 2265 status = ocrdma_build_write(qp, hdr, wr); 2266 break; 2267 case IB_WR_RDMA_READ: 2268 ocrdma_build_read(qp, hdr, wr); 2269 break; 2270 case IB_WR_LOCAL_INV: 2271 hdr->cw |= 2272 (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT); 2273 hdr->cw |= ((sizeof(struct ocrdma_hdr_wqe) + 2274 sizeof(struct ocrdma_sge)) / 2275 OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT; 2276 hdr->lkey = wr->ex.invalidate_rkey; 2277 break; 2278 case IB_WR_REG_MR: 2279 status = ocrdma_build_reg(qp, hdr, reg_wr(wr)); 2280 break; 2281 default: 2282 status = -EINVAL; 2283 break; 2284 } 2285 if (status) { 2286 *bad_wr = wr; 2287 break; 2288 } 2289 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) 2290 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1; 2291 else 2292 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0; 2293 qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id; 2294 ocrdma_cpu_to_le32(hdr, ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) & 2295 OCRDMA_WQE_SIZE_MASK) * OCRDMA_WQE_STRIDE); 2296 /* make sure wqe is written before adapter can access it */ 2297 wmb(); 2298 /* inform hw to start processing it */ 2299 ocrdma_ring_sq_db(qp); 2300 2301 /* update pointer, counter for next wr */ 2302 ocrdma_hwq_inc_head(&qp->sq); 2303 wr = wr->next; 2304 } 2305 spin_unlock_irqrestore(&qp->q_lock, flags); 2306 return status; 2307 } 2308 2309 static void ocrdma_ring_rq_db(struct ocrdma_qp *qp) 2310 { 2311 u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT); 2312 2313 iowrite32(val, qp->rq_db); 2314 } 2315 2316 static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe, struct ib_recv_wr *wr, 2317 u16 tag) 2318 { 2319 u32 wqe_size = 0; 2320 struct ocrdma_sge *sge; 2321 if (wr->num_sge) 2322 wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe); 2323 else 2324 wqe_size = sizeof(*sge) + sizeof(*rqe); 2325 2326 rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) << 2327 OCRDMA_WQE_SIZE_SHIFT); 2328 rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT); 2329 rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 2330 rqe->total_len = 0; 2331 rqe->rsvd_tag = tag; 2332 sge = (struct ocrdma_sge *)(rqe + 1); 2333 ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list); 2334 ocrdma_cpu_to_le32(rqe, wqe_size); 2335 } 2336 2337 int ocrdma_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, 2338 struct ib_recv_wr **bad_wr) 2339 { 2340 int status = 0; 2341 unsigned long flags; 2342 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 2343 struct ocrdma_hdr_wqe *rqe; 2344 2345 spin_lock_irqsave(&qp->q_lock, flags); 2346 if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) { 2347 spin_unlock_irqrestore(&qp->q_lock, flags); 2348 *bad_wr = wr; 2349 return -EINVAL; 2350 } 2351 while (wr) { 2352 if (ocrdma_hwq_free_cnt(&qp->rq) == 0 || 2353 wr->num_sge > qp->rq.max_sges) { 2354 *bad_wr = wr; 2355 status = -ENOMEM; 2356 break; 2357 } 2358 rqe = ocrdma_hwq_head(&qp->rq); 2359 ocrdma_build_rqe(rqe, wr, 0); 2360 2361 qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id; 2362 /* make sure rqe is written before adapter can access it */ 2363 wmb(); 2364 2365 /* inform hw to start processing it */ 2366 ocrdma_ring_rq_db(qp); 2367 2368 /* update pointer, counter for next wr */ 2369 ocrdma_hwq_inc_head(&qp->rq); 2370 wr = wr->next; 2371 } 2372 spin_unlock_irqrestore(&qp->q_lock, flags); 2373 return status; 2374 } 2375 2376 /* cqe for srq's rqe can potentially arrive out of order. 2377 * index gives the entry in the shadow table where to store 2378 * the wr_id. tag/index is returned in cqe to reference back 2379 * for a given rqe. 2380 */ 2381 static int ocrdma_srq_get_idx(struct ocrdma_srq *srq) 2382 { 2383 int row = 0; 2384 int indx = 0; 2385 2386 for (row = 0; row < srq->bit_fields_len; row++) { 2387 if (srq->idx_bit_fields[row]) { 2388 indx = ffs(srq->idx_bit_fields[row]); 2389 indx = (row * 32) + (indx - 1); 2390 BUG_ON(indx >= srq->rq.max_cnt); 2391 ocrdma_srq_toggle_bit(srq, indx); 2392 break; 2393 } 2394 } 2395 2396 BUG_ON(row == srq->bit_fields_len); 2397 return indx + 1; /* Use from index 1 */ 2398 } 2399 2400 static void ocrdma_ring_srq_db(struct ocrdma_srq *srq) 2401 { 2402 u32 val = srq->rq.dbid | (1 << 16); 2403 2404 iowrite32(val, srq->db + OCRDMA_DB_GEN2_SRQ_OFFSET); 2405 } 2406 2407 int ocrdma_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, 2408 struct ib_recv_wr **bad_wr) 2409 { 2410 int status = 0; 2411 unsigned long flags; 2412 struct ocrdma_srq *srq; 2413 struct ocrdma_hdr_wqe *rqe; 2414 u16 tag; 2415 2416 srq = get_ocrdma_srq(ibsrq); 2417 2418 spin_lock_irqsave(&srq->q_lock, flags); 2419 while (wr) { 2420 if (ocrdma_hwq_free_cnt(&srq->rq) == 0 || 2421 wr->num_sge > srq->rq.max_sges) { 2422 status = -ENOMEM; 2423 *bad_wr = wr; 2424 break; 2425 } 2426 tag = ocrdma_srq_get_idx(srq); 2427 rqe = ocrdma_hwq_head(&srq->rq); 2428 ocrdma_build_rqe(rqe, wr, tag); 2429 2430 srq->rqe_wr_id_tbl[tag] = wr->wr_id; 2431 /* make sure rqe is written before adapter can perform DMA */ 2432 wmb(); 2433 /* inform hw to start processing it */ 2434 ocrdma_ring_srq_db(srq); 2435 /* update pointer, counter for next wr */ 2436 ocrdma_hwq_inc_head(&srq->rq); 2437 wr = wr->next; 2438 } 2439 spin_unlock_irqrestore(&srq->q_lock, flags); 2440 return status; 2441 } 2442 2443 static enum ib_wc_status ocrdma_to_ibwc_err(u16 status) 2444 { 2445 enum ib_wc_status ibwc_status; 2446 2447 switch (status) { 2448 case OCRDMA_CQE_GENERAL_ERR: 2449 ibwc_status = IB_WC_GENERAL_ERR; 2450 break; 2451 case OCRDMA_CQE_LOC_LEN_ERR: 2452 ibwc_status = IB_WC_LOC_LEN_ERR; 2453 break; 2454 case OCRDMA_CQE_LOC_QP_OP_ERR: 2455 ibwc_status = IB_WC_LOC_QP_OP_ERR; 2456 break; 2457 case OCRDMA_CQE_LOC_EEC_OP_ERR: 2458 ibwc_status = IB_WC_LOC_EEC_OP_ERR; 2459 break; 2460 case OCRDMA_CQE_LOC_PROT_ERR: 2461 ibwc_status = IB_WC_LOC_PROT_ERR; 2462 break; 2463 case OCRDMA_CQE_WR_FLUSH_ERR: 2464 ibwc_status = IB_WC_WR_FLUSH_ERR; 2465 break; 2466 case OCRDMA_CQE_MW_BIND_ERR: 2467 ibwc_status = IB_WC_MW_BIND_ERR; 2468 break; 2469 case OCRDMA_CQE_BAD_RESP_ERR: 2470 ibwc_status = IB_WC_BAD_RESP_ERR; 2471 break; 2472 case OCRDMA_CQE_LOC_ACCESS_ERR: 2473 ibwc_status = IB_WC_LOC_ACCESS_ERR; 2474 break; 2475 case OCRDMA_CQE_REM_INV_REQ_ERR: 2476 ibwc_status = IB_WC_REM_INV_REQ_ERR; 2477 break; 2478 case OCRDMA_CQE_REM_ACCESS_ERR: 2479 ibwc_status = IB_WC_REM_ACCESS_ERR; 2480 break; 2481 case OCRDMA_CQE_REM_OP_ERR: 2482 ibwc_status = IB_WC_REM_OP_ERR; 2483 break; 2484 case OCRDMA_CQE_RETRY_EXC_ERR: 2485 ibwc_status = IB_WC_RETRY_EXC_ERR; 2486 break; 2487 case OCRDMA_CQE_RNR_RETRY_EXC_ERR: 2488 ibwc_status = IB_WC_RNR_RETRY_EXC_ERR; 2489 break; 2490 case OCRDMA_CQE_LOC_RDD_VIOL_ERR: 2491 ibwc_status = IB_WC_LOC_RDD_VIOL_ERR; 2492 break; 2493 case OCRDMA_CQE_REM_INV_RD_REQ_ERR: 2494 ibwc_status = IB_WC_REM_INV_RD_REQ_ERR; 2495 break; 2496 case OCRDMA_CQE_REM_ABORT_ERR: 2497 ibwc_status = IB_WC_REM_ABORT_ERR; 2498 break; 2499 case OCRDMA_CQE_INV_EECN_ERR: 2500 ibwc_status = IB_WC_INV_EECN_ERR; 2501 break; 2502 case OCRDMA_CQE_INV_EEC_STATE_ERR: 2503 ibwc_status = IB_WC_INV_EEC_STATE_ERR; 2504 break; 2505 case OCRDMA_CQE_FATAL_ERR: 2506 ibwc_status = IB_WC_FATAL_ERR; 2507 break; 2508 case OCRDMA_CQE_RESP_TIMEOUT_ERR: 2509 ibwc_status = IB_WC_RESP_TIMEOUT_ERR; 2510 break; 2511 default: 2512 ibwc_status = IB_WC_GENERAL_ERR; 2513 break; 2514 } 2515 return ibwc_status; 2516 } 2517 2518 static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc, 2519 u32 wqe_idx) 2520 { 2521 struct ocrdma_hdr_wqe *hdr; 2522 struct ocrdma_sge *rw; 2523 int opcode; 2524 2525 hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx); 2526 2527 ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid; 2528 /* Undo the hdr->cw swap */ 2529 opcode = le32_to_cpu(hdr->cw) & OCRDMA_WQE_OPCODE_MASK; 2530 switch (opcode) { 2531 case OCRDMA_WRITE: 2532 ibwc->opcode = IB_WC_RDMA_WRITE; 2533 break; 2534 case OCRDMA_READ: 2535 rw = (struct ocrdma_sge *)(hdr + 1); 2536 ibwc->opcode = IB_WC_RDMA_READ; 2537 ibwc->byte_len = rw->len; 2538 break; 2539 case OCRDMA_SEND: 2540 ibwc->opcode = IB_WC_SEND; 2541 break; 2542 case OCRDMA_FR_MR: 2543 ibwc->opcode = IB_WC_REG_MR; 2544 break; 2545 case OCRDMA_LKEY_INV: 2546 ibwc->opcode = IB_WC_LOCAL_INV; 2547 break; 2548 default: 2549 ibwc->status = IB_WC_GENERAL_ERR; 2550 pr_err("%s() invalid opcode received = 0x%x\n", 2551 __func__, hdr->cw & OCRDMA_WQE_OPCODE_MASK); 2552 break; 2553 } 2554 } 2555 2556 static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp, 2557 struct ocrdma_cqe *cqe) 2558 { 2559 if (is_cqe_for_sq(cqe)) { 2560 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2561 cqe->flags_status_srcqpn) & 2562 ~OCRDMA_CQE_STATUS_MASK); 2563 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2564 cqe->flags_status_srcqpn) | 2565 (OCRDMA_CQE_WR_FLUSH_ERR << 2566 OCRDMA_CQE_STATUS_SHIFT)); 2567 } else { 2568 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 2569 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2570 cqe->flags_status_srcqpn) & 2571 ~OCRDMA_CQE_UD_STATUS_MASK); 2572 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2573 cqe->flags_status_srcqpn) | 2574 (OCRDMA_CQE_WR_FLUSH_ERR << 2575 OCRDMA_CQE_UD_STATUS_SHIFT)); 2576 } else { 2577 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2578 cqe->flags_status_srcqpn) & 2579 ~OCRDMA_CQE_STATUS_MASK); 2580 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2581 cqe->flags_status_srcqpn) | 2582 (OCRDMA_CQE_WR_FLUSH_ERR << 2583 OCRDMA_CQE_STATUS_SHIFT)); 2584 } 2585 } 2586 } 2587 2588 static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2589 struct ocrdma_qp *qp, int status) 2590 { 2591 bool expand = false; 2592 2593 ibwc->byte_len = 0; 2594 ibwc->qp = &qp->ibqp; 2595 ibwc->status = ocrdma_to_ibwc_err(status); 2596 2597 ocrdma_flush_qp(qp); 2598 ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL); 2599 2600 /* if wqe/rqe pending for which cqe needs to be returned, 2601 * trigger inflating it. 2602 */ 2603 if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) { 2604 expand = true; 2605 ocrdma_set_cqe_status_flushed(qp, cqe); 2606 } 2607 return expand; 2608 } 2609 2610 static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2611 struct ocrdma_qp *qp, int status) 2612 { 2613 ibwc->opcode = IB_WC_RECV; 2614 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2615 ocrdma_hwq_inc_tail(&qp->rq); 2616 2617 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); 2618 } 2619 2620 static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2621 struct ocrdma_qp *qp, int status) 2622 { 2623 ocrdma_update_wc(qp, ibwc, qp->sq.tail); 2624 ocrdma_hwq_inc_tail(&qp->sq); 2625 2626 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); 2627 } 2628 2629 2630 static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp, 2631 struct ocrdma_cqe *cqe, struct ib_wc *ibwc, 2632 bool *polled, bool *stop) 2633 { 2634 bool expand; 2635 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 2636 int status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2637 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2638 if (status < OCRDMA_MAX_CQE_ERR) 2639 atomic_inc(&dev->cqe_err_stats[status]); 2640 2641 /* when hw sq is empty, but rq is not empty, so we continue 2642 * to keep the cqe in order to get the cq event again. 2643 */ 2644 if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) { 2645 /* when cq for rq and sq is same, it is safe to return 2646 * flush cqe for RQEs. 2647 */ 2648 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { 2649 *polled = true; 2650 status = OCRDMA_CQE_WR_FLUSH_ERR; 2651 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); 2652 } else { 2653 /* stop processing further cqe as this cqe is used for 2654 * triggering cq event on buddy cq of RQ. 2655 * When QP is destroyed, this cqe will be removed 2656 * from the cq's hardware q. 2657 */ 2658 *polled = false; 2659 *stop = true; 2660 expand = false; 2661 } 2662 } else if (is_hw_sq_empty(qp)) { 2663 /* Do nothing */ 2664 expand = false; 2665 *polled = false; 2666 *stop = false; 2667 } else { 2668 *polled = true; 2669 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); 2670 } 2671 return expand; 2672 } 2673 2674 static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp, 2675 struct ocrdma_cqe *cqe, 2676 struct ib_wc *ibwc, bool *polled) 2677 { 2678 bool expand = false; 2679 int tail = qp->sq.tail; 2680 u32 wqe_idx; 2681 2682 if (!qp->wqe_wr_id_tbl[tail].signaled) { 2683 *polled = false; /* WC cannot be consumed yet */ 2684 } else { 2685 ibwc->status = IB_WC_SUCCESS; 2686 ibwc->wc_flags = 0; 2687 ibwc->qp = &qp->ibqp; 2688 ocrdma_update_wc(qp, ibwc, tail); 2689 *polled = true; 2690 } 2691 wqe_idx = (le32_to_cpu(cqe->wq.wqeidx) & 2692 OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx; 2693 if (tail != wqe_idx) 2694 expand = true; /* Coalesced CQE can't be consumed yet */ 2695 2696 ocrdma_hwq_inc_tail(&qp->sq); 2697 return expand; 2698 } 2699 2700 static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2701 struct ib_wc *ibwc, bool *polled, bool *stop) 2702 { 2703 int status; 2704 bool expand; 2705 2706 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2707 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2708 2709 if (status == OCRDMA_CQE_SUCCESS) 2710 expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled); 2711 else 2712 expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop); 2713 return expand; 2714 } 2715 2716 static int ocrdma_update_ud_rcqe(struct ocrdma_dev *dev, struct ib_wc *ibwc, 2717 struct ocrdma_cqe *cqe) 2718 { 2719 int status; 2720 u16 hdr_type = 0; 2721 2722 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2723 OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT; 2724 ibwc->src_qp = le32_to_cpu(cqe->flags_status_srcqpn) & 2725 OCRDMA_CQE_SRCQP_MASK; 2726 ibwc->pkey_index = 0; 2727 ibwc->wc_flags = IB_WC_GRH; 2728 ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >> 2729 OCRDMA_CQE_UD_XFER_LEN_SHIFT) & 2730 OCRDMA_CQE_UD_XFER_LEN_MASK; 2731 2732 if (ocrdma_is_udp_encap_supported(dev)) { 2733 hdr_type = (le32_to_cpu(cqe->ud.rxlen_pkey) >> 2734 OCRDMA_CQE_UD_L3TYPE_SHIFT) & 2735 OCRDMA_CQE_UD_L3TYPE_MASK; 2736 ibwc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 2737 ibwc->network_hdr_type = hdr_type; 2738 } 2739 2740 return status; 2741 } 2742 2743 static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc, 2744 struct ocrdma_cqe *cqe, 2745 struct ocrdma_qp *qp) 2746 { 2747 unsigned long flags; 2748 struct ocrdma_srq *srq; 2749 u32 wqe_idx; 2750 2751 srq = get_ocrdma_srq(qp->ibqp.srq); 2752 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >> 2753 OCRDMA_CQE_BUFTAG_SHIFT) & srq->rq.max_wqe_idx; 2754 BUG_ON(wqe_idx < 1); 2755 2756 ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx]; 2757 spin_lock_irqsave(&srq->q_lock, flags); 2758 ocrdma_srq_toggle_bit(srq, wqe_idx - 1); 2759 spin_unlock_irqrestore(&srq->q_lock, flags); 2760 ocrdma_hwq_inc_tail(&srq->rq); 2761 } 2762 2763 static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2764 struct ib_wc *ibwc, bool *polled, bool *stop, 2765 int status) 2766 { 2767 bool expand; 2768 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 2769 2770 if (status < OCRDMA_MAX_CQE_ERR) 2771 atomic_inc(&dev->cqe_err_stats[status]); 2772 2773 /* when hw_rq is empty, but wq is not empty, so continue 2774 * to keep the cqe to get the cq event again. 2775 */ 2776 if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) { 2777 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { 2778 *polled = true; 2779 status = OCRDMA_CQE_WR_FLUSH_ERR; 2780 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); 2781 } else { 2782 *polled = false; 2783 *stop = true; 2784 expand = false; 2785 } 2786 } else if (is_hw_rq_empty(qp)) { 2787 /* Do nothing */ 2788 expand = false; 2789 *polled = false; 2790 *stop = false; 2791 } else { 2792 *polled = true; 2793 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); 2794 } 2795 return expand; 2796 } 2797 2798 static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp, 2799 struct ocrdma_cqe *cqe, struct ib_wc *ibwc) 2800 { 2801 struct ocrdma_dev *dev; 2802 2803 dev = get_ocrdma_dev(qp->ibqp.device); 2804 ibwc->opcode = IB_WC_RECV; 2805 ibwc->qp = &qp->ibqp; 2806 ibwc->status = IB_WC_SUCCESS; 2807 2808 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) 2809 ocrdma_update_ud_rcqe(dev, ibwc, cqe); 2810 else 2811 ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen); 2812 2813 if (is_cqe_imm(cqe)) { 2814 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt)); 2815 ibwc->wc_flags |= IB_WC_WITH_IMM; 2816 } else if (is_cqe_wr_imm(cqe)) { 2817 ibwc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 2818 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt)); 2819 ibwc->wc_flags |= IB_WC_WITH_IMM; 2820 } else if (is_cqe_invalidated(cqe)) { 2821 ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt); 2822 ibwc->wc_flags |= IB_WC_WITH_INVALIDATE; 2823 } 2824 if (qp->ibqp.srq) { 2825 ocrdma_update_free_srq_cqe(ibwc, cqe, qp); 2826 } else { 2827 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2828 ocrdma_hwq_inc_tail(&qp->rq); 2829 } 2830 } 2831 2832 static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2833 struct ib_wc *ibwc, bool *polled, bool *stop) 2834 { 2835 int status; 2836 bool expand = false; 2837 2838 ibwc->wc_flags = 0; 2839 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 2840 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2841 OCRDMA_CQE_UD_STATUS_MASK) >> 2842 OCRDMA_CQE_UD_STATUS_SHIFT; 2843 } else { 2844 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2845 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2846 } 2847 2848 if (status == OCRDMA_CQE_SUCCESS) { 2849 *polled = true; 2850 ocrdma_poll_success_rcqe(qp, cqe, ibwc); 2851 } else { 2852 expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop, 2853 status); 2854 } 2855 return expand; 2856 } 2857 2858 static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe, 2859 u16 cur_getp) 2860 { 2861 if (cq->phase_change) { 2862 if (cur_getp == 0) 2863 cq->phase = (~cq->phase & OCRDMA_CQE_VALID); 2864 } else { 2865 /* clear valid bit */ 2866 cqe->flags_status_srcqpn = 0; 2867 } 2868 } 2869 2870 static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries, 2871 struct ib_wc *ibwc) 2872 { 2873 u16 qpn = 0; 2874 int i = 0; 2875 bool expand = false; 2876 int polled_hw_cqes = 0; 2877 struct ocrdma_qp *qp = NULL; 2878 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device); 2879 struct ocrdma_cqe *cqe; 2880 u16 cur_getp; bool polled = false; bool stop = false; 2881 2882 cur_getp = cq->getp; 2883 while (num_entries) { 2884 cqe = cq->va + cur_getp; 2885 /* check whether valid cqe or not */ 2886 if (!is_cqe_valid(cq, cqe)) 2887 break; 2888 qpn = (le32_to_cpu(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK); 2889 /* ignore discarded cqe */ 2890 if (qpn == 0) 2891 goto skip_cqe; 2892 qp = dev->qp_tbl[qpn]; 2893 BUG_ON(qp == NULL); 2894 2895 if (is_cqe_for_sq(cqe)) { 2896 expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled, 2897 &stop); 2898 } else { 2899 expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled, 2900 &stop); 2901 } 2902 if (expand) 2903 goto expand_cqe; 2904 if (stop) 2905 goto stop_cqe; 2906 /* clear qpn to avoid duplicate processing by discard_cqe() */ 2907 cqe->cmn.qpn = 0; 2908 skip_cqe: 2909 polled_hw_cqes += 1; 2910 cur_getp = (cur_getp + 1) % cq->max_hw_cqe; 2911 ocrdma_change_cq_phase(cq, cqe, cur_getp); 2912 expand_cqe: 2913 if (polled) { 2914 num_entries -= 1; 2915 i += 1; 2916 ibwc = ibwc + 1; 2917 polled = false; 2918 } 2919 } 2920 stop_cqe: 2921 cq->getp = cur_getp; 2922 2923 if (polled_hw_cqes) 2924 ocrdma_ring_cq_db(dev, cq->id, false, false, polled_hw_cqes); 2925 2926 return i; 2927 } 2928 2929 /* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */ 2930 static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries, 2931 struct ocrdma_qp *qp, struct ib_wc *ibwc) 2932 { 2933 int err_cqes = 0; 2934 2935 while (num_entries) { 2936 if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp)) 2937 break; 2938 if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) { 2939 ocrdma_update_wc(qp, ibwc, qp->sq.tail); 2940 ocrdma_hwq_inc_tail(&qp->sq); 2941 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) { 2942 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2943 ocrdma_hwq_inc_tail(&qp->rq); 2944 } else { 2945 return err_cqes; 2946 } 2947 ibwc->byte_len = 0; 2948 ibwc->status = IB_WC_WR_FLUSH_ERR; 2949 ibwc = ibwc + 1; 2950 err_cqes += 1; 2951 num_entries -= 1; 2952 } 2953 return err_cqes; 2954 } 2955 2956 int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 2957 { 2958 int cqes_to_poll = num_entries; 2959 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 2960 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 2961 int num_os_cqe = 0, err_cqes = 0; 2962 struct ocrdma_qp *qp; 2963 unsigned long flags; 2964 2965 /* poll cqes from adapter CQ */ 2966 spin_lock_irqsave(&cq->cq_lock, flags); 2967 num_os_cqe = ocrdma_poll_hwcq(cq, cqes_to_poll, wc); 2968 spin_unlock_irqrestore(&cq->cq_lock, flags); 2969 cqes_to_poll -= num_os_cqe; 2970 2971 if (cqes_to_poll) { 2972 wc = wc + num_os_cqe; 2973 /* adapter returns single error cqe when qp moves to 2974 * error state. So insert error cqes with wc_status as 2975 * FLUSHED for pending WQEs and RQEs of QP's SQ and RQ 2976 * respectively which uses this CQ. 2977 */ 2978 spin_lock_irqsave(&dev->flush_q_lock, flags); 2979 list_for_each_entry(qp, &cq->sq_head, sq_entry) { 2980 if (cqes_to_poll == 0) 2981 break; 2982 err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc); 2983 cqes_to_poll -= err_cqes; 2984 num_os_cqe += err_cqes; 2985 wc = wc + err_cqes; 2986 } 2987 spin_unlock_irqrestore(&dev->flush_q_lock, flags); 2988 } 2989 return num_os_cqe; 2990 } 2991 2992 int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags) 2993 { 2994 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 2995 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 2996 u16 cq_id; 2997 unsigned long flags; 2998 bool arm_needed = false, sol_needed = false; 2999 3000 cq_id = cq->id; 3001 3002 spin_lock_irqsave(&cq->cq_lock, flags); 3003 if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED) 3004 arm_needed = true; 3005 if (cq_flags & IB_CQ_SOLICITED) 3006 sol_needed = true; 3007 3008 ocrdma_ring_cq_db(dev, cq_id, arm_needed, sol_needed, 0); 3009 spin_unlock_irqrestore(&cq->cq_lock, flags); 3010 3011 return 0; 3012 } 3013 3014 struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, 3015 enum ib_mr_type mr_type, 3016 u32 max_num_sg) 3017 { 3018 int status; 3019 struct ocrdma_mr *mr; 3020 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 3021 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 3022 3023 if (mr_type != IB_MR_TYPE_MEM_REG) 3024 return ERR_PTR(-EINVAL); 3025 3026 if (max_num_sg > dev->attr.max_pages_per_frmr) 3027 return ERR_PTR(-EINVAL); 3028 3029 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3030 if (!mr) 3031 return ERR_PTR(-ENOMEM); 3032 3033 mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); 3034 if (!mr->pages) { 3035 status = -ENOMEM; 3036 goto pl_err; 3037 } 3038 3039 status = ocrdma_get_pbl_info(dev, mr, max_num_sg); 3040 if (status) 3041 goto pbl_err; 3042 mr->hwmr.fr_mr = 1; 3043 mr->hwmr.remote_rd = 0; 3044 mr->hwmr.remote_wr = 0; 3045 mr->hwmr.local_rd = 0; 3046 mr->hwmr.local_wr = 0; 3047 mr->hwmr.mw_bind = 0; 3048 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr); 3049 if (status) 3050 goto pbl_err; 3051 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0); 3052 if (status) 3053 goto mbx_err; 3054 mr->ibmr.rkey = mr->hwmr.lkey; 3055 mr->ibmr.lkey = mr->hwmr.lkey; 3056 dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] = 3057 (unsigned long) mr; 3058 return &mr->ibmr; 3059 mbx_err: 3060 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 3061 pbl_err: 3062 kfree(mr->pages); 3063 pl_err: 3064 kfree(mr); 3065 return ERR_PTR(-ENOMEM); 3066 } 3067 3068 static int ocrdma_set_page(struct ib_mr *ibmr, u64 addr) 3069 { 3070 struct ocrdma_mr *mr = get_ocrdma_mr(ibmr); 3071 3072 if (unlikely(mr->npages == mr->hwmr.num_pbes)) 3073 return -ENOMEM; 3074 3075 mr->pages[mr->npages++] = addr; 3076 3077 return 0; 3078 } 3079 3080 int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 3081 unsigned int *sg_offset) 3082 { 3083 struct ocrdma_mr *mr = get_ocrdma_mr(ibmr); 3084 3085 mr->npages = 0; 3086 3087 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ocrdma_set_page); 3088 } 3089