1 /* This file is part of the Emulex RoCE Device Driver for 2 * RoCE (RDMA over Converged Ethernet) adapters. 3 * Copyright (C) 2012-2015 Emulex. All rights reserved. 4 * EMULEX and SLI are trademarks of Emulex. 5 * www.emulex.com 6 * 7 * This software is available to you under a choice of one of two licenses. 8 * You may choose to be licensed under the terms of the GNU General Public 9 * License (GPL) Version 2, available from the file COPYING in the main 10 * directory of this source tree, or the BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 16 * - Redistributions of source code must retain the above copyright notice, 17 * this list of conditions and the following disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 30 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 31 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 32 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 33 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * Contact Information: 36 * linux-drivers@emulex.com 37 * 38 * Emulex 39 * 3333 Susan Street 40 * Costa Mesa, CA 92626 41 */ 42 43 #include <linux/dma-mapping.h> 44 #include <rdma/ib_verbs.h> 45 #include <rdma/ib_user_verbs.h> 46 #include <rdma/iw_cm.h> 47 #include <rdma/ib_umem.h> 48 #include <rdma/ib_addr.h> 49 #include <rdma/ib_cache.h> 50 #include <rdma/uverbs_ioctl.h> 51 52 #include "ocrdma.h" 53 #include "ocrdma_hw.h" 54 #include "ocrdma_verbs.h" 55 #include <rdma/ocrdma-abi.h> 56 57 int ocrdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 58 { 59 if (index > 0) 60 return -EINVAL; 61 62 *pkey = 0xffff; 63 return 0; 64 } 65 66 int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 67 struct ib_udata *uhw) 68 { 69 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 70 71 if (uhw->inlen || uhw->outlen) 72 return -EINVAL; 73 74 memset(attr, 0, sizeof *attr); 75 memcpy(&attr->fw_ver, &dev->attr.fw_ver[0], 76 min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver))); 77 ocrdma_get_guid(dev, (u8 *)&attr->sys_image_guid); 78 attr->max_mr_size = dev->attr.max_mr_size; 79 attr->page_size_cap = 0xffff000; 80 attr->vendor_id = dev->nic_info.pdev->vendor; 81 attr->vendor_part_id = dev->nic_info.pdev->device; 82 attr->hw_ver = dev->asic_id; 83 attr->max_qp = dev->attr.max_qp; 84 attr->max_ah = OCRDMA_MAX_AH; 85 attr->max_qp_wr = dev->attr.max_wqe; 86 87 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 88 IB_DEVICE_RC_RNR_NAK_GEN | 89 IB_DEVICE_SHUTDOWN_PORT | 90 IB_DEVICE_SYS_IMAGE_GUID | 91 IB_DEVICE_LOCAL_DMA_LKEY | 92 IB_DEVICE_MEM_MGT_EXTENSIONS; 93 attr->max_send_sge = dev->attr.max_send_sge; 94 attr->max_recv_sge = dev->attr.max_recv_sge; 95 attr->max_sge_rd = dev->attr.max_rdma_sge; 96 attr->max_cq = dev->attr.max_cq; 97 attr->max_cqe = dev->attr.max_cqe; 98 attr->max_mr = dev->attr.max_mr; 99 attr->max_mw = dev->attr.max_mw; 100 attr->max_pd = dev->attr.max_pd; 101 attr->atomic_cap = 0; 102 attr->max_fmr = 0; 103 attr->max_map_per_fmr = 0; 104 attr->max_qp_rd_atom = 105 min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp); 106 attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp; 107 attr->max_srq = dev->attr.max_srq; 108 attr->max_srq_sge = dev->attr.max_srq_sge; 109 attr->max_srq_wr = dev->attr.max_rqe; 110 attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay; 111 attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr; 112 attr->max_pkeys = 1; 113 return 0; 114 } 115 116 static inline void get_link_speed_and_width(struct ocrdma_dev *dev, 117 u8 *ib_speed, u8 *ib_width) 118 { 119 int status; 120 u8 speed; 121 122 status = ocrdma_mbx_get_link_speed(dev, &speed, NULL); 123 if (status) 124 speed = OCRDMA_PHYS_LINK_SPEED_ZERO; 125 126 switch (speed) { 127 case OCRDMA_PHYS_LINK_SPEED_1GBPS: 128 *ib_speed = IB_SPEED_SDR; 129 *ib_width = IB_WIDTH_1X; 130 break; 131 132 case OCRDMA_PHYS_LINK_SPEED_10GBPS: 133 *ib_speed = IB_SPEED_QDR; 134 *ib_width = IB_WIDTH_1X; 135 break; 136 137 case OCRDMA_PHYS_LINK_SPEED_20GBPS: 138 *ib_speed = IB_SPEED_DDR; 139 *ib_width = IB_WIDTH_4X; 140 break; 141 142 case OCRDMA_PHYS_LINK_SPEED_40GBPS: 143 *ib_speed = IB_SPEED_QDR; 144 *ib_width = IB_WIDTH_4X; 145 break; 146 147 default: 148 /* Unsupported */ 149 *ib_speed = IB_SPEED_SDR; 150 *ib_width = IB_WIDTH_1X; 151 } 152 } 153 154 int ocrdma_query_port(struct ib_device *ibdev, 155 u8 port, struct ib_port_attr *props) 156 { 157 enum ib_port_state port_state; 158 struct ocrdma_dev *dev; 159 struct net_device *netdev; 160 161 /* props being zeroed by the caller, avoid zeroing it here */ 162 dev = get_ocrdma_dev(ibdev); 163 netdev = dev->nic_info.netdev; 164 if (netif_running(netdev) && netif_oper_up(netdev)) { 165 port_state = IB_PORT_ACTIVE; 166 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 167 } else { 168 port_state = IB_PORT_DOWN; 169 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 170 } 171 props->max_mtu = IB_MTU_4096; 172 props->active_mtu = iboe_get_mtu(netdev->mtu); 173 props->lid = 0; 174 props->lmc = 0; 175 props->sm_lid = 0; 176 props->sm_sl = 0; 177 props->state = port_state; 178 props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 179 IB_PORT_DEVICE_MGMT_SUP | 180 IB_PORT_VENDOR_CLASS_SUP; 181 props->ip_gids = true; 182 props->gid_tbl_len = OCRDMA_MAX_SGID; 183 props->pkey_tbl_len = 1; 184 props->bad_pkey_cntr = 0; 185 props->qkey_viol_cntr = 0; 186 get_link_speed_and_width(dev, &props->active_speed, 187 &props->active_width); 188 props->max_msg_sz = 0x80000000; 189 props->max_vl_num = 4; 190 return 0; 191 } 192 193 static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 194 unsigned long len) 195 { 196 struct ocrdma_mm *mm; 197 198 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 199 if (mm == NULL) 200 return -ENOMEM; 201 mm->key.phy_addr = phy_addr; 202 mm->key.len = len; 203 INIT_LIST_HEAD(&mm->entry); 204 205 mutex_lock(&uctx->mm_list_lock); 206 list_add_tail(&mm->entry, &uctx->mm_head); 207 mutex_unlock(&uctx->mm_list_lock); 208 return 0; 209 } 210 211 static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 212 unsigned long len) 213 { 214 struct ocrdma_mm *mm, *tmp; 215 216 mutex_lock(&uctx->mm_list_lock); 217 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 218 if (len != mm->key.len && phy_addr != mm->key.phy_addr) 219 continue; 220 221 list_del(&mm->entry); 222 kfree(mm); 223 break; 224 } 225 mutex_unlock(&uctx->mm_list_lock); 226 } 227 228 static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 229 unsigned long len) 230 { 231 bool found = false; 232 struct ocrdma_mm *mm; 233 234 mutex_lock(&uctx->mm_list_lock); 235 list_for_each_entry(mm, &uctx->mm_head, entry) { 236 if (len != mm->key.len && phy_addr != mm->key.phy_addr) 237 continue; 238 239 found = true; 240 break; 241 } 242 mutex_unlock(&uctx->mm_list_lock); 243 return found; 244 } 245 246 247 static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool) 248 { 249 u16 pd_bitmap_idx = 0; 250 const unsigned long *pd_bitmap; 251 252 if (dpp_pool) { 253 pd_bitmap = dev->pd_mgr->pd_dpp_bitmap; 254 pd_bitmap_idx = find_first_zero_bit(pd_bitmap, 255 dev->pd_mgr->max_dpp_pd); 256 __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_dpp_bitmap); 257 dev->pd_mgr->pd_dpp_count++; 258 if (dev->pd_mgr->pd_dpp_count > dev->pd_mgr->pd_dpp_thrsh) 259 dev->pd_mgr->pd_dpp_thrsh = dev->pd_mgr->pd_dpp_count; 260 } else { 261 pd_bitmap = dev->pd_mgr->pd_norm_bitmap; 262 pd_bitmap_idx = find_first_zero_bit(pd_bitmap, 263 dev->pd_mgr->max_normal_pd); 264 __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_norm_bitmap); 265 dev->pd_mgr->pd_norm_count++; 266 if (dev->pd_mgr->pd_norm_count > dev->pd_mgr->pd_norm_thrsh) 267 dev->pd_mgr->pd_norm_thrsh = dev->pd_mgr->pd_norm_count; 268 } 269 return pd_bitmap_idx; 270 } 271 272 static int _ocrdma_pd_mgr_put_bitmap(struct ocrdma_dev *dev, u16 pd_id, 273 bool dpp_pool) 274 { 275 u16 pd_count; 276 u16 pd_bit_index; 277 278 pd_count = dpp_pool ? dev->pd_mgr->pd_dpp_count : 279 dev->pd_mgr->pd_norm_count; 280 if (pd_count == 0) 281 return -EINVAL; 282 283 if (dpp_pool) { 284 pd_bit_index = pd_id - dev->pd_mgr->pd_dpp_start; 285 if (pd_bit_index >= dev->pd_mgr->max_dpp_pd) { 286 return -EINVAL; 287 } else { 288 __clear_bit(pd_bit_index, dev->pd_mgr->pd_dpp_bitmap); 289 dev->pd_mgr->pd_dpp_count--; 290 } 291 } else { 292 pd_bit_index = pd_id - dev->pd_mgr->pd_norm_start; 293 if (pd_bit_index >= dev->pd_mgr->max_normal_pd) { 294 return -EINVAL; 295 } else { 296 __clear_bit(pd_bit_index, dev->pd_mgr->pd_norm_bitmap); 297 dev->pd_mgr->pd_norm_count--; 298 } 299 } 300 301 return 0; 302 } 303 304 static int ocrdma_put_pd_num(struct ocrdma_dev *dev, u16 pd_id, 305 bool dpp_pool) 306 { 307 int status; 308 309 mutex_lock(&dev->dev_lock); 310 status = _ocrdma_pd_mgr_put_bitmap(dev, pd_id, dpp_pool); 311 mutex_unlock(&dev->dev_lock); 312 return status; 313 } 314 315 static int ocrdma_get_pd_num(struct ocrdma_dev *dev, struct ocrdma_pd *pd) 316 { 317 u16 pd_idx = 0; 318 int status = 0; 319 320 mutex_lock(&dev->dev_lock); 321 if (pd->dpp_enabled) { 322 /* try allocating DPP PD, if not available then normal PD */ 323 if (dev->pd_mgr->pd_dpp_count < dev->pd_mgr->max_dpp_pd) { 324 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, true); 325 pd->id = dev->pd_mgr->pd_dpp_start + pd_idx; 326 pd->dpp_page = dev->pd_mgr->dpp_page_index + pd_idx; 327 } else if (dev->pd_mgr->pd_norm_count < 328 dev->pd_mgr->max_normal_pd) { 329 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false); 330 pd->id = dev->pd_mgr->pd_norm_start + pd_idx; 331 pd->dpp_enabled = false; 332 } else { 333 status = -EINVAL; 334 } 335 } else { 336 if (dev->pd_mgr->pd_norm_count < dev->pd_mgr->max_normal_pd) { 337 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false); 338 pd->id = dev->pd_mgr->pd_norm_start + pd_idx; 339 } else { 340 status = -EINVAL; 341 } 342 } 343 mutex_unlock(&dev->dev_lock); 344 return status; 345 } 346 347 /* 348 * NOTE: 349 * 350 * ocrdma_ucontext must be used here because this function is also 351 * called from ocrdma_alloc_ucontext where ib_udata does not have 352 * valid ib_ucontext pointer. ib_uverbs_get_context does not call 353 * uobj_{alloc|get_xxx} helpers which are used to store the 354 * ib_ucontext in uverbs_attr_bundle wrapping the ib_udata. so 355 * ib_udata does NOT imply valid ib_ucontext here! 356 */ 357 static int _ocrdma_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd, 358 struct ocrdma_ucontext *uctx, 359 struct ib_udata *udata) 360 { 361 int status; 362 363 if (udata && uctx && dev->attr.max_dpp_pds) { 364 pd->dpp_enabled = 365 ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R; 366 pd->num_dpp_qp = 367 pd->dpp_enabled ? (dev->nic_info.db_page_size / 368 dev->attr.wqe_size) : 0; 369 } 370 371 if (dev->pd_mgr->pd_prealloc_valid) 372 return ocrdma_get_pd_num(dev, pd); 373 374 retry: 375 status = ocrdma_mbx_alloc_pd(dev, pd); 376 if (status) { 377 if (pd->dpp_enabled) { 378 pd->dpp_enabled = false; 379 pd->num_dpp_qp = 0; 380 goto retry; 381 } 382 return status; 383 } 384 385 return 0; 386 } 387 388 static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx, 389 struct ocrdma_pd *pd) 390 { 391 return (uctx->cntxt_pd == pd); 392 } 393 394 static void _ocrdma_dealloc_pd(struct ocrdma_dev *dev, 395 struct ocrdma_pd *pd) 396 { 397 if (dev->pd_mgr->pd_prealloc_valid) 398 ocrdma_put_pd_num(dev, pd->id, pd->dpp_enabled); 399 else 400 ocrdma_mbx_dealloc_pd(dev, pd); 401 } 402 403 static int ocrdma_alloc_ucontext_pd(struct ocrdma_dev *dev, 404 struct ocrdma_ucontext *uctx, 405 struct ib_udata *udata) 406 { 407 struct ib_device *ibdev = &dev->ibdev; 408 struct ib_pd *pd; 409 int status; 410 411 pd = rdma_zalloc_drv_obj(ibdev, ib_pd); 412 if (!pd) 413 return -ENOMEM; 414 415 pd->device = ibdev; 416 uctx->cntxt_pd = get_ocrdma_pd(pd); 417 418 status = _ocrdma_alloc_pd(dev, uctx->cntxt_pd, uctx, udata); 419 if (status) { 420 kfree(uctx->cntxt_pd); 421 goto err; 422 } 423 424 uctx->cntxt_pd->uctx = uctx; 425 uctx->cntxt_pd->ibpd.device = &dev->ibdev; 426 err: 427 return status; 428 } 429 430 static void ocrdma_dealloc_ucontext_pd(struct ocrdma_ucontext *uctx) 431 { 432 struct ocrdma_pd *pd = uctx->cntxt_pd; 433 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device); 434 435 if (uctx->pd_in_use) { 436 pr_err("%s(%d) Freeing in use pdid=0x%x.\n", 437 __func__, dev->id, pd->id); 438 } 439 kfree(uctx->cntxt_pd); 440 uctx->cntxt_pd = NULL; 441 _ocrdma_dealloc_pd(dev, pd); 442 } 443 444 static struct ocrdma_pd *ocrdma_get_ucontext_pd(struct ocrdma_ucontext *uctx) 445 { 446 struct ocrdma_pd *pd = NULL; 447 448 mutex_lock(&uctx->mm_list_lock); 449 if (!uctx->pd_in_use) { 450 uctx->pd_in_use = true; 451 pd = uctx->cntxt_pd; 452 } 453 mutex_unlock(&uctx->mm_list_lock); 454 455 return pd; 456 } 457 458 static void ocrdma_release_ucontext_pd(struct ocrdma_ucontext *uctx) 459 { 460 mutex_lock(&uctx->mm_list_lock); 461 uctx->pd_in_use = false; 462 mutex_unlock(&uctx->mm_list_lock); 463 } 464 465 int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 466 { 467 struct ib_device *ibdev = uctx->device; 468 int status; 469 struct ocrdma_ucontext *ctx = get_ocrdma_ucontext(uctx); 470 struct ocrdma_alloc_ucontext_resp resp = {}; 471 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 472 struct pci_dev *pdev = dev->nic_info.pdev; 473 u32 map_len = roundup(sizeof(u32) * 2048, PAGE_SIZE); 474 475 if (!udata) 476 return -EFAULT; 477 INIT_LIST_HEAD(&ctx->mm_head); 478 mutex_init(&ctx->mm_list_lock); 479 480 ctx->ah_tbl.va = dma_alloc_coherent(&pdev->dev, map_len, 481 &ctx->ah_tbl.pa, GFP_KERNEL); 482 if (!ctx->ah_tbl.va) 483 return -ENOMEM; 484 485 ctx->ah_tbl.len = map_len; 486 487 resp.ah_tbl_len = ctx->ah_tbl.len; 488 resp.ah_tbl_page = virt_to_phys(ctx->ah_tbl.va); 489 490 status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len); 491 if (status) 492 goto map_err; 493 494 status = ocrdma_alloc_ucontext_pd(dev, ctx, udata); 495 if (status) 496 goto pd_err; 497 498 resp.dev_id = dev->id; 499 resp.max_inline_data = dev->attr.max_inline_data; 500 resp.wqe_size = dev->attr.wqe_size; 501 resp.rqe_size = dev->attr.rqe_size; 502 resp.dpp_wqe_size = dev->attr.wqe_size; 503 504 memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver)); 505 status = ib_copy_to_udata(udata, &resp, sizeof(resp)); 506 if (status) 507 goto cpy_err; 508 return 0; 509 510 cpy_err: 511 ocrdma_dealloc_ucontext_pd(ctx); 512 pd_err: 513 ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len); 514 map_err: 515 dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va, 516 ctx->ah_tbl.pa); 517 return status; 518 } 519 520 void ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx) 521 { 522 struct ocrdma_mm *mm, *tmp; 523 struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx); 524 struct ocrdma_dev *dev = get_ocrdma_dev(ibctx->device); 525 struct pci_dev *pdev = dev->nic_info.pdev; 526 527 ocrdma_dealloc_ucontext_pd(uctx); 528 529 ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len); 530 dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va, 531 uctx->ah_tbl.pa); 532 533 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 534 list_del(&mm->entry); 535 kfree(mm); 536 } 537 } 538 539 int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 540 { 541 struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context); 542 struct ocrdma_dev *dev = get_ocrdma_dev(context->device); 543 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 544 u64 unmapped_db = (u64) dev->nic_info.unmapped_db; 545 unsigned long len = (vma->vm_end - vma->vm_start); 546 int status; 547 bool found; 548 549 if (vma->vm_start & (PAGE_SIZE - 1)) 550 return -EINVAL; 551 found = ocrdma_search_mmap(ucontext, vma->vm_pgoff << PAGE_SHIFT, len); 552 if (!found) 553 return -EINVAL; 554 555 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + 556 dev->nic_info.db_total_size)) && 557 (len <= dev->nic_info.db_page_size)) { 558 if (vma->vm_flags & VM_READ) 559 return -EPERM; 560 561 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 562 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 563 len, vma->vm_page_prot); 564 } else if (dev->nic_info.dpp_unmapped_len && 565 (vm_page >= (u64) dev->nic_info.dpp_unmapped_addr) && 566 (vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr + 567 dev->nic_info.dpp_unmapped_len)) && 568 (len <= dev->nic_info.dpp_unmapped_len)) { 569 if (vma->vm_flags & VM_READ) 570 return -EPERM; 571 572 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 573 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 574 len, vma->vm_page_prot); 575 } else { 576 status = remap_pfn_range(vma, vma->vm_start, 577 vma->vm_pgoff, len, vma->vm_page_prot); 578 } 579 return status; 580 } 581 582 static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd, 583 struct ib_udata *udata) 584 { 585 int status; 586 u64 db_page_addr; 587 u64 dpp_page_addr = 0; 588 u32 db_page_size; 589 struct ocrdma_alloc_pd_uresp rsp; 590 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 591 udata, struct ocrdma_ucontext, ibucontext); 592 593 memset(&rsp, 0, sizeof(rsp)); 594 rsp.id = pd->id; 595 rsp.dpp_enabled = pd->dpp_enabled; 596 db_page_addr = ocrdma_get_db_addr(dev, pd->id); 597 db_page_size = dev->nic_info.db_page_size; 598 599 status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size); 600 if (status) 601 return status; 602 603 if (pd->dpp_enabled) { 604 dpp_page_addr = dev->nic_info.dpp_unmapped_addr + 605 (pd->id * PAGE_SIZE); 606 status = ocrdma_add_mmap(uctx, dpp_page_addr, 607 PAGE_SIZE); 608 if (status) 609 goto dpp_map_err; 610 rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr); 611 rsp.dpp_page_addr_lo = dpp_page_addr; 612 } 613 614 status = ib_copy_to_udata(udata, &rsp, sizeof(rsp)); 615 if (status) 616 goto ucopy_err; 617 618 pd->uctx = uctx; 619 return 0; 620 621 ucopy_err: 622 if (pd->dpp_enabled) 623 ocrdma_del_mmap(pd->uctx, dpp_page_addr, PAGE_SIZE); 624 dpp_map_err: 625 ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size); 626 return status; 627 } 628 629 int ocrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 630 { 631 struct ib_device *ibdev = ibpd->device; 632 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 633 struct ocrdma_pd *pd; 634 int status; 635 u8 is_uctx_pd = false; 636 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 637 udata, struct ocrdma_ucontext, ibucontext); 638 639 if (udata) { 640 pd = ocrdma_get_ucontext_pd(uctx); 641 if (pd) { 642 is_uctx_pd = true; 643 goto pd_mapping; 644 } 645 } 646 647 pd = get_ocrdma_pd(ibpd); 648 status = _ocrdma_alloc_pd(dev, pd, uctx, udata); 649 if (status) 650 goto exit; 651 652 pd_mapping: 653 if (udata) { 654 status = ocrdma_copy_pd_uresp(dev, pd, udata); 655 if (status) 656 goto err; 657 } 658 return 0; 659 660 err: 661 if (is_uctx_pd) 662 ocrdma_release_ucontext_pd(uctx); 663 else 664 _ocrdma_dealloc_pd(dev, pd); 665 exit: 666 return status; 667 } 668 669 void ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 670 { 671 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 672 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 673 struct ocrdma_ucontext *uctx = NULL; 674 u64 usr_db; 675 676 uctx = pd->uctx; 677 if (uctx) { 678 u64 dpp_db = dev->nic_info.dpp_unmapped_addr + 679 (pd->id * PAGE_SIZE); 680 if (pd->dpp_enabled) 681 ocrdma_del_mmap(pd->uctx, dpp_db, PAGE_SIZE); 682 usr_db = ocrdma_get_db_addr(dev, pd->id); 683 ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size); 684 685 if (is_ucontext_pd(uctx, pd)) { 686 ocrdma_release_ucontext_pd(uctx); 687 return; 688 } 689 } 690 _ocrdma_dealloc_pd(dev, pd); 691 } 692 693 static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 694 u32 pdid, int acc, u32 num_pbls, u32 addr_check) 695 { 696 int status; 697 698 mr->hwmr.fr_mr = 0; 699 mr->hwmr.local_rd = 1; 700 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 701 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 702 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 703 mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0; 704 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 705 mr->hwmr.num_pbls = num_pbls; 706 707 status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check); 708 if (status) 709 return status; 710 711 mr->ibmr.lkey = mr->hwmr.lkey; 712 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd) 713 mr->ibmr.rkey = mr->hwmr.lkey; 714 return 0; 715 } 716 717 struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc) 718 { 719 int status; 720 struct ocrdma_mr *mr; 721 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 722 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 723 724 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { 725 pr_err("%s err, invalid access rights\n", __func__); 726 return ERR_PTR(-EINVAL); 727 } 728 729 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 730 if (!mr) 731 return ERR_PTR(-ENOMEM); 732 733 status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0, 734 OCRDMA_ADDR_CHECK_DISABLE); 735 if (status) { 736 kfree(mr); 737 return ERR_PTR(status); 738 } 739 740 return &mr->ibmr; 741 } 742 743 static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev, 744 struct ocrdma_hw_mr *mr) 745 { 746 struct pci_dev *pdev = dev->nic_info.pdev; 747 int i = 0; 748 749 if (mr->pbl_table) { 750 for (i = 0; i < mr->num_pbls; i++) { 751 if (!mr->pbl_table[i].va) 752 continue; 753 dma_free_coherent(&pdev->dev, mr->pbl_size, 754 mr->pbl_table[i].va, 755 mr->pbl_table[i].pa); 756 } 757 kfree(mr->pbl_table); 758 mr->pbl_table = NULL; 759 } 760 } 761 762 static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 763 u32 num_pbes) 764 { 765 u32 num_pbls = 0; 766 u32 idx = 0; 767 int status = 0; 768 u32 pbl_size; 769 770 do { 771 pbl_size = OCRDMA_MIN_HPAGE_SIZE * (1 << idx); 772 if (pbl_size > MAX_OCRDMA_PBL_SIZE) { 773 status = -EFAULT; 774 break; 775 } 776 num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64))); 777 num_pbls = num_pbls / (pbl_size / sizeof(u64)); 778 idx++; 779 } while (num_pbls >= dev->attr.max_num_mr_pbl); 780 781 mr->hwmr.num_pbes = num_pbes; 782 mr->hwmr.num_pbls = num_pbls; 783 mr->hwmr.pbl_size = pbl_size; 784 return status; 785 } 786 787 static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr) 788 { 789 int status = 0; 790 int i; 791 u32 dma_len = mr->pbl_size; 792 struct pci_dev *pdev = dev->nic_info.pdev; 793 void *va; 794 dma_addr_t pa; 795 796 mr->pbl_table = kcalloc(mr->num_pbls, sizeof(struct ocrdma_pbl), 797 GFP_KERNEL); 798 799 if (!mr->pbl_table) 800 return -ENOMEM; 801 802 for (i = 0; i < mr->num_pbls; i++) { 803 va = dma_alloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL); 804 if (!va) { 805 ocrdma_free_mr_pbl_tbl(dev, mr); 806 status = -ENOMEM; 807 break; 808 } 809 mr->pbl_table[i].va = va; 810 mr->pbl_table[i].pa = pa; 811 } 812 return status; 813 } 814 815 static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 816 u32 num_pbes) 817 { 818 struct ocrdma_pbe *pbe; 819 struct sg_dma_page_iter sg_iter; 820 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; 821 struct ib_umem *umem = mr->umem; 822 int pbe_cnt, total_num_pbes = 0; 823 u64 pg_addr; 824 825 if (!mr->hwmr.num_pbes) 826 return; 827 828 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 829 pbe_cnt = 0; 830 831 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { 832 /* store the page address in pbe */ 833 pg_addr = sg_page_iter_dma_address(&sg_iter); 834 pbe->pa_lo = cpu_to_le32(pg_addr); 835 pbe->pa_hi = cpu_to_le32(upper_32_bits(pg_addr)); 836 pbe_cnt += 1; 837 total_num_pbes += 1; 838 pbe++; 839 840 /* if done building pbes, issue the mbx cmd. */ 841 if (total_num_pbes == num_pbes) 842 return; 843 844 /* if the given pbl is full storing the pbes, 845 * move to next pbl. 846 */ 847 if (pbe_cnt == (mr->hwmr.pbl_size / sizeof(u64))) { 848 pbl_tbl++; 849 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 850 pbe_cnt = 0; 851 } 852 } 853 } 854 855 struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 856 u64 usr_addr, int acc, struct ib_udata *udata) 857 { 858 int status = -ENOMEM; 859 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 860 struct ocrdma_mr *mr; 861 struct ocrdma_pd *pd; 862 u32 num_pbes; 863 864 pd = get_ocrdma_pd(ibpd); 865 866 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 867 return ERR_PTR(-EINVAL); 868 869 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 870 if (!mr) 871 return ERR_PTR(status); 872 mr->umem = ib_umem_get(ibpd->device, start, len, acc); 873 if (IS_ERR(mr->umem)) { 874 status = -EFAULT; 875 goto umem_err; 876 } 877 num_pbes = ib_umem_page_count(mr->umem); 878 status = ocrdma_get_pbl_info(dev, mr, num_pbes); 879 if (status) 880 goto umem_err; 881 882 mr->hwmr.pbe_size = PAGE_SIZE; 883 mr->hwmr.fbo = ib_umem_offset(mr->umem); 884 mr->hwmr.va = usr_addr; 885 mr->hwmr.len = len; 886 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 887 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 888 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 889 mr->hwmr.local_rd = 1; 890 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 891 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr); 892 if (status) 893 goto umem_err; 894 build_user_pbes(dev, mr, num_pbes); 895 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc); 896 if (status) 897 goto mbx_err; 898 mr->ibmr.lkey = mr->hwmr.lkey; 899 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd) 900 mr->ibmr.rkey = mr->hwmr.lkey; 901 902 return &mr->ibmr; 903 904 mbx_err: 905 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 906 umem_err: 907 kfree(mr); 908 return ERR_PTR(status); 909 } 910 911 int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 912 { 913 struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr); 914 struct ocrdma_dev *dev = get_ocrdma_dev(ib_mr->device); 915 916 (void) ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey); 917 918 kfree(mr->pages); 919 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 920 921 /* it could be user registered memory. */ 922 ib_umem_release(mr->umem); 923 kfree(mr); 924 925 /* Don't stop cleanup, in case FW is unresponsive */ 926 if (dev->mqe_ctx.fw_error_state) { 927 pr_err("%s(%d) fw not responding.\n", 928 __func__, dev->id); 929 } 930 return 0; 931 } 932 933 static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq, 934 struct ib_udata *udata) 935 { 936 int status; 937 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 938 udata, struct ocrdma_ucontext, ibucontext); 939 struct ocrdma_create_cq_uresp uresp; 940 941 /* this must be user flow! */ 942 if (!udata) 943 return -EINVAL; 944 945 memset(&uresp, 0, sizeof(uresp)); 946 uresp.cq_id = cq->id; 947 uresp.page_size = PAGE_ALIGN(cq->len); 948 uresp.num_pages = 1; 949 uresp.max_hw_cqe = cq->max_hw_cqe; 950 uresp.page_addr[0] = virt_to_phys(cq->va); 951 uresp.db_page_addr = ocrdma_get_db_addr(dev, uctx->cntxt_pd->id); 952 uresp.db_page_size = dev->nic_info.db_page_size; 953 uresp.phase_change = cq->phase_change ? 1 : 0; 954 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 955 if (status) { 956 pr_err("%s(%d) copy error cqid=0x%x.\n", 957 __func__, dev->id, cq->id); 958 goto err; 959 } 960 status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); 961 if (status) 962 goto err; 963 status = ocrdma_add_mmap(uctx, uresp.page_addr[0], uresp.page_size); 964 if (status) { 965 ocrdma_del_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); 966 goto err; 967 } 968 cq->ucontext = uctx; 969 err: 970 return status; 971 } 972 973 int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 974 struct ib_udata *udata) 975 { 976 struct ib_device *ibdev = ibcq->device; 977 int entries = attr->cqe; 978 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 979 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 980 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 981 udata, struct ocrdma_ucontext, ibucontext); 982 u16 pd_id = 0; 983 int status; 984 struct ocrdma_create_cq_ureq ureq; 985 986 if (attr->flags) 987 return -EINVAL; 988 989 if (udata) { 990 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 991 return -EFAULT; 992 } else 993 ureq.dpp_cq = 0; 994 995 spin_lock_init(&cq->cq_lock); 996 spin_lock_init(&cq->comp_handler_lock); 997 INIT_LIST_HEAD(&cq->sq_head); 998 INIT_LIST_HEAD(&cq->rq_head); 999 1000 if (udata) 1001 pd_id = uctx->cntxt_pd->id; 1002 1003 status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id); 1004 if (status) 1005 return status; 1006 1007 if (udata) { 1008 status = ocrdma_copy_cq_uresp(dev, cq, udata); 1009 if (status) 1010 goto ctx_err; 1011 } 1012 cq->phase = OCRDMA_CQE_VALID; 1013 dev->cq_tbl[cq->id] = cq; 1014 return 0; 1015 1016 ctx_err: 1017 ocrdma_mbx_destroy_cq(dev, cq); 1018 return status; 1019 } 1020 1021 int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt, 1022 struct ib_udata *udata) 1023 { 1024 int status = 0; 1025 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 1026 1027 if (new_cnt < 1 || new_cnt > cq->max_hw_cqe) { 1028 status = -EINVAL; 1029 return status; 1030 } 1031 ibcq->cqe = new_cnt; 1032 return status; 1033 } 1034 1035 static void ocrdma_flush_cq(struct ocrdma_cq *cq) 1036 { 1037 int cqe_cnt; 1038 int valid_count = 0; 1039 unsigned long flags; 1040 1041 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device); 1042 struct ocrdma_cqe *cqe = NULL; 1043 1044 cqe = cq->va; 1045 cqe_cnt = cq->cqe_cnt; 1046 1047 /* Last irq might have scheduled a polling thread 1048 * sync-up with it before hard flushing. 1049 */ 1050 spin_lock_irqsave(&cq->cq_lock, flags); 1051 while (cqe_cnt) { 1052 if (is_cqe_valid(cq, cqe)) 1053 valid_count++; 1054 cqe++; 1055 cqe_cnt--; 1056 } 1057 ocrdma_ring_cq_db(dev, cq->id, false, false, valid_count); 1058 spin_unlock_irqrestore(&cq->cq_lock, flags); 1059 } 1060 1061 void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1062 { 1063 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 1064 struct ocrdma_eq *eq = NULL; 1065 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 1066 int pdid = 0; 1067 u32 irq, indx; 1068 1069 dev->cq_tbl[cq->id] = NULL; 1070 indx = ocrdma_get_eq_table_index(dev, cq->eqn); 1071 1072 eq = &dev->eq_tbl[indx]; 1073 irq = ocrdma_get_irq(dev, eq); 1074 synchronize_irq(irq); 1075 ocrdma_flush_cq(cq); 1076 1077 ocrdma_mbx_destroy_cq(dev, cq); 1078 if (cq->ucontext) { 1079 pdid = cq->ucontext->cntxt_pd->id; 1080 ocrdma_del_mmap(cq->ucontext, (u64) cq->pa, 1081 PAGE_ALIGN(cq->len)); 1082 ocrdma_del_mmap(cq->ucontext, 1083 ocrdma_get_db_addr(dev, pdid), 1084 dev->nic_info.db_page_size); 1085 } 1086 } 1087 1088 static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) 1089 { 1090 int status = -EINVAL; 1091 1092 if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) { 1093 dev->qp_tbl[qp->id] = qp; 1094 status = 0; 1095 } 1096 return status; 1097 } 1098 1099 static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) 1100 { 1101 dev->qp_tbl[qp->id] = NULL; 1102 } 1103 1104 static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev, 1105 struct ib_qp_init_attr *attrs, 1106 struct ib_udata *udata) 1107 { 1108 if ((attrs->qp_type != IB_QPT_GSI) && 1109 (attrs->qp_type != IB_QPT_RC) && 1110 (attrs->qp_type != IB_QPT_UC) && 1111 (attrs->qp_type != IB_QPT_UD)) { 1112 pr_err("%s(%d) unsupported qp type=0x%x requested\n", 1113 __func__, dev->id, attrs->qp_type); 1114 return -EINVAL; 1115 } 1116 /* Skip the check for QP1 to support CM size of 128 */ 1117 if ((attrs->qp_type != IB_QPT_GSI) && 1118 (attrs->cap.max_send_wr > dev->attr.max_wqe)) { 1119 pr_err("%s(%d) unsupported send_wr=0x%x requested\n", 1120 __func__, dev->id, attrs->cap.max_send_wr); 1121 pr_err("%s(%d) supported send_wr=0x%x\n", 1122 __func__, dev->id, dev->attr.max_wqe); 1123 return -EINVAL; 1124 } 1125 if (!attrs->srq && (attrs->cap.max_recv_wr > dev->attr.max_rqe)) { 1126 pr_err("%s(%d) unsupported recv_wr=0x%x requested\n", 1127 __func__, dev->id, attrs->cap.max_recv_wr); 1128 pr_err("%s(%d) supported recv_wr=0x%x\n", 1129 __func__, dev->id, dev->attr.max_rqe); 1130 return -EINVAL; 1131 } 1132 if (attrs->cap.max_inline_data > dev->attr.max_inline_data) { 1133 pr_err("%s(%d) unsupported inline data size=0x%x requested\n", 1134 __func__, dev->id, attrs->cap.max_inline_data); 1135 pr_err("%s(%d) supported inline data size=0x%x\n", 1136 __func__, dev->id, dev->attr.max_inline_data); 1137 return -EINVAL; 1138 } 1139 if (attrs->cap.max_send_sge > dev->attr.max_send_sge) { 1140 pr_err("%s(%d) unsupported send_sge=0x%x requested\n", 1141 __func__, dev->id, attrs->cap.max_send_sge); 1142 pr_err("%s(%d) supported send_sge=0x%x\n", 1143 __func__, dev->id, dev->attr.max_send_sge); 1144 return -EINVAL; 1145 } 1146 if (attrs->cap.max_recv_sge > dev->attr.max_recv_sge) { 1147 pr_err("%s(%d) unsupported recv_sge=0x%x requested\n", 1148 __func__, dev->id, attrs->cap.max_recv_sge); 1149 pr_err("%s(%d) supported recv_sge=0x%x\n", 1150 __func__, dev->id, dev->attr.max_recv_sge); 1151 return -EINVAL; 1152 } 1153 /* unprivileged user space cannot create special QP */ 1154 if (udata && attrs->qp_type == IB_QPT_GSI) { 1155 pr_err 1156 ("%s(%d) Userspace can't create special QPs of type=0x%x\n", 1157 __func__, dev->id, attrs->qp_type); 1158 return -EINVAL; 1159 } 1160 /* allow creating only one GSI type of QP */ 1161 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) { 1162 pr_err("%s(%d) GSI special QPs already created.\n", 1163 __func__, dev->id); 1164 return -EINVAL; 1165 } 1166 /* verify consumer QPs are not trying to use GSI QP's CQ */ 1167 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) { 1168 if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) || 1169 (dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) { 1170 pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n", 1171 __func__, dev->id); 1172 return -EINVAL; 1173 } 1174 } 1175 return 0; 1176 } 1177 1178 static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp, 1179 struct ib_udata *udata, int dpp_offset, 1180 int dpp_credit_lmt, int srq) 1181 { 1182 int status; 1183 u64 usr_db; 1184 struct ocrdma_create_qp_uresp uresp; 1185 struct ocrdma_pd *pd = qp->pd; 1186 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device); 1187 1188 memset(&uresp, 0, sizeof(uresp)); 1189 usr_db = dev->nic_info.unmapped_db + 1190 (pd->id * dev->nic_info.db_page_size); 1191 uresp.qp_id = qp->id; 1192 uresp.sq_dbid = qp->sq.dbid; 1193 uresp.num_sq_pages = 1; 1194 uresp.sq_page_size = PAGE_ALIGN(qp->sq.len); 1195 uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va); 1196 uresp.num_wqe_allocated = qp->sq.max_cnt; 1197 if (!srq) { 1198 uresp.rq_dbid = qp->rq.dbid; 1199 uresp.num_rq_pages = 1; 1200 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len); 1201 uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va); 1202 uresp.num_rqe_allocated = qp->rq.max_cnt; 1203 } 1204 uresp.db_page_addr = usr_db; 1205 uresp.db_page_size = dev->nic_info.db_page_size; 1206 uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET; 1207 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET; 1208 uresp.db_shift = OCRDMA_DB_RQ_SHIFT; 1209 1210 if (qp->dpp_enabled) { 1211 uresp.dpp_credit = dpp_credit_lmt; 1212 uresp.dpp_offset = dpp_offset; 1213 } 1214 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1215 if (status) { 1216 pr_err("%s(%d) user copy error.\n", __func__, dev->id); 1217 goto err; 1218 } 1219 status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0], 1220 uresp.sq_page_size); 1221 if (status) 1222 goto err; 1223 1224 if (!srq) { 1225 status = ocrdma_add_mmap(pd->uctx, uresp.rq_page_addr[0], 1226 uresp.rq_page_size); 1227 if (status) 1228 goto rq_map_err; 1229 } 1230 return status; 1231 rq_map_err: 1232 ocrdma_del_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size); 1233 err: 1234 return status; 1235 } 1236 1237 static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp, 1238 struct ocrdma_pd *pd) 1239 { 1240 if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) { 1241 qp->sq_db = dev->nic_info.db + 1242 (pd->id * dev->nic_info.db_page_size) + 1243 OCRDMA_DB_GEN2_SQ_OFFSET; 1244 qp->rq_db = dev->nic_info.db + 1245 (pd->id * dev->nic_info.db_page_size) + 1246 OCRDMA_DB_GEN2_RQ_OFFSET; 1247 } else { 1248 qp->sq_db = dev->nic_info.db + 1249 (pd->id * dev->nic_info.db_page_size) + 1250 OCRDMA_DB_SQ_OFFSET; 1251 qp->rq_db = dev->nic_info.db + 1252 (pd->id * dev->nic_info.db_page_size) + 1253 OCRDMA_DB_RQ_OFFSET; 1254 } 1255 } 1256 1257 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp) 1258 { 1259 qp->wqe_wr_id_tbl = 1260 kcalloc(qp->sq.max_cnt, sizeof(*(qp->wqe_wr_id_tbl)), 1261 GFP_KERNEL); 1262 if (qp->wqe_wr_id_tbl == NULL) 1263 return -ENOMEM; 1264 qp->rqe_wr_id_tbl = 1265 kcalloc(qp->rq.max_cnt, sizeof(u64), GFP_KERNEL); 1266 if (qp->rqe_wr_id_tbl == NULL) 1267 return -ENOMEM; 1268 1269 return 0; 1270 } 1271 1272 static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp, 1273 struct ocrdma_pd *pd, 1274 struct ib_qp_init_attr *attrs) 1275 { 1276 qp->pd = pd; 1277 spin_lock_init(&qp->q_lock); 1278 INIT_LIST_HEAD(&qp->sq_entry); 1279 INIT_LIST_HEAD(&qp->rq_entry); 1280 1281 qp->qp_type = attrs->qp_type; 1282 qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR; 1283 qp->max_inline_data = attrs->cap.max_inline_data; 1284 qp->sq.max_sges = attrs->cap.max_send_sge; 1285 qp->rq.max_sges = attrs->cap.max_recv_sge; 1286 qp->state = OCRDMA_QPS_RST; 1287 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1288 } 1289 1290 static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev, 1291 struct ib_qp_init_attr *attrs) 1292 { 1293 if (attrs->qp_type == IB_QPT_GSI) { 1294 dev->gsi_qp_created = 1; 1295 dev->gsi_sqcq = get_ocrdma_cq(attrs->send_cq); 1296 dev->gsi_rqcq = get_ocrdma_cq(attrs->recv_cq); 1297 } 1298 } 1299 1300 struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd, 1301 struct ib_qp_init_attr *attrs, 1302 struct ib_udata *udata) 1303 { 1304 int status; 1305 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 1306 struct ocrdma_qp *qp; 1307 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 1308 struct ocrdma_create_qp_ureq ureq; 1309 u16 dpp_credit_lmt, dpp_offset; 1310 1311 status = ocrdma_check_qp_params(ibpd, dev, attrs, udata); 1312 if (status) 1313 goto gen_err; 1314 1315 memset(&ureq, 0, sizeof(ureq)); 1316 if (udata) { 1317 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 1318 return ERR_PTR(-EFAULT); 1319 } 1320 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1321 if (!qp) { 1322 status = -ENOMEM; 1323 goto gen_err; 1324 } 1325 ocrdma_set_qp_init_params(qp, pd, attrs); 1326 if (udata == NULL) 1327 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 | 1328 OCRDMA_QP_FAST_REG); 1329 1330 mutex_lock(&dev->dev_lock); 1331 status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq, 1332 ureq.dpp_cq_id, 1333 &dpp_offset, &dpp_credit_lmt); 1334 if (status) 1335 goto mbx_err; 1336 1337 /* user space QP's wr_id table are managed in library */ 1338 if (udata == NULL) { 1339 status = ocrdma_alloc_wr_id_tbl(qp); 1340 if (status) 1341 goto map_err; 1342 } 1343 1344 status = ocrdma_add_qpn_map(dev, qp); 1345 if (status) 1346 goto map_err; 1347 ocrdma_set_qp_db(dev, qp, pd); 1348 if (udata) { 1349 status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset, 1350 dpp_credit_lmt, 1351 (attrs->srq != NULL)); 1352 if (status) 1353 goto cpy_err; 1354 } 1355 ocrdma_store_gsi_qp_cq(dev, attrs); 1356 qp->ibqp.qp_num = qp->id; 1357 mutex_unlock(&dev->dev_lock); 1358 return &qp->ibqp; 1359 1360 cpy_err: 1361 ocrdma_del_qpn_map(dev, qp); 1362 map_err: 1363 ocrdma_mbx_destroy_qp(dev, qp); 1364 mbx_err: 1365 mutex_unlock(&dev->dev_lock); 1366 kfree(qp->wqe_wr_id_tbl); 1367 kfree(qp->rqe_wr_id_tbl); 1368 kfree(qp); 1369 pr_err("%s(%d) error=%d\n", __func__, dev->id, status); 1370 gen_err: 1371 return ERR_PTR(status); 1372 } 1373 1374 int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1375 int attr_mask) 1376 { 1377 int status = 0; 1378 struct ocrdma_qp *qp; 1379 struct ocrdma_dev *dev; 1380 enum ib_qp_state old_qps; 1381 1382 qp = get_ocrdma_qp(ibqp); 1383 dev = get_ocrdma_dev(ibqp->device); 1384 if (attr_mask & IB_QP_STATE) 1385 status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps); 1386 /* if new and previous states are same hw doesn't need to 1387 * know about it. 1388 */ 1389 if (status < 0) 1390 return status; 1391 return ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask); 1392 } 1393 1394 int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1395 int attr_mask, struct ib_udata *udata) 1396 { 1397 unsigned long flags; 1398 int status = -EINVAL; 1399 struct ocrdma_qp *qp; 1400 struct ocrdma_dev *dev; 1401 enum ib_qp_state old_qps, new_qps; 1402 1403 qp = get_ocrdma_qp(ibqp); 1404 dev = get_ocrdma_dev(ibqp->device); 1405 1406 /* syncronize with multiple context trying to change, retrive qps */ 1407 mutex_lock(&dev->dev_lock); 1408 /* syncronize with wqe, rqe posting and cqe processing contexts */ 1409 spin_lock_irqsave(&qp->q_lock, flags); 1410 old_qps = get_ibqp_state(qp->state); 1411 if (attr_mask & IB_QP_STATE) 1412 new_qps = attr->qp_state; 1413 else 1414 new_qps = old_qps; 1415 spin_unlock_irqrestore(&qp->q_lock, flags); 1416 1417 if (!ib_modify_qp_is_ok(old_qps, new_qps, ibqp->qp_type, attr_mask)) { 1418 pr_err("%s(%d) invalid attribute mask=0x%x specified for\n" 1419 "qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n", 1420 __func__, dev->id, attr_mask, qp->id, ibqp->qp_type, 1421 old_qps, new_qps); 1422 goto param_err; 1423 } 1424 1425 status = _ocrdma_modify_qp(ibqp, attr, attr_mask); 1426 if (status > 0) 1427 status = 0; 1428 param_err: 1429 mutex_unlock(&dev->dev_lock); 1430 return status; 1431 } 1432 1433 static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu) 1434 { 1435 switch (mtu) { 1436 case 256: 1437 return IB_MTU_256; 1438 case 512: 1439 return IB_MTU_512; 1440 case 1024: 1441 return IB_MTU_1024; 1442 case 2048: 1443 return IB_MTU_2048; 1444 case 4096: 1445 return IB_MTU_4096; 1446 default: 1447 return IB_MTU_1024; 1448 } 1449 } 1450 1451 static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags) 1452 { 1453 int ib_qp_acc_flags = 0; 1454 1455 if (qp_cap_flags & OCRDMA_QP_INB_WR) 1456 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 1457 if (qp_cap_flags & OCRDMA_QP_INB_RD) 1458 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 1459 return ib_qp_acc_flags; 1460 } 1461 1462 int ocrdma_query_qp(struct ib_qp *ibqp, 1463 struct ib_qp_attr *qp_attr, 1464 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 1465 { 1466 int status; 1467 u32 qp_state; 1468 struct ocrdma_qp_params params; 1469 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 1470 struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device); 1471 1472 memset(¶ms, 0, sizeof(params)); 1473 mutex_lock(&dev->dev_lock); 1474 status = ocrdma_mbx_query_qp(dev, qp, ¶ms); 1475 mutex_unlock(&dev->dev_lock); 1476 if (status) 1477 goto mbx_err; 1478 if (qp->qp_type == IB_QPT_UD) 1479 qp_attr->qkey = params.qkey; 1480 qp_attr->path_mtu = 1481 ocrdma_mtu_int_to_enum(params.path_mtu_pkey_indx & 1482 OCRDMA_QP_PARAMS_PATH_MTU_MASK) >> 1483 OCRDMA_QP_PARAMS_PATH_MTU_SHIFT; 1484 qp_attr->path_mig_state = IB_MIG_MIGRATED; 1485 qp_attr->rq_psn = params.hop_lmt_rq_psn & OCRDMA_QP_PARAMS_RQ_PSN_MASK; 1486 qp_attr->sq_psn = params.tclass_sq_psn & OCRDMA_QP_PARAMS_SQ_PSN_MASK; 1487 qp_attr->dest_qp_num = 1488 params.ack_to_rnr_rtc_dest_qpn & OCRDMA_QP_PARAMS_DEST_QPN_MASK; 1489 1490 qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags); 1491 qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1; 1492 qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1; 1493 qp_attr->cap.max_send_sge = qp->sq.max_sges; 1494 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 1495 qp_attr->cap.max_inline_data = qp->max_inline_data; 1496 qp_init_attr->cap = qp_attr->cap; 1497 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 1498 1499 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 1500 params.rnt_rc_sl_fl & 1501 OCRDMA_QP_PARAMS_FLOW_LABEL_MASK, 1502 qp->sgid_idx, 1503 (params.hop_lmt_rq_psn & 1504 OCRDMA_QP_PARAMS_HOP_LMT_MASK) >> 1505 OCRDMA_QP_PARAMS_HOP_LMT_SHIFT, 1506 (params.tclass_sq_psn & 1507 OCRDMA_QP_PARAMS_TCLASS_MASK) >> 1508 OCRDMA_QP_PARAMS_TCLASS_SHIFT); 1509 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid[0]); 1510 1511 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 1512 rdma_ah_set_sl(&qp_attr->ah_attr, (params.rnt_rc_sl_fl & 1513 OCRDMA_QP_PARAMS_SL_MASK) >> 1514 OCRDMA_QP_PARAMS_SL_SHIFT); 1515 qp_attr->timeout = (params.ack_to_rnr_rtc_dest_qpn & 1516 OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK) >> 1517 OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT; 1518 qp_attr->rnr_retry = (params.ack_to_rnr_rtc_dest_qpn & 1519 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK) >> 1520 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT; 1521 qp_attr->retry_cnt = 1522 (params.rnt_rc_sl_fl & OCRDMA_QP_PARAMS_RETRY_CNT_MASK) >> 1523 OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT; 1524 qp_attr->min_rnr_timer = 0; 1525 qp_attr->pkey_index = 0; 1526 qp_attr->port_num = 1; 1527 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 1528 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 1529 qp_attr->alt_pkey_index = 0; 1530 qp_attr->alt_port_num = 0; 1531 qp_attr->alt_timeout = 0; 1532 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 1533 qp_state = (params.max_sge_recv_flags & OCRDMA_QP_PARAMS_STATE_MASK) >> 1534 OCRDMA_QP_PARAMS_STATE_SHIFT; 1535 qp_attr->qp_state = get_ibqp_state(qp_state); 1536 qp_attr->cur_qp_state = qp_attr->qp_state; 1537 qp_attr->sq_draining = (qp_state == OCRDMA_QPS_SQ_DRAINING) ? 1 : 0; 1538 qp_attr->max_dest_rd_atomic = 1539 params.max_ord_ird >> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT; 1540 qp_attr->max_rd_atomic = 1541 params.max_ord_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK; 1542 qp_attr->en_sqd_async_notify = (params.max_sge_recv_flags & 1543 OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC) ? 1 : 0; 1544 /* Sync driver QP state with FW */ 1545 ocrdma_qp_state_change(qp, qp_attr->qp_state, NULL); 1546 mbx_err: 1547 return status; 1548 } 1549 1550 static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, unsigned int idx) 1551 { 1552 unsigned int i = idx / 32; 1553 u32 mask = (1U << (idx % 32)); 1554 1555 srq->idx_bit_fields[i] ^= mask; 1556 } 1557 1558 static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q) 1559 { 1560 return ((q->max_wqe_idx - q->head) + q->tail) % q->max_cnt; 1561 } 1562 1563 static int is_hw_sq_empty(struct ocrdma_qp *qp) 1564 { 1565 return (qp->sq.tail == qp->sq.head); 1566 } 1567 1568 static int is_hw_rq_empty(struct ocrdma_qp *qp) 1569 { 1570 return (qp->rq.tail == qp->rq.head); 1571 } 1572 1573 static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q) 1574 { 1575 return q->va + (q->head * q->entry_size); 1576 } 1577 1578 static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q, 1579 u32 idx) 1580 { 1581 return q->va + (idx * q->entry_size); 1582 } 1583 1584 static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q) 1585 { 1586 q->head = (q->head + 1) & q->max_wqe_idx; 1587 } 1588 1589 static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q) 1590 { 1591 q->tail = (q->tail + 1) & q->max_wqe_idx; 1592 } 1593 1594 /* discard the cqe for a given QP */ 1595 static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq) 1596 { 1597 unsigned long cq_flags; 1598 unsigned long flags; 1599 int discard_cnt = 0; 1600 u32 cur_getp, stop_getp; 1601 struct ocrdma_cqe *cqe; 1602 u32 qpn = 0, wqe_idx = 0; 1603 1604 spin_lock_irqsave(&cq->cq_lock, cq_flags); 1605 1606 /* traverse through the CQEs in the hw CQ, 1607 * find the matching CQE for a given qp, 1608 * mark the matching one discarded by clearing qpn. 1609 * ring the doorbell in the poll_cq() as 1610 * we don't complete out of order cqe. 1611 */ 1612 1613 cur_getp = cq->getp; 1614 /* find upto when do we reap the cq. */ 1615 stop_getp = cur_getp; 1616 do { 1617 if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp))) 1618 break; 1619 1620 cqe = cq->va + cur_getp; 1621 /* if (a) done reaping whole hw cq, or 1622 * (b) qp_xq becomes empty. 1623 * then exit 1624 */ 1625 qpn = cqe->cmn.qpn & OCRDMA_CQE_QPN_MASK; 1626 /* if previously discarded cqe found, skip that too. */ 1627 /* check for matching qp */ 1628 if (qpn == 0 || qpn != qp->id) 1629 goto skip_cqe; 1630 1631 if (is_cqe_for_sq(cqe)) { 1632 ocrdma_hwq_inc_tail(&qp->sq); 1633 } else { 1634 if (qp->srq) { 1635 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >> 1636 OCRDMA_CQE_BUFTAG_SHIFT) & 1637 qp->srq->rq.max_wqe_idx; 1638 BUG_ON(wqe_idx < 1); 1639 spin_lock_irqsave(&qp->srq->q_lock, flags); 1640 ocrdma_hwq_inc_tail(&qp->srq->rq); 1641 ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1); 1642 spin_unlock_irqrestore(&qp->srq->q_lock, flags); 1643 1644 } else { 1645 ocrdma_hwq_inc_tail(&qp->rq); 1646 } 1647 } 1648 /* mark cqe discarded so that it is not picked up later 1649 * in the poll_cq(). 1650 */ 1651 discard_cnt += 1; 1652 cqe->cmn.qpn = 0; 1653 skip_cqe: 1654 cur_getp = (cur_getp + 1) % cq->max_hw_cqe; 1655 } while (cur_getp != stop_getp); 1656 spin_unlock_irqrestore(&cq->cq_lock, cq_flags); 1657 } 1658 1659 void ocrdma_del_flush_qp(struct ocrdma_qp *qp) 1660 { 1661 int found = false; 1662 unsigned long flags; 1663 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 1664 /* sync with any active CQ poll */ 1665 1666 spin_lock_irqsave(&dev->flush_q_lock, flags); 1667 found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp); 1668 if (found) 1669 list_del(&qp->sq_entry); 1670 if (!qp->srq) { 1671 found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp); 1672 if (found) 1673 list_del(&qp->rq_entry); 1674 } 1675 spin_unlock_irqrestore(&dev->flush_q_lock, flags); 1676 } 1677 1678 int ocrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 1679 { 1680 struct ocrdma_pd *pd; 1681 struct ocrdma_qp *qp; 1682 struct ocrdma_dev *dev; 1683 struct ib_qp_attr attrs; 1684 int attr_mask; 1685 unsigned long flags; 1686 1687 qp = get_ocrdma_qp(ibqp); 1688 dev = get_ocrdma_dev(ibqp->device); 1689 1690 pd = qp->pd; 1691 1692 /* change the QP state to ERROR */ 1693 if (qp->state != OCRDMA_QPS_RST) { 1694 attrs.qp_state = IB_QPS_ERR; 1695 attr_mask = IB_QP_STATE; 1696 _ocrdma_modify_qp(ibqp, &attrs, attr_mask); 1697 } 1698 /* ensure that CQEs for newly created QP (whose id may be same with 1699 * one which just getting destroyed are same), dont get 1700 * discarded until the old CQEs are discarded. 1701 */ 1702 mutex_lock(&dev->dev_lock); 1703 (void) ocrdma_mbx_destroy_qp(dev, qp); 1704 1705 /* 1706 * acquire CQ lock while destroy is in progress, in order to 1707 * protect against proessing in-flight CQEs for this QP. 1708 */ 1709 spin_lock_irqsave(&qp->sq_cq->cq_lock, flags); 1710 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) { 1711 spin_lock(&qp->rq_cq->cq_lock); 1712 ocrdma_del_qpn_map(dev, qp); 1713 spin_unlock(&qp->rq_cq->cq_lock); 1714 } else { 1715 ocrdma_del_qpn_map(dev, qp); 1716 } 1717 spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags); 1718 1719 if (!pd->uctx) { 1720 ocrdma_discard_cqes(qp, qp->sq_cq); 1721 ocrdma_discard_cqes(qp, qp->rq_cq); 1722 } 1723 mutex_unlock(&dev->dev_lock); 1724 1725 if (pd->uctx) { 1726 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa, 1727 PAGE_ALIGN(qp->sq.len)); 1728 if (!qp->srq) 1729 ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa, 1730 PAGE_ALIGN(qp->rq.len)); 1731 } 1732 1733 ocrdma_del_flush_qp(qp); 1734 1735 kfree(qp->wqe_wr_id_tbl); 1736 kfree(qp->rqe_wr_id_tbl); 1737 kfree(qp); 1738 return 0; 1739 } 1740 1741 static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq, 1742 struct ib_udata *udata) 1743 { 1744 int status; 1745 struct ocrdma_create_srq_uresp uresp; 1746 1747 memset(&uresp, 0, sizeof(uresp)); 1748 uresp.rq_dbid = srq->rq.dbid; 1749 uresp.num_rq_pages = 1; 1750 uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va); 1751 uresp.rq_page_size = srq->rq.len; 1752 uresp.db_page_addr = dev->nic_info.unmapped_db + 1753 (srq->pd->id * dev->nic_info.db_page_size); 1754 uresp.db_page_size = dev->nic_info.db_page_size; 1755 uresp.num_rqe_allocated = srq->rq.max_cnt; 1756 if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) { 1757 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET; 1758 uresp.db_shift = 24; 1759 } else { 1760 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET; 1761 uresp.db_shift = 16; 1762 } 1763 1764 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1765 if (status) 1766 return status; 1767 status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0], 1768 uresp.rq_page_size); 1769 if (status) 1770 return status; 1771 return status; 1772 } 1773 1774 int ocrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, 1775 struct ib_udata *udata) 1776 { 1777 int status; 1778 struct ocrdma_pd *pd = get_ocrdma_pd(ibsrq->pd); 1779 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device); 1780 struct ocrdma_srq *srq = get_ocrdma_srq(ibsrq); 1781 1782 if (init_attr->attr.max_sge > dev->attr.max_recv_sge) 1783 return -EINVAL; 1784 if (init_attr->attr.max_wr > dev->attr.max_rqe) 1785 return -EINVAL; 1786 1787 spin_lock_init(&srq->q_lock); 1788 srq->pd = pd; 1789 srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size); 1790 status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd); 1791 if (status) 1792 return status; 1793 1794 if (!udata) { 1795 srq->rqe_wr_id_tbl = kcalloc(srq->rq.max_cnt, sizeof(u64), 1796 GFP_KERNEL); 1797 if (!srq->rqe_wr_id_tbl) { 1798 status = -ENOMEM; 1799 goto arm_err; 1800 } 1801 1802 srq->bit_fields_len = (srq->rq.max_cnt / 32) + 1803 (srq->rq.max_cnt % 32 ? 1 : 0); 1804 srq->idx_bit_fields = 1805 kmalloc_array(srq->bit_fields_len, sizeof(u32), 1806 GFP_KERNEL); 1807 if (!srq->idx_bit_fields) { 1808 status = -ENOMEM; 1809 goto arm_err; 1810 } 1811 memset(srq->idx_bit_fields, 0xff, 1812 srq->bit_fields_len * sizeof(u32)); 1813 } 1814 1815 if (init_attr->attr.srq_limit) { 1816 status = ocrdma_mbx_modify_srq(srq, &init_attr->attr); 1817 if (status) 1818 goto arm_err; 1819 } 1820 1821 if (udata) { 1822 status = ocrdma_copy_srq_uresp(dev, srq, udata); 1823 if (status) 1824 goto arm_err; 1825 } 1826 1827 return 0; 1828 1829 arm_err: 1830 ocrdma_mbx_destroy_srq(dev, srq); 1831 kfree(srq->rqe_wr_id_tbl); 1832 kfree(srq->idx_bit_fields); 1833 return status; 1834 } 1835 1836 int ocrdma_modify_srq(struct ib_srq *ibsrq, 1837 struct ib_srq_attr *srq_attr, 1838 enum ib_srq_attr_mask srq_attr_mask, 1839 struct ib_udata *udata) 1840 { 1841 int status; 1842 struct ocrdma_srq *srq; 1843 1844 srq = get_ocrdma_srq(ibsrq); 1845 if (srq_attr_mask & IB_SRQ_MAX_WR) 1846 status = -EINVAL; 1847 else 1848 status = ocrdma_mbx_modify_srq(srq, srq_attr); 1849 return status; 1850 } 1851 1852 int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 1853 { 1854 int status; 1855 struct ocrdma_srq *srq; 1856 1857 srq = get_ocrdma_srq(ibsrq); 1858 status = ocrdma_mbx_query_srq(srq, srq_attr); 1859 return status; 1860 } 1861 1862 void ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 1863 { 1864 struct ocrdma_srq *srq; 1865 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device); 1866 1867 srq = get_ocrdma_srq(ibsrq); 1868 1869 ocrdma_mbx_destroy_srq(dev, srq); 1870 1871 if (srq->pd->uctx) 1872 ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa, 1873 PAGE_ALIGN(srq->rq.len)); 1874 1875 kfree(srq->idx_bit_fields); 1876 kfree(srq->rqe_wr_id_tbl); 1877 } 1878 1879 /* unprivileged verbs and their support functions. */ 1880 static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp, 1881 struct ocrdma_hdr_wqe *hdr, 1882 const struct ib_send_wr *wr) 1883 { 1884 struct ocrdma_ewqe_ud_hdr *ud_hdr = 1885 (struct ocrdma_ewqe_ud_hdr *)(hdr + 1); 1886 struct ocrdma_ah *ah = get_ocrdma_ah(ud_wr(wr)->ah); 1887 1888 ud_hdr->rsvd_dest_qpn = ud_wr(wr)->remote_qpn; 1889 if (qp->qp_type == IB_QPT_GSI) 1890 ud_hdr->qkey = qp->qkey; 1891 else 1892 ud_hdr->qkey = ud_wr(wr)->remote_qkey; 1893 ud_hdr->rsvd_ahid = ah->id; 1894 ud_hdr->hdr_type = ah->hdr_type; 1895 if (ah->av->valid & OCRDMA_AV_VLAN_VALID) 1896 hdr->cw |= (OCRDMA_FLAG_AH_VLAN_PR << OCRDMA_WQE_FLAGS_SHIFT); 1897 } 1898 1899 static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr, 1900 struct ocrdma_sge *sge, int num_sge, 1901 struct ib_sge *sg_list) 1902 { 1903 int i; 1904 1905 for (i = 0; i < num_sge; i++) { 1906 sge[i].lrkey = sg_list[i].lkey; 1907 sge[i].addr_lo = sg_list[i].addr; 1908 sge[i].addr_hi = upper_32_bits(sg_list[i].addr); 1909 sge[i].len = sg_list[i].length; 1910 hdr->total_len += sg_list[i].length; 1911 } 1912 if (num_sge == 0) 1913 memset(sge, 0, sizeof(*sge)); 1914 } 1915 1916 static inline uint32_t ocrdma_sglist_len(struct ib_sge *sg_list, int num_sge) 1917 { 1918 uint32_t total_len = 0, i; 1919 1920 for (i = 0; i < num_sge; i++) 1921 total_len += sg_list[i].length; 1922 return total_len; 1923 } 1924 1925 1926 static int ocrdma_build_inline_sges(struct ocrdma_qp *qp, 1927 struct ocrdma_hdr_wqe *hdr, 1928 struct ocrdma_sge *sge, 1929 const struct ib_send_wr *wr, u32 wqe_size) 1930 { 1931 int i; 1932 char *dpp_addr; 1933 1934 if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) { 1935 hdr->total_len = ocrdma_sglist_len(wr->sg_list, wr->num_sge); 1936 if (unlikely(hdr->total_len > qp->max_inline_data)) { 1937 pr_err("%s() supported_len=0x%x,\n" 1938 " unsupported len req=0x%x\n", __func__, 1939 qp->max_inline_data, hdr->total_len); 1940 return -EINVAL; 1941 } 1942 dpp_addr = (char *)sge; 1943 for (i = 0; i < wr->num_sge; i++) { 1944 memcpy(dpp_addr, 1945 (void *)(unsigned long)wr->sg_list[i].addr, 1946 wr->sg_list[i].length); 1947 dpp_addr += wr->sg_list[i].length; 1948 } 1949 1950 wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES); 1951 if (0 == hdr->total_len) 1952 wqe_size += sizeof(struct ocrdma_sge); 1953 hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT); 1954 } else { 1955 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list); 1956 if (wr->num_sge) 1957 wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge)); 1958 else 1959 wqe_size += sizeof(struct ocrdma_sge); 1960 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 1961 } 1962 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 1963 return 0; 1964 } 1965 1966 static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 1967 const struct ib_send_wr *wr) 1968 { 1969 int status; 1970 struct ocrdma_sge *sge; 1971 u32 wqe_size = sizeof(*hdr); 1972 1973 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 1974 ocrdma_build_ud_hdr(qp, hdr, wr); 1975 sge = (struct ocrdma_sge *)(hdr + 2); 1976 wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr); 1977 } else { 1978 sge = (struct ocrdma_sge *)(hdr + 1); 1979 } 1980 1981 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); 1982 return status; 1983 } 1984 1985 static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 1986 const struct ib_send_wr *wr) 1987 { 1988 int status; 1989 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1); 1990 struct ocrdma_sge *sge = ext_rw + 1; 1991 u32 wqe_size = sizeof(*hdr) + sizeof(*ext_rw); 1992 1993 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); 1994 if (status) 1995 return status; 1996 ext_rw->addr_lo = rdma_wr(wr)->remote_addr; 1997 ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr); 1998 ext_rw->lrkey = rdma_wr(wr)->rkey; 1999 ext_rw->len = hdr->total_len; 2000 return 0; 2001 } 2002 2003 static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 2004 const struct ib_send_wr *wr) 2005 { 2006 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1); 2007 struct ocrdma_sge *sge = ext_rw + 1; 2008 u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) + 2009 sizeof(struct ocrdma_hdr_wqe); 2010 2011 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list); 2012 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 2013 hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT); 2014 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 2015 2016 ext_rw->addr_lo = rdma_wr(wr)->remote_addr; 2017 ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr); 2018 ext_rw->lrkey = rdma_wr(wr)->rkey; 2019 ext_rw->len = hdr->total_len; 2020 } 2021 2022 static int get_encoded_page_size(int pg_sz) 2023 { 2024 /* Max size is 256M 4096 << 16 */ 2025 int i = 0; 2026 for (; i < 17; i++) 2027 if (pg_sz == (4096 << i)) 2028 break; 2029 return i; 2030 } 2031 2032 static int ocrdma_build_reg(struct ocrdma_qp *qp, 2033 struct ocrdma_hdr_wqe *hdr, 2034 const struct ib_reg_wr *wr) 2035 { 2036 u64 fbo; 2037 struct ocrdma_ewqe_fr *fast_reg = (struct ocrdma_ewqe_fr *)(hdr + 1); 2038 struct ocrdma_mr *mr = get_ocrdma_mr(wr->mr); 2039 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; 2040 struct ocrdma_pbe *pbe; 2041 u32 wqe_size = sizeof(*fast_reg) + sizeof(*hdr); 2042 int num_pbes = 0, i; 2043 2044 wqe_size = roundup(wqe_size, OCRDMA_WQE_ALIGN_BYTES); 2045 2046 hdr->cw |= (OCRDMA_FR_MR << OCRDMA_WQE_OPCODE_SHIFT); 2047 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 2048 2049 if (wr->access & IB_ACCESS_LOCAL_WRITE) 2050 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_LOCAL_WR; 2051 if (wr->access & IB_ACCESS_REMOTE_WRITE) 2052 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_WR; 2053 if (wr->access & IB_ACCESS_REMOTE_READ) 2054 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_RD; 2055 hdr->lkey = wr->key; 2056 hdr->total_len = mr->ibmr.length; 2057 2058 fbo = mr->ibmr.iova - mr->pages[0]; 2059 2060 fast_reg->va_hi = upper_32_bits(mr->ibmr.iova); 2061 fast_reg->va_lo = (u32) (mr->ibmr.iova & 0xffffffff); 2062 fast_reg->fbo_hi = upper_32_bits(fbo); 2063 fast_reg->fbo_lo = (u32) fbo & 0xffffffff; 2064 fast_reg->num_sges = mr->npages; 2065 fast_reg->size_sge = get_encoded_page_size(mr->ibmr.page_size); 2066 2067 pbe = pbl_tbl->va; 2068 for (i = 0; i < mr->npages; i++) { 2069 u64 buf_addr = mr->pages[i]; 2070 2071 pbe->pa_lo = cpu_to_le32((u32) (buf_addr & PAGE_MASK)); 2072 pbe->pa_hi = cpu_to_le32((u32) upper_32_bits(buf_addr)); 2073 num_pbes += 1; 2074 pbe++; 2075 2076 /* if the pbl is full storing the pbes, 2077 * move to next pbl. 2078 */ 2079 if (num_pbes == (mr->hwmr.pbl_size/sizeof(u64))) { 2080 pbl_tbl++; 2081 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 2082 } 2083 } 2084 2085 return 0; 2086 } 2087 2088 static void ocrdma_ring_sq_db(struct ocrdma_qp *qp) 2089 { 2090 u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT); 2091 2092 iowrite32(val, qp->sq_db); 2093 } 2094 2095 int ocrdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 2096 const struct ib_send_wr **bad_wr) 2097 { 2098 int status = 0; 2099 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 2100 struct ocrdma_hdr_wqe *hdr; 2101 unsigned long flags; 2102 2103 spin_lock_irqsave(&qp->q_lock, flags); 2104 if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) { 2105 spin_unlock_irqrestore(&qp->q_lock, flags); 2106 *bad_wr = wr; 2107 return -EINVAL; 2108 } 2109 2110 while (wr) { 2111 if (qp->qp_type == IB_QPT_UD && 2112 (wr->opcode != IB_WR_SEND && 2113 wr->opcode != IB_WR_SEND_WITH_IMM)) { 2114 *bad_wr = wr; 2115 status = -EINVAL; 2116 break; 2117 } 2118 if (ocrdma_hwq_free_cnt(&qp->sq) == 0 || 2119 wr->num_sge > qp->sq.max_sges) { 2120 *bad_wr = wr; 2121 status = -ENOMEM; 2122 break; 2123 } 2124 hdr = ocrdma_hwq_head(&qp->sq); 2125 hdr->cw = 0; 2126 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) 2127 hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT); 2128 if (wr->send_flags & IB_SEND_FENCE) 2129 hdr->cw |= 2130 (OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT); 2131 if (wr->send_flags & IB_SEND_SOLICITED) 2132 hdr->cw |= 2133 (OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT); 2134 hdr->total_len = 0; 2135 switch (wr->opcode) { 2136 case IB_WR_SEND_WITH_IMM: 2137 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT); 2138 hdr->immdt = ntohl(wr->ex.imm_data); 2139 /* fall through */ 2140 case IB_WR_SEND: 2141 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT); 2142 ocrdma_build_send(qp, hdr, wr); 2143 break; 2144 case IB_WR_SEND_WITH_INV: 2145 hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT); 2146 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT); 2147 hdr->lkey = wr->ex.invalidate_rkey; 2148 status = ocrdma_build_send(qp, hdr, wr); 2149 break; 2150 case IB_WR_RDMA_WRITE_WITH_IMM: 2151 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT); 2152 hdr->immdt = ntohl(wr->ex.imm_data); 2153 /* fall through */ 2154 case IB_WR_RDMA_WRITE: 2155 hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT); 2156 status = ocrdma_build_write(qp, hdr, wr); 2157 break; 2158 case IB_WR_RDMA_READ: 2159 ocrdma_build_read(qp, hdr, wr); 2160 break; 2161 case IB_WR_LOCAL_INV: 2162 hdr->cw |= 2163 (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT); 2164 hdr->cw |= ((sizeof(struct ocrdma_hdr_wqe) + 2165 sizeof(struct ocrdma_sge)) / 2166 OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT; 2167 hdr->lkey = wr->ex.invalidate_rkey; 2168 break; 2169 case IB_WR_REG_MR: 2170 status = ocrdma_build_reg(qp, hdr, reg_wr(wr)); 2171 break; 2172 default: 2173 status = -EINVAL; 2174 break; 2175 } 2176 if (status) { 2177 *bad_wr = wr; 2178 break; 2179 } 2180 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) 2181 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1; 2182 else 2183 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0; 2184 qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id; 2185 ocrdma_cpu_to_le32(hdr, ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) & 2186 OCRDMA_WQE_SIZE_MASK) * OCRDMA_WQE_STRIDE); 2187 /* make sure wqe is written before adapter can access it */ 2188 wmb(); 2189 /* inform hw to start processing it */ 2190 ocrdma_ring_sq_db(qp); 2191 2192 /* update pointer, counter for next wr */ 2193 ocrdma_hwq_inc_head(&qp->sq); 2194 wr = wr->next; 2195 } 2196 spin_unlock_irqrestore(&qp->q_lock, flags); 2197 return status; 2198 } 2199 2200 static void ocrdma_ring_rq_db(struct ocrdma_qp *qp) 2201 { 2202 u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT); 2203 2204 iowrite32(val, qp->rq_db); 2205 } 2206 2207 static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe, 2208 const struct ib_recv_wr *wr, u16 tag) 2209 { 2210 u32 wqe_size = 0; 2211 struct ocrdma_sge *sge; 2212 if (wr->num_sge) 2213 wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe); 2214 else 2215 wqe_size = sizeof(*sge) + sizeof(*rqe); 2216 2217 rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) << 2218 OCRDMA_WQE_SIZE_SHIFT); 2219 rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT); 2220 rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 2221 rqe->total_len = 0; 2222 rqe->rsvd_tag = tag; 2223 sge = (struct ocrdma_sge *)(rqe + 1); 2224 ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list); 2225 ocrdma_cpu_to_le32(rqe, wqe_size); 2226 } 2227 2228 int ocrdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 2229 const struct ib_recv_wr **bad_wr) 2230 { 2231 int status = 0; 2232 unsigned long flags; 2233 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 2234 struct ocrdma_hdr_wqe *rqe; 2235 2236 spin_lock_irqsave(&qp->q_lock, flags); 2237 if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) { 2238 spin_unlock_irqrestore(&qp->q_lock, flags); 2239 *bad_wr = wr; 2240 return -EINVAL; 2241 } 2242 while (wr) { 2243 if (ocrdma_hwq_free_cnt(&qp->rq) == 0 || 2244 wr->num_sge > qp->rq.max_sges) { 2245 *bad_wr = wr; 2246 status = -ENOMEM; 2247 break; 2248 } 2249 rqe = ocrdma_hwq_head(&qp->rq); 2250 ocrdma_build_rqe(rqe, wr, 0); 2251 2252 qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id; 2253 /* make sure rqe is written before adapter can access it */ 2254 wmb(); 2255 2256 /* inform hw to start processing it */ 2257 ocrdma_ring_rq_db(qp); 2258 2259 /* update pointer, counter for next wr */ 2260 ocrdma_hwq_inc_head(&qp->rq); 2261 wr = wr->next; 2262 } 2263 spin_unlock_irqrestore(&qp->q_lock, flags); 2264 return status; 2265 } 2266 2267 /* cqe for srq's rqe can potentially arrive out of order. 2268 * index gives the entry in the shadow table where to store 2269 * the wr_id. tag/index is returned in cqe to reference back 2270 * for a given rqe. 2271 */ 2272 static int ocrdma_srq_get_idx(struct ocrdma_srq *srq) 2273 { 2274 int row = 0; 2275 int indx = 0; 2276 2277 for (row = 0; row < srq->bit_fields_len; row++) { 2278 if (srq->idx_bit_fields[row]) { 2279 indx = ffs(srq->idx_bit_fields[row]); 2280 indx = (row * 32) + (indx - 1); 2281 BUG_ON(indx >= srq->rq.max_cnt); 2282 ocrdma_srq_toggle_bit(srq, indx); 2283 break; 2284 } 2285 } 2286 2287 BUG_ON(row == srq->bit_fields_len); 2288 return indx + 1; /* Use from index 1 */ 2289 } 2290 2291 static void ocrdma_ring_srq_db(struct ocrdma_srq *srq) 2292 { 2293 u32 val = srq->rq.dbid | (1 << 16); 2294 2295 iowrite32(val, srq->db + OCRDMA_DB_GEN2_SRQ_OFFSET); 2296 } 2297 2298 int ocrdma_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 2299 const struct ib_recv_wr **bad_wr) 2300 { 2301 int status = 0; 2302 unsigned long flags; 2303 struct ocrdma_srq *srq; 2304 struct ocrdma_hdr_wqe *rqe; 2305 u16 tag; 2306 2307 srq = get_ocrdma_srq(ibsrq); 2308 2309 spin_lock_irqsave(&srq->q_lock, flags); 2310 while (wr) { 2311 if (ocrdma_hwq_free_cnt(&srq->rq) == 0 || 2312 wr->num_sge > srq->rq.max_sges) { 2313 status = -ENOMEM; 2314 *bad_wr = wr; 2315 break; 2316 } 2317 tag = ocrdma_srq_get_idx(srq); 2318 rqe = ocrdma_hwq_head(&srq->rq); 2319 ocrdma_build_rqe(rqe, wr, tag); 2320 2321 srq->rqe_wr_id_tbl[tag] = wr->wr_id; 2322 /* make sure rqe is written before adapter can perform DMA */ 2323 wmb(); 2324 /* inform hw to start processing it */ 2325 ocrdma_ring_srq_db(srq); 2326 /* update pointer, counter for next wr */ 2327 ocrdma_hwq_inc_head(&srq->rq); 2328 wr = wr->next; 2329 } 2330 spin_unlock_irqrestore(&srq->q_lock, flags); 2331 return status; 2332 } 2333 2334 static enum ib_wc_status ocrdma_to_ibwc_err(u16 status) 2335 { 2336 enum ib_wc_status ibwc_status; 2337 2338 switch (status) { 2339 case OCRDMA_CQE_GENERAL_ERR: 2340 ibwc_status = IB_WC_GENERAL_ERR; 2341 break; 2342 case OCRDMA_CQE_LOC_LEN_ERR: 2343 ibwc_status = IB_WC_LOC_LEN_ERR; 2344 break; 2345 case OCRDMA_CQE_LOC_QP_OP_ERR: 2346 ibwc_status = IB_WC_LOC_QP_OP_ERR; 2347 break; 2348 case OCRDMA_CQE_LOC_EEC_OP_ERR: 2349 ibwc_status = IB_WC_LOC_EEC_OP_ERR; 2350 break; 2351 case OCRDMA_CQE_LOC_PROT_ERR: 2352 ibwc_status = IB_WC_LOC_PROT_ERR; 2353 break; 2354 case OCRDMA_CQE_WR_FLUSH_ERR: 2355 ibwc_status = IB_WC_WR_FLUSH_ERR; 2356 break; 2357 case OCRDMA_CQE_MW_BIND_ERR: 2358 ibwc_status = IB_WC_MW_BIND_ERR; 2359 break; 2360 case OCRDMA_CQE_BAD_RESP_ERR: 2361 ibwc_status = IB_WC_BAD_RESP_ERR; 2362 break; 2363 case OCRDMA_CQE_LOC_ACCESS_ERR: 2364 ibwc_status = IB_WC_LOC_ACCESS_ERR; 2365 break; 2366 case OCRDMA_CQE_REM_INV_REQ_ERR: 2367 ibwc_status = IB_WC_REM_INV_REQ_ERR; 2368 break; 2369 case OCRDMA_CQE_REM_ACCESS_ERR: 2370 ibwc_status = IB_WC_REM_ACCESS_ERR; 2371 break; 2372 case OCRDMA_CQE_REM_OP_ERR: 2373 ibwc_status = IB_WC_REM_OP_ERR; 2374 break; 2375 case OCRDMA_CQE_RETRY_EXC_ERR: 2376 ibwc_status = IB_WC_RETRY_EXC_ERR; 2377 break; 2378 case OCRDMA_CQE_RNR_RETRY_EXC_ERR: 2379 ibwc_status = IB_WC_RNR_RETRY_EXC_ERR; 2380 break; 2381 case OCRDMA_CQE_LOC_RDD_VIOL_ERR: 2382 ibwc_status = IB_WC_LOC_RDD_VIOL_ERR; 2383 break; 2384 case OCRDMA_CQE_REM_INV_RD_REQ_ERR: 2385 ibwc_status = IB_WC_REM_INV_RD_REQ_ERR; 2386 break; 2387 case OCRDMA_CQE_REM_ABORT_ERR: 2388 ibwc_status = IB_WC_REM_ABORT_ERR; 2389 break; 2390 case OCRDMA_CQE_INV_EECN_ERR: 2391 ibwc_status = IB_WC_INV_EECN_ERR; 2392 break; 2393 case OCRDMA_CQE_INV_EEC_STATE_ERR: 2394 ibwc_status = IB_WC_INV_EEC_STATE_ERR; 2395 break; 2396 case OCRDMA_CQE_FATAL_ERR: 2397 ibwc_status = IB_WC_FATAL_ERR; 2398 break; 2399 case OCRDMA_CQE_RESP_TIMEOUT_ERR: 2400 ibwc_status = IB_WC_RESP_TIMEOUT_ERR; 2401 break; 2402 default: 2403 ibwc_status = IB_WC_GENERAL_ERR; 2404 break; 2405 } 2406 return ibwc_status; 2407 } 2408 2409 static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc, 2410 u32 wqe_idx) 2411 { 2412 struct ocrdma_hdr_wqe *hdr; 2413 struct ocrdma_sge *rw; 2414 int opcode; 2415 2416 hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx); 2417 2418 ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid; 2419 /* Undo the hdr->cw swap */ 2420 opcode = le32_to_cpu(hdr->cw) & OCRDMA_WQE_OPCODE_MASK; 2421 switch (opcode) { 2422 case OCRDMA_WRITE: 2423 ibwc->opcode = IB_WC_RDMA_WRITE; 2424 break; 2425 case OCRDMA_READ: 2426 rw = (struct ocrdma_sge *)(hdr + 1); 2427 ibwc->opcode = IB_WC_RDMA_READ; 2428 ibwc->byte_len = rw->len; 2429 break; 2430 case OCRDMA_SEND: 2431 ibwc->opcode = IB_WC_SEND; 2432 break; 2433 case OCRDMA_FR_MR: 2434 ibwc->opcode = IB_WC_REG_MR; 2435 break; 2436 case OCRDMA_LKEY_INV: 2437 ibwc->opcode = IB_WC_LOCAL_INV; 2438 break; 2439 default: 2440 ibwc->status = IB_WC_GENERAL_ERR; 2441 pr_err("%s() invalid opcode received = 0x%x\n", 2442 __func__, hdr->cw & OCRDMA_WQE_OPCODE_MASK); 2443 break; 2444 } 2445 } 2446 2447 static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp, 2448 struct ocrdma_cqe *cqe) 2449 { 2450 if (is_cqe_for_sq(cqe)) { 2451 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2452 cqe->flags_status_srcqpn) & 2453 ~OCRDMA_CQE_STATUS_MASK); 2454 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2455 cqe->flags_status_srcqpn) | 2456 (OCRDMA_CQE_WR_FLUSH_ERR << 2457 OCRDMA_CQE_STATUS_SHIFT)); 2458 } else { 2459 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 2460 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2461 cqe->flags_status_srcqpn) & 2462 ~OCRDMA_CQE_UD_STATUS_MASK); 2463 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2464 cqe->flags_status_srcqpn) | 2465 (OCRDMA_CQE_WR_FLUSH_ERR << 2466 OCRDMA_CQE_UD_STATUS_SHIFT)); 2467 } else { 2468 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2469 cqe->flags_status_srcqpn) & 2470 ~OCRDMA_CQE_STATUS_MASK); 2471 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2472 cqe->flags_status_srcqpn) | 2473 (OCRDMA_CQE_WR_FLUSH_ERR << 2474 OCRDMA_CQE_STATUS_SHIFT)); 2475 } 2476 } 2477 } 2478 2479 static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2480 struct ocrdma_qp *qp, int status) 2481 { 2482 bool expand = false; 2483 2484 ibwc->byte_len = 0; 2485 ibwc->qp = &qp->ibqp; 2486 ibwc->status = ocrdma_to_ibwc_err(status); 2487 2488 ocrdma_flush_qp(qp); 2489 ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL); 2490 2491 /* if wqe/rqe pending for which cqe needs to be returned, 2492 * trigger inflating it. 2493 */ 2494 if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) { 2495 expand = true; 2496 ocrdma_set_cqe_status_flushed(qp, cqe); 2497 } 2498 return expand; 2499 } 2500 2501 static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2502 struct ocrdma_qp *qp, int status) 2503 { 2504 ibwc->opcode = IB_WC_RECV; 2505 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2506 ocrdma_hwq_inc_tail(&qp->rq); 2507 2508 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); 2509 } 2510 2511 static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2512 struct ocrdma_qp *qp, int status) 2513 { 2514 ocrdma_update_wc(qp, ibwc, qp->sq.tail); 2515 ocrdma_hwq_inc_tail(&qp->sq); 2516 2517 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); 2518 } 2519 2520 2521 static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp, 2522 struct ocrdma_cqe *cqe, struct ib_wc *ibwc, 2523 bool *polled, bool *stop) 2524 { 2525 bool expand; 2526 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 2527 int status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2528 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2529 if (status < OCRDMA_MAX_CQE_ERR) 2530 atomic_inc(&dev->cqe_err_stats[status]); 2531 2532 /* when hw sq is empty, but rq is not empty, so we continue 2533 * to keep the cqe in order to get the cq event again. 2534 */ 2535 if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) { 2536 /* when cq for rq and sq is same, it is safe to return 2537 * flush cqe for RQEs. 2538 */ 2539 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { 2540 *polled = true; 2541 status = OCRDMA_CQE_WR_FLUSH_ERR; 2542 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); 2543 } else { 2544 /* stop processing further cqe as this cqe is used for 2545 * triggering cq event on buddy cq of RQ. 2546 * When QP is destroyed, this cqe will be removed 2547 * from the cq's hardware q. 2548 */ 2549 *polled = false; 2550 *stop = true; 2551 expand = false; 2552 } 2553 } else if (is_hw_sq_empty(qp)) { 2554 /* Do nothing */ 2555 expand = false; 2556 *polled = false; 2557 *stop = false; 2558 } else { 2559 *polled = true; 2560 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); 2561 } 2562 return expand; 2563 } 2564 2565 static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp, 2566 struct ocrdma_cqe *cqe, 2567 struct ib_wc *ibwc, bool *polled) 2568 { 2569 bool expand = false; 2570 int tail = qp->sq.tail; 2571 u32 wqe_idx; 2572 2573 if (!qp->wqe_wr_id_tbl[tail].signaled) { 2574 *polled = false; /* WC cannot be consumed yet */ 2575 } else { 2576 ibwc->status = IB_WC_SUCCESS; 2577 ibwc->wc_flags = 0; 2578 ibwc->qp = &qp->ibqp; 2579 ocrdma_update_wc(qp, ibwc, tail); 2580 *polled = true; 2581 } 2582 wqe_idx = (le32_to_cpu(cqe->wq.wqeidx) & 2583 OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx; 2584 if (tail != wqe_idx) 2585 expand = true; /* Coalesced CQE can't be consumed yet */ 2586 2587 ocrdma_hwq_inc_tail(&qp->sq); 2588 return expand; 2589 } 2590 2591 static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2592 struct ib_wc *ibwc, bool *polled, bool *stop) 2593 { 2594 int status; 2595 bool expand; 2596 2597 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2598 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2599 2600 if (status == OCRDMA_CQE_SUCCESS) 2601 expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled); 2602 else 2603 expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop); 2604 return expand; 2605 } 2606 2607 static int ocrdma_update_ud_rcqe(struct ocrdma_dev *dev, struct ib_wc *ibwc, 2608 struct ocrdma_cqe *cqe) 2609 { 2610 int status; 2611 u16 hdr_type = 0; 2612 2613 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2614 OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT; 2615 ibwc->src_qp = le32_to_cpu(cqe->flags_status_srcqpn) & 2616 OCRDMA_CQE_SRCQP_MASK; 2617 ibwc->pkey_index = 0; 2618 ibwc->wc_flags = IB_WC_GRH; 2619 ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >> 2620 OCRDMA_CQE_UD_XFER_LEN_SHIFT) & 2621 OCRDMA_CQE_UD_XFER_LEN_MASK; 2622 2623 if (ocrdma_is_udp_encap_supported(dev)) { 2624 hdr_type = (le32_to_cpu(cqe->ud.rxlen_pkey) >> 2625 OCRDMA_CQE_UD_L3TYPE_SHIFT) & 2626 OCRDMA_CQE_UD_L3TYPE_MASK; 2627 ibwc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 2628 ibwc->network_hdr_type = hdr_type; 2629 } 2630 2631 return status; 2632 } 2633 2634 static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc, 2635 struct ocrdma_cqe *cqe, 2636 struct ocrdma_qp *qp) 2637 { 2638 unsigned long flags; 2639 struct ocrdma_srq *srq; 2640 u32 wqe_idx; 2641 2642 srq = get_ocrdma_srq(qp->ibqp.srq); 2643 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >> 2644 OCRDMA_CQE_BUFTAG_SHIFT) & srq->rq.max_wqe_idx; 2645 BUG_ON(wqe_idx < 1); 2646 2647 ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx]; 2648 spin_lock_irqsave(&srq->q_lock, flags); 2649 ocrdma_srq_toggle_bit(srq, wqe_idx - 1); 2650 spin_unlock_irqrestore(&srq->q_lock, flags); 2651 ocrdma_hwq_inc_tail(&srq->rq); 2652 } 2653 2654 static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2655 struct ib_wc *ibwc, bool *polled, bool *stop, 2656 int status) 2657 { 2658 bool expand; 2659 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 2660 2661 if (status < OCRDMA_MAX_CQE_ERR) 2662 atomic_inc(&dev->cqe_err_stats[status]); 2663 2664 /* when hw_rq is empty, but wq is not empty, so continue 2665 * to keep the cqe to get the cq event again. 2666 */ 2667 if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) { 2668 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { 2669 *polled = true; 2670 status = OCRDMA_CQE_WR_FLUSH_ERR; 2671 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); 2672 } else { 2673 *polled = false; 2674 *stop = true; 2675 expand = false; 2676 } 2677 } else if (is_hw_rq_empty(qp)) { 2678 /* Do nothing */ 2679 expand = false; 2680 *polled = false; 2681 *stop = false; 2682 } else { 2683 *polled = true; 2684 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); 2685 } 2686 return expand; 2687 } 2688 2689 static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp, 2690 struct ocrdma_cqe *cqe, struct ib_wc *ibwc) 2691 { 2692 struct ocrdma_dev *dev; 2693 2694 dev = get_ocrdma_dev(qp->ibqp.device); 2695 ibwc->opcode = IB_WC_RECV; 2696 ibwc->qp = &qp->ibqp; 2697 ibwc->status = IB_WC_SUCCESS; 2698 2699 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) 2700 ocrdma_update_ud_rcqe(dev, ibwc, cqe); 2701 else 2702 ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen); 2703 2704 if (is_cqe_imm(cqe)) { 2705 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt)); 2706 ibwc->wc_flags |= IB_WC_WITH_IMM; 2707 } else if (is_cqe_wr_imm(cqe)) { 2708 ibwc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 2709 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt)); 2710 ibwc->wc_flags |= IB_WC_WITH_IMM; 2711 } else if (is_cqe_invalidated(cqe)) { 2712 ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt); 2713 ibwc->wc_flags |= IB_WC_WITH_INVALIDATE; 2714 } 2715 if (qp->ibqp.srq) { 2716 ocrdma_update_free_srq_cqe(ibwc, cqe, qp); 2717 } else { 2718 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2719 ocrdma_hwq_inc_tail(&qp->rq); 2720 } 2721 } 2722 2723 static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2724 struct ib_wc *ibwc, bool *polled, bool *stop) 2725 { 2726 int status; 2727 bool expand = false; 2728 2729 ibwc->wc_flags = 0; 2730 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 2731 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2732 OCRDMA_CQE_UD_STATUS_MASK) >> 2733 OCRDMA_CQE_UD_STATUS_SHIFT; 2734 } else { 2735 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2736 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2737 } 2738 2739 if (status == OCRDMA_CQE_SUCCESS) { 2740 *polled = true; 2741 ocrdma_poll_success_rcqe(qp, cqe, ibwc); 2742 } else { 2743 expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop, 2744 status); 2745 } 2746 return expand; 2747 } 2748 2749 static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe, 2750 u16 cur_getp) 2751 { 2752 if (cq->phase_change) { 2753 if (cur_getp == 0) 2754 cq->phase = (~cq->phase & OCRDMA_CQE_VALID); 2755 } else { 2756 /* clear valid bit */ 2757 cqe->flags_status_srcqpn = 0; 2758 } 2759 } 2760 2761 static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries, 2762 struct ib_wc *ibwc) 2763 { 2764 u16 qpn = 0; 2765 int i = 0; 2766 bool expand = false; 2767 int polled_hw_cqes = 0; 2768 struct ocrdma_qp *qp = NULL; 2769 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device); 2770 struct ocrdma_cqe *cqe; 2771 u16 cur_getp; bool polled = false; bool stop = false; 2772 2773 cur_getp = cq->getp; 2774 while (num_entries) { 2775 cqe = cq->va + cur_getp; 2776 /* check whether valid cqe or not */ 2777 if (!is_cqe_valid(cq, cqe)) 2778 break; 2779 qpn = (le32_to_cpu(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK); 2780 /* ignore discarded cqe */ 2781 if (qpn == 0) 2782 goto skip_cqe; 2783 qp = dev->qp_tbl[qpn]; 2784 BUG_ON(qp == NULL); 2785 2786 if (is_cqe_for_sq(cqe)) { 2787 expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled, 2788 &stop); 2789 } else { 2790 expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled, 2791 &stop); 2792 } 2793 if (expand) 2794 goto expand_cqe; 2795 if (stop) 2796 goto stop_cqe; 2797 /* clear qpn to avoid duplicate processing by discard_cqe() */ 2798 cqe->cmn.qpn = 0; 2799 skip_cqe: 2800 polled_hw_cqes += 1; 2801 cur_getp = (cur_getp + 1) % cq->max_hw_cqe; 2802 ocrdma_change_cq_phase(cq, cqe, cur_getp); 2803 expand_cqe: 2804 if (polled) { 2805 num_entries -= 1; 2806 i += 1; 2807 ibwc = ibwc + 1; 2808 polled = false; 2809 } 2810 } 2811 stop_cqe: 2812 cq->getp = cur_getp; 2813 2814 if (polled_hw_cqes) 2815 ocrdma_ring_cq_db(dev, cq->id, false, false, polled_hw_cqes); 2816 2817 return i; 2818 } 2819 2820 /* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */ 2821 static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries, 2822 struct ocrdma_qp *qp, struct ib_wc *ibwc) 2823 { 2824 int err_cqes = 0; 2825 2826 while (num_entries) { 2827 if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp)) 2828 break; 2829 if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) { 2830 ocrdma_update_wc(qp, ibwc, qp->sq.tail); 2831 ocrdma_hwq_inc_tail(&qp->sq); 2832 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) { 2833 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2834 ocrdma_hwq_inc_tail(&qp->rq); 2835 } else { 2836 return err_cqes; 2837 } 2838 ibwc->byte_len = 0; 2839 ibwc->status = IB_WC_WR_FLUSH_ERR; 2840 ibwc = ibwc + 1; 2841 err_cqes += 1; 2842 num_entries -= 1; 2843 } 2844 return err_cqes; 2845 } 2846 2847 int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 2848 { 2849 int cqes_to_poll = num_entries; 2850 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 2851 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 2852 int num_os_cqe = 0, err_cqes = 0; 2853 struct ocrdma_qp *qp; 2854 unsigned long flags; 2855 2856 /* poll cqes from adapter CQ */ 2857 spin_lock_irqsave(&cq->cq_lock, flags); 2858 num_os_cqe = ocrdma_poll_hwcq(cq, cqes_to_poll, wc); 2859 spin_unlock_irqrestore(&cq->cq_lock, flags); 2860 cqes_to_poll -= num_os_cqe; 2861 2862 if (cqes_to_poll) { 2863 wc = wc + num_os_cqe; 2864 /* adapter returns single error cqe when qp moves to 2865 * error state. So insert error cqes with wc_status as 2866 * FLUSHED for pending WQEs and RQEs of QP's SQ and RQ 2867 * respectively which uses this CQ. 2868 */ 2869 spin_lock_irqsave(&dev->flush_q_lock, flags); 2870 list_for_each_entry(qp, &cq->sq_head, sq_entry) { 2871 if (cqes_to_poll == 0) 2872 break; 2873 err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc); 2874 cqes_to_poll -= err_cqes; 2875 num_os_cqe += err_cqes; 2876 wc = wc + err_cqes; 2877 } 2878 spin_unlock_irqrestore(&dev->flush_q_lock, flags); 2879 } 2880 return num_os_cqe; 2881 } 2882 2883 int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags) 2884 { 2885 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 2886 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 2887 u16 cq_id; 2888 unsigned long flags; 2889 bool arm_needed = false, sol_needed = false; 2890 2891 cq_id = cq->id; 2892 2893 spin_lock_irqsave(&cq->cq_lock, flags); 2894 if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED) 2895 arm_needed = true; 2896 if (cq_flags & IB_CQ_SOLICITED) 2897 sol_needed = true; 2898 2899 ocrdma_ring_cq_db(dev, cq_id, arm_needed, sol_needed, 0); 2900 spin_unlock_irqrestore(&cq->cq_lock, flags); 2901 2902 return 0; 2903 } 2904 2905 struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 2906 u32 max_num_sg, struct ib_udata *udata) 2907 { 2908 int status; 2909 struct ocrdma_mr *mr; 2910 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 2911 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 2912 2913 if (mr_type != IB_MR_TYPE_MEM_REG) 2914 return ERR_PTR(-EINVAL); 2915 2916 if (max_num_sg > dev->attr.max_pages_per_frmr) 2917 return ERR_PTR(-EINVAL); 2918 2919 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2920 if (!mr) 2921 return ERR_PTR(-ENOMEM); 2922 2923 mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); 2924 if (!mr->pages) { 2925 status = -ENOMEM; 2926 goto pl_err; 2927 } 2928 2929 status = ocrdma_get_pbl_info(dev, mr, max_num_sg); 2930 if (status) 2931 goto pbl_err; 2932 mr->hwmr.fr_mr = 1; 2933 mr->hwmr.remote_rd = 0; 2934 mr->hwmr.remote_wr = 0; 2935 mr->hwmr.local_rd = 0; 2936 mr->hwmr.local_wr = 0; 2937 mr->hwmr.mw_bind = 0; 2938 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr); 2939 if (status) 2940 goto pbl_err; 2941 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0); 2942 if (status) 2943 goto mbx_err; 2944 mr->ibmr.rkey = mr->hwmr.lkey; 2945 mr->ibmr.lkey = mr->hwmr.lkey; 2946 dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] = 2947 (unsigned long) mr; 2948 return &mr->ibmr; 2949 mbx_err: 2950 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 2951 pbl_err: 2952 kfree(mr->pages); 2953 pl_err: 2954 kfree(mr); 2955 return ERR_PTR(-ENOMEM); 2956 } 2957 2958 static int ocrdma_set_page(struct ib_mr *ibmr, u64 addr) 2959 { 2960 struct ocrdma_mr *mr = get_ocrdma_mr(ibmr); 2961 2962 if (unlikely(mr->npages == mr->hwmr.num_pbes)) 2963 return -ENOMEM; 2964 2965 mr->pages[mr->npages++] = addr; 2966 2967 return 0; 2968 } 2969 2970 int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 2971 unsigned int *sg_offset) 2972 { 2973 struct ocrdma_mr *mr = get_ocrdma_mr(ibmr); 2974 2975 mr->npages = 0; 2976 2977 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ocrdma_set_page); 2978 } 2979