1 /* 2 * Broadcom NetXtreme-E RoCE driver. 3 * 4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term 5 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Description: IB Verbs interpreter 37 */ 38 39 #include <linux/interrupt.h> 40 #include <linux/types.h> 41 #include <linux/pci.h> 42 #include <linux/netdevice.h> 43 #include <linux/if_ether.h> 44 #include <net/addrconf.h> 45 46 #include <rdma/ib_verbs.h> 47 #include <rdma/ib_user_verbs.h> 48 #include <rdma/ib_umem.h> 49 #include <rdma/ib_addr.h> 50 #include <rdma/ib_mad.h> 51 #include <rdma/ib_cache.h> 52 #include <rdma/uverbs_ioctl.h> 53 54 #include "bnxt_ulp.h" 55 56 #include "roce_hsi.h" 57 #include "qplib_res.h" 58 #include "qplib_sp.h" 59 #include "qplib_fp.h" 60 #include "qplib_rcfw.h" 61 62 #include "bnxt_re.h" 63 #include "ib_verbs.h" 64 #include <rdma/bnxt_re-abi.h> 65 66 static int __from_ib_access_flags(int iflags) 67 { 68 int qflags = 0; 69 70 if (iflags & IB_ACCESS_LOCAL_WRITE) 71 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 72 if (iflags & IB_ACCESS_REMOTE_READ) 73 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ; 74 if (iflags & IB_ACCESS_REMOTE_WRITE) 75 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE; 76 if (iflags & IB_ACCESS_REMOTE_ATOMIC) 77 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC; 78 if (iflags & IB_ACCESS_MW_BIND) 79 qflags |= BNXT_QPLIB_ACCESS_MW_BIND; 80 if (iflags & IB_ZERO_BASED) 81 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED; 82 if (iflags & IB_ACCESS_ON_DEMAND) 83 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND; 84 return qflags; 85 }; 86 87 static enum ib_access_flags __to_ib_access_flags(int qflags) 88 { 89 enum ib_access_flags iflags = 0; 90 91 if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE) 92 iflags |= IB_ACCESS_LOCAL_WRITE; 93 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE) 94 iflags |= IB_ACCESS_REMOTE_WRITE; 95 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ) 96 iflags |= IB_ACCESS_REMOTE_READ; 97 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC) 98 iflags |= IB_ACCESS_REMOTE_ATOMIC; 99 if (qflags & BNXT_QPLIB_ACCESS_MW_BIND) 100 iflags |= IB_ACCESS_MW_BIND; 101 if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED) 102 iflags |= IB_ZERO_BASED; 103 if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND) 104 iflags |= IB_ACCESS_ON_DEMAND; 105 return iflags; 106 }; 107 108 static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list, 109 struct bnxt_qplib_sge *sg_list, int num) 110 { 111 int i, total = 0; 112 113 for (i = 0; i < num; i++) { 114 sg_list[i].addr = ib_sg_list[i].addr; 115 sg_list[i].lkey = ib_sg_list[i].lkey; 116 sg_list[i].size = ib_sg_list[i].length; 117 total += sg_list[i].size; 118 } 119 return total; 120 } 121 122 /* Device */ 123 int bnxt_re_query_device(struct ib_device *ibdev, 124 struct ib_device_attr *ib_attr, 125 struct ib_udata *udata) 126 { 127 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 128 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 129 130 memset(ib_attr, 0, sizeof(*ib_attr)); 131 memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, 132 min(sizeof(dev_attr->fw_ver), 133 sizeof(ib_attr->fw_ver))); 134 addrconf_addr_eui48((u8 *)&ib_attr->sys_image_guid, 135 rdev->netdev->dev_addr); 136 ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE; 137 ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_SUPPORTED; 138 139 ib_attr->vendor_id = rdev->en_dev->pdev->vendor; 140 ib_attr->vendor_part_id = rdev->en_dev->pdev->device; 141 ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; 142 ib_attr->max_qp = dev_attr->max_qp; 143 ib_attr->max_qp_wr = dev_attr->max_qp_wqes; 144 ib_attr->device_cap_flags = 145 IB_DEVICE_CURR_QP_STATE_MOD 146 | IB_DEVICE_RC_RNR_NAK_GEN 147 | IB_DEVICE_SHUTDOWN_PORT 148 | IB_DEVICE_SYS_IMAGE_GUID 149 | IB_DEVICE_RESIZE_MAX_WR 150 | IB_DEVICE_PORT_ACTIVE_EVENT 151 | IB_DEVICE_N_NOTIFY_CQ 152 | IB_DEVICE_MEM_WINDOW 153 | IB_DEVICE_MEM_WINDOW_TYPE_2B 154 | IB_DEVICE_MEM_MGT_EXTENSIONS; 155 ib_attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 156 ib_attr->max_send_sge = dev_attr->max_qp_sges; 157 ib_attr->max_recv_sge = dev_attr->max_qp_sges; 158 ib_attr->max_sge_rd = dev_attr->max_qp_sges; 159 ib_attr->max_cq = dev_attr->max_cq; 160 ib_attr->max_cqe = dev_attr->max_cq_wqes; 161 ib_attr->max_mr = dev_attr->max_mr; 162 ib_attr->max_pd = dev_attr->max_pd; 163 ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; 164 ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom; 165 ib_attr->atomic_cap = IB_ATOMIC_NONE; 166 ib_attr->masked_atomic_cap = IB_ATOMIC_NONE; 167 if (dev_attr->is_atomic) { 168 ib_attr->atomic_cap = IB_ATOMIC_GLOB; 169 ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB; 170 } 171 172 ib_attr->max_ee_rd_atom = 0; 173 ib_attr->max_res_rd_atom = 0; 174 ib_attr->max_ee_init_rd_atom = 0; 175 ib_attr->max_ee = 0; 176 ib_attr->max_rdd = 0; 177 ib_attr->max_mw = dev_attr->max_mw; 178 ib_attr->max_raw_ipv6_qp = 0; 179 ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; 180 ib_attr->max_mcast_grp = 0; 181 ib_attr->max_mcast_qp_attach = 0; 182 ib_attr->max_total_mcast_qp_attach = 0; 183 ib_attr->max_ah = dev_attr->max_ah; 184 185 ib_attr->max_srq = dev_attr->max_srq; 186 ib_attr->max_srq_wr = dev_attr->max_srq_wqes; 187 ib_attr->max_srq_sge = dev_attr->max_srq_sges; 188 189 ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; 190 191 ib_attr->max_pkeys = 1; 192 ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY; 193 return 0; 194 } 195 196 /* Port */ 197 int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num, 198 struct ib_port_attr *port_attr) 199 { 200 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 201 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 202 203 memset(port_attr, 0, sizeof(*port_attr)); 204 205 if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) { 206 port_attr->state = IB_PORT_ACTIVE; 207 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 208 } else { 209 port_attr->state = IB_PORT_DOWN; 210 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 211 } 212 port_attr->max_mtu = IB_MTU_4096; 213 port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu); 214 port_attr->gid_tbl_len = dev_attr->max_sgid; 215 port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 216 IB_PORT_DEVICE_MGMT_SUP | 217 IB_PORT_VENDOR_CLASS_SUP; 218 port_attr->ip_gids = true; 219 220 port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW; 221 port_attr->bad_pkey_cntr = 0; 222 port_attr->qkey_viol_cntr = 0; 223 port_attr->pkey_tbl_len = dev_attr->max_pkey; 224 port_attr->lid = 0; 225 port_attr->sm_lid = 0; 226 port_attr->lmc = 0; 227 port_attr->max_vl_num = 4; 228 port_attr->sm_sl = 0; 229 port_attr->subnet_timeout = 0; 230 port_attr->init_type_reply = 0; 231 port_attr->active_speed = rdev->active_speed; 232 port_attr->active_width = rdev->active_width; 233 234 return 0; 235 } 236 237 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num, 238 struct ib_port_immutable *immutable) 239 { 240 struct ib_port_attr port_attr; 241 242 if (bnxt_re_query_port(ibdev, port_num, &port_attr)) 243 return -EINVAL; 244 245 immutable->pkey_tbl_len = port_attr.pkey_tbl_len; 246 immutable->gid_tbl_len = port_attr.gid_tbl_len; 247 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; 248 immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP; 249 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 250 return 0; 251 } 252 253 void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str) 254 { 255 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 256 257 snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d", 258 rdev->dev_attr.fw_ver[0], rdev->dev_attr.fw_ver[1], 259 rdev->dev_attr.fw_ver[2], rdev->dev_attr.fw_ver[3]); 260 } 261 262 int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num, 263 u16 index, u16 *pkey) 264 { 265 if (index > 0) 266 return -EINVAL; 267 268 *pkey = IB_DEFAULT_PKEY_FULL; 269 270 return 0; 271 } 272 273 int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num, 274 int index, union ib_gid *gid) 275 { 276 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 277 int rc = 0; 278 279 /* Ignore port_num */ 280 memset(gid, 0, sizeof(*gid)); 281 rc = bnxt_qplib_get_sgid(&rdev->qplib_res, 282 &rdev->qplib_res.sgid_tbl, index, 283 (struct bnxt_qplib_gid *)gid); 284 return rc; 285 } 286 287 int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context) 288 { 289 int rc = 0; 290 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 291 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); 292 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 293 struct bnxt_qplib_gid *gid_to_del; 294 u16 vlan_id = 0xFFFF; 295 296 /* Delete the entry from the hardware */ 297 ctx = *context; 298 if (!ctx) 299 return -EINVAL; 300 301 if (sgid_tbl && sgid_tbl->active) { 302 if (ctx->idx >= sgid_tbl->max) 303 return -EINVAL; 304 gid_to_del = &sgid_tbl->tbl[ctx->idx].gid; 305 vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id; 306 /* DEL_GID is called in WQ context(netdevice_event_work_handler) 307 * or via the ib_unregister_device path. In the former case QP1 308 * may not be destroyed yet, in which case just return as FW 309 * needs that entry to be present and will fail it's deletion. 310 * We could get invoked again after QP1 is destroyed OR get an 311 * ADD_GID call with a different GID value for the same index 312 * where we issue MODIFY_GID cmd to update the GID entry -- TBD 313 */ 314 if (ctx->idx == 0 && 315 rdma_link_local_addr((struct in6_addr *)gid_to_del) && 316 ctx->refcnt == 1 && rdev->gsi_ctx.gsi_sqp) { 317 ibdev_dbg(&rdev->ibdev, 318 "Trying to delete GID0 while QP1 is alive\n"); 319 return -EFAULT; 320 } 321 ctx->refcnt--; 322 if (!ctx->refcnt) { 323 rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, 324 vlan_id, true); 325 if (rc) { 326 ibdev_err(&rdev->ibdev, 327 "Failed to remove GID: %#x", rc); 328 } else { 329 ctx_tbl = sgid_tbl->ctx; 330 ctx_tbl[ctx->idx] = NULL; 331 kfree(ctx); 332 } 333 } 334 } else { 335 return -EINVAL; 336 } 337 return rc; 338 } 339 340 int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context) 341 { 342 int rc; 343 u32 tbl_idx = 0; 344 u16 vlan_id = 0xFFFF; 345 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 346 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); 347 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 348 349 rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL); 350 if (rc) 351 return rc; 352 353 rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid, 354 rdev->qplib_res.netdev->dev_addr, 355 vlan_id, true, &tbl_idx); 356 if (rc == -EALREADY) { 357 ctx_tbl = sgid_tbl->ctx; 358 ctx_tbl[tbl_idx]->refcnt++; 359 *context = ctx_tbl[tbl_idx]; 360 return 0; 361 } 362 363 if (rc < 0) { 364 ibdev_err(&rdev->ibdev, "Failed to add GID: %#x", rc); 365 return rc; 366 } 367 368 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 369 if (!ctx) 370 return -ENOMEM; 371 ctx_tbl = sgid_tbl->ctx; 372 ctx->idx = tbl_idx; 373 ctx->refcnt = 1; 374 ctx_tbl[tbl_idx] = ctx; 375 *context = ctx; 376 377 return rc; 378 } 379 380 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, 381 u32 port_num) 382 { 383 return IB_LINK_LAYER_ETHERNET; 384 } 385 386 #define BNXT_RE_FENCE_PBL_SIZE DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE) 387 388 static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd) 389 { 390 struct bnxt_re_fence_data *fence = &pd->fence; 391 struct ib_mr *ib_mr = &fence->mr->ib_mr; 392 struct bnxt_qplib_swqe *wqe = &fence->bind_wqe; 393 394 memset(wqe, 0, sizeof(*wqe)); 395 wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW; 396 wqe->wr_id = BNXT_QPLIB_FENCE_WRID; 397 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 398 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 399 wqe->bind.zero_based = false; 400 wqe->bind.parent_l_key = ib_mr->lkey; 401 wqe->bind.va = (u64)(unsigned long)fence->va; 402 wqe->bind.length = fence->size; 403 wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ); 404 wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1; 405 406 /* Save the initial rkey in fence structure for now; 407 * wqe->bind.r_key will be set at (re)bind time. 408 */ 409 fence->bind_rkey = ib_inc_rkey(fence->mw->rkey); 410 } 411 412 static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp) 413 { 414 struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp, 415 qplib_qp); 416 struct ib_pd *ib_pd = qp->ib_qp.pd; 417 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 418 struct bnxt_re_fence_data *fence = &pd->fence; 419 struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe; 420 struct bnxt_qplib_swqe wqe; 421 int rc; 422 423 memcpy(&wqe, fence_wqe, sizeof(wqe)); 424 wqe.bind.r_key = fence->bind_rkey; 425 fence->bind_rkey = ib_inc_rkey(fence->bind_rkey); 426 427 ibdev_dbg(&qp->rdev->ibdev, 428 "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n", 429 wqe.bind.r_key, qp->qplib_qp.id, pd); 430 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 431 if (rc) { 432 ibdev_err(&qp->rdev->ibdev, "Failed to bind fence-WQE\n"); 433 return rc; 434 } 435 bnxt_qplib_post_send_db(&qp->qplib_qp); 436 437 return rc; 438 } 439 440 static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd) 441 { 442 struct bnxt_re_fence_data *fence = &pd->fence; 443 struct bnxt_re_dev *rdev = pd->rdev; 444 struct device *dev = &rdev->en_dev->pdev->dev; 445 struct bnxt_re_mr *mr = fence->mr; 446 447 if (fence->mw) { 448 bnxt_re_dealloc_mw(fence->mw); 449 fence->mw = NULL; 450 } 451 if (mr) { 452 if (mr->ib_mr.rkey) 453 bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr, 454 true); 455 if (mr->ib_mr.lkey) 456 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 457 kfree(mr); 458 fence->mr = NULL; 459 } 460 if (fence->dma_addr) { 461 dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES, 462 DMA_BIDIRECTIONAL); 463 fence->dma_addr = 0; 464 } 465 } 466 467 static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd) 468 { 469 int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND; 470 struct bnxt_re_fence_data *fence = &pd->fence; 471 struct bnxt_re_dev *rdev = pd->rdev; 472 struct device *dev = &rdev->en_dev->pdev->dev; 473 struct bnxt_re_mr *mr = NULL; 474 dma_addr_t dma_addr = 0; 475 struct ib_mw *mw; 476 int rc; 477 478 dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES, 479 DMA_BIDIRECTIONAL); 480 rc = dma_mapping_error(dev, dma_addr); 481 if (rc) { 482 ibdev_err(&rdev->ibdev, "Failed to dma-map fence-MR-mem\n"); 483 rc = -EIO; 484 fence->dma_addr = 0; 485 goto fail; 486 } 487 fence->dma_addr = dma_addr; 488 489 /* Allocate a MR */ 490 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 491 if (!mr) { 492 rc = -ENOMEM; 493 goto fail; 494 } 495 fence->mr = mr; 496 mr->rdev = rdev; 497 mr->qplib_mr.pd = &pd->qplib_pd; 498 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 499 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 500 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 501 if (rc) { 502 ibdev_err(&rdev->ibdev, "Failed to alloc fence-HW-MR\n"); 503 goto fail; 504 } 505 506 /* Register MR */ 507 mr->ib_mr.lkey = mr->qplib_mr.lkey; 508 mr->qplib_mr.va = (u64)(unsigned long)fence->va; 509 mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES; 510 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 511 BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE); 512 if (rc) { 513 ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n"); 514 goto fail; 515 } 516 mr->ib_mr.rkey = mr->qplib_mr.rkey; 517 518 /* Create a fence MW only for kernel consumers */ 519 mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL); 520 if (IS_ERR(mw)) { 521 ibdev_err(&rdev->ibdev, 522 "Failed to create fence-MW for PD: %p\n", pd); 523 rc = PTR_ERR(mw); 524 goto fail; 525 } 526 fence->mw = mw; 527 528 bnxt_re_create_fence_wqe(pd); 529 return 0; 530 531 fail: 532 bnxt_re_destroy_fence_mr(pd); 533 return rc; 534 } 535 536 /* Protection Domains */ 537 int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) 538 { 539 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 540 struct bnxt_re_dev *rdev = pd->rdev; 541 542 bnxt_re_destroy_fence_mr(pd); 543 544 if (pd->qplib_pd.id) { 545 if (!bnxt_qplib_dealloc_pd(&rdev->qplib_res, 546 &rdev->qplib_res.pd_tbl, 547 &pd->qplib_pd)) 548 atomic_dec(&rdev->pd_count); 549 } 550 return 0; 551 } 552 553 int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 554 { 555 struct ib_device *ibdev = ibpd->device; 556 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 557 struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context( 558 udata, struct bnxt_re_ucontext, ib_uctx); 559 struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd); 560 int rc; 561 562 pd->rdev = rdev; 563 if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) { 564 ibdev_err(&rdev->ibdev, "Failed to allocate HW PD"); 565 rc = -ENOMEM; 566 goto fail; 567 } 568 569 if (udata) { 570 struct bnxt_re_pd_resp resp; 571 572 if (!ucntx->dpi.dbr) { 573 /* Allocate DPI in alloc_pd to avoid failing of 574 * ibv_devinfo and family of application when DPIs 575 * are depleted. 576 */ 577 if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl, 578 &ucntx->dpi, ucntx)) { 579 rc = -ENOMEM; 580 goto dbfail; 581 } 582 } 583 584 resp.pdid = pd->qplib_pd.id; 585 /* Still allow mapping this DBR to the new user PD. */ 586 resp.dpi = ucntx->dpi.dpi; 587 resp.dbr = (u64)ucntx->dpi.umdbr; 588 589 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 590 if (rc) { 591 ibdev_err(&rdev->ibdev, 592 "Failed to copy user response\n"); 593 goto dbfail; 594 } 595 } 596 597 if (!udata) 598 if (bnxt_re_create_fence_mr(pd)) 599 ibdev_warn(&rdev->ibdev, 600 "Failed to create Fence-MR\n"); 601 atomic_inc(&rdev->pd_count); 602 603 return 0; 604 dbfail: 605 bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, 606 &pd->qplib_pd); 607 fail: 608 return rc; 609 } 610 611 /* Address Handles */ 612 int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags) 613 { 614 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 615 struct bnxt_re_dev *rdev = ah->rdev; 616 617 bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, 618 !(flags & RDMA_DESTROY_AH_SLEEPABLE)); 619 atomic_dec(&rdev->ah_count); 620 621 return 0; 622 } 623 624 static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype) 625 { 626 u8 nw_type; 627 628 switch (ntype) { 629 case RDMA_NETWORK_IPV4: 630 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4; 631 break; 632 case RDMA_NETWORK_IPV6: 633 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6; 634 break; 635 default: 636 nw_type = CMDQ_CREATE_AH_TYPE_V1; 637 break; 638 } 639 return nw_type; 640 } 641 642 int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr, 643 struct ib_udata *udata) 644 { 645 struct ib_pd *ib_pd = ib_ah->pd; 646 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 647 struct rdma_ah_attr *ah_attr = init_attr->ah_attr; 648 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); 649 struct bnxt_re_dev *rdev = pd->rdev; 650 const struct ib_gid_attr *sgid_attr; 651 struct bnxt_re_gid_ctx *ctx; 652 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 653 u8 nw_type; 654 int rc; 655 656 if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) { 657 ibdev_err(&rdev->ibdev, "Failed to alloc AH: GRH not set"); 658 return -EINVAL; 659 } 660 661 ah->rdev = rdev; 662 ah->qplib_ah.pd = &pd->qplib_pd; 663 664 /* Supply the configuration for the HW */ 665 memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw, 666 sizeof(union ib_gid)); 667 sgid_attr = grh->sgid_attr; 668 /* Get the HW context of the GID. The reference 669 * of GID table entry is already taken by the caller. 670 */ 671 ctx = rdma_read_gid_hw_context(sgid_attr); 672 ah->qplib_ah.sgid_index = ctx->idx; 673 ah->qplib_ah.host_sgid_index = grh->sgid_index; 674 ah->qplib_ah.traffic_class = grh->traffic_class; 675 ah->qplib_ah.flow_label = grh->flow_label; 676 ah->qplib_ah.hop_limit = grh->hop_limit; 677 ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr); 678 679 /* Get network header type for this GID */ 680 nw_type = rdma_gid_attr_network_type(sgid_attr); 681 ah->qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(nw_type); 682 683 memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN); 684 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, 685 !(init_attr->flags & 686 RDMA_CREATE_AH_SLEEPABLE)); 687 if (rc) { 688 ibdev_err(&rdev->ibdev, "Failed to allocate HW AH"); 689 return rc; 690 } 691 692 /* Write AVID to shared page. */ 693 if (udata) { 694 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context( 695 udata, struct bnxt_re_ucontext, ib_uctx); 696 unsigned long flag; 697 u32 *wrptr; 698 699 spin_lock_irqsave(&uctx->sh_lock, flag); 700 wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT); 701 *wrptr = ah->qplib_ah.id; 702 wmb(); /* make sure cache is updated. */ 703 spin_unlock_irqrestore(&uctx->sh_lock, flag); 704 } 705 atomic_inc(&rdev->ah_count); 706 707 return 0; 708 } 709 710 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) 711 { 712 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 713 714 ah_attr->type = ib_ah->type; 715 rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl); 716 memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN); 717 rdma_ah_set_grh(ah_attr, NULL, 0, 718 ah->qplib_ah.host_sgid_index, 719 0, ah->qplib_ah.traffic_class); 720 rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data); 721 rdma_ah_set_port_num(ah_attr, 1); 722 rdma_ah_set_static_rate(ah_attr, 0); 723 return 0; 724 } 725 726 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp) 727 __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock) 728 { 729 unsigned long flags; 730 731 spin_lock_irqsave(&qp->scq->cq_lock, flags); 732 if (qp->rcq != qp->scq) 733 spin_lock(&qp->rcq->cq_lock); 734 else 735 __acquire(&qp->rcq->cq_lock); 736 737 return flags; 738 } 739 740 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, 741 unsigned long flags) 742 __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock) 743 { 744 if (qp->rcq != qp->scq) 745 spin_unlock(&qp->rcq->cq_lock); 746 else 747 __release(&qp->rcq->cq_lock); 748 spin_unlock_irqrestore(&qp->scq->cq_lock, flags); 749 } 750 751 static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp) 752 { 753 struct bnxt_re_qp *gsi_sqp; 754 struct bnxt_re_ah *gsi_sah; 755 struct bnxt_re_dev *rdev; 756 int rc = 0; 757 758 rdev = qp->rdev; 759 gsi_sqp = rdev->gsi_ctx.gsi_sqp; 760 gsi_sah = rdev->gsi_ctx.gsi_sah; 761 762 ibdev_dbg(&rdev->ibdev, "Destroy the shadow AH\n"); 763 bnxt_qplib_destroy_ah(&rdev->qplib_res, 764 &gsi_sah->qplib_ah, 765 true); 766 atomic_dec(&rdev->ah_count); 767 bnxt_qplib_clean_qp(&qp->qplib_qp); 768 769 ibdev_dbg(&rdev->ibdev, "Destroy the shadow QP\n"); 770 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp); 771 if (rc) { 772 ibdev_err(&rdev->ibdev, "Destroy Shadow QP failed"); 773 goto fail; 774 } 775 bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp); 776 777 /* remove from active qp list */ 778 mutex_lock(&rdev->qp_lock); 779 list_del(&gsi_sqp->list); 780 mutex_unlock(&rdev->qp_lock); 781 atomic_dec(&rdev->qp_count); 782 783 kfree(rdev->gsi_ctx.sqp_tbl); 784 kfree(gsi_sah); 785 kfree(gsi_sqp); 786 rdev->gsi_ctx.gsi_sqp = NULL; 787 rdev->gsi_ctx.gsi_sah = NULL; 788 rdev->gsi_ctx.sqp_tbl = NULL; 789 790 return 0; 791 fail: 792 return rc; 793 } 794 795 /* Queue Pairs */ 796 int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) 797 { 798 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 799 struct bnxt_re_dev *rdev = qp->rdev; 800 unsigned int flags; 801 int rc; 802 803 bnxt_qplib_flush_cqn_wq(&qp->qplib_qp); 804 805 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 806 if (rc) { 807 ibdev_err(&rdev->ibdev, "Failed to destroy HW QP"); 808 return rc; 809 } 810 811 if (rdma_is_kernel_res(&qp->ib_qp.res)) { 812 flags = bnxt_re_lock_cqs(qp); 813 bnxt_qplib_clean_qp(&qp->qplib_qp); 814 bnxt_re_unlock_cqs(qp, flags); 815 } 816 817 bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); 818 819 if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) { 820 rc = bnxt_re_destroy_gsi_sqp(qp); 821 if (rc) 822 return rc; 823 } 824 825 mutex_lock(&rdev->qp_lock); 826 list_del(&qp->list); 827 mutex_unlock(&rdev->qp_lock); 828 atomic_dec(&rdev->qp_count); 829 830 ib_umem_release(qp->rumem); 831 ib_umem_release(qp->sumem); 832 833 return 0; 834 } 835 836 static u8 __from_ib_qp_type(enum ib_qp_type type) 837 { 838 switch (type) { 839 case IB_QPT_GSI: 840 return CMDQ_CREATE_QP1_TYPE_GSI; 841 case IB_QPT_RC: 842 return CMDQ_CREATE_QP_TYPE_RC; 843 case IB_QPT_UD: 844 return CMDQ_CREATE_QP_TYPE_UD; 845 default: 846 return IB_QPT_MAX; 847 } 848 } 849 850 static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp, 851 int rsge, int max) 852 { 853 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 854 rsge = max; 855 return bnxt_re_get_rwqe_size(rsge); 856 } 857 858 static u16 bnxt_re_get_wqe_size(int ilsize, int nsge) 859 { 860 u16 wqe_size, calc_ils; 861 862 wqe_size = bnxt_re_get_swqe_size(nsge); 863 if (ilsize) { 864 calc_ils = sizeof(struct sq_send_hdr) + ilsize; 865 wqe_size = max_t(u16, calc_ils, wqe_size); 866 wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr)); 867 } 868 return wqe_size; 869 } 870 871 static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp, 872 struct ib_qp_init_attr *init_attr) 873 { 874 struct bnxt_qplib_dev_attr *dev_attr; 875 struct bnxt_qplib_qp *qplqp; 876 struct bnxt_re_dev *rdev; 877 struct bnxt_qplib_q *sq; 878 int align, ilsize; 879 880 rdev = qp->rdev; 881 qplqp = &qp->qplib_qp; 882 sq = &qplqp->sq; 883 dev_attr = &rdev->dev_attr; 884 885 align = sizeof(struct sq_send_hdr); 886 ilsize = ALIGN(init_attr->cap.max_inline_data, align); 887 888 sq->wqe_size = bnxt_re_get_wqe_size(ilsize, sq->max_sge); 889 if (sq->wqe_size > bnxt_re_get_swqe_size(dev_attr->max_qp_sges)) 890 return -EINVAL; 891 /* For gen p4 and gen p5 backward compatibility mode 892 * wqe size is fixed to 128 bytes 893 */ 894 if (sq->wqe_size < bnxt_re_get_swqe_size(dev_attr->max_qp_sges) && 895 qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 896 sq->wqe_size = bnxt_re_get_swqe_size(dev_attr->max_qp_sges); 897 898 if (init_attr->cap.max_inline_data) { 899 qplqp->max_inline_data = sq->wqe_size - 900 sizeof(struct sq_send_hdr); 901 init_attr->cap.max_inline_data = qplqp->max_inline_data; 902 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 903 sq->max_sge = qplqp->max_inline_data / 904 sizeof(struct sq_sge); 905 } 906 907 return 0; 908 } 909 910 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, 911 struct bnxt_re_qp *qp, struct ib_udata *udata) 912 { 913 struct bnxt_qplib_qp *qplib_qp; 914 struct bnxt_re_ucontext *cntx; 915 struct bnxt_re_qp_req ureq; 916 int bytes = 0, psn_sz; 917 struct ib_umem *umem; 918 int psn_nume; 919 920 qplib_qp = &qp->qplib_qp; 921 cntx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, 922 ib_uctx); 923 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 924 return -EFAULT; 925 926 bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); 927 /* Consider mapping PSN search memory only for RC QPs. */ 928 if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { 929 psn_sz = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ? 930 sizeof(struct sq_psn_search_ext) : 931 sizeof(struct sq_psn_search); 932 psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? 933 qplib_qp->sq.max_wqe : 934 ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / 935 sizeof(struct bnxt_qplib_sge)); 936 bytes += (psn_nume * psn_sz); 937 } 938 939 bytes = PAGE_ALIGN(bytes); 940 umem = ib_umem_get(&rdev->ibdev, ureq.qpsva, bytes, 941 IB_ACCESS_LOCAL_WRITE); 942 if (IS_ERR(umem)) 943 return PTR_ERR(umem); 944 945 qp->sumem = umem; 946 qplib_qp->sq.sg_info.umem = umem; 947 qplib_qp->sq.sg_info.pgsize = PAGE_SIZE; 948 qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT; 949 qplib_qp->qp_handle = ureq.qp_handle; 950 951 if (!qp->qplib_qp.srq) { 952 bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); 953 bytes = PAGE_ALIGN(bytes); 954 umem = ib_umem_get(&rdev->ibdev, ureq.qprva, bytes, 955 IB_ACCESS_LOCAL_WRITE); 956 if (IS_ERR(umem)) 957 goto rqfail; 958 qp->rumem = umem; 959 qplib_qp->rq.sg_info.umem = umem; 960 qplib_qp->rq.sg_info.pgsize = PAGE_SIZE; 961 qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT; 962 } 963 964 qplib_qp->dpi = &cntx->dpi; 965 return 0; 966 rqfail: 967 ib_umem_release(qp->sumem); 968 qp->sumem = NULL; 969 memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info)); 970 971 return PTR_ERR(umem); 972 } 973 974 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah 975 (struct bnxt_re_pd *pd, 976 struct bnxt_qplib_res *qp1_res, 977 struct bnxt_qplib_qp *qp1_qp) 978 { 979 struct bnxt_re_dev *rdev = pd->rdev; 980 struct bnxt_re_ah *ah; 981 union ib_gid sgid; 982 int rc; 983 984 ah = kzalloc(sizeof(*ah), GFP_KERNEL); 985 if (!ah) 986 return NULL; 987 988 ah->rdev = rdev; 989 ah->qplib_ah.pd = &pd->qplib_pd; 990 991 rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); 992 if (rc) 993 goto fail; 994 995 /* supply the dgid data same as sgid */ 996 memcpy(ah->qplib_ah.dgid.data, &sgid.raw, 997 sizeof(union ib_gid)); 998 ah->qplib_ah.sgid_index = 0; 999 1000 ah->qplib_ah.traffic_class = 0; 1001 ah->qplib_ah.flow_label = 0; 1002 ah->qplib_ah.hop_limit = 1; 1003 ah->qplib_ah.sl = 0; 1004 /* Have DMAC same as SMAC */ 1005 ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr); 1006 1007 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, false); 1008 if (rc) { 1009 ibdev_err(&rdev->ibdev, 1010 "Failed to allocate HW AH for Shadow QP"); 1011 goto fail; 1012 } 1013 atomic_inc(&rdev->ah_count); 1014 1015 return ah; 1016 1017 fail: 1018 kfree(ah); 1019 return NULL; 1020 } 1021 1022 static struct bnxt_re_qp *bnxt_re_create_shadow_qp 1023 (struct bnxt_re_pd *pd, 1024 struct bnxt_qplib_res *qp1_res, 1025 struct bnxt_qplib_qp *qp1_qp) 1026 { 1027 struct bnxt_re_dev *rdev = pd->rdev; 1028 struct bnxt_re_qp *qp; 1029 int rc; 1030 1031 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1032 if (!qp) 1033 return NULL; 1034 1035 qp->rdev = rdev; 1036 1037 /* Initialize the shadow QP structure from the QP1 values */ 1038 ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); 1039 1040 qp->qplib_qp.pd = &pd->qplib_pd; 1041 qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); 1042 qp->qplib_qp.type = IB_QPT_UD; 1043 1044 qp->qplib_qp.max_inline_data = 0; 1045 qp->qplib_qp.sig_type = true; 1046 1047 /* Shadow QP SQ depth should be same as QP1 RQ depth */ 1048 qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6); 1049 qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; 1050 qp->qplib_qp.sq.max_sge = 2; 1051 /* Q full delta can be 1 since it is internal QP */ 1052 qp->qplib_qp.sq.q_full_delta = 1; 1053 qp->qplib_qp.sq.sg_info.pgsize = PAGE_SIZE; 1054 qp->qplib_qp.sq.sg_info.pgshft = PAGE_SHIFT; 1055 1056 qp->qplib_qp.scq = qp1_qp->scq; 1057 qp->qplib_qp.rcq = qp1_qp->rcq; 1058 1059 qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6); 1060 qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; 1061 qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; 1062 /* Q full delta can be 1 since it is internal QP */ 1063 qp->qplib_qp.rq.q_full_delta = 1; 1064 qp->qplib_qp.rq.sg_info.pgsize = PAGE_SIZE; 1065 qp->qplib_qp.rq.sg_info.pgshft = PAGE_SHIFT; 1066 1067 qp->qplib_qp.mtu = qp1_qp->mtu; 1068 1069 qp->qplib_qp.sq_hdr_buf_size = 0; 1070 qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; 1071 qp->qplib_qp.dpi = &rdev->dpi_privileged; 1072 1073 rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); 1074 if (rc) 1075 goto fail; 1076 1077 spin_lock_init(&qp->sq_lock); 1078 INIT_LIST_HEAD(&qp->list); 1079 mutex_lock(&rdev->qp_lock); 1080 list_add_tail(&qp->list, &rdev->qp_list); 1081 atomic_inc(&rdev->qp_count); 1082 mutex_unlock(&rdev->qp_lock); 1083 return qp; 1084 fail: 1085 kfree(qp); 1086 return NULL; 1087 } 1088 1089 static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, 1090 struct ib_qp_init_attr *init_attr) 1091 { 1092 struct bnxt_qplib_dev_attr *dev_attr; 1093 struct bnxt_qplib_qp *qplqp; 1094 struct bnxt_re_dev *rdev; 1095 struct bnxt_qplib_q *rq; 1096 int entries; 1097 1098 rdev = qp->rdev; 1099 qplqp = &qp->qplib_qp; 1100 rq = &qplqp->rq; 1101 dev_attr = &rdev->dev_attr; 1102 1103 if (init_attr->srq) { 1104 struct bnxt_re_srq *srq; 1105 1106 srq = container_of(init_attr->srq, struct bnxt_re_srq, ib_srq); 1107 qplqp->srq = &srq->qplib_srq; 1108 rq->max_wqe = 0; 1109 } else { 1110 rq->max_sge = init_attr->cap.max_recv_sge; 1111 if (rq->max_sge > dev_attr->max_qp_sges) 1112 rq->max_sge = dev_attr->max_qp_sges; 1113 init_attr->cap.max_recv_sge = rq->max_sge; 1114 rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge, 1115 dev_attr->max_qp_sges); 1116 /* Allocate 1 more than what's provided so posting max doesn't 1117 * mean empty. 1118 */ 1119 entries = roundup_pow_of_two(init_attr->cap.max_recv_wr + 1); 1120 rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1); 1121 rq->q_full_delta = 0; 1122 rq->sg_info.pgsize = PAGE_SIZE; 1123 rq->sg_info.pgshft = PAGE_SHIFT; 1124 } 1125 1126 return 0; 1127 } 1128 1129 static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp) 1130 { 1131 struct bnxt_qplib_dev_attr *dev_attr; 1132 struct bnxt_qplib_qp *qplqp; 1133 struct bnxt_re_dev *rdev; 1134 1135 rdev = qp->rdev; 1136 qplqp = &qp->qplib_qp; 1137 dev_attr = &rdev->dev_attr; 1138 1139 if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) { 1140 qplqp->rq.max_sge = dev_attr->max_qp_sges; 1141 if (qplqp->rq.max_sge > dev_attr->max_qp_sges) 1142 qplqp->rq.max_sge = dev_attr->max_qp_sges; 1143 qplqp->rq.max_sge = 6; 1144 } 1145 } 1146 1147 static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, 1148 struct ib_qp_init_attr *init_attr, 1149 struct ib_udata *udata) 1150 { 1151 struct bnxt_qplib_dev_attr *dev_attr; 1152 struct bnxt_qplib_qp *qplqp; 1153 struct bnxt_re_dev *rdev; 1154 struct bnxt_qplib_q *sq; 1155 int entries; 1156 int diff; 1157 int rc; 1158 1159 rdev = qp->rdev; 1160 qplqp = &qp->qplib_qp; 1161 sq = &qplqp->sq; 1162 dev_attr = &rdev->dev_attr; 1163 1164 sq->max_sge = init_attr->cap.max_send_sge; 1165 if (sq->max_sge > dev_attr->max_qp_sges) { 1166 sq->max_sge = dev_attr->max_qp_sges; 1167 init_attr->cap.max_send_sge = sq->max_sge; 1168 } 1169 1170 rc = bnxt_re_setup_swqe_size(qp, init_attr); 1171 if (rc) 1172 return rc; 1173 1174 entries = init_attr->cap.max_send_wr; 1175 /* Allocate 128 + 1 more than what's provided */ 1176 diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ? 1177 0 : BNXT_QPLIB_RESERVED_QP_WRS; 1178 entries = roundup_pow_of_two(entries + diff + 1); 1179 sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1); 1180 sq->q_full_delta = diff + 1; 1181 /* 1182 * Reserving one slot for Phantom WQE. Application can 1183 * post one extra entry in this case. But allowing this to avoid 1184 * unexpected Queue full condition 1185 */ 1186 qplqp->sq.q_full_delta -= 1; 1187 qplqp->sq.sg_info.pgsize = PAGE_SIZE; 1188 qplqp->sq.sg_info.pgshft = PAGE_SHIFT; 1189 1190 return 0; 1191 } 1192 1193 static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp, 1194 struct ib_qp_init_attr *init_attr) 1195 { 1196 struct bnxt_qplib_dev_attr *dev_attr; 1197 struct bnxt_qplib_qp *qplqp; 1198 struct bnxt_re_dev *rdev; 1199 int entries; 1200 1201 rdev = qp->rdev; 1202 qplqp = &qp->qplib_qp; 1203 dev_attr = &rdev->dev_attr; 1204 1205 if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) { 1206 entries = roundup_pow_of_two(init_attr->cap.max_send_wr + 1); 1207 qplqp->sq.max_wqe = min_t(u32, entries, 1208 dev_attr->max_qp_wqes + 1); 1209 qplqp->sq.q_full_delta = qplqp->sq.max_wqe - 1210 init_attr->cap.max_send_wr; 1211 qplqp->sq.max_sge++; /* Need one extra sge to put UD header */ 1212 if (qplqp->sq.max_sge > dev_attr->max_qp_sges) 1213 qplqp->sq.max_sge = dev_attr->max_qp_sges; 1214 } 1215 } 1216 1217 static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev, 1218 struct ib_qp_init_attr *init_attr) 1219 { 1220 struct bnxt_qplib_chip_ctx *chip_ctx; 1221 int qptype; 1222 1223 chip_ctx = rdev->chip_ctx; 1224 1225 qptype = __from_ib_qp_type(init_attr->qp_type); 1226 if (qptype == IB_QPT_MAX) { 1227 ibdev_err(&rdev->ibdev, "QP type 0x%x not supported", qptype); 1228 qptype = -EOPNOTSUPP; 1229 goto out; 1230 } 1231 1232 if (bnxt_qplib_is_chip_gen_p5(chip_ctx) && 1233 init_attr->qp_type == IB_QPT_GSI) 1234 qptype = CMDQ_CREATE_QP_TYPE_GSI; 1235 out: 1236 return qptype; 1237 } 1238 1239 static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, 1240 struct ib_qp_init_attr *init_attr, 1241 struct ib_udata *udata) 1242 { 1243 struct bnxt_qplib_dev_attr *dev_attr; 1244 struct bnxt_qplib_qp *qplqp; 1245 struct bnxt_re_dev *rdev; 1246 struct bnxt_re_cq *cq; 1247 int rc = 0, qptype; 1248 1249 rdev = qp->rdev; 1250 qplqp = &qp->qplib_qp; 1251 dev_attr = &rdev->dev_attr; 1252 1253 /* Setup misc params */ 1254 ether_addr_copy(qplqp->smac, rdev->netdev->dev_addr); 1255 qplqp->pd = &pd->qplib_pd; 1256 qplqp->qp_handle = (u64)qplqp; 1257 qplqp->max_inline_data = init_attr->cap.max_inline_data; 1258 qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1259 true : false); 1260 qptype = bnxt_re_init_qp_type(rdev, init_attr); 1261 if (qptype < 0) { 1262 rc = qptype; 1263 goto out; 1264 } 1265 qplqp->type = (u8)qptype; 1266 qplqp->wqe_mode = rdev->chip_ctx->modes.wqe_mode; 1267 1268 if (init_attr->qp_type == IB_QPT_RC) { 1269 qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom; 1270 qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; 1271 } 1272 qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); 1273 qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */ 1274 if (init_attr->create_flags) { 1275 ibdev_dbg(&rdev->ibdev, 1276 "QP create flags 0x%x not supported", 1277 init_attr->create_flags); 1278 return -EOPNOTSUPP; 1279 } 1280 1281 /* Setup CQs */ 1282 if (init_attr->send_cq) { 1283 cq = container_of(init_attr->send_cq, struct bnxt_re_cq, ib_cq); 1284 qplqp->scq = &cq->qplib_cq; 1285 qp->scq = cq; 1286 } 1287 1288 if (init_attr->recv_cq) { 1289 cq = container_of(init_attr->recv_cq, struct bnxt_re_cq, ib_cq); 1290 qplqp->rcq = &cq->qplib_cq; 1291 qp->rcq = cq; 1292 } 1293 1294 /* Setup RQ/SRQ */ 1295 rc = bnxt_re_init_rq_attr(qp, init_attr); 1296 if (rc) 1297 goto out; 1298 if (init_attr->qp_type == IB_QPT_GSI) 1299 bnxt_re_adjust_gsi_rq_attr(qp); 1300 1301 /* Setup SQ */ 1302 rc = bnxt_re_init_sq_attr(qp, init_attr, udata); 1303 if (rc) 1304 goto out; 1305 if (init_attr->qp_type == IB_QPT_GSI) 1306 bnxt_re_adjust_gsi_sq_attr(qp, init_attr); 1307 1308 if (udata) /* This will update DPI and qp_handle */ 1309 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); 1310 out: 1311 return rc; 1312 } 1313 1314 static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp, 1315 struct bnxt_re_pd *pd) 1316 { 1317 struct bnxt_re_sqp_entries *sqp_tbl; 1318 struct bnxt_re_dev *rdev; 1319 struct bnxt_re_qp *sqp; 1320 struct bnxt_re_ah *sah; 1321 int rc = 0; 1322 1323 rdev = qp->rdev; 1324 /* Create a shadow QP to handle the QP1 traffic */ 1325 sqp_tbl = kcalloc(BNXT_RE_MAX_GSI_SQP_ENTRIES, sizeof(*sqp_tbl), 1326 GFP_KERNEL); 1327 if (!sqp_tbl) 1328 return -ENOMEM; 1329 rdev->gsi_ctx.sqp_tbl = sqp_tbl; 1330 1331 sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp); 1332 if (!sqp) { 1333 rc = -ENODEV; 1334 ibdev_err(&rdev->ibdev, "Failed to create Shadow QP for QP1"); 1335 goto out; 1336 } 1337 rdev->gsi_ctx.gsi_sqp = sqp; 1338 1339 sqp->rcq = qp->rcq; 1340 sqp->scq = qp->scq; 1341 sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, 1342 &qp->qplib_qp); 1343 if (!sah) { 1344 bnxt_qplib_destroy_qp(&rdev->qplib_res, 1345 &sqp->qplib_qp); 1346 rc = -ENODEV; 1347 ibdev_err(&rdev->ibdev, 1348 "Failed to create AH entry for ShadowQP"); 1349 goto out; 1350 } 1351 rdev->gsi_ctx.gsi_sah = sah; 1352 1353 return 0; 1354 out: 1355 kfree(sqp_tbl); 1356 return rc; 1357 } 1358 1359 static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, 1360 struct ib_qp_init_attr *init_attr) 1361 { 1362 struct bnxt_re_dev *rdev; 1363 struct bnxt_qplib_qp *qplqp; 1364 int rc = 0; 1365 1366 rdev = qp->rdev; 1367 qplqp = &qp->qplib_qp; 1368 1369 qplqp->rq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; 1370 qplqp->sq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2; 1371 1372 rc = bnxt_qplib_create_qp1(&rdev->qplib_res, qplqp); 1373 if (rc) { 1374 ibdev_err(&rdev->ibdev, "create HW QP1 failed!"); 1375 goto out; 1376 } 1377 1378 rc = bnxt_re_create_shadow_gsi(qp, pd); 1379 out: 1380 return rc; 1381 } 1382 1383 static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev, 1384 struct ib_qp_init_attr *init_attr, 1385 struct bnxt_qplib_dev_attr *dev_attr) 1386 { 1387 bool rc = true; 1388 1389 if (init_attr->cap.max_send_wr > dev_attr->max_qp_wqes || 1390 init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes || 1391 init_attr->cap.max_send_sge > dev_attr->max_qp_sges || 1392 init_attr->cap.max_recv_sge > dev_attr->max_qp_sges || 1393 init_attr->cap.max_inline_data > dev_attr->max_inline_data) { 1394 ibdev_err(&rdev->ibdev, 1395 "Create QP failed - max exceeded! 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x", 1396 init_attr->cap.max_send_wr, dev_attr->max_qp_wqes, 1397 init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes, 1398 init_attr->cap.max_send_sge, dev_attr->max_qp_sges, 1399 init_attr->cap.max_recv_sge, dev_attr->max_qp_sges, 1400 init_attr->cap.max_inline_data, 1401 dev_attr->max_inline_data); 1402 rc = false; 1403 } 1404 return rc; 1405 } 1406 1407 int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, 1408 struct ib_udata *udata) 1409 { 1410 struct ib_pd *ib_pd = ib_qp->pd; 1411 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 1412 struct bnxt_re_dev *rdev = pd->rdev; 1413 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 1414 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 1415 int rc; 1416 1417 rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); 1418 if (!rc) { 1419 rc = -EINVAL; 1420 goto fail; 1421 } 1422 1423 qp->rdev = rdev; 1424 rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata); 1425 if (rc) 1426 goto fail; 1427 1428 if (qp_init_attr->qp_type == IB_QPT_GSI && 1429 !(bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx))) { 1430 rc = bnxt_re_create_gsi_qp(qp, pd, qp_init_attr); 1431 if (rc == -ENODEV) 1432 goto qp_destroy; 1433 if (rc) 1434 goto fail; 1435 } else { 1436 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); 1437 if (rc) { 1438 ibdev_err(&rdev->ibdev, "Failed to create HW QP"); 1439 goto free_umem; 1440 } 1441 if (udata) { 1442 struct bnxt_re_qp_resp resp; 1443 1444 resp.qpid = qp->qplib_qp.id; 1445 resp.rsvd = 0; 1446 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 1447 if (rc) { 1448 ibdev_err(&rdev->ibdev, "Failed to copy QP udata"); 1449 goto qp_destroy; 1450 } 1451 } 1452 } 1453 1454 qp->ib_qp.qp_num = qp->qplib_qp.id; 1455 if (qp_init_attr->qp_type == IB_QPT_GSI) 1456 rdev->gsi_ctx.gsi_qp = qp; 1457 spin_lock_init(&qp->sq_lock); 1458 spin_lock_init(&qp->rq_lock); 1459 INIT_LIST_HEAD(&qp->list); 1460 mutex_lock(&rdev->qp_lock); 1461 list_add_tail(&qp->list, &rdev->qp_list); 1462 mutex_unlock(&rdev->qp_lock); 1463 atomic_inc(&rdev->qp_count); 1464 1465 return 0; 1466 qp_destroy: 1467 bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 1468 free_umem: 1469 ib_umem_release(qp->rumem); 1470 ib_umem_release(qp->sumem); 1471 fail: 1472 return rc; 1473 } 1474 1475 static u8 __from_ib_qp_state(enum ib_qp_state state) 1476 { 1477 switch (state) { 1478 case IB_QPS_RESET: 1479 return CMDQ_MODIFY_QP_NEW_STATE_RESET; 1480 case IB_QPS_INIT: 1481 return CMDQ_MODIFY_QP_NEW_STATE_INIT; 1482 case IB_QPS_RTR: 1483 return CMDQ_MODIFY_QP_NEW_STATE_RTR; 1484 case IB_QPS_RTS: 1485 return CMDQ_MODIFY_QP_NEW_STATE_RTS; 1486 case IB_QPS_SQD: 1487 return CMDQ_MODIFY_QP_NEW_STATE_SQD; 1488 case IB_QPS_SQE: 1489 return CMDQ_MODIFY_QP_NEW_STATE_SQE; 1490 case IB_QPS_ERR: 1491 default: 1492 return CMDQ_MODIFY_QP_NEW_STATE_ERR; 1493 } 1494 } 1495 1496 static enum ib_qp_state __to_ib_qp_state(u8 state) 1497 { 1498 switch (state) { 1499 case CMDQ_MODIFY_QP_NEW_STATE_RESET: 1500 return IB_QPS_RESET; 1501 case CMDQ_MODIFY_QP_NEW_STATE_INIT: 1502 return IB_QPS_INIT; 1503 case CMDQ_MODIFY_QP_NEW_STATE_RTR: 1504 return IB_QPS_RTR; 1505 case CMDQ_MODIFY_QP_NEW_STATE_RTS: 1506 return IB_QPS_RTS; 1507 case CMDQ_MODIFY_QP_NEW_STATE_SQD: 1508 return IB_QPS_SQD; 1509 case CMDQ_MODIFY_QP_NEW_STATE_SQE: 1510 return IB_QPS_SQE; 1511 case CMDQ_MODIFY_QP_NEW_STATE_ERR: 1512 default: 1513 return IB_QPS_ERR; 1514 } 1515 } 1516 1517 static u32 __from_ib_mtu(enum ib_mtu mtu) 1518 { 1519 switch (mtu) { 1520 case IB_MTU_256: 1521 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256; 1522 case IB_MTU_512: 1523 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512; 1524 case IB_MTU_1024: 1525 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024; 1526 case IB_MTU_2048: 1527 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 1528 case IB_MTU_4096: 1529 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096; 1530 default: 1531 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 1532 } 1533 } 1534 1535 static enum ib_mtu __to_ib_mtu(u32 mtu) 1536 { 1537 switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) { 1538 case CMDQ_MODIFY_QP_PATH_MTU_MTU_256: 1539 return IB_MTU_256; 1540 case CMDQ_MODIFY_QP_PATH_MTU_MTU_512: 1541 return IB_MTU_512; 1542 case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024: 1543 return IB_MTU_1024; 1544 case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048: 1545 return IB_MTU_2048; 1546 case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096: 1547 return IB_MTU_4096; 1548 default: 1549 return IB_MTU_2048; 1550 } 1551 } 1552 1553 /* Shared Receive Queues */ 1554 int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) 1555 { 1556 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1557 ib_srq); 1558 struct bnxt_re_dev *rdev = srq->rdev; 1559 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; 1560 struct bnxt_qplib_nq *nq = NULL; 1561 1562 if (qplib_srq->cq) 1563 nq = qplib_srq->cq->nq; 1564 bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); 1565 ib_umem_release(srq->umem); 1566 atomic_dec(&rdev->srq_count); 1567 if (nq) 1568 nq->budget--; 1569 return 0; 1570 } 1571 1572 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, 1573 struct bnxt_re_pd *pd, 1574 struct bnxt_re_srq *srq, 1575 struct ib_udata *udata) 1576 { 1577 struct bnxt_re_srq_req ureq; 1578 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; 1579 struct ib_umem *umem; 1580 int bytes = 0; 1581 struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context( 1582 udata, struct bnxt_re_ucontext, ib_uctx); 1583 1584 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 1585 return -EFAULT; 1586 1587 bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size); 1588 bytes = PAGE_ALIGN(bytes); 1589 umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes, 1590 IB_ACCESS_LOCAL_WRITE); 1591 if (IS_ERR(umem)) 1592 return PTR_ERR(umem); 1593 1594 srq->umem = umem; 1595 qplib_srq->sg_info.umem = umem; 1596 qplib_srq->sg_info.pgsize = PAGE_SIZE; 1597 qplib_srq->sg_info.pgshft = PAGE_SHIFT; 1598 qplib_srq->srq_handle = ureq.srq_handle; 1599 qplib_srq->dpi = &cntx->dpi; 1600 1601 return 0; 1602 } 1603 1604 int bnxt_re_create_srq(struct ib_srq *ib_srq, 1605 struct ib_srq_init_attr *srq_init_attr, 1606 struct ib_udata *udata) 1607 { 1608 struct bnxt_qplib_dev_attr *dev_attr; 1609 struct bnxt_qplib_nq *nq = NULL; 1610 struct bnxt_re_dev *rdev; 1611 struct bnxt_re_srq *srq; 1612 struct bnxt_re_pd *pd; 1613 struct ib_pd *ib_pd; 1614 int rc, entries; 1615 1616 ib_pd = ib_srq->pd; 1617 pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 1618 rdev = pd->rdev; 1619 dev_attr = &rdev->dev_attr; 1620 srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq); 1621 1622 if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) { 1623 ibdev_err(&rdev->ibdev, "Create CQ failed - max exceeded"); 1624 rc = -EINVAL; 1625 goto exit; 1626 } 1627 1628 if (srq_init_attr->srq_type != IB_SRQT_BASIC) { 1629 rc = -EOPNOTSUPP; 1630 goto exit; 1631 } 1632 1633 srq->rdev = rdev; 1634 srq->qplib_srq.pd = &pd->qplib_pd; 1635 srq->qplib_srq.dpi = &rdev->dpi_privileged; 1636 /* Allocate 1 more than what's provided so posting max doesn't 1637 * mean empty 1638 */ 1639 entries = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1); 1640 if (entries > dev_attr->max_srq_wqes + 1) 1641 entries = dev_attr->max_srq_wqes + 1; 1642 srq->qplib_srq.max_wqe = entries; 1643 1644 srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge; 1645 /* 128 byte wqe size for SRQ . So use max sges */ 1646 srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr->max_srq_sges); 1647 srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit; 1648 srq->srq_limit = srq_init_attr->attr.srq_limit; 1649 srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id; 1650 nq = &rdev->nq[0]; 1651 1652 if (udata) { 1653 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata); 1654 if (rc) 1655 goto fail; 1656 } 1657 1658 rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq); 1659 if (rc) { 1660 ibdev_err(&rdev->ibdev, "Create HW SRQ failed!"); 1661 goto fail; 1662 } 1663 1664 if (udata) { 1665 struct bnxt_re_srq_resp resp; 1666 1667 resp.srqid = srq->qplib_srq.id; 1668 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 1669 if (rc) { 1670 ibdev_err(&rdev->ibdev, "SRQ copy to udata failed!"); 1671 bnxt_qplib_destroy_srq(&rdev->qplib_res, 1672 &srq->qplib_srq); 1673 goto fail; 1674 } 1675 } 1676 if (nq) 1677 nq->budget++; 1678 atomic_inc(&rdev->srq_count); 1679 spin_lock_init(&srq->lock); 1680 1681 return 0; 1682 1683 fail: 1684 ib_umem_release(srq->umem); 1685 exit: 1686 return rc; 1687 } 1688 1689 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, 1690 enum ib_srq_attr_mask srq_attr_mask, 1691 struct ib_udata *udata) 1692 { 1693 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1694 ib_srq); 1695 struct bnxt_re_dev *rdev = srq->rdev; 1696 int rc; 1697 1698 switch (srq_attr_mask) { 1699 case IB_SRQ_MAX_WR: 1700 /* SRQ resize is not supported */ 1701 break; 1702 case IB_SRQ_LIMIT: 1703 /* Change the SRQ threshold */ 1704 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) 1705 return -EINVAL; 1706 1707 srq->qplib_srq.threshold = srq_attr->srq_limit; 1708 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq); 1709 if (rc) { 1710 ibdev_err(&rdev->ibdev, "Modify HW SRQ failed!"); 1711 return rc; 1712 } 1713 /* On success, update the shadow */ 1714 srq->srq_limit = srq_attr->srq_limit; 1715 /* No need to Build and send response back to udata */ 1716 break; 1717 default: 1718 ibdev_err(&rdev->ibdev, 1719 "Unsupported srq_attr_mask 0x%x", srq_attr_mask); 1720 return -EINVAL; 1721 } 1722 return 0; 1723 } 1724 1725 int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr) 1726 { 1727 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1728 ib_srq); 1729 struct bnxt_re_srq tsrq; 1730 struct bnxt_re_dev *rdev = srq->rdev; 1731 int rc; 1732 1733 /* Get live SRQ attr */ 1734 tsrq.qplib_srq.id = srq->qplib_srq.id; 1735 rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq); 1736 if (rc) { 1737 ibdev_err(&rdev->ibdev, "Query HW SRQ failed!"); 1738 return rc; 1739 } 1740 srq_attr->max_wr = srq->qplib_srq.max_wqe; 1741 srq_attr->max_sge = srq->qplib_srq.max_sge; 1742 srq_attr->srq_limit = tsrq.qplib_srq.threshold; 1743 1744 return 0; 1745 } 1746 1747 int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, 1748 const struct ib_recv_wr **bad_wr) 1749 { 1750 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1751 ib_srq); 1752 struct bnxt_qplib_swqe wqe; 1753 unsigned long flags; 1754 int rc = 0; 1755 1756 spin_lock_irqsave(&srq->lock, flags); 1757 while (wr) { 1758 /* Transcribe each ib_recv_wr to qplib_swqe */ 1759 wqe.num_sge = wr->num_sge; 1760 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); 1761 wqe.wr_id = wr->wr_id; 1762 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 1763 1764 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe); 1765 if (rc) { 1766 *bad_wr = wr; 1767 break; 1768 } 1769 wr = wr->next; 1770 } 1771 spin_unlock_irqrestore(&srq->lock, flags); 1772 1773 return rc; 1774 } 1775 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, 1776 struct bnxt_re_qp *qp1_qp, 1777 int qp_attr_mask) 1778 { 1779 struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp; 1780 int rc = 0; 1781 1782 if (qp_attr_mask & IB_QP_STATE) { 1783 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 1784 qp->qplib_qp.state = qp1_qp->qplib_qp.state; 1785 } 1786 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 1787 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 1788 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; 1789 } 1790 1791 if (qp_attr_mask & IB_QP_QKEY) { 1792 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 1793 /* Using a Random QKEY */ 1794 qp->qplib_qp.qkey = 0x81818181; 1795 } 1796 if (qp_attr_mask & IB_QP_SQ_PSN) { 1797 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 1798 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; 1799 } 1800 1801 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 1802 if (rc) 1803 ibdev_err(&rdev->ibdev, "Failed to modify Shadow QP for QP1"); 1804 return rc; 1805 } 1806 1807 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 1808 int qp_attr_mask, struct ib_udata *udata) 1809 { 1810 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 1811 struct bnxt_re_dev *rdev = qp->rdev; 1812 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 1813 enum ib_qp_state curr_qp_state, new_qp_state; 1814 int rc, entries; 1815 unsigned int flags; 1816 u8 nw_type; 1817 1818 if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1819 return -EOPNOTSUPP; 1820 1821 qp->qplib_qp.modify_flags = 0; 1822 if (qp_attr_mask & IB_QP_STATE) { 1823 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); 1824 new_qp_state = qp_attr->qp_state; 1825 if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state, 1826 ib_qp->qp_type, qp_attr_mask)) { 1827 ibdev_err(&rdev->ibdev, 1828 "Invalid attribute mask: %#x specified ", 1829 qp_attr_mask); 1830 ibdev_err(&rdev->ibdev, 1831 "for qpn: %#x type: %#x", 1832 ib_qp->qp_num, ib_qp->qp_type); 1833 ibdev_err(&rdev->ibdev, 1834 "curr_qp_state=0x%x, new_qp_state=0x%x\n", 1835 curr_qp_state, new_qp_state); 1836 return -EINVAL; 1837 } 1838 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 1839 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); 1840 1841 if (!qp->sumem && 1842 qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { 1843 ibdev_dbg(&rdev->ibdev, 1844 "Move QP = %p to flush list\n", qp); 1845 flags = bnxt_re_lock_cqs(qp); 1846 bnxt_qplib_add_flush_qp(&qp->qplib_qp); 1847 bnxt_re_unlock_cqs(qp, flags); 1848 } 1849 if (!qp->sumem && 1850 qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { 1851 ibdev_dbg(&rdev->ibdev, 1852 "Move QP = %p out of flush list\n", qp); 1853 flags = bnxt_re_lock_cqs(qp); 1854 bnxt_qplib_clean_qp(&qp->qplib_qp); 1855 bnxt_re_unlock_cqs(qp, flags); 1856 } 1857 } 1858 if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { 1859 qp->qplib_qp.modify_flags |= 1860 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY; 1861 qp->qplib_qp.en_sqd_async_notify = true; 1862 } 1863 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) { 1864 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; 1865 qp->qplib_qp.access = 1866 __from_ib_access_flags(qp_attr->qp_access_flags); 1867 /* LOCAL_WRITE access must be set to allow RC receive */ 1868 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 1869 /* Temp: Set all params on QP as of now */ 1870 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE; 1871 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ; 1872 } 1873 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 1874 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 1875 qp->qplib_qp.pkey_index = qp_attr->pkey_index; 1876 } 1877 if (qp_attr_mask & IB_QP_QKEY) { 1878 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 1879 qp->qplib_qp.qkey = qp_attr->qkey; 1880 } 1881 if (qp_attr_mask & IB_QP_AV) { 1882 const struct ib_global_route *grh = 1883 rdma_ah_read_grh(&qp_attr->ah_attr); 1884 const struct ib_gid_attr *sgid_attr; 1885 struct bnxt_re_gid_ctx *ctx; 1886 1887 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | 1888 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | 1889 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | 1890 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | 1891 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | 1892 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | 1893 CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; 1894 memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw, 1895 sizeof(qp->qplib_qp.ah.dgid.data)); 1896 qp->qplib_qp.ah.flow_label = grh->flow_label; 1897 sgid_attr = grh->sgid_attr; 1898 /* Get the HW context of the GID. The reference 1899 * of GID table entry is already taken by the caller. 1900 */ 1901 ctx = rdma_read_gid_hw_context(sgid_attr); 1902 qp->qplib_qp.ah.sgid_index = ctx->idx; 1903 qp->qplib_qp.ah.host_sgid_index = grh->sgid_index; 1904 qp->qplib_qp.ah.hop_limit = grh->hop_limit; 1905 qp->qplib_qp.ah.traffic_class = grh->traffic_class; 1906 qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr); 1907 ether_addr_copy(qp->qplib_qp.ah.dmac, 1908 qp_attr->ah_attr.roce.dmac); 1909 1910 rc = rdma_read_gid_l2_fields(sgid_attr, NULL, 1911 &qp->qplib_qp.smac[0]); 1912 if (rc) 1913 return rc; 1914 1915 nw_type = rdma_gid_attr_network_type(sgid_attr); 1916 switch (nw_type) { 1917 case RDMA_NETWORK_IPV4: 1918 qp->qplib_qp.nw_type = 1919 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4; 1920 break; 1921 case RDMA_NETWORK_IPV6: 1922 qp->qplib_qp.nw_type = 1923 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6; 1924 break; 1925 default: 1926 qp->qplib_qp.nw_type = 1927 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1; 1928 break; 1929 } 1930 } 1931 1932 if (qp_attr_mask & IB_QP_PATH_MTU) { 1933 qp->qplib_qp.modify_flags |= 1934 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 1935 qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu); 1936 qp->qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr->path_mtu); 1937 } else if (qp_attr->qp_state == IB_QPS_RTR) { 1938 qp->qplib_qp.modify_flags |= 1939 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 1940 qp->qplib_qp.path_mtu = 1941 __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu)); 1942 qp->qplib_qp.mtu = 1943 ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); 1944 } 1945 1946 if (qp_attr_mask & IB_QP_TIMEOUT) { 1947 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; 1948 qp->qplib_qp.timeout = qp_attr->timeout; 1949 } 1950 if (qp_attr_mask & IB_QP_RETRY_CNT) { 1951 qp->qplib_qp.modify_flags |= 1952 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT; 1953 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; 1954 } 1955 if (qp_attr_mask & IB_QP_RNR_RETRY) { 1956 qp->qplib_qp.modify_flags |= 1957 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY; 1958 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; 1959 } 1960 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) { 1961 qp->qplib_qp.modify_flags |= 1962 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER; 1963 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; 1964 } 1965 if (qp_attr_mask & IB_QP_RQ_PSN) { 1966 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; 1967 qp->qplib_qp.rq.psn = qp_attr->rq_psn; 1968 } 1969 if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1970 qp->qplib_qp.modify_flags |= 1971 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC; 1972 /* Cap the max_rd_atomic to device max */ 1973 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic, 1974 dev_attr->max_qp_rd_atom); 1975 } 1976 if (qp_attr_mask & IB_QP_SQ_PSN) { 1977 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 1978 qp->qplib_qp.sq.psn = qp_attr->sq_psn; 1979 } 1980 if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1981 if (qp_attr->max_dest_rd_atomic > 1982 dev_attr->max_qp_init_rd_atom) { 1983 ibdev_err(&rdev->ibdev, 1984 "max_dest_rd_atomic requested%d is > dev_max%d", 1985 qp_attr->max_dest_rd_atomic, 1986 dev_attr->max_qp_init_rd_atom); 1987 return -EINVAL; 1988 } 1989 1990 qp->qplib_qp.modify_flags |= 1991 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC; 1992 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; 1993 } 1994 if (qp_attr_mask & IB_QP_CAP) { 1995 qp->qplib_qp.modify_flags |= 1996 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE | 1997 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE | 1998 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE | 1999 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE | 2000 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA; 2001 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || 2002 (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || 2003 (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || 2004 (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || 2005 (qp_attr->cap.max_inline_data >= 2006 dev_attr->max_inline_data)) { 2007 ibdev_err(&rdev->ibdev, 2008 "Create QP failed - max exceeded"); 2009 return -EINVAL; 2010 } 2011 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr); 2012 qp->qplib_qp.sq.max_wqe = min_t(u32, entries, 2013 dev_attr->max_qp_wqes + 1); 2014 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - 2015 qp_attr->cap.max_send_wr; 2016 /* 2017 * Reserving one slot for Phantom WQE. Some application can 2018 * post one extra entry in this case. Allowing this to avoid 2019 * unexpected Queue full condition 2020 */ 2021 qp->qplib_qp.sq.q_full_delta -= 1; 2022 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; 2023 if (qp->qplib_qp.rq.max_wqe) { 2024 entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr); 2025 qp->qplib_qp.rq.max_wqe = 2026 min_t(u32, entries, dev_attr->max_qp_wqes + 1); 2027 qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - 2028 qp_attr->cap.max_recv_wr; 2029 qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; 2030 } else { 2031 /* SRQ was used prior, just ignore the RQ caps */ 2032 } 2033 } 2034 if (qp_attr_mask & IB_QP_DEST_QPN) { 2035 qp->qplib_qp.modify_flags |= 2036 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID; 2037 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; 2038 } 2039 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 2040 if (rc) { 2041 ibdev_err(&rdev->ibdev, "Failed to modify HW QP"); 2042 return rc; 2043 } 2044 if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) 2045 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); 2046 return rc; 2047 } 2048 2049 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 2050 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) 2051 { 2052 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 2053 struct bnxt_re_dev *rdev = qp->rdev; 2054 struct bnxt_qplib_qp *qplib_qp; 2055 int rc; 2056 2057 qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL); 2058 if (!qplib_qp) 2059 return -ENOMEM; 2060 2061 qplib_qp->id = qp->qplib_qp.id; 2062 qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; 2063 2064 rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp); 2065 if (rc) { 2066 ibdev_err(&rdev->ibdev, "Failed to query HW QP"); 2067 goto out; 2068 } 2069 qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state); 2070 qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state); 2071 qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0; 2072 qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access); 2073 qp_attr->pkey_index = qplib_qp->pkey_index; 2074 qp_attr->qkey = qplib_qp->qkey; 2075 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2076 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->ah.flow_label, 2077 qplib_qp->ah.host_sgid_index, 2078 qplib_qp->ah.hop_limit, 2079 qplib_qp->ah.traffic_class); 2080 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data); 2081 rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl); 2082 ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac); 2083 qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu); 2084 qp_attr->timeout = qplib_qp->timeout; 2085 qp_attr->retry_cnt = qplib_qp->retry_cnt; 2086 qp_attr->rnr_retry = qplib_qp->rnr_retry; 2087 qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer; 2088 qp_attr->rq_psn = qplib_qp->rq.psn; 2089 qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic; 2090 qp_attr->sq_psn = qplib_qp->sq.psn; 2091 qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic; 2092 qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR : 2093 IB_SIGNAL_REQ_WR; 2094 qp_attr->dest_qp_num = qplib_qp->dest_qpn; 2095 2096 qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; 2097 qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; 2098 qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; 2099 qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; 2100 qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; 2101 qp_init_attr->cap = qp_attr->cap; 2102 2103 out: 2104 kfree(qplib_qp); 2105 return rc; 2106 } 2107 2108 /* Routine for sending QP1 packets for RoCE V1 an V2 2109 */ 2110 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp, 2111 const struct ib_send_wr *wr, 2112 struct bnxt_qplib_swqe *wqe, 2113 int payload_size) 2114 { 2115 struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, 2116 ib_ah); 2117 struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; 2118 const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr; 2119 struct bnxt_qplib_sge sge; 2120 u8 nw_type; 2121 u16 ether_type; 2122 union ib_gid dgid; 2123 bool is_eth = false; 2124 bool is_vlan = false; 2125 bool is_grh = false; 2126 bool is_udp = false; 2127 u8 ip_version = 0; 2128 u16 vlan_id = 0xFFFF; 2129 void *buf; 2130 int i, rc = 0; 2131 2132 memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); 2133 2134 rc = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, NULL); 2135 if (rc) 2136 return rc; 2137 2138 /* Get network header type for this GID */ 2139 nw_type = rdma_gid_attr_network_type(sgid_attr); 2140 switch (nw_type) { 2141 case RDMA_NETWORK_IPV4: 2142 nw_type = BNXT_RE_ROCEV2_IPV4_PACKET; 2143 break; 2144 case RDMA_NETWORK_IPV6: 2145 nw_type = BNXT_RE_ROCEV2_IPV6_PACKET; 2146 break; 2147 default: 2148 nw_type = BNXT_RE_ROCE_V1_PACKET; 2149 break; 2150 } 2151 memcpy(&dgid.raw, &qplib_ah->dgid, 16); 2152 is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP; 2153 if (is_udp) { 2154 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) { 2155 ip_version = 4; 2156 ether_type = ETH_P_IP; 2157 } else { 2158 ip_version = 6; 2159 ether_type = ETH_P_IPV6; 2160 } 2161 is_grh = false; 2162 } else { 2163 ether_type = ETH_P_IBOE; 2164 is_grh = true; 2165 } 2166 2167 is_eth = true; 2168 is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false; 2169 2170 ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh, 2171 ip_version, is_udp, 0, &qp->qp1_hdr); 2172 2173 /* ETH */ 2174 ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac); 2175 ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac); 2176 2177 /* For vlan, check the sgid for vlan existence */ 2178 2179 if (!is_vlan) { 2180 qp->qp1_hdr.eth.type = cpu_to_be16(ether_type); 2181 } else { 2182 qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type); 2183 qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id); 2184 } 2185 2186 if (is_grh || (ip_version == 6)) { 2187 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw, 2188 sizeof(sgid_attr->gid)); 2189 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, 2190 sizeof(sgid_attr->gid)); 2191 qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit; 2192 } 2193 2194 if (ip_version == 4) { 2195 qp->qp1_hdr.ip4.tos = 0; 2196 qp->qp1_hdr.ip4.id = 0; 2197 qp->qp1_hdr.ip4.frag_off = htons(IP_DF); 2198 qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit; 2199 2200 memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4); 2201 memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4); 2202 qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr); 2203 } 2204 2205 if (is_udp) { 2206 qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT); 2207 qp->qp1_hdr.udp.sport = htons(0x8CD1); 2208 qp->qp1_hdr.udp.csum = 0; 2209 } 2210 2211 /* BTH */ 2212 if (wr->opcode == IB_WR_SEND_WITH_IMM) { 2213 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; 2214 qp->qp1_hdr.immediate_present = 1; 2215 } else { 2216 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; 2217 } 2218 if (wr->send_flags & IB_SEND_SOLICITED) 2219 qp->qp1_hdr.bth.solicited_event = 1; 2220 /* pad_count */ 2221 qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; 2222 2223 /* P_key for QP1 is for all members */ 2224 qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); 2225 qp->qp1_hdr.bth.destination_qpn = IB_QP1; 2226 qp->qp1_hdr.bth.ack_req = 0; 2227 qp->send_psn++; 2228 qp->send_psn &= BTH_PSN_MASK; 2229 qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); 2230 /* DETH */ 2231 /* Use the priviledged Q_Key for QP1 */ 2232 qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); 2233 qp->qp1_hdr.deth.source_qpn = IB_QP1; 2234 2235 /* Pack the QP1 to the transmit buffer */ 2236 buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); 2237 if (buf) { 2238 ib_ud_header_pack(&qp->qp1_hdr, buf); 2239 for (i = wqe->num_sge; i; i--) { 2240 wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; 2241 wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; 2242 wqe->sg_list[i].size = wqe->sg_list[i - 1].size; 2243 } 2244 2245 /* 2246 * Max Header buf size for IPV6 RoCE V2 is 86, 2247 * which is same as the QP1 SQ header buffer. 2248 * Header buf size for IPV4 RoCE V2 can be 66. 2249 * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20). 2250 * Subtract 20 bytes from QP1 SQ header buf size 2251 */ 2252 if (is_udp && ip_version == 4) 2253 sge.size -= 20; 2254 /* 2255 * Max Header buf size for RoCE V1 is 78. 2256 * ETH(14) + VLAN(4) + GRH(40) + BTH(20). 2257 * Subtract 8 bytes from QP1 SQ header buf size 2258 */ 2259 if (!is_udp) 2260 sge.size -= 8; 2261 2262 /* Subtract 4 bytes for non vlan packets */ 2263 if (!is_vlan) 2264 sge.size -= 4; 2265 2266 wqe->sg_list[0].addr = sge.addr; 2267 wqe->sg_list[0].lkey = sge.lkey; 2268 wqe->sg_list[0].size = sge.size; 2269 wqe->num_sge++; 2270 2271 } else { 2272 ibdev_err(&qp->rdev->ibdev, "QP1 buffer is empty!"); 2273 rc = -ENOMEM; 2274 } 2275 return rc; 2276 } 2277 2278 /* For the MAD layer, it only provides the recv SGE the size of 2279 * ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH, 2280 * nor RoCE iCRC. The Cu+ solution must provide buffer for the entire 2281 * receive packet (334 bytes) with no VLAN and then copy the GRH 2282 * and the MAD datagram out to the provided SGE. 2283 */ 2284 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, 2285 const struct ib_recv_wr *wr, 2286 struct bnxt_qplib_swqe *wqe, 2287 int payload_size) 2288 { 2289 struct bnxt_re_sqp_entries *sqp_entry; 2290 struct bnxt_qplib_sge ref, sge; 2291 struct bnxt_re_dev *rdev; 2292 u32 rq_prod_index; 2293 2294 rdev = qp->rdev; 2295 2296 rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); 2297 2298 if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) 2299 return -ENOMEM; 2300 2301 /* Create 1 SGE to receive the entire 2302 * ethernet packet 2303 */ 2304 /* Save the reference from ULP */ 2305 ref.addr = wqe->sg_list[0].addr; 2306 ref.lkey = wqe->sg_list[0].lkey; 2307 ref.size = wqe->sg_list[0].size; 2308 2309 sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index]; 2310 2311 /* SGE 1 */ 2312 wqe->sg_list[0].addr = sge.addr; 2313 wqe->sg_list[0].lkey = sge.lkey; 2314 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; 2315 sge.size -= wqe->sg_list[0].size; 2316 2317 sqp_entry->sge.addr = ref.addr; 2318 sqp_entry->sge.lkey = ref.lkey; 2319 sqp_entry->sge.size = ref.size; 2320 /* Store the wrid for reporting completion */ 2321 sqp_entry->wrid = wqe->wr_id; 2322 /* change the wqe->wrid to table index */ 2323 wqe->wr_id = rq_prod_index; 2324 return 0; 2325 } 2326 2327 static int is_ud_qp(struct bnxt_re_qp *qp) 2328 { 2329 return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD || 2330 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI); 2331 } 2332 2333 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, 2334 const struct ib_send_wr *wr, 2335 struct bnxt_qplib_swqe *wqe) 2336 { 2337 struct bnxt_re_ah *ah = NULL; 2338 2339 if (is_ud_qp(qp)) { 2340 ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah); 2341 wqe->send.q_key = ud_wr(wr)->remote_qkey; 2342 wqe->send.dst_qp = ud_wr(wr)->remote_qpn; 2343 wqe->send.avid = ah->qplib_ah.id; 2344 } 2345 switch (wr->opcode) { 2346 case IB_WR_SEND: 2347 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; 2348 break; 2349 case IB_WR_SEND_WITH_IMM: 2350 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; 2351 wqe->send.imm_data = wr->ex.imm_data; 2352 break; 2353 case IB_WR_SEND_WITH_INV: 2354 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; 2355 wqe->send.inv_key = wr->ex.invalidate_rkey; 2356 break; 2357 default: 2358 return -EINVAL; 2359 } 2360 if (wr->send_flags & IB_SEND_SIGNALED) 2361 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2362 if (wr->send_flags & IB_SEND_FENCE) 2363 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2364 if (wr->send_flags & IB_SEND_SOLICITED) 2365 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2366 if (wr->send_flags & IB_SEND_INLINE) 2367 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 2368 2369 return 0; 2370 } 2371 2372 static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr, 2373 struct bnxt_qplib_swqe *wqe) 2374 { 2375 switch (wr->opcode) { 2376 case IB_WR_RDMA_WRITE: 2377 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; 2378 break; 2379 case IB_WR_RDMA_WRITE_WITH_IMM: 2380 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; 2381 wqe->rdma.imm_data = wr->ex.imm_data; 2382 break; 2383 case IB_WR_RDMA_READ: 2384 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; 2385 wqe->rdma.inv_key = wr->ex.invalidate_rkey; 2386 break; 2387 default: 2388 return -EINVAL; 2389 } 2390 wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; 2391 wqe->rdma.r_key = rdma_wr(wr)->rkey; 2392 if (wr->send_flags & IB_SEND_SIGNALED) 2393 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2394 if (wr->send_flags & IB_SEND_FENCE) 2395 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2396 if (wr->send_flags & IB_SEND_SOLICITED) 2397 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2398 if (wr->send_flags & IB_SEND_INLINE) 2399 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 2400 2401 return 0; 2402 } 2403 2404 static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr, 2405 struct bnxt_qplib_swqe *wqe) 2406 { 2407 switch (wr->opcode) { 2408 case IB_WR_ATOMIC_CMP_AND_SWP: 2409 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; 2410 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 2411 wqe->atomic.swap_data = atomic_wr(wr)->swap; 2412 break; 2413 case IB_WR_ATOMIC_FETCH_AND_ADD: 2414 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; 2415 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 2416 break; 2417 default: 2418 return -EINVAL; 2419 } 2420 wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; 2421 wqe->atomic.r_key = atomic_wr(wr)->rkey; 2422 if (wr->send_flags & IB_SEND_SIGNALED) 2423 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2424 if (wr->send_flags & IB_SEND_FENCE) 2425 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2426 if (wr->send_flags & IB_SEND_SOLICITED) 2427 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2428 return 0; 2429 } 2430 2431 static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr, 2432 struct bnxt_qplib_swqe *wqe) 2433 { 2434 wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; 2435 wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; 2436 2437 /* Need unconditional fence for local invalidate 2438 * opcode to work as expected. 2439 */ 2440 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2441 2442 if (wr->send_flags & IB_SEND_SIGNALED) 2443 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2444 if (wr->send_flags & IB_SEND_SOLICITED) 2445 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2446 2447 return 0; 2448 } 2449 2450 static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr, 2451 struct bnxt_qplib_swqe *wqe) 2452 { 2453 struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr); 2454 struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; 2455 int access = wr->access; 2456 2457 wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; 2458 wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; 2459 wqe->frmr.page_list = mr->pages; 2460 wqe->frmr.page_list_len = mr->npages; 2461 wqe->frmr.levels = qplib_frpl->hwq.level; 2462 wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; 2463 2464 /* Need unconditional fence for reg_mr 2465 * opcode to function as expected. 2466 */ 2467 2468 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2469 2470 if (wr->wr.send_flags & IB_SEND_SIGNALED) 2471 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2472 2473 if (access & IB_ACCESS_LOCAL_WRITE) 2474 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; 2475 if (access & IB_ACCESS_REMOTE_READ) 2476 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; 2477 if (access & IB_ACCESS_REMOTE_WRITE) 2478 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; 2479 if (access & IB_ACCESS_REMOTE_ATOMIC) 2480 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; 2481 if (access & IB_ACCESS_MW_BIND) 2482 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; 2483 2484 wqe->frmr.l_key = wr->key; 2485 wqe->frmr.length = wr->mr->length; 2486 wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K); 2487 wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K); 2488 wqe->frmr.va = wr->mr->iova; 2489 return 0; 2490 } 2491 2492 static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev, 2493 const struct ib_send_wr *wr, 2494 struct bnxt_qplib_swqe *wqe) 2495 { 2496 /* Copy the inline data to the data field */ 2497 u8 *in_data; 2498 u32 i, sge_len; 2499 void *sge_addr; 2500 2501 in_data = wqe->inline_data; 2502 for (i = 0; i < wr->num_sge; i++) { 2503 sge_addr = (void *)(unsigned long) 2504 wr->sg_list[i].addr; 2505 sge_len = wr->sg_list[i].length; 2506 2507 if ((sge_len + wqe->inline_len) > 2508 BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { 2509 ibdev_err(&rdev->ibdev, 2510 "Inline data size requested > supported value"); 2511 return -EINVAL; 2512 } 2513 sge_len = wr->sg_list[i].length; 2514 2515 memcpy(in_data, sge_addr, sge_len); 2516 in_data += wr->sg_list[i].length; 2517 wqe->inline_len += wr->sg_list[i].length; 2518 } 2519 return wqe->inline_len; 2520 } 2521 2522 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev, 2523 const struct ib_send_wr *wr, 2524 struct bnxt_qplib_swqe *wqe) 2525 { 2526 int payload_sz = 0; 2527 2528 if (wr->send_flags & IB_SEND_INLINE) 2529 payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe); 2530 else 2531 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list, 2532 wqe->num_sge); 2533 2534 return payload_sz; 2535 } 2536 2537 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp) 2538 { 2539 if ((qp->ib_qp.qp_type == IB_QPT_UD || 2540 qp->ib_qp.qp_type == IB_QPT_GSI || 2541 qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) && 2542 qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) { 2543 int qp_attr_mask; 2544 struct ib_qp_attr qp_attr; 2545 2546 qp_attr_mask = IB_QP_STATE; 2547 qp_attr.qp_state = IB_QPS_RTS; 2548 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL); 2549 qp->qplib_qp.wqe_cnt = 0; 2550 } 2551 } 2552 2553 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, 2554 struct bnxt_re_qp *qp, 2555 const struct ib_send_wr *wr) 2556 { 2557 int rc = 0, payload_sz = 0; 2558 unsigned long flags; 2559 2560 spin_lock_irqsave(&qp->sq_lock, flags); 2561 while (wr) { 2562 struct bnxt_qplib_swqe wqe = {}; 2563 2564 /* Common */ 2565 wqe.num_sge = wr->num_sge; 2566 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 2567 ibdev_err(&rdev->ibdev, 2568 "Limit exceeded for Send SGEs"); 2569 rc = -EINVAL; 2570 goto bad; 2571 } 2572 2573 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); 2574 if (payload_sz < 0) { 2575 rc = -EINVAL; 2576 goto bad; 2577 } 2578 wqe.wr_id = wr->wr_id; 2579 2580 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; 2581 2582 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 2583 if (!rc) 2584 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 2585 bad: 2586 if (rc) { 2587 ibdev_err(&rdev->ibdev, 2588 "Post send failed opcode = %#x rc = %d", 2589 wr->opcode, rc); 2590 break; 2591 } 2592 wr = wr->next; 2593 } 2594 bnxt_qplib_post_send_db(&qp->qplib_qp); 2595 bnxt_ud_qp_hw_stall_workaround(qp); 2596 spin_unlock_irqrestore(&qp->sq_lock, flags); 2597 return rc; 2598 } 2599 2600 int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, 2601 const struct ib_send_wr **bad_wr) 2602 { 2603 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 2604 struct bnxt_qplib_swqe wqe; 2605 int rc = 0, payload_sz = 0; 2606 unsigned long flags; 2607 2608 spin_lock_irqsave(&qp->sq_lock, flags); 2609 while (wr) { 2610 /* House keeping */ 2611 memset(&wqe, 0, sizeof(wqe)); 2612 2613 /* Common */ 2614 wqe.num_sge = wr->num_sge; 2615 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 2616 ibdev_err(&qp->rdev->ibdev, 2617 "Limit exceeded for Send SGEs"); 2618 rc = -EINVAL; 2619 goto bad; 2620 } 2621 2622 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); 2623 if (payload_sz < 0) { 2624 rc = -EINVAL; 2625 goto bad; 2626 } 2627 wqe.wr_id = wr->wr_id; 2628 2629 switch (wr->opcode) { 2630 case IB_WR_SEND: 2631 case IB_WR_SEND_WITH_IMM: 2632 if (qp->qplib_qp.type == CMDQ_CREATE_QP1_TYPE_GSI) { 2633 rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe, 2634 payload_sz); 2635 if (rc) 2636 goto bad; 2637 wqe.rawqp1.lflags |= 2638 SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; 2639 } 2640 switch (wr->send_flags) { 2641 case IB_SEND_IP_CSUM: 2642 wqe.rawqp1.lflags |= 2643 SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; 2644 break; 2645 default: 2646 break; 2647 } 2648 fallthrough; 2649 case IB_WR_SEND_WITH_INV: 2650 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 2651 break; 2652 case IB_WR_RDMA_WRITE: 2653 case IB_WR_RDMA_WRITE_WITH_IMM: 2654 case IB_WR_RDMA_READ: 2655 rc = bnxt_re_build_rdma_wqe(wr, &wqe); 2656 break; 2657 case IB_WR_ATOMIC_CMP_AND_SWP: 2658 case IB_WR_ATOMIC_FETCH_AND_ADD: 2659 rc = bnxt_re_build_atomic_wqe(wr, &wqe); 2660 break; 2661 case IB_WR_RDMA_READ_WITH_INV: 2662 ibdev_err(&qp->rdev->ibdev, 2663 "RDMA Read with Invalidate is not supported"); 2664 rc = -EINVAL; 2665 goto bad; 2666 case IB_WR_LOCAL_INV: 2667 rc = bnxt_re_build_inv_wqe(wr, &wqe); 2668 break; 2669 case IB_WR_REG_MR: 2670 rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); 2671 break; 2672 default: 2673 /* Unsupported WRs */ 2674 ibdev_err(&qp->rdev->ibdev, 2675 "WR (%#x) is not supported", wr->opcode); 2676 rc = -EINVAL; 2677 goto bad; 2678 } 2679 if (!rc) 2680 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 2681 bad: 2682 if (rc) { 2683 ibdev_err(&qp->rdev->ibdev, 2684 "post_send failed op:%#x qps = %#x rc = %d\n", 2685 wr->opcode, qp->qplib_qp.state, rc); 2686 *bad_wr = wr; 2687 break; 2688 } 2689 wr = wr->next; 2690 } 2691 bnxt_qplib_post_send_db(&qp->qplib_qp); 2692 bnxt_ud_qp_hw_stall_workaround(qp); 2693 spin_unlock_irqrestore(&qp->sq_lock, flags); 2694 2695 return rc; 2696 } 2697 2698 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, 2699 struct bnxt_re_qp *qp, 2700 const struct ib_recv_wr *wr) 2701 { 2702 struct bnxt_qplib_swqe wqe; 2703 int rc = 0; 2704 2705 memset(&wqe, 0, sizeof(wqe)); 2706 while (wr) { 2707 /* House keeping */ 2708 memset(&wqe, 0, sizeof(wqe)); 2709 2710 /* Common */ 2711 wqe.num_sge = wr->num_sge; 2712 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 2713 ibdev_err(&rdev->ibdev, 2714 "Limit exceeded for Receive SGEs"); 2715 rc = -EINVAL; 2716 break; 2717 } 2718 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); 2719 wqe.wr_id = wr->wr_id; 2720 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 2721 2722 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 2723 if (rc) 2724 break; 2725 2726 wr = wr->next; 2727 } 2728 if (!rc) 2729 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2730 return rc; 2731 } 2732 2733 int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, 2734 const struct ib_recv_wr **bad_wr) 2735 { 2736 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 2737 struct bnxt_qplib_swqe wqe; 2738 int rc = 0, payload_sz = 0; 2739 unsigned long flags; 2740 u32 count = 0; 2741 2742 spin_lock_irqsave(&qp->rq_lock, flags); 2743 while (wr) { 2744 /* House keeping */ 2745 memset(&wqe, 0, sizeof(wqe)); 2746 2747 /* Common */ 2748 wqe.num_sge = wr->num_sge; 2749 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 2750 ibdev_err(&qp->rdev->ibdev, 2751 "Limit exceeded for Receive SGEs"); 2752 rc = -EINVAL; 2753 *bad_wr = wr; 2754 break; 2755 } 2756 2757 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, 2758 wr->num_sge); 2759 wqe.wr_id = wr->wr_id; 2760 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 2761 2762 if (ib_qp->qp_type == IB_QPT_GSI && 2763 qp->qplib_qp.type != CMDQ_CREATE_QP_TYPE_GSI) 2764 rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe, 2765 payload_sz); 2766 if (!rc) 2767 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 2768 if (rc) { 2769 *bad_wr = wr; 2770 break; 2771 } 2772 2773 /* Ring DB if the RQEs posted reaches a threshold value */ 2774 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) { 2775 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2776 count = 0; 2777 } 2778 2779 wr = wr->next; 2780 } 2781 2782 if (count) 2783 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2784 2785 spin_unlock_irqrestore(&qp->rq_lock, flags); 2786 2787 return rc; 2788 } 2789 2790 /* Completion Queues */ 2791 int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 2792 { 2793 struct bnxt_re_cq *cq; 2794 struct bnxt_qplib_nq *nq; 2795 struct bnxt_re_dev *rdev; 2796 2797 cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 2798 rdev = cq->rdev; 2799 nq = cq->qplib_cq.nq; 2800 2801 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); 2802 ib_umem_release(cq->umem); 2803 2804 atomic_dec(&rdev->cq_count); 2805 nq->budget--; 2806 kfree(cq->cql); 2807 return 0; 2808 } 2809 2810 int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 2811 struct ib_udata *udata) 2812 { 2813 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev); 2814 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 2815 struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq); 2816 int rc, entries; 2817 int cqe = attr->cqe; 2818 struct bnxt_qplib_nq *nq = NULL; 2819 unsigned int nq_alloc_cnt; 2820 2821 if (attr->flags) 2822 return -EOPNOTSUPP; 2823 2824 /* Validate CQ fields */ 2825 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { 2826 ibdev_err(&rdev->ibdev, "Failed to create CQ -max exceeded"); 2827 return -EINVAL; 2828 } 2829 2830 cq->rdev = rdev; 2831 cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq); 2832 2833 entries = roundup_pow_of_two(cqe + 1); 2834 if (entries > dev_attr->max_cq_wqes + 1) 2835 entries = dev_attr->max_cq_wqes + 1; 2836 2837 cq->qplib_cq.sg_info.pgsize = PAGE_SIZE; 2838 cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT; 2839 if (udata) { 2840 struct bnxt_re_cq_req req; 2841 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context( 2842 udata, struct bnxt_re_ucontext, ib_uctx); 2843 if (ib_copy_from_udata(&req, udata, sizeof(req))) { 2844 rc = -EFAULT; 2845 goto fail; 2846 } 2847 2848 cq->umem = ib_umem_get(&rdev->ibdev, req.cq_va, 2849 entries * sizeof(struct cq_base), 2850 IB_ACCESS_LOCAL_WRITE); 2851 if (IS_ERR(cq->umem)) { 2852 rc = PTR_ERR(cq->umem); 2853 goto fail; 2854 } 2855 cq->qplib_cq.sg_info.umem = cq->umem; 2856 cq->qplib_cq.dpi = &uctx->dpi; 2857 } else { 2858 cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL); 2859 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), 2860 GFP_KERNEL); 2861 if (!cq->cql) { 2862 rc = -ENOMEM; 2863 goto fail; 2864 } 2865 2866 cq->qplib_cq.dpi = &rdev->dpi_privileged; 2867 } 2868 /* 2869 * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a 2870 * used for getting the NQ index. 2871 */ 2872 nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt); 2873 nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)]; 2874 cq->qplib_cq.max_wqe = entries; 2875 cq->qplib_cq.cnq_hw_ring_id = nq->ring_id; 2876 cq->qplib_cq.nq = nq; 2877 2878 rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq); 2879 if (rc) { 2880 ibdev_err(&rdev->ibdev, "Failed to create HW CQ"); 2881 goto fail; 2882 } 2883 2884 cq->ib_cq.cqe = entries; 2885 cq->cq_period = cq->qplib_cq.period; 2886 nq->budget++; 2887 2888 atomic_inc(&rdev->cq_count); 2889 spin_lock_init(&cq->cq_lock); 2890 2891 if (udata) { 2892 struct bnxt_re_cq_resp resp; 2893 2894 resp.cqid = cq->qplib_cq.id; 2895 resp.tail = cq->qplib_cq.hwq.cons; 2896 resp.phase = cq->qplib_cq.period; 2897 resp.rsvd = 0; 2898 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 2899 if (rc) { 2900 ibdev_err(&rdev->ibdev, "Failed to copy CQ udata"); 2901 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); 2902 goto c2fail; 2903 } 2904 } 2905 2906 return 0; 2907 2908 c2fail: 2909 ib_umem_release(cq->umem); 2910 fail: 2911 kfree(cq->cql); 2912 return rc; 2913 } 2914 2915 static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq) 2916 { 2917 struct bnxt_re_dev *rdev = cq->rdev; 2918 2919 bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq); 2920 2921 cq->qplib_cq.max_wqe = cq->resize_cqe; 2922 if (cq->resize_umem) { 2923 ib_umem_release(cq->umem); 2924 cq->umem = cq->resize_umem; 2925 cq->resize_umem = NULL; 2926 cq->resize_cqe = 0; 2927 } 2928 } 2929 2930 int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata) 2931 { 2932 struct bnxt_qplib_sg_info sg_info = {}; 2933 struct bnxt_qplib_dpi *orig_dpi = NULL; 2934 struct bnxt_qplib_dev_attr *dev_attr; 2935 struct bnxt_re_ucontext *uctx = NULL; 2936 struct bnxt_re_resize_cq_req req; 2937 struct bnxt_re_dev *rdev; 2938 struct bnxt_re_cq *cq; 2939 int rc, entries; 2940 2941 cq = container_of(ibcq, struct bnxt_re_cq, ib_cq); 2942 rdev = cq->rdev; 2943 dev_attr = &rdev->dev_attr; 2944 if (!ibcq->uobject) { 2945 ibdev_err(&rdev->ibdev, "Kernel CQ Resize not supported"); 2946 return -EOPNOTSUPP; 2947 } 2948 2949 if (cq->resize_umem) { 2950 ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - Busy", 2951 cq->qplib_cq.id); 2952 return -EBUSY; 2953 } 2954 2955 /* Check the requested cq depth out of supported depth */ 2956 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { 2957 ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - out of range cqe %d", 2958 cq->qplib_cq.id, cqe); 2959 return -EINVAL; 2960 } 2961 2962 entries = roundup_pow_of_two(cqe + 1); 2963 if (entries > dev_attr->max_cq_wqes + 1) 2964 entries = dev_attr->max_cq_wqes + 1; 2965 2966 uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, 2967 ib_uctx); 2968 /* uverbs consumer */ 2969 if (ib_copy_from_udata(&req, udata, sizeof(req))) { 2970 rc = -EFAULT; 2971 goto fail; 2972 } 2973 2974 cq->resize_umem = ib_umem_get(&rdev->ibdev, req.cq_va, 2975 entries * sizeof(struct cq_base), 2976 IB_ACCESS_LOCAL_WRITE); 2977 if (IS_ERR(cq->resize_umem)) { 2978 rc = PTR_ERR(cq->resize_umem); 2979 cq->resize_umem = NULL; 2980 ibdev_err(&rdev->ibdev, "%s: ib_umem_get failed! rc = %d\n", 2981 __func__, rc); 2982 goto fail; 2983 } 2984 cq->resize_cqe = entries; 2985 memcpy(&sg_info, &cq->qplib_cq.sg_info, sizeof(sg_info)); 2986 orig_dpi = cq->qplib_cq.dpi; 2987 2988 cq->qplib_cq.sg_info.umem = cq->resize_umem; 2989 cq->qplib_cq.sg_info.pgsize = PAGE_SIZE; 2990 cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT; 2991 cq->qplib_cq.dpi = &uctx->dpi; 2992 2993 rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries); 2994 if (rc) { 2995 ibdev_err(&rdev->ibdev, "Resize HW CQ %#x failed!", 2996 cq->qplib_cq.id); 2997 goto fail; 2998 } 2999 3000 cq->ib_cq.cqe = cq->resize_cqe; 3001 3002 return 0; 3003 3004 fail: 3005 if (cq->resize_umem) { 3006 ib_umem_release(cq->resize_umem); 3007 cq->resize_umem = NULL; 3008 cq->resize_cqe = 0; 3009 memcpy(&cq->qplib_cq.sg_info, &sg_info, sizeof(sg_info)); 3010 cq->qplib_cq.dpi = orig_dpi; 3011 } 3012 return rc; 3013 } 3014 3015 static u8 __req_to_ib_wc_status(u8 qstatus) 3016 { 3017 switch (qstatus) { 3018 case CQ_REQ_STATUS_OK: 3019 return IB_WC_SUCCESS; 3020 case CQ_REQ_STATUS_BAD_RESPONSE_ERR: 3021 return IB_WC_BAD_RESP_ERR; 3022 case CQ_REQ_STATUS_LOCAL_LENGTH_ERR: 3023 return IB_WC_LOC_LEN_ERR; 3024 case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR: 3025 return IB_WC_LOC_QP_OP_ERR; 3026 case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR: 3027 return IB_WC_LOC_PROT_ERR; 3028 case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR: 3029 return IB_WC_GENERAL_ERR; 3030 case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR: 3031 return IB_WC_REM_INV_REQ_ERR; 3032 case CQ_REQ_STATUS_REMOTE_ACCESS_ERR: 3033 return IB_WC_REM_ACCESS_ERR; 3034 case CQ_REQ_STATUS_REMOTE_OPERATION_ERR: 3035 return IB_WC_REM_OP_ERR; 3036 case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR: 3037 return IB_WC_RNR_RETRY_EXC_ERR; 3038 case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR: 3039 return IB_WC_RETRY_EXC_ERR; 3040 case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR: 3041 return IB_WC_WR_FLUSH_ERR; 3042 default: 3043 return IB_WC_GENERAL_ERR; 3044 } 3045 return 0; 3046 } 3047 3048 static u8 __rawqp1_to_ib_wc_status(u8 qstatus) 3049 { 3050 switch (qstatus) { 3051 case CQ_RES_RAWETH_QP1_STATUS_OK: 3052 return IB_WC_SUCCESS; 3053 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR: 3054 return IB_WC_LOC_ACCESS_ERR; 3055 case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR: 3056 return IB_WC_LOC_LEN_ERR; 3057 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR: 3058 return IB_WC_LOC_PROT_ERR; 3059 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR: 3060 return IB_WC_LOC_QP_OP_ERR; 3061 case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR: 3062 return IB_WC_GENERAL_ERR; 3063 case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR: 3064 return IB_WC_WR_FLUSH_ERR; 3065 case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR: 3066 return IB_WC_WR_FLUSH_ERR; 3067 default: 3068 return IB_WC_GENERAL_ERR; 3069 } 3070 } 3071 3072 static u8 __rc_to_ib_wc_status(u8 qstatus) 3073 { 3074 switch (qstatus) { 3075 case CQ_RES_RC_STATUS_OK: 3076 return IB_WC_SUCCESS; 3077 case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR: 3078 return IB_WC_LOC_ACCESS_ERR; 3079 case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR: 3080 return IB_WC_LOC_LEN_ERR; 3081 case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR: 3082 return IB_WC_LOC_PROT_ERR; 3083 case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR: 3084 return IB_WC_LOC_QP_OP_ERR; 3085 case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR: 3086 return IB_WC_GENERAL_ERR; 3087 case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR: 3088 return IB_WC_REM_INV_REQ_ERR; 3089 case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR: 3090 return IB_WC_WR_FLUSH_ERR; 3091 case CQ_RES_RC_STATUS_HW_FLUSH_ERR: 3092 return IB_WC_WR_FLUSH_ERR; 3093 default: 3094 return IB_WC_GENERAL_ERR; 3095 } 3096 } 3097 3098 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) 3099 { 3100 switch (cqe->type) { 3101 case BNXT_QPLIB_SWQE_TYPE_SEND: 3102 wc->opcode = IB_WC_SEND; 3103 break; 3104 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: 3105 wc->opcode = IB_WC_SEND; 3106 wc->wc_flags |= IB_WC_WITH_IMM; 3107 break; 3108 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: 3109 wc->opcode = IB_WC_SEND; 3110 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3111 break; 3112 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: 3113 wc->opcode = IB_WC_RDMA_WRITE; 3114 break; 3115 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: 3116 wc->opcode = IB_WC_RDMA_WRITE; 3117 wc->wc_flags |= IB_WC_WITH_IMM; 3118 break; 3119 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: 3120 wc->opcode = IB_WC_RDMA_READ; 3121 break; 3122 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: 3123 wc->opcode = IB_WC_COMP_SWAP; 3124 break; 3125 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: 3126 wc->opcode = IB_WC_FETCH_ADD; 3127 break; 3128 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: 3129 wc->opcode = IB_WC_LOCAL_INV; 3130 break; 3131 case BNXT_QPLIB_SWQE_TYPE_REG_MR: 3132 wc->opcode = IB_WC_REG_MR; 3133 break; 3134 default: 3135 wc->opcode = IB_WC_SEND; 3136 break; 3137 } 3138 3139 wc->status = __req_to_ib_wc_status(cqe->status); 3140 } 3141 3142 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, 3143 u16 raweth_qp1_flags2) 3144 { 3145 bool is_ipv6 = false, is_ipv4 = false; 3146 3147 /* raweth_qp1_flags Bit 9-6 indicates itype */ 3148 if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 3149 != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 3150 return -1; 3151 3152 if (raweth_qp1_flags2 & 3153 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC && 3154 raweth_qp1_flags2 & 3155 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) { 3156 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ 3157 (raweth_qp1_flags2 & 3158 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ? 3159 (is_ipv6 = true) : (is_ipv4 = true); 3160 return ((is_ipv6) ? 3161 BNXT_RE_ROCEV2_IPV6_PACKET : 3162 BNXT_RE_ROCEV2_IPV4_PACKET); 3163 } else { 3164 return BNXT_RE_ROCE_V1_PACKET; 3165 } 3166 } 3167 3168 static int bnxt_re_to_ib_nw_type(int nw_type) 3169 { 3170 u8 nw_hdr_type = 0xFF; 3171 3172 switch (nw_type) { 3173 case BNXT_RE_ROCE_V1_PACKET: 3174 nw_hdr_type = RDMA_NETWORK_ROCE_V1; 3175 break; 3176 case BNXT_RE_ROCEV2_IPV4_PACKET: 3177 nw_hdr_type = RDMA_NETWORK_IPV4; 3178 break; 3179 case BNXT_RE_ROCEV2_IPV6_PACKET: 3180 nw_hdr_type = RDMA_NETWORK_IPV6; 3181 break; 3182 } 3183 return nw_hdr_type; 3184 } 3185 3186 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev, 3187 void *rq_hdr_buf) 3188 { 3189 u8 *tmp_buf = NULL; 3190 struct ethhdr *eth_hdr; 3191 u16 eth_type; 3192 bool rc = false; 3193 3194 tmp_buf = (u8 *)rq_hdr_buf; 3195 /* 3196 * If dest mac is not same as I/F mac, this could be a 3197 * loopback address or multicast address, check whether 3198 * it is a loopback packet 3199 */ 3200 if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) { 3201 tmp_buf += 4; 3202 /* Check the ether type */ 3203 eth_hdr = (struct ethhdr *)tmp_buf; 3204 eth_type = ntohs(eth_hdr->h_proto); 3205 switch (eth_type) { 3206 case ETH_P_IBOE: 3207 rc = true; 3208 break; 3209 case ETH_P_IP: 3210 case ETH_P_IPV6: { 3211 u32 len; 3212 struct udphdr *udp_hdr; 3213 3214 len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) : 3215 sizeof(struct ipv6hdr)); 3216 tmp_buf += sizeof(struct ethhdr) + len; 3217 udp_hdr = (struct udphdr *)tmp_buf; 3218 if (ntohs(udp_hdr->dest) == 3219 ROCE_V2_UDP_DPORT) 3220 rc = true; 3221 break; 3222 } 3223 default: 3224 break; 3225 } 3226 } 3227 3228 return rc; 3229 } 3230 3231 static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *gsi_qp, 3232 struct bnxt_qplib_cqe *cqe) 3233 { 3234 struct bnxt_re_dev *rdev = gsi_qp->rdev; 3235 struct bnxt_re_sqp_entries *sqp_entry = NULL; 3236 struct bnxt_re_qp *gsi_sqp = rdev->gsi_ctx.gsi_sqp; 3237 struct bnxt_re_ah *gsi_sah; 3238 struct ib_send_wr *swr; 3239 struct ib_ud_wr udwr; 3240 struct ib_recv_wr rwr; 3241 int pkt_type = 0; 3242 u32 tbl_idx; 3243 void *rq_hdr_buf; 3244 dma_addr_t rq_hdr_buf_map; 3245 dma_addr_t shrq_hdr_buf_map; 3246 u32 offset = 0; 3247 u32 skip_bytes = 0; 3248 struct ib_sge s_sge[2]; 3249 struct ib_sge r_sge[2]; 3250 int rc; 3251 3252 memset(&udwr, 0, sizeof(udwr)); 3253 memset(&rwr, 0, sizeof(rwr)); 3254 memset(&s_sge, 0, sizeof(s_sge)); 3255 memset(&r_sge, 0, sizeof(r_sge)); 3256 3257 swr = &udwr.wr; 3258 tbl_idx = cqe->wr_id; 3259 3260 rq_hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf + 3261 (tbl_idx * gsi_qp->qplib_qp.rq_hdr_buf_size); 3262 rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp, 3263 tbl_idx); 3264 3265 /* Shadow QP header buffer */ 3266 shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp, 3267 tbl_idx); 3268 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; 3269 3270 /* Store this cqe */ 3271 memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); 3272 sqp_entry->qp1_qp = gsi_qp; 3273 3274 /* Find packet type from the cqe */ 3275 3276 pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, 3277 cqe->raweth_qp1_flags2); 3278 if (pkt_type < 0) { 3279 ibdev_err(&rdev->ibdev, "Invalid packet\n"); 3280 return -EINVAL; 3281 } 3282 3283 /* Adjust the offset for the user buffer and post in the rq */ 3284 3285 if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET) 3286 offset = 20; 3287 3288 /* 3289 * QP1 loopback packet has 4 bytes of internal header before 3290 * ether header. Skip these four bytes. 3291 */ 3292 if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf)) 3293 skip_bytes = 4; 3294 3295 /* First send SGE . Skip the ether header*/ 3296 s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 3297 + skip_bytes; 3298 s_sge[0].lkey = 0xFFFFFFFF; 3299 s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 : 3300 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; 3301 3302 /* Second Send SGE */ 3303 s_sge[1].addr = s_sge[0].addr + s_sge[0].length + 3304 BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE; 3305 if (pkt_type != BNXT_RE_ROCE_V1_PACKET) 3306 s_sge[1].addr += 8; 3307 s_sge[1].lkey = 0xFFFFFFFF; 3308 s_sge[1].length = 256; 3309 3310 /* First recv SGE */ 3311 3312 r_sge[0].addr = shrq_hdr_buf_map; 3313 r_sge[0].lkey = 0xFFFFFFFF; 3314 r_sge[0].length = 40; 3315 3316 r_sge[1].addr = sqp_entry->sge.addr + offset; 3317 r_sge[1].lkey = sqp_entry->sge.lkey; 3318 r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; 3319 3320 /* Create receive work request */ 3321 rwr.num_sge = 2; 3322 rwr.sg_list = r_sge; 3323 rwr.wr_id = tbl_idx; 3324 rwr.next = NULL; 3325 3326 rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr); 3327 if (rc) { 3328 ibdev_err(&rdev->ibdev, 3329 "Failed to post Rx buffers to shadow QP"); 3330 return -ENOMEM; 3331 } 3332 3333 swr->num_sge = 2; 3334 swr->sg_list = s_sge; 3335 swr->wr_id = tbl_idx; 3336 swr->opcode = IB_WR_SEND; 3337 swr->next = NULL; 3338 gsi_sah = rdev->gsi_ctx.gsi_sah; 3339 udwr.ah = &gsi_sah->ib_ah; 3340 udwr.remote_qpn = gsi_sqp->qplib_qp.id; 3341 udwr.remote_qkey = gsi_sqp->qplib_qp.qkey; 3342 3343 /* post data received in the send queue */ 3344 rc = bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr); 3345 3346 return 0; 3347 } 3348 3349 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc, 3350 struct bnxt_qplib_cqe *cqe) 3351 { 3352 wc->opcode = IB_WC_RECV; 3353 wc->status = __rawqp1_to_ib_wc_status(cqe->status); 3354 wc->wc_flags |= IB_WC_GRH; 3355 } 3356 3357 static bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev, 3358 u16 vlan_id) 3359 { 3360 /* 3361 * Check if the vlan is configured in the host. If not configured, it 3362 * can be a transparent VLAN. So dont report the vlan id. 3363 */ 3364 if (!__vlan_find_dev_deep_rcu(rdev->netdev, 3365 htons(ETH_P_8021Q), vlan_id)) 3366 return false; 3367 return true; 3368 } 3369 3370 static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe, 3371 u16 *vid, u8 *sl) 3372 { 3373 bool ret = false; 3374 u32 metadata; 3375 u16 tpid; 3376 3377 metadata = orig_cqe->raweth_qp1_metadata; 3378 if (orig_cqe->raweth_qp1_flags2 & 3379 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) { 3380 tpid = ((metadata & 3381 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >> 3382 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); 3383 if (tpid == ETH_P_8021Q) { 3384 *vid = metadata & 3385 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK; 3386 *sl = (metadata & 3387 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >> 3388 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT; 3389 ret = true; 3390 } 3391 } 3392 3393 return ret; 3394 } 3395 3396 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc, 3397 struct bnxt_qplib_cqe *cqe) 3398 { 3399 wc->opcode = IB_WC_RECV; 3400 wc->status = __rc_to_ib_wc_status(cqe->status); 3401 3402 if (cqe->flags & CQ_RES_RC_FLAGS_IMM) 3403 wc->wc_flags |= IB_WC_WITH_IMM; 3404 if (cqe->flags & CQ_RES_RC_FLAGS_INV) 3405 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3406 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == 3407 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) 3408 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 3409 } 3410 3411 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp, 3412 struct ib_wc *wc, 3413 struct bnxt_qplib_cqe *cqe) 3414 { 3415 struct bnxt_re_dev *rdev = gsi_sqp->rdev; 3416 struct bnxt_re_qp *gsi_qp = NULL; 3417 struct bnxt_qplib_cqe *orig_cqe = NULL; 3418 struct bnxt_re_sqp_entries *sqp_entry = NULL; 3419 int nw_type; 3420 u32 tbl_idx; 3421 u16 vlan_id; 3422 u8 sl; 3423 3424 tbl_idx = cqe->wr_id; 3425 3426 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; 3427 gsi_qp = sqp_entry->qp1_qp; 3428 orig_cqe = &sqp_entry->cqe; 3429 3430 wc->wr_id = sqp_entry->wrid; 3431 wc->byte_len = orig_cqe->length; 3432 wc->qp = &gsi_qp->ib_qp; 3433 3434 wc->ex.imm_data = orig_cqe->immdata; 3435 wc->src_qp = orig_cqe->src_qp; 3436 memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); 3437 if (bnxt_re_is_vlan_pkt(orig_cqe, &vlan_id, &sl)) { 3438 if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { 3439 wc->vlan_id = vlan_id; 3440 wc->sl = sl; 3441 wc->wc_flags |= IB_WC_WITH_VLAN; 3442 } 3443 } 3444 wc->port_num = 1; 3445 wc->vendor_err = orig_cqe->status; 3446 3447 wc->opcode = IB_WC_RECV; 3448 wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); 3449 wc->wc_flags |= IB_WC_GRH; 3450 3451 nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, 3452 orig_cqe->raweth_qp1_flags2); 3453 if (nw_type >= 0) { 3454 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type); 3455 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 3456 } 3457 } 3458 3459 static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp, 3460 struct ib_wc *wc, 3461 struct bnxt_qplib_cqe *cqe) 3462 { 3463 struct bnxt_re_dev *rdev; 3464 u16 vlan_id = 0; 3465 u8 nw_type; 3466 3467 rdev = qp->rdev; 3468 wc->opcode = IB_WC_RECV; 3469 wc->status = __rc_to_ib_wc_status(cqe->status); 3470 3471 if (cqe->flags & CQ_RES_UD_FLAGS_IMM) 3472 wc->wc_flags |= IB_WC_WITH_IMM; 3473 /* report only on GSI QP for Thor */ 3474 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI) { 3475 wc->wc_flags |= IB_WC_GRH; 3476 memcpy(wc->smac, cqe->smac, ETH_ALEN); 3477 wc->wc_flags |= IB_WC_WITH_SMAC; 3478 if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) { 3479 vlan_id = (cqe->cfa_meta & 0xFFF); 3480 } 3481 /* Mark only if vlan_id is non zero */ 3482 if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { 3483 wc->vlan_id = vlan_id; 3484 wc->wc_flags |= IB_WC_WITH_VLAN; 3485 } 3486 nw_type = (cqe->flags & CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK) >> 3487 CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT; 3488 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type); 3489 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 3490 } 3491 3492 } 3493 3494 static int send_phantom_wqe(struct bnxt_re_qp *qp) 3495 { 3496 struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp; 3497 unsigned long flags; 3498 int rc = 0; 3499 3500 spin_lock_irqsave(&qp->sq_lock, flags); 3501 3502 rc = bnxt_re_bind_fence_mw(lib_qp); 3503 if (!rc) { 3504 lib_qp->sq.phantom_wqe_cnt++; 3505 ibdev_dbg(&qp->rdev->ibdev, 3506 "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n", 3507 lib_qp->id, lib_qp->sq.hwq.prod, 3508 HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq), 3509 lib_qp->sq.phantom_wqe_cnt); 3510 } 3511 3512 spin_unlock_irqrestore(&qp->sq_lock, flags); 3513 return rc; 3514 } 3515 3516 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) 3517 { 3518 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 3519 struct bnxt_re_qp *qp, *sh_qp; 3520 struct bnxt_qplib_cqe *cqe; 3521 int i, ncqe, budget; 3522 struct bnxt_qplib_q *sq; 3523 struct bnxt_qplib_qp *lib_qp; 3524 u32 tbl_idx; 3525 struct bnxt_re_sqp_entries *sqp_entry = NULL; 3526 unsigned long flags; 3527 3528 /* User CQ; the only processing we do is to 3529 * complete any pending CQ resize operation. 3530 */ 3531 if (cq->umem) { 3532 if (cq->resize_umem) 3533 bnxt_re_resize_cq_complete(cq); 3534 return 0; 3535 } 3536 3537 spin_lock_irqsave(&cq->cq_lock, flags); 3538 budget = min_t(u32, num_entries, cq->max_cql); 3539 num_entries = budget; 3540 if (!cq->cql) { 3541 ibdev_err(&cq->rdev->ibdev, "POLL CQ : no CQL to use"); 3542 goto exit; 3543 } 3544 cqe = &cq->cql[0]; 3545 while (budget) { 3546 lib_qp = NULL; 3547 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp); 3548 if (lib_qp) { 3549 sq = &lib_qp->sq; 3550 if (sq->send_phantom) { 3551 qp = container_of(lib_qp, 3552 struct bnxt_re_qp, qplib_qp); 3553 if (send_phantom_wqe(qp) == -ENOMEM) 3554 ibdev_err(&cq->rdev->ibdev, 3555 "Phantom failed! Scheduled to send again\n"); 3556 else 3557 sq->send_phantom = false; 3558 } 3559 } 3560 if (ncqe < budget) 3561 ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq, 3562 cqe + ncqe, 3563 budget - ncqe); 3564 3565 if (!ncqe) 3566 break; 3567 3568 for (i = 0; i < ncqe; i++, cqe++) { 3569 /* Transcribe each qplib_wqe back to ib_wc */ 3570 memset(wc, 0, sizeof(*wc)); 3571 3572 wc->wr_id = cqe->wr_id; 3573 wc->byte_len = cqe->length; 3574 qp = container_of 3575 ((struct bnxt_qplib_qp *) 3576 (unsigned long)(cqe->qp_handle), 3577 struct bnxt_re_qp, qplib_qp); 3578 wc->qp = &qp->ib_qp; 3579 wc->ex.imm_data = cqe->immdata; 3580 wc->src_qp = cqe->src_qp; 3581 memcpy(wc->smac, cqe->smac, ETH_ALEN); 3582 wc->port_num = 1; 3583 wc->vendor_err = cqe->status; 3584 3585 switch (cqe->opcode) { 3586 case CQ_BASE_CQE_TYPE_REQ: 3587 sh_qp = qp->rdev->gsi_ctx.gsi_sqp; 3588 if (sh_qp && 3589 qp->qplib_qp.id == sh_qp->qplib_qp.id) { 3590 /* Handle this completion with 3591 * the stored completion 3592 */ 3593 memset(wc, 0, sizeof(*wc)); 3594 continue; 3595 } 3596 bnxt_re_process_req_wc(wc, cqe); 3597 break; 3598 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 3599 if (!cqe->status) { 3600 int rc = 0; 3601 3602 rc = bnxt_re_process_raw_qp_pkt_rx 3603 (qp, cqe); 3604 if (!rc) { 3605 memset(wc, 0, sizeof(*wc)); 3606 continue; 3607 } 3608 cqe->status = -1; 3609 } 3610 /* Errors need not be looped back. 3611 * But change the wr_id to the one 3612 * stored in the table 3613 */ 3614 tbl_idx = cqe->wr_id; 3615 sqp_entry = &cq->rdev->gsi_ctx.sqp_tbl[tbl_idx]; 3616 wc->wr_id = sqp_entry->wrid; 3617 bnxt_re_process_res_rawqp1_wc(wc, cqe); 3618 break; 3619 case CQ_BASE_CQE_TYPE_RES_RC: 3620 bnxt_re_process_res_rc_wc(wc, cqe); 3621 break; 3622 case CQ_BASE_CQE_TYPE_RES_UD: 3623 sh_qp = qp->rdev->gsi_ctx.gsi_sqp; 3624 if (sh_qp && 3625 qp->qplib_qp.id == sh_qp->qplib_qp.id) { 3626 /* Handle this completion with 3627 * the stored completion 3628 */ 3629 if (cqe->status) { 3630 continue; 3631 } else { 3632 bnxt_re_process_res_shadow_qp_wc 3633 (qp, wc, cqe); 3634 break; 3635 } 3636 } 3637 bnxt_re_process_res_ud_wc(qp, wc, cqe); 3638 break; 3639 default: 3640 ibdev_err(&cq->rdev->ibdev, 3641 "POLL CQ : type 0x%x not handled", 3642 cqe->opcode); 3643 continue; 3644 } 3645 wc++; 3646 budget--; 3647 } 3648 } 3649 exit: 3650 spin_unlock_irqrestore(&cq->cq_lock, flags); 3651 return num_entries - budget; 3652 } 3653 3654 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq, 3655 enum ib_cq_notify_flags ib_cqn_flags) 3656 { 3657 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 3658 int type = 0, rc = 0; 3659 unsigned long flags; 3660 3661 spin_lock_irqsave(&cq->cq_lock, flags); 3662 /* Trigger on the very next completion */ 3663 if (ib_cqn_flags & IB_CQ_NEXT_COMP) 3664 type = DBC_DBC_TYPE_CQ_ARMALL; 3665 /* Trigger on the next solicited completion */ 3666 else if (ib_cqn_flags & IB_CQ_SOLICITED) 3667 type = DBC_DBC_TYPE_CQ_ARMSE; 3668 3669 /* Poll to see if there are missed events */ 3670 if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) && 3671 !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) { 3672 rc = 1; 3673 goto exit; 3674 } 3675 bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); 3676 3677 exit: 3678 spin_unlock_irqrestore(&cq->cq_lock, flags); 3679 return rc; 3680 } 3681 3682 /* Memory Regions */ 3683 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) 3684 { 3685 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3686 struct bnxt_re_dev *rdev = pd->rdev; 3687 struct bnxt_re_mr *mr; 3688 int rc; 3689 3690 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3691 if (!mr) 3692 return ERR_PTR(-ENOMEM); 3693 3694 mr->rdev = rdev; 3695 mr->qplib_mr.pd = &pd->qplib_pd; 3696 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 3697 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 3698 3699 /* Allocate and register 0 as the address */ 3700 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3701 if (rc) 3702 goto fail; 3703 3704 mr->qplib_mr.hwq.level = PBL_LVL_MAX; 3705 mr->qplib_mr.total_size = -1; /* Infinte length */ 3706 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 0, 3707 PAGE_SIZE); 3708 if (rc) 3709 goto fail_mr; 3710 3711 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3712 if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | 3713 IB_ACCESS_REMOTE_ATOMIC)) 3714 mr->ib_mr.rkey = mr->ib_mr.lkey; 3715 atomic_inc(&rdev->mr_count); 3716 3717 return &mr->ib_mr; 3718 3719 fail_mr: 3720 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3721 fail: 3722 kfree(mr); 3723 return ERR_PTR(rc); 3724 } 3725 3726 int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 3727 { 3728 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3729 struct bnxt_re_dev *rdev = mr->rdev; 3730 int rc; 3731 3732 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3733 if (rc) { 3734 ibdev_err(&rdev->ibdev, "Dereg MR failed: %#x\n", rc); 3735 return rc; 3736 } 3737 3738 if (mr->pages) { 3739 rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, 3740 &mr->qplib_frpl); 3741 kfree(mr->pages); 3742 mr->npages = 0; 3743 mr->pages = NULL; 3744 } 3745 ib_umem_release(mr->ib_umem); 3746 3747 kfree(mr); 3748 atomic_dec(&rdev->mr_count); 3749 return rc; 3750 } 3751 3752 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) 3753 { 3754 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3755 3756 if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) 3757 return -ENOMEM; 3758 3759 mr->pages[mr->npages++] = addr; 3760 return 0; 3761 } 3762 3763 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, 3764 unsigned int *sg_offset) 3765 { 3766 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3767 3768 mr->npages = 0; 3769 return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page); 3770 } 3771 3772 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, 3773 u32 max_num_sg) 3774 { 3775 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3776 struct bnxt_re_dev *rdev = pd->rdev; 3777 struct bnxt_re_mr *mr = NULL; 3778 int rc; 3779 3780 if (type != IB_MR_TYPE_MEM_REG) { 3781 ibdev_dbg(&rdev->ibdev, "MR type 0x%x not supported", type); 3782 return ERR_PTR(-EINVAL); 3783 } 3784 if (max_num_sg > MAX_PBL_LVL_1_PGS) 3785 return ERR_PTR(-EINVAL); 3786 3787 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3788 if (!mr) 3789 return ERR_PTR(-ENOMEM); 3790 3791 mr->rdev = rdev; 3792 mr->qplib_mr.pd = &pd->qplib_pd; 3793 mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; 3794 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 3795 3796 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3797 if (rc) 3798 goto bail; 3799 3800 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3801 mr->ib_mr.rkey = mr->ib_mr.lkey; 3802 3803 mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); 3804 if (!mr->pages) { 3805 rc = -ENOMEM; 3806 goto fail; 3807 } 3808 rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, 3809 &mr->qplib_frpl, max_num_sg); 3810 if (rc) { 3811 ibdev_err(&rdev->ibdev, 3812 "Failed to allocate HW FR page list"); 3813 goto fail_mr; 3814 } 3815 3816 atomic_inc(&rdev->mr_count); 3817 return &mr->ib_mr; 3818 3819 fail_mr: 3820 kfree(mr->pages); 3821 fail: 3822 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3823 bail: 3824 kfree(mr); 3825 return ERR_PTR(rc); 3826 } 3827 3828 struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, 3829 struct ib_udata *udata) 3830 { 3831 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3832 struct bnxt_re_dev *rdev = pd->rdev; 3833 struct bnxt_re_mw *mw; 3834 int rc; 3835 3836 mw = kzalloc(sizeof(*mw), GFP_KERNEL); 3837 if (!mw) 3838 return ERR_PTR(-ENOMEM); 3839 mw->rdev = rdev; 3840 mw->qplib_mw.pd = &pd->qplib_pd; 3841 3842 mw->qplib_mw.type = (type == IB_MW_TYPE_1 ? 3843 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 : 3844 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B); 3845 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw); 3846 if (rc) { 3847 ibdev_err(&rdev->ibdev, "Allocate MW failed!"); 3848 goto fail; 3849 } 3850 mw->ib_mw.rkey = mw->qplib_mw.rkey; 3851 3852 atomic_inc(&rdev->mw_count); 3853 return &mw->ib_mw; 3854 3855 fail: 3856 kfree(mw); 3857 return ERR_PTR(rc); 3858 } 3859 3860 int bnxt_re_dealloc_mw(struct ib_mw *ib_mw) 3861 { 3862 struct bnxt_re_mw *mw = container_of(ib_mw, struct bnxt_re_mw, ib_mw); 3863 struct bnxt_re_dev *rdev = mw->rdev; 3864 int rc; 3865 3866 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw); 3867 if (rc) { 3868 ibdev_err(&rdev->ibdev, "Free MW failed: %#x\n", rc); 3869 return rc; 3870 } 3871 3872 kfree(mw); 3873 atomic_dec(&rdev->mw_count); 3874 return rc; 3875 } 3876 3877 /* uverbs */ 3878 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, 3879 u64 virt_addr, int mr_access_flags, 3880 struct ib_udata *udata) 3881 { 3882 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3883 struct bnxt_re_dev *rdev = pd->rdev; 3884 struct bnxt_re_mr *mr; 3885 struct ib_umem *umem; 3886 unsigned long page_size; 3887 int umem_pgs, rc; 3888 3889 if (length > BNXT_RE_MAX_MR_SIZE) { 3890 ibdev_err(&rdev->ibdev, "MR Size: %lld > Max supported:%lld\n", 3891 length, BNXT_RE_MAX_MR_SIZE); 3892 return ERR_PTR(-ENOMEM); 3893 } 3894 3895 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3896 if (!mr) 3897 return ERR_PTR(-ENOMEM); 3898 3899 mr->rdev = rdev; 3900 mr->qplib_mr.pd = &pd->qplib_pd; 3901 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 3902 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; 3903 3904 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3905 if (rc) { 3906 ibdev_err(&rdev->ibdev, "Failed to allocate MR"); 3907 goto free_mr; 3908 } 3909 /* The fixed portion of the rkey is the same as the lkey */ 3910 mr->ib_mr.rkey = mr->qplib_mr.rkey; 3911 3912 umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags); 3913 if (IS_ERR(umem)) { 3914 ibdev_err(&rdev->ibdev, "Failed to get umem"); 3915 rc = -EFAULT; 3916 goto free_mrw; 3917 } 3918 mr->ib_umem = umem; 3919 3920 mr->qplib_mr.va = virt_addr; 3921 page_size = ib_umem_find_best_pgsz( 3922 umem, BNXT_RE_PAGE_SIZE_SUPPORTED, virt_addr); 3923 if (!page_size) { 3924 ibdev_err(&rdev->ibdev, "umem page size unsupported!"); 3925 rc = -EFAULT; 3926 goto free_umem; 3927 } 3928 mr->qplib_mr.total_size = length; 3929 3930 umem_pgs = ib_umem_num_dma_blocks(umem, page_size); 3931 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, umem, 3932 umem_pgs, page_size); 3933 if (rc) { 3934 ibdev_err(&rdev->ibdev, "Failed to register user MR"); 3935 goto free_umem; 3936 } 3937 3938 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3939 mr->ib_mr.rkey = mr->qplib_mr.lkey; 3940 atomic_inc(&rdev->mr_count); 3941 3942 return &mr->ib_mr; 3943 free_umem: 3944 ib_umem_release(umem); 3945 free_mrw: 3946 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3947 free_mr: 3948 kfree(mr); 3949 return ERR_PTR(rc); 3950 } 3951 3952 int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata) 3953 { 3954 struct ib_device *ibdev = ctx->device; 3955 struct bnxt_re_ucontext *uctx = 3956 container_of(ctx, struct bnxt_re_ucontext, ib_uctx); 3957 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 3958 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 3959 struct bnxt_re_uctx_resp resp = {}; 3960 u32 chip_met_rev_num = 0; 3961 int rc; 3962 3963 ibdev_dbg(ibdev, "ABI version requested %u", ibdev->ops.uverbs_abi_ver); 3964 3965 if (ibdev->ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) { 3966 ibdev_dbg(ibdev, " is different from the device %d ", 3967 BNXT_RE_ABI_VERSION); 3968 return -EPERM; 3969 } 3970 3971 uctx->rdev = rdev; 3972 3973 uctx->shpg = (void *)__get_free_page(GFP_KERNEL); 3974 if (!uctx->shpg) { 3975 rc = -ENOMEM; 3976 goto fail; 3977 } 3978 spin_lock_init(&uctx->sh_lock); 3979 3980 resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX; 3981 chip_met_rev_num = rdev->chip_ctx->chip_num; 3982 chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_rev & 0xFF) << 3983 BNXT_RE_CHIP_ID0_CHIP_REV_SFT; 3984 chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_metal & 0xFF) << 3985 BNXT_RE_CHIP_ID0_CHIP_MET_SFT; 3986 resp.chip_id0 = chip_met_rev_num; 3987 /*Temp, Use xa_alloc instead */ 3988 resp.dev_id = rdev->en_dev->pdev->devfn; 3989 resp.max_qp = rdev->qplib_ctx.qpc_count; 3990 resp.pg_size = PAGE_SIZE; 3991 resp.cqe_sz = sizeof(struct cq_base); 3992 resp.max_cqd = dev_attr->max_cq_wqes; 3993 3994 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_HAVE_MODE; 3995 resp.mode = rdev->chip_ctx->modes.wqe_mode; 3996 3997 rc = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp))); 3998 if (rc) { 3999 ibdev_err(ibdev, "Failed to copy user context"); 4000 rc = -EFAULT; 4001 goto cfail; 4002 } 4003 4004 return 0; 4005 cfail: 4006 free_page((unsigned long)uctx->shpg); 4007 uctx->shpg = NULL; 4008 fail: 4009 return rc; 4010 } 4011 4012 void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx) 4013 { 4014 struct bnxt_re_ucontext *uctx = container_of(ib_uctx, 4015 struct bnxt_re_ucontext, 4016 ib_uctx); 4017 4018 struct bnxt_re_dev *rdev = uctx->rdev; 4019 4020 if (uctx->shpg) 4021 free_page((unsigned long)uctx->shpg); 4022 4023 if (uctx->dpi.dbr) { 4024 /* Free DPI only if this is the first PD allocated by the 4025 * application and mark the context dpi as NULL 4026 */ 4027 bnxt_qplib_dealloc_dpi(&rdev->qplib_res, 4028 &rdev->qplib_res.dpi_tbl, &uctx->dpi); 4029 uctx->dpi.dbr = NULL; 4030 } 4031 } 4032 4033 /* Helper function to mmap the virtual memory from user app */ 4034 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) 4035 { 4036 struct bnxt_re_ucontext *uctx = container_of(ib_uctx, 4037 struct bnxt_re_ucontext, 4038 ib_uctx); 4039 struct bnxt_re_dev *rdev = uctx->rdev; 4040 u64 pfn; 4041 4042 if (vma->vm_end - vma->vm_start != PAGE_SIZE) 4043 return -EINVAL; 4044 4045 if (vma->vm_pgoff) { 4046 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 4047 if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 4048 PAGE_SIZE, vma->vm_page_prot)) { 4049 ibdev_err(&rdev->ibdev, "Failed to map DPI"); 4050 return -EAGAIN; 4051 } 4052 } else { 4053 pfn = virt_to_phys(uctx->shpg) >> PAGE_SHIFT; 4054 if (remap_pfn_range(vma, vma->vm_start, 4055 pfn, PAGE_SIZE, vma->vm_page_prot)) { 4056 ibdev_err(&rdev->ibdev, "Failed to map shared page"); 4057 return -EAGAIN; 4058 } 4059 } 4060 4061 return 0; 4062 } 4063