1 /* 2 * Broadcom NetXtreme-E RoCE driver. 3 * 4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term 5 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Description: IB Verbs interpreter 37 */ 38 39 #include <linux/interrupt.h> 40 #include <linux/types.h> 41 #include <linux/pci.h> 42 #include <linux/netdevice.h> 43 #include <linux/if_ether.h> 44 #include <net/addrconf.h> 45 46 #include <rdma/ib_verbs.h> 47 #include <rdma/ib_user_verbs.h> 48 #include <rdma/ib_umem.h> 49 #include <rdma/ib_addr.h> 50 #include <rdma/ib_mad.h> 51 #include <rdma/ib_cache.h> 52 #include <rdma/uverbs_ioctl.h> 53 54 #include "bnxt_ulp.h" 55 56 #include "roce_hsi.h" 57 #include "qplib_res.h" 58 #include "qplib_sp.h" 59 #include "qplib_fp.h" 60 #include "qplib_rcfw.h" 61 62 #include "bnxt_re.h" 63 #include "ib_verbs.h" 64 65 #include <rdma/uverbs_types.h> 66 #include <rdma/uverbs_std_types.h> 67 68 #include <rdma/ib_user_ioctl_cmds.h> 69 70 #define UVERBS_MODULE_NAME bnxt_re 71 #include <rdma/uverbs_named_ioctl.h> 72 73 #include <rdma/bnxt_re-abi.h> 74 75 static int __from_ib_access_flags(int iflags) 76 { 77 int qflags = 0; 78 79 if (iflags & IB_ACCESS_LOCAL_WRITE) 80 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 81 if (iflags & IB_ACCESS_REMOTE_READ) 82 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ; 83 if (iflags & IB_ACCESS_REMOTE_WRITE) 84 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE; 85 if (iflags & IB_ACCESS_REMOTE_ATOMIC) 86 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC; 87 if (iflags & IB_ACCESS_MW_BIND) 88 qflags |= BNXT_QPLIB_ACCESS_MW_BIND; 89 if (iflags & IB_ZERO_BASED) 90 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED; 91 if (iflags & IB_ACCESS_ON_DEMAND) 92 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND; 93 return qflags; 94 }; 95 96 static enum ib_access_flags __to_ib_access_flags(int qflags) 97 { 98 enum ib_access_flags iflags = 0; 99 100 if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE) 101 iflags |= IB_ACCESS_LOCAL_WRITE; 102 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE) 103 iflags |= IB_ACCESS_REMOTE_WRITE; 104 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ) 105 iflags |= IB_ACCESS_REMOTE_READ; 106 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC) 107 iflags |= IB_ACCESS_REMOTE_ATOMIC; 108 if (qflags & BNXT_QPLIB_ACCESS_MW_BIND) 109 iflags |= IB_ACCESS_MW_BIND; 110 if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED) 111 iflags |= IB_ZERO_BASED; 112 if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND) 113 iflags |= IB_ACCESS_ON_DEMAND; 114 return iflags; 115 }; 116 117 static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list, 118 struct bnxt_qplib_sge *sg_list, int num) 119 { 120 int i, total = 0; 121 122 for (i = 0; i < num; i++) { 123 sg_list[i].addr = ib_sg_list[i].addr; 124 sg_list[i].lkey = ib_sg_list[i].lkey; 125 sg_list[i].size = ib_sg_list[i].length; 126 total += sg_list[i].size; 127 } 128 return total; 129 } 130 131 /* Device */ 132 int bnxt_re_query_device(struct ib_device *ibdev, 133 struct ib_device_attr *ib_attr, 134 struct ib_udata *udata) 135 { 136 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 137 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 138 139 memset(ib_attr, 0, sizeof(*ib_attr)); 140 memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, 141 min(sizeof(dev_attr->fw_ver), 142 sizeof(ib_attr->fw_ver))); 143 addrconf_addr_eui48((u8 *)&ib_attr->sys_image_guid, 144 rdev->netdev->dev_addr); 145 ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE; 146 ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_SUPPORTED; 147 148 ib_attr->vendor_id = rdev->en_dev->pdev->vendor; 149 ib_attr->vendor_part_id = rdev->en_dev->pdev->device; 150 ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; 151 ib_attr->max_qp = dev_attr->max_qp; 152 ib_attr->max_qp_wr = dev_attr->max_qp_wqes; 153 ib_attr->device_cap_flags = 154 IB_DEVICE_CURR_QP_STATE_MOD 155 | IB_DEVICE_RC_RNR_NAK_GEN 156 | IB_DEVICE_SHUTDOWN_PORT 157 | IB_DEVICE_SYS_IMAGE_GUID 158 | IB_DEVICE_RESIZE_MAX_WR 159 | IB_DEVICE_PORT_ACTIVE_EVENT 160 | IB_DEVICE_N_NOTIFY_CQ 161 | IB_DEVICE_MEM_WINDOW 162 | IB_DEVICE_MEM_WINDOW_TYPE_2B 163 | IB_DEVICE_MEM_MGT_EXTENSIONS; 164 ib_attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 165 ib_attr->max_send_sge = dev_attr->max_qp_sges; 166 ib_attr->max_recv_sge = dev_attr->max_qp_sges; 167 ib_attr->max_sge_rd = dev_attr->max_qp_sges; 168 ib_attr->max_cq = dev_attr->max_cq; 169 ib_attr->max_cqe = dev_attr->max_cq_wqes; 170 ib_attr->max_mr = dev_attr->max_mr; 171 ib_attr->max_pd = dev_attr->max_pd; 172 ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; 173 ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom; 174 ib_attr->atomic_cap = IB_ATOMIC_NONE; 175 ib_attr->masked_atomic_cap = IB_ATOMIC_NONE; 176 if (dev_attr->is_atomic) { 177 ib_attr->atomic_cap = IB_ATOMIC_GLOB; 178 ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB; 179 } 180 181 ib_attr->max_ee_rd_atom = 0; 182 ib_attr->max_res_rd_atom = 0; 183 ib_attr->max_ee_init_rd_atom = 0; 184 ib_attr->max_ee = 0; 185 ib_attr->max_rdd = 0; 186 ib_attr->max_mw = dev_attr->max_mw; 187 ib_attr->max_raw_ipv6_qp = 0; 188 ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; 189 ib_attr->max_mcast_grp = 0; 190 ib_attr->max_mcast_qp_attach = 0; 191 ib_attr->max_total_mcast_qp_attach = 0; 192 ib_attr->max_ah = dev_attr->max_ah; 193 194 ib_attr->max_srq = dev_attr->max_srq; 195 ib_attr->max_srq_wr = dev_attr->max_srq_wqes; 196 ib_attr->max_srq_sge = dev_attr->max_srq_sges; 197 198 ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; 199 200 ib_attr->max_pkeys = 1; 201 ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY; 202 return 0; 203 } 204 205 /* Port */ 206 int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num, 207 struct ib_port_attr *port_attr) 208 { 209 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 210 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 211 int rc; 212 213 memset(port_attr, 0, sizeof(*port_attr)); 214 215 if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) { 216 port_attr->state = IB_PORT_ACTIVE; 217 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 218 } else { 219 port_attr->state = IB_PORT_DOWN; 220 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 221 } 222 port_attr->max_mtu = IB_MTU_4096; 223 port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu); 224 port_attr->gid_tbl_len = dev_attr->max_sgid; 225 port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 226 IB_PORT_DEVICE_MGMT_SUP | 227 IB_PORT_VENDOR_CLASS_SUP; 228 port_attr->ip_gids = true; 229 230 port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW; 231 port_attr->bad_pkey_cntr = 0; 232 port_attr->qkey_viol_cntr = 0; 233 port_attr->pkey_tbl_len = dev_attr->max_pkey; 234 port_attr->lid = 0; 235 port_attr->sm_lid = 0; 236 port_attr->lmc = 0; 237 port_attr->max_vl_num = 4; 238 port_attr->sm_sl = 0; 239 port_attr->subnet_timeout = 0; 240 port_attr->init_type_reply = 0; 241 rc = ib_get_eth_speed(&rdev->ibdev, port_num, &port_attr->active_speed, 242 &port_attr->active_width); 243 244 return rc; 245 } 246 247 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num, 248 struct ib_port_immutable *immutable) 249 { 250 struct ib_port_attr port_attr; 251 252 if (bnxt_re_query_port(ibdev, port_num, &port_attr)) 253 return -EINVAL; 254 255 immutable->pkey_tbl_len = port_attr.pkey_tbl_len; 256 immutable->gid_tbl_len = port_attr.gid_tbl_len; 257 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; 258 immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP; 259 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 260 return 0; 261 } 262 263 void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str) 264 { 265 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 266 267 snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d", 268 rdev->dev_attr.fw_ver[0], rdev->dev_attr.fw_ver[1], 269 rdev->dev_attr.fw_ver[2], rdev->dev_attr.fw_ver[3]); 270 } 271 272 int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num, 273 u16 index, u16 *pkey) 274 { 275 if (index > 0) 276 return -EINVAL; 277 278 *pkey = IB_DEFAULT_PKEY_FULL; 279 280 return 0; 281 } 282 283 int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num, 284 int index, union ib_gid *gid) 285 { 286 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 287 int rc; 288 289 /* Ignore port_num */ 290 memset(gid, 0, sizeof(*gid)); 291 rc = bnxt_qplib_get_sgid(&rdev->qplib_res, 292 &rdev->qplib_res.sgid_tbl, index, 293 (struct bnxt_qplib_gid *)gid); 294 return rc; 295 } 296 297 int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context) 298 { 299 int rc = 0; 300 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 301 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); 302 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 303 struct bnxt_qplib_gid *gid_to_del; 304 u16 vlan_id = 0xFFFF; 305 306 /* Delete the entry from the hardware */ 307 ctx = *context; 308 if (!ctx) 309 return -EINVAL; 310 311 if (sgid_tbl && sgid_tbl->active) { 312 if (ctx->idx >= sgid_tbl->max) 313 return -EINVAL; 314 gid_to_del = &sgid_tbl->tbl[ctx->idx].gid; 315 vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id; 316 /* DEL_GID is called in WQ context(netdevice_event_work_handler) 317 * or via the ib_unregister_device path. In the former case QP1 318 * may not be destroyed yet, in which case just return as FW 319 * needs that entry to be present and will fail it's deletion. 320 * We could get invoked again after QP1 is destroyed OR get an 321 * ADD_GID call with a different GID value for the same index 322 * where we issue MODIFY_GID cmd to update the GID entry -- TBD 323 */ 324 if (ctx->idx == 0 && 325 rdma_link_local_addr((struct in6_addr *)gid_to_del) && 326 ctx->refcnt == 1 && rdev->gsi_ctx.gsi_sqp) { 327 ibdev_dbg(&rdev->ibdev, 328 "Trying to delete GID0 while QP1 is alive\n"); 329 return -EFAULT; 330 } 331 ctx->refcnt--; 332 if (!ctx->refcnt) { 333 rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, 334 vlan_id, true); 335 if (rc) { 336 ibdev_err(&rdev->ibdev, 337 "Failed to remove GID: %#x", rc); 338 } else { 339 ctx_tbl = sgid_tbl->ctx; 340 ctx_tbl[ctx->idx] = NULL; 341 kfree(ctx); 342 } 343 } 344 } else { 345 return -EINVAL; 346 } 347 return rc; 348 } 349 350 int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context) 351 { 352 int rc; 353 u32 tbl_idx = 0; 354 u16 vlan_id = 0xFFFF; 355 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 356 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); 357 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 358 359 rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL); 360 if (rc) 361 return rc; 362 363 rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid, 364 rdev->qplib_res.netdev->dev_addr, 365 vlan_id, true, &tbl_idx); 366 if (rc == -EALREADY) { 367 ctx_tbl = sgid_tbl->ctx; 368 ctx_tbl[tbl_idx]->refcnt++; 369 *context = ctx_tbl[tbl_idx]; 370 return 0; 371 } 372 373 if (rc < 0) { 374 ibdev_err(&rdev->ibdev, "Failed to add GID: %#x", rc); 375 return rc; 376 } 377 378 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 379 if (!ctx) 380 return -ENOMEM; 381 ctx_tbl = sgid_tbl->ctx; 382 ctx->idx = tbl_idx; 383 ctx->refcnt = 1; 384 ctx_tbl[tbl_idx] = ctx; 385 *context = ctx; 386 387 return rc; 388 } 389 390 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, 391 u32 port_num) 392 { 393 return IB_LINK_LAYER_ETHERNET; 394 } 395 396 #define BNXT_RE_FENCE_PBL_SIZE DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE) 397 398 static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd) 399 { 400 struct bnxt_re_fence_data *fence = &pd->fence; 401 struct ib_mr *ib_mr = &fence->mr->ib_mr; 402 struct bnxt_qplib_swqe *wqe = &fence->bind_wqe; 403 struct bnxt_re_dev *rdev = pd->rdev; 404 405 if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx)) 406 return; 407 408 memset(wqe, 0, sizeof(*wqe)); 409 wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW; 410 wqe->wr_id = BNXT_QPLIB_FENCE_WRID; 411 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 412 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 413 wqe->bind.zero_based = false; 414 wqe->bind.parent_l_key = ib_mr->lkey; 415 wqe->bind.va = (u64)(unsigned long)fence->va; 416 wqe->bind.length = fence->size; 417 wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ); 418 wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1; 419 420 /* Save the initial rkey in fence structure for now; 421 * wqe->bind.r_key will be set at (re)bind time. 422 */ 423 fence->bind_rkey = ib_inc_rkey(fence->mw->rkey); 424 } 425 426 static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp) 427 { 428 struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp, 429 qplib_qp); 430 struct ib_pd *ib_pd = qp->ib_qp.pd; 431 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 432 struct bnxt_re_fence_data *fence = &pd->fence; 433 struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe; 434 struct bnxt_qplib_swqe wqe; 435 int rc; 436 437 memcpy(&wqe, fence_wqe, sizeof(wqe)); 438 wqe.bind.r_key = fence->bind_rkey; 439 fence->bind_rkey = ib_inc_rkey(fence->bind_rkey); 440 441 ibdev_dbg(&qp->rdev->ibdev, 442 "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n", 443 wqe.bind.r_key, qp->qplib_qp.id, pd); 444 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 445 if (rc) { 446 ibdev_err(&qp->rdev->ibdev, "Failed to bind fence-WQE\n"); 447 return rc; 448 } 449 bnxt_qplib_post_send_db(&qp->qplib_qp); 450 451 return rc; 452 } 453 454 static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd) 455 { 456 struct bnxt_re_fence_data *fence = &pd->fence; 457 struct bnxt_re_dev *rdev = pd->rdev; 458 struct device *dev = &rdev->en_dev->pdev->dev; 459 struct bnxt_re_mr *mr = fence->mr; 460 461 if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx)) 462 return; 463 464 if (fence->mw) { 465 bnxt_re_dealloc_mw(fence->mw); 466 fence->mw = NULL; 467 } 468 if (mr) { 469 if (mr->ib_mr.rkey) 470 bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr, 471 true); 472 if (mr->ib_mr.lkey) 473 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 474 kfree(mr); 475 fence->mr = NULL; 476 } 477 if (fence->dma_addr) { 478 dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES, 479 DMA_BIDIRECTIONAL); 480 fence->dma_addr = 0; 481 } 482 } 483 484 static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd) 485 { 486 int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND; 487 struct bnxt_re_fence_data *fence = &pd->fence; 488 struct bnxt_re_dev *rdev = pd->rdev; 489 struct device *dev = &rdev->en_dev->pdev->dev; 490 struct bnxt_re_mr *mr = NULL; 491 dma_addr_t dma_addr = 0; 492 struct ib_mw *mw; 493 int rc; 494 495 if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx)) 496 return 0; 497 498 dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES, 499 DMA_BIDIRECTIONAL); 500 rc = dma_mapping_error(dev, dma_addr); 501 if (rc) { 502 ibdev_err(&rdev->ibdev, "Failed to dma-map fence-MR-mem\n"); 503 rc = -EIO; 504 fence->dma_addr = 0; 505 goto fail; 506 } 507 fence->dma_addr = dma_addr; 508 509 /* Allocate a MR */ 510 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 511 if (!mr) { 512 rc = -ENOMEM; 513 goto fail; 514 } 515 fence->mr = mr; 516 mr->rdev = rdev; 517 mr->qplib_mr.pd = &pd->qplib_pd; 518 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 519 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 520 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 521 if (rc) { 522 ibdev_err(&rdev->ibdev, "Failed to alloc fence-HW-MR\n"); 523 goto fail; 524 } 525 526 /* Register MR */ 527 mr->ib_mr.lkey = mr->qplib_mr.lkey; 528 mr->qplib_mr.va = (u64)(unsigned long)fence->va; 529 mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES; 530 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 531 BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE); 532 if (rc) { 533 ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n"); 534 goto fail; 535 } 536 mr->ib_mr.rkey = mr->qplib_mr.rkey; 537 538 /* Create a fence MW only for kernel consumers */ 539 mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL); 540 if (IS_ERR(mw)) { 541 ibdev_err(&rdev->ibdev, 542 "Failed to create fence-MW for PD: %p\n", pd); 543 rc = PTR_ERR(mw); 544 goto fail; 545 } 546 fence->mw = mw; 547 548 bnxt_re_create_fence_wqe(pd); 549 return 0; 550 551 fail: 552 bnxt_re_destroy_fence_mr(pd); 553 return rc; 554 } 555 556 static struct bnxt_re_user_mmap_entry* 557 bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset, 558 enum bnxt_re_mmap_flag mmap_flag, u64 *offset) 559 { 560 struct bnxt_re_user_mmap_entry *entry; 561 int ret; 562 563 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 564 if (!entry) 565 return NULL; 566 567 entry->mem_offset = mem_offset; 568 entry->mmap_flag = mmap_flag; 569 entry->uctx = uctx; 570 571 switch (mmap_flag) { 572 case BNXT_RE_MMAP_SH_PAGE: 573 ret = rdma_user_mmap_entry_insert_exact(&uctx->ib_uctx, 574 &entry->rdma_entry, PAGE_SIZE, 0); 575 break; 576 case BNXT_RE_MMAP_UC_DB: 577 case BNXT_RE_MMAP_WC_DB: 578 case BNXT_RE_MMAP_DBR_BAR: 579 case BNXT_RE_MMAP_DBR_PAGE: 580 ret = rdma_user_mmap_entry_insert(&uctx->ib_uctx, 581 &entry->rdma_entry, PAGE_SIZE); 582 break; 583 default: 584 ret = -EINVAL; 585 break; 586 } 587 588 if (ret) { 589 kfree(entry); 590 return NULL; 591 } 592 if (offset) 593 *offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 594 595 return entry; 596 } 597 598 /* Protection Domains */ 599 int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) 600 { 601 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 602 struct bnxt_re_dev *rdev = pd->rdev; 603 604 if (udata) { 605 rdma_user_mmap_entry_remove(pd->pd_db_mmap); 606 pd->pd_db_mmap = NULL; 607 } 608 609 bnxt_re_destroy_fence_mr(pd); 610 611 if (pd->qplib_pd.id) { 612 if (!bnxt_qplib_dealloc_pd(&rdev->qplib_res, 613 &rdev->qplib_res.pd_tbl, 614 &pd->qplib_pd)) 615 atomic_dec(&rdev->stats.res.pd_count); 616 } 617 return 0; 618 } 619 620 int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 621 { 622 struct ib_device *ibdev = ibpd->device; 623 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 624 struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context( 625 udata, struct bnxt_re_ucontext, ib_uctx); 626 struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd); 627 struct bnxt_re_user_mmap_entry *entry = NULL; 628 u32 active_pds; 629 int rc = 0; 630 631 pd->rdev = rdev; 632 if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) { 633 ibdev_err(&rdev->ibdev, "Failed to allocate HW PD"); 634 rc = -ENOMEM; 635 goto fail; 636 } 637 638 if (udata) { 639 struct bnxt_re_pd_resp resp = {}; 640 641 if (!ucntx->dpi.dbr) { 642 /* Allocate DPI in alloc_pd to avoid failing of 643 * ibv_devinfo and family of application when DPIs 644 * are depleted. 645 */ 646 if (bnxt_qplib_alloc_dpi(&rdev->qplib_res, 647 &ucntx->dpi, ucntx, BNXT_QPLIB_DPI_TYPE_UC)) { 648 rc = -ENOMEM; 649 goto dbfail; 650 } 651 } 652 653 resp.pdid = pd->qplib_pd.id; 654 /* Still allow mapping this DBR to the new user PD. */ 655 resp.dpi = ucntx->dpi.dpi; 656 657 entry = bnxt_re_mmap_entry_insert(ucntx, (u64)ucntx->dpi.umdbr, 658 BNXT_RE_MMAP_UC_DB, &resp.dbr); 659 660 if (!entry) { 661 rc = -ENOMEM; 662 goto dbfail; 663 } 664 665 pd->pd_db_mmap = &entry->rdma_entry; 666 667 rc = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); 668 if (rc) { 669 rdma_user_mmap_entry_remove(pd->pd_db_mmap); 670 rc = -EFAULT; 671 goto dbfail; 672 } 673 } 674 675 if (!udata) 676 if (bnxt_re_create_fence_mr(pd)) 677 ibdev_warn(&rdev->ibdev, 678 "Failed to create Fence-MR\n"); 679 active_pds = atomic_inc_return(&rdev->stats.res.pd_count); 680 if (active_pds > rdev->stats.res.pd_watermark) 681 rdev->stats.res.pd_watermark = active_pds; 682 683 return 0; 684 dbfail: 685 bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, 686 &pd->qplib_pd); 687 fail: 688 return rc; 689 } 690 691 /* Address Handles */ 692 int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags) 693 { 694 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 695 struct bnxt_re_dev *rdev = ah->rdev; 696 bool block = true; 697 int rc; 698 699 block = !(flags & RDMA_DESTROY_AH_SLEEPABLE); 700 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block); 701 if (BNXT_RE_CHECK_RC(rc)) { 702 if (rc == -ETIMEDOUT) 703 rc = 0; 704 else 705 goto fail; 706 } 707 atomic_dec(&rdev->stats.res.ah_count); 708 fail: 709 return rc; 710 } 711 712 static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype) 713 { 714 u8 nw_type; 715 716 switch (ntype) { 717 case RDMA_NETWORK_IPV4: 718 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4; 719 break; 720 case RDMA_NETWORK_IPV6: 721 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6; 722 break; 723 default: 724 nw_type = CMDQ_CREATE_AH_TYPE_V1; 725 break; 726 } 727 return nw_type; 728 } 729 730 int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr, 731 struct ib_udata *udata) 732 { 733 struct ib_pd *ib_pd = ib_ah->pd; 734 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 735 struct rdma_ah_attr *ah_attr = init_attr->ah_attr; 736 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); 737 struct bnxt_re_dev *rdev = pd->rdev; 738 const struct ib_gid_attr *sgid_attr; 739 struct bnxt_re_gid_ctx *ctx; 740 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 741 u32 active_ahs; 742 u8 nw_type; 743 int rc; 744 745 if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) { 746 ibdev_err(&rdev->ibdev, "Failed to alloc AH: GRH not set"); 747 return -EINVAL; 748 } 749 750 ah->rdev = rdev; 751 ah->qplib_ah.pd = &pd->qplib_pd; 752 753 /* Supply the configuration for the HW */ 754 memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw, 755 sizeof(union ib_gid)); 756 sgid_attr = grh->sgid_attr; 757 /* Get the HW context of the GID. The reference 758 * of GID table entry is already taken by the caller. 759 */ 760 ctx = rdma_read_gid_hw_context(sgid_attr); 761 ah->qplib_ah.sgid_index = ctx->idx; 762 ah->qplib_ah.host_sgid_index = grh->sgid_index; 763 ah->qplib_ah.traffic_class = grh->traffic_class; 764 ah->qplib_ah.flow_label = grh->flow_label; 765 ah->qplib_ah.hop_limit = grh->hop_limit; 766 ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr); 767 768 /* Get network header type for this GID */ 769 nw_type = rdma_gid_attr_network_type(sgid_attr); 770 ah->qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(nw_type); 771 772 memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN); 773 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, 774 !(init_attr->flags & 775 RDMA_CREATE_AH_SLEEPABLE)); 776 if (rc) { 777 ibdev_err(&rdev->ibdev, "Failed to allocate HW AH"); 778 return rc; 779 } 780 781 /* Write AVID to shared page. */ 782 if (udata) { 783 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context( 784 udata, struct bnxt_re_ucontext, ib_uctx); 785 unsigned long flag; 786 u32 *wrptr; 787 788 spin_lock_irqsave(&uctx->sh_lock, flag); 789 wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT); 790 *wrptr = ah->qplib_ah.id; 791 wmb(); /* make sure cache is updated. */ 792 spin_unlock_irqrestore(&uctx->sh_lock, flag); 793 } 794 active_ahs = atomic_inc_return(&rdev->stats.res.ah_count); 795 if (active_ahs > rdev->stats.res.ah_watermark) 796 rdev->stats.res.ah_watermark = active_ahs; 797 798 return 0; 799 } 800 801 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) 802 { 803 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 804 805 ah_attr->type = ib_ah->type; 806 rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl); 807 memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN); 808 rdma_ah_set_grh(ah_attr, NULL, 0, 809 ah->qplib_ah.host_sgid_index, 810 0, ah->qplib_ah.traffic_class); 811 rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data); 812 rdma_ah_set_port_num(ah_attr, 1); 813 rdma_ah_set_static_rate(ah_attr, 0); 814 return 0; 815 } 816 817 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp) 818 __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock) 819 { 820 unsigned long flags; 821 822 spin_lock_irqsave(&qp->scq->cq_lock, flags); 823 if (qp->rcq != qp->scq) 824 spin_lock(&qp->rcq->cq_lock); 825 else 826 __acquire(&qp->rcq->cq_lock); 827 828 return flags; 829 } 830 831 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, 832 unsigned long flags) 833 __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock) 834 { 835 if (qp->rcq != qp->scq) 836 spin_unlock(&qp->rcq->cq_lock); 837 else 838 __release(&qp->rcq->cq_lock); 839 spin_unlock_irqrestore(&qp->scq->cq_lock, flags); 840 } 841 842 static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp) 843 { 844 struct bnxt_re_qp *gsi_sqp; 845 struct bnxt_re_ah *gsi_sah; 846 struct bnxt_re_dev *rdev; 847 int rc; 848 849 rdev = qp->rdev; 850 gsi_sqp = rdev->gsi_ctx.gsi_sqp; 851 gsi_sah = rdev->gsi_ctx.gsi_sah; 852 853 ibdev_dbg(&rdev->ibdev, "Destroy the shadow AH\n"); 854 bnxt_qplib_destroy_ah(&rdev->qplib_res, 855 &gsi_sah->qplib_ah, 856 true); 857 atomic_dec(&rdev->stats.res.ah_count); 858 bnxt_qplib_clean_qp(&qp->qplib_qp); 859 860 ibdev_dbg(&rdev->ibdev, "Destroy the shadow QP\n"); 861 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp); 862 if (rc) { 863 ibdev_err(&rdev->ibdev, "Destroy Shadow QP failed"); 864 goto fail; 865 } 866 bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp); 867 868 /* remove from active qp list */ 869 mutex_lock(&rdev->qp_lock); 870 list_del(&gsi_sqp->list); 871 mutex_unlock(&rdev->qp_lock); 872 atomic_dec(&rdev->stats.res.qp_count); 873 874 kfree(rdev->gsi_ctx.sqp_tbl); 875 kfree(gsi_sah); 876 kfree(gsi_sqp); 877 rdev->gsi_ctx.gsi_sqp = NULL; 878 rdev->gsi_ctx.gsi_sah = NULL; 879 rdev->gsi_ctx.sqp_tbl = NULL; 880 881 return 0; 882 fail: 883 return rc; 884 } 885 886 /* Queue Pairs */ 887 int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) 888 { 889 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 890 struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; 891 struct bnxt_re_dev *rdev = qp->rdev; 892 struct bnxt_qplib_nq *scq_nq = NULL; 893 struct bnxt_qplib_nq *rcq_nq = NULL; 894 unsigned int flags; 895 int rc; 896 897 bnxt_qplib_flush_cqn_wq(&qp->qplib_qp); 898 899 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 900 if (rc) { 901 ibdev_err(&rdev->ibdev, "Failed to destroy HW QP"); 902 return rc; 903 } 904 905 if (rdma_is_kernel_res(&qp->ib_qp.res)) { 906 flags = bnxt_re_lock_cqs(qp); 907 bnxt_qplib_clean_qp(&qp->qplib_qp); 908 bnxt_re_unlock_cqs(qp, flags); 909 } 910 911 bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); 912 913 if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) { 914 rc = bnxt_re_destroy_gsi_sqp(qp); 915 if (rc) 916 return rc; 917 } 918 919 mutex_lock(&rdev->qp_lock); 920 list_del(&qp->list); 921 mutex_unlock(&rdev->qp_lock); 922 atomic_dec(&rdev->stats.res.qp_count); 923 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC) 924 atomic_dec(&rdev->stats.res.rc_qp_count); 925 else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD) 926 atomic_dec(&rdev->stats.res.ud_qp_count); 927 928 ib_umem_release(qp->rumem); 929 ib_umem_release(qp->sumem); 930 931 /* Flush all the entries of notification queue associated with 932 * given qp. 933 */ 934 scq_nq = qplib_qp->scq->nq; 935 rcq_nq = qplib_qp->rcq->nq; 936 bnxt_re_synchronize_nq(scq_nq); 937 if (scq_nq != rcq_nq) 938 bnxt_re_synchronize_nq(rcq_nq); 939 940 return 0; 941 } 942 943 static u8 __from_ib_qp_type(enum ib_qp_type type) 944 { 945 switch (type) { 946 case IB_QPT_GSI: 947 return CMDQ_CREATE_QP1_TYPE_GSI; 948 case IB_QPT_RC: 949 return CMDQ_CREATE_QP_TYPE_RC; 950 case IB_QPT_UD: 951 return CMDQ_CREATE_QP_TYPE_UD; 952 default: 953 return IB_QPT_MAX; 954 } 955 } 956 957 static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp, 958 int rsge, int max) 959 { 960 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 961 rsge = max; 962 return bnxt_re_get_rwqe_size(rsge); 963 } 964 965 static u16 bnxt_re_get_wqe_size(int ilsize, int nsge) 966 { 967 u16 wqe_size, calc_ils; 968 969 wqe_size = bnxt_re_get_swqe_size(nsge); 970 if (ilsize) { 971 calc_ils = sizeof(struct sq_send_hdr) + ilsize; 972 wqe_size = max_t(u16, calc_ils, wqe_size); 973 wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr)); 974 } 975 return wqe_size; 976 } 977 978 static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp, 979 struct ib_qp_init_attr *init_attr) 980 { 981 struct bnxt_qplib_dev_attr *dev_attr; 982 struct bnxt_qplib_qp *qplqp; 983 struct bnxt_re_dev *rdev; 984 struct bnxt_qplib_q *sq; 985 int align, ilsize; 986 987 rdev = qp->rdev; 988 qplqp = &qp->qplib_qp; 989 sq = &qplqp->sq; 990 dev_attr = &rdev->dev_attr; 991 992 align = sizeof(struct sq_send_hdr); 993 ilsize = ALIGN(init_attr->cap.max_inline_data, align); 994 995 sq->wqe_size = bnxt_re_get_wqe_size(ilsize, sq->max_sge); 996 if (sq->wqe_size > bnxt_re_get_swqe_size(dev_attr->max_qp_sges)) 997 return -EINVAL; 998 /* For gen p4 and gen p5 backward compatibility mode 999 * wqe size is fixed to 128 bytes 1000 */ 1001 if (sq->wqe_size < bnxt_re_get_swqe_size(dev_attr->max_qp_sges) && 1002 qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 1003 sq->wqe_size = bnxt_re_get_swqe_size(dev_attr->max_qp_sges); 1004 1005 if (init_attr->cap.max_inline_data) { 1006 qplqp->max_inline_data = sq->wqe_size - 1007 sizeof(struct sq_send_hdr); 1008 init_attr->cap.max_inline_data = qplqp->max_inline_data; 1009 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 1010 sq->max_sge = qplqp->max_inline_data / 1011 sizeof(struct sq_sge); 1012 } 1013 1014 return 0; 1015 } 1016 1017 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, 1018 struct bnxt_re_qp *qp, struct ib_udata *udata) 1019 { 1020 struct bnxt_qplib_qp *qplib_qp; 1021 struct bnxt_re_ucontext *cntx; 1022 struct bnxt_re_qp_req ureq; 1023 int bytes = 0, psn_sz; 1024 struct ib_umem *umem; 1025 int psn_nume; 1026 1027 qplib_qp = &qp->qplib_qp; 1028 cntx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, 1029 ib_uctx); 1030 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 1031 return -EFAULT; 1032 1033 bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); 1034 /* Consider mapping PSN search memory only for RC QPs. */ 1035 if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { 1036 psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ? 1037 sizeof(struct sq_psn_search_ext) : 1038 sizeof(struct sq_psn_search); 1039 psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? 1040 qplib_qp->sq.max_wqe : 1041 ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / 1042 sizeof(struct bnxt_qplib_sge)); 1043 bytes += (psn_nume * psn_sz); 1044 } 1045 1046 bytes = PAGE_ALIGN(bytes); 1047 umem = ib_umem_get(&rdev->ibdev, ureq.qpsva, bytes, 1048 IB_ACCESS_LOCAL_WRITE); 1049 if (IS_ERR(umem)) 1050 return PTR_ERR(umem); 1051 1052 qp->sumem = umem; 1053 qplib_qp->sq.sg_info.umem = umem; 1054 qplib_qp->sq.sg_info.pgsize = PAGE_SIZE; 1055 qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT; 1056 qplib_qp->qp_handle = ureq.qp_handle; 1057 1058 if (!qp->qplib_qp.srq) { 1059 bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); 1060 bytes = PAGE_ALIGN(bytes); 1061 umem = ib_umem_get(&rdev->ibdev, ureq.qprva, bytes, 1062 IB_ACCESS_LOCAL_WRITE); 1063 if (IS_ERR(umem)) 1064 goto rqfail; 1065 qp->rumem = umem; 1066 qplib_qp->rq.sg_info.umem = umem; 1067 qplib_qp->rq.sg_info.pgsize = PAGE_SIZE; 1068 qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT; 1069 } 1070 1071 qplib_qp->dpi = &cntx->dpi; 1072 return 0; 1073 rqfail: 1074 ib_umem_release(qp->sumem); 1075 qp->sumem = NULL; 1076 memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info)); 1077 1078 return PTR_ERR(umem); 1079 } 1080 1081 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah 1082 (struct bnxt_re_pd *pd, 1083 struct bnxt_qplib_res *qp1_res, 1084 struct bnxt_qplib_qp *qp1_qp) 1085 { 1086 struct bnxt_re_dev *rdev = pd->rdev; 1087 struct bnxt_re_ah *ah; 1088 union ib_gid sgid; 1089 int rc; 1090 1091 ah = kzalloc(sizeof(*ah), GFP_KERNEL); 1092 if (!ah) 1093 return NULL; 1094 1095 ah->rdev = rdev; 1096 ah->qplib_ah.pd = &pd->qplib_pd; 1097 1098 rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); 1099 if (rc) 1100 goto fail; 1101 1102 /* supply the dgid data same as sgid */ 1103 memcpy(ah->qplib_ah.dgid.data, &sgid.raw, 1104 sizeof(union ib_gid)); 1105 ah->qplib_ah.sgid_index = 0; 1106 1107 ah->qplib_ah.traffic_class = 0; 1108 ah->qplib_ah.flow_label = 0; 1109 ah->qplib_ah.hop_limit = 1; 1110 ah->qplib_ah.sl = 0; 1111 /* Have DMAC same as SMAC */ 1112 ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr); 1113 1114 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, false); 1115 if (rc) { 1116 ibdev_err(&rdev->ibdev, 1117 "Failed to allocate HW AH for Shadow QP"); 1118 goto fail; 1119 } 1120 atomic_inc(&rdev->stats.res.ah_count); 1121 1122 return ah; 1123 1124 fail: 1125 kfree(ah); 1126 return NULL; 1127 } 1128 1129 static struct bnxt_re_qp *bnxt_re_create_shadow_qp 1130 (struct bnxt_re_pd *pd, 1131 struct bnxt_qplib_res *qp1_res, 1132 struct bnxt_qplib_qp *qp1_qp) 1133 { 1134 struct bnxt_re_dev *rdev = pd->rdev; 1135 struct bnxt_re_qp *qp; 1136 int rc; 1137 1138 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1139 if (!qp) 1140 return NULL; 1141 1142 qp->rdev = rdev; 1143 1144 /* Initialize the shadow QP structure from the QP1 values */ 1145 ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); 1146 1147 qp->qplib_qp.pd = &pd->qplib_pd; 1148 qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); 1149 qp->qplib_qp.type = IB_QPT_UD; 1150 1151 qp->qplib_qp.max_inline_data = 0; 1152 qp->qplib_qp.sig_type = true; 1153 1154 /* Shadow QP SQ depth should be same as QP1 RQ depth */ 1155 qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6); 1156 qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; 1157 qp->qplib_qp.sq.max_sge = 2; 1158 /* Q full delta can be 1 since it is internal QP */ 1159 qp->qplib_qp.sq.q_full_delta = 1; 1160 qp->qplib_qp.sq.sg_info.pgsize = PAGE_SIZE; 1161 qp->qplib_qp.sq.sg_info.pgshft = PAGE_SHIFT; 1162 1163 qp->qplib_qp.scq = qp1_qp->scq; 1164 qp->qplib_qp.rcq = qp1_qp->rcq; 1165 1166 qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6); 1167 qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; 1168 qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; 1169 /* Q full delta can be 1 since it is internal QP */ 1170 qp->qplib_qp.rq.q_full_delta = 1; 1171 qp->qplib_qp.rq.sg_info.pgsize = PAGE_SIZE; 1172 qp->qplib_qp.rq.sg_info.pgshft = PAGE_SHIFT; 1173 1174 qp->qplib_qp.mtu = qp1_qp->mtu; 1175 1176 qp->qplib_qp.sq_hdr_buf_size = 0; 1177 qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; 1178 qp->qplib_qp.dpi = &rdev->dpi_privileged; 1179 1180 rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); 1181 if (rc) 1182 goto fail; 1183 1184 spin_lock_init(&qp->sq_lock); 1185 INIT_LIST_HEAD(&qp->list); 1186 mutex_lock(&rdev->qp_lock); 1187 list_add_tail(&qp->list, &rdev->qp_list); 1188 atomic_inc(&rdev->stats.res.qp_count); 1189 mutex_unlock(&rdev->qp_lock); 1190 return qp; 1191 fail: 1192 kfree(qp); 1193 return NULL; 1194 } 1195 1196 static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, 1197 struct ib_qp_init_attr *init_attr, 1198 struct bnxt_re_ucontext *uctx) 1199 { 1200 struct bnxt_qplib_dev_attr *dev_attr; 1201 struct bnxt_qplib_qp *qplqp; 1202 struct bnxt_re_dev *rdev; 1203 struct bnxt_qplib_q *rq; 1204 int entries; 1205 1206 rdev = qp->rdev; 1207 qplqp = &qp->qplib_qp; 1208 rq = &qplqp->rq; 1209 dev_attr = &rdev->dev_attr; 1210 1211 if (init_attr->srq) { 1212 struct bnxt_re_srq *srq; 1213 1214 srq = container_of(init_attr->srq, struct bnxt_re_srq, ib_srq); 1215 qplqp->srq = &srq->qplib_srq; 1216 rq->max_wqe = 0; 1217 } else { 1218 rq->max_sge = init_attr->cap.max_recv_sge; 1219 if (rq->max_sge > dev_attr->max_qp_sges) 1220 rq->max_sge = dev_attr->max_qp_sges; 1221 init_attr->cap.max_recv_sge = rq->max_sge; 1222 rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge, 1223 dev_attr->max_qp_sges); 1224 /* Allocate 1 more than what's provided so posting max doesn't 1225 * mean empty. 1226 */ 1227 entries = bnxt_re_init_depth(init_attr->cap.max_recv_wr + 1, uctx); 1228 rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1); 1229 rq->q_full_delta = 0; 1230 rq->sg_info.pgsize = PAGE_SIZE; 1231 rq->sg_info.pgshft = PAGE_SHIFT; 1232 } 1233 1234 return 0; 1235 } 1236 1237 static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp) 1238 { 1239 struct bnxt_qplib_dev_attr *dev_attr; 1240 struct bnxt_qplib_qp *qplqp; 1241 struct bnxt_re_dev *rdev; 1242 1243 rdev = qp->rdev; 1244 qplqp = &qp->qplib_qp; 1245 dev_attr = &rdev->dev_attr; 1246 1247 if (!bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx)) { 1248 qplqp->rq.max_sge = dev_attr->max_qp_sges; 1249 if (qplqp->rq.max_sge > dev_attr->max_qp_sges) 1250 qplqp->rq.max_sge = dev_attr->max_qp_sges; 1251 qplqp->rq.max_sge = 6; 1252 } 1253 } 1254 1255 static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, 1256 struct ib_qp_init_attr *init_attr, 1257 struct bnxt_re_ucontext *uctx) 1258 { 1259 struct bnxt_qplib_dev_attr *dev_attr; 1260 struct bnxt_qplib_qp *qplqp; 1261 struct bnxt_re_dev *rdev; 1262 struct bnxt_qplib_q *sq; 1263 int entries; 1264 int diff; 1265 int rc; 1266 1267 rdev = qp->rdev; 1268 qplqp = &qp->qplib_qp; 1269 sq = &qplqp->sq; 1270 dev_attr = &rdev->dev_attr; 1271 1272 sq->max_sge = init_attr->cap.max_send_sge; 1273 if (sq->max_sge > dev_attr->max_qp_sges) { 1274 sq->max_sge = dev_attr->max_qp_sges; 1275 init_attr->cap.max_send_sge = sq->max_sge; 1276 } 1277 1278 rc = bnxt_re_setup_swqe_size(qp, init_attr); 1279 if (rc) 1280 return rc; 1281 1282 entries = init_attr->cap.max_send_wr; 1283 /* Allocate 128 + 1 more than what's provided */ 1284 diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ? 1285 0 : BNXT_QPLIB_RESERVED_QP_WRS; 1286 entries = bnxt_re_init_depth(entries + diff + 1, uctx); 1287 sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1); 1288 sq->q_full_delta = diff + 1; 1289 /* 1290 * Reserving one slot for Phantom WQE. Application can 1291 * post one extra entry in this case. But allowing this to avoid 1292 * unexpected Queue full condition 1293 */ 1294 qplqp->sq.q_full_delta -= 1; 1295 qplqp->sq.sg_info.pgsize = PAGE_SIZE; 1296 qplqp->sq.sg_info.pgshft = PAGE_SHIFT; 1297 1298 return 0; 1299 } 1300 1301 static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp, 1302 struct ib_qp_init_attr *init_attr, 1303 struct bnxt_re_ucontext *uctx) 1304 { 1305 struct bnxt_qplib_dev_attr *dev_attr; 1306 struct bnxt_qplib_qp *qplqp; 1307 struct bnxt_re_dev *rdev; 1308 int entries; 1309 1310 rdev = qp->rdev; 1311 qplqp = &qp->qplib_qp; 1312 dev_attr = &rdev->dev_attr; 1313 1314 if (!bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx)) { 1315 entries = bnxt_re_init_depth(init_attr->cap.max_send_wr + 1, uctx); 1316 qplqp->sq.max_wqe = min_t(u32, entries, 1317 dev_attr->max_qp_wqes + 1); 1318 qplqp->sq.q_full_delta = qplqp->sq.max_wqe - 1319 init_attr->cap.max_send_wr; 1320 qplqp->sq.max_sge++; /* Need one extra sge to put UD header */ 1321 if (qplqp->sq.max_sge > dev_attr->max_qp_sges) 1322 qplqp->sq.max_sge = dev_attr->max_qp_sges; 1323 } 1324 } 1325 1326 static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev, 1327 struct ib_qp_init_attr *init_attr) 1328 { 1329 struct bnxt_qplib_chip_ctx *chip_ctx; 1330 int qptype; 1331 1332 chip_ctx = rdev->chip_ctx; 1333 1334 qptype = __from_ib_qp_type(init_attr->qp_type); 1335 if (qptype == IB_QPT_MAX) { 1336 ibdev_err(&rdev->ibdev, "QP type 0x%x not supported", qptype); 1337 qptype = -EOPNOTSUPP; 1338 goto out; 1339 } 1340 1341 if (bnxt_qplib_is_chip_gen_p5_p7(chip_ctx) && 1342 init_attr->qp_type == IB_QPT_GSI) 1343 qptype = CMDQ_CREATE_QP_TYPE_GSI; 1344 out: 1345 return qptype; 1346 } 1347 1348 static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, 1349 struct ib_qp_init_attr *init_attr, 1350 struct ib_udata *udata) 1351 { 1352 struct bnxt_qplib_dev_attr *dev_attr; 1353 struct bnxt_re_ucontext *uctx; 1354 struct bnxt_qplib_qp *qplqp; 1355 struct bnxt_re_dev *rdev; 1356 struct bnxt_re_cq *cq; 1357 int rc = 0, qptype; 1358 1359 rdev = qp->rdev; 1360 qplqp = &qp->qplib_qp; 1361 dev_attr = &rdev->dev_attr; 1362 1363 uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx); 1364 /* Setup misc params */ 1365 ether_addr_copy(qplqp->smac, rdev->netdev->dev_addr); 1366 qplqp->pd = &pd->qplib_pd; 1367 qplqp->qp_handle = (u64)qplqp; 1368 qplqp->max_inline_data = init_attr->cap.max_inline_data; 1369 qplqp->sig_type = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 1370 qptype = bnxt_re_init_qp_type(rdev, init_attr); 1371 if (qptype < 0) { 1372 rc = qptype; 1373 goto out; 1374 } 1375 qplqp->type = (u8)qptype; 1376 qplqp->wqe_mode = rdev->chip_ctx->modes.wqe_mode; 1377 1378 if (init_attr->qp_type == IB_QPT_RC) { 1379 qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom; 1380 qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; 1381 } 1382 qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); 1383 qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */ 1384 if (init_attr->create_flags) { 1385 ibdev_dbg(&rdev->ibdev, 1386 "QP create flags 0x%x not supported", 1387 init_attr->create_flags); 1388 return -EOPNOTSUPP; 1389 } 1390 1391 /* Setup CQs */ 1392 if (init_attr->send_cq) { 1393 cq = container_of(init_attr->send_cq, struct bnxt_re_cq, ib_cq); 1394 qplqp->scq = &cq->qplib_cq; 1395 qp->scq = cq; 1396 } 1397 1398 if (init_attr->recv_cq) { 1399 cq = container_of(init_attr->recv_cq, struct bnxt_re_cq, ib_cq); 1400 qplqp->rcq = &cq->qplib_cq; 1401 qp->rcq = cq; 1402 } 1403 1404 /* Setup RQ/SRQ */ 1405 rc = bnxt_re_init_rq_attr(qp, init_attr, uctx); 1406 if (rc) 1407 goto out; 1408 if (init_attr->qp_type == IB_QPT_GSI) 1409 bnxt_re_adjust_gsi_rq_attr(qp); 1410 1411 /* Setup SQ */ 1412 rc = bnxt_re_init_sq_attr(qp, init_attr, uctx); 1413 if (rc) 1414 goto out; 1415 if (init_attr->qp_type == IB_QPT_GSI) 1416 bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx); 1417 1418 if (udata) /* This will update DPI and qp_handle */ 1419 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); 1420 out: 1421 return rc; 1422 } 1423 1424 static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp, 1425 struct bnxt_re_pd *pd) 1426 { 1427 struct bnxt_re_sqp_entries *sqp_tbl; 1428 struct bnxt_re_dev *rdev; 1429 struct bnxt_re_qp *sqp; 1430 struct bnxt_re_ah *sah; 1431 int rc = 0; 1432 1433 rdev = qp->rdev; 1434 /* Create a shadow QP to handle the QP1 traffic */ 1435 sqp_tbl = kcalloc(BNXT_RE_MAX_GSI_SQP_ENTRIES, sizeof(*sqp_tbl), 1436 GFP_KERNEL); 1437 if (!sqp_tbl) 1438 return -ENOMEM; 1439 rdev->gsi_ctx.sqp_tbl = sqp_tbl; 1440 1441 sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp); 1442 if (!sqp) { 1443 rc = -ENODEV; 1444 ibdev_err(&rdev->ibdev, "Failed to create Shadow QP for QP1"); 1445 goto out; 1446 } 1447 rdev->gsi_ctx.gsi_sqp = sqp; 1448 1449 sqp->rcq = qp->rcq; 1450 sqp->scq = qp->scq; 1451 sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, 1452 &qp->qplib_qp); 1453 if (!sah) { 1454 bnxt_qplib_destroy_qp(&rdev->qplib_res, 1455 &sqp->qplib_qp); 1456 rc = -ENODEV; 1457 ibdev_err(&rdev->ibdev, 1458 "Failed to create AH entry for ShadowQP"); 1459 goto out; 1460 } 1461 rdev->gsi_ctx.gsi_sah = sah; 1462 1463 return 0; 1464 out: 1465 kfree(sqp_tbl); 1466 return rc; 1467 } 1468 1469 static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, 1470 struct ib_qp_init_attr *init_attr) 1471 { 1472 struct bnxt_re_dev *rdev; 1473 struct bnxt_qplib_qp *qplqp; 1474 int rc; 1475 1476 rdev = qp->rdev; 1477 qplqp = &qp->qplib_qp; 1478 1479 qplqp->rq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; 1480 qplqp->sq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2; 1481 1482 rc = bnxt_qplib_create_qp1(&rdev->qplib_res, qplqp); 1483 if (rc) { 1484 ibdev_err(&rdev->ibdev, "create HW QP1 failed!"); 1485 goto out; 1486 } 1487 1488 rc = bnxt_re_create_shadow_gsi(qp, pd); 1489 out: 1490 return rc; 1491 } 1492 1493 static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev, 1494 struct ib_qp_init_attr *init_attr, 1495 struct bnxt_qplib_dev_attr *dev_attr) 1496 { 1497 bool rc = true; 1498 1499 if (init_attr->cap.max_send_wr > dev_attr->max_qp_wqes || 1500 init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes || 1501 init_attr->cap.max_send_sge > dev_attr->max_qp_sges || 1502 init_attr->cap.max_recv_sge > dev_attr->max_qp_sges || 1503 init_attr->cap.max_inline_data > dev_attr->max_inline_data) { 1504 ibdev_err(&rdev->ibdev, 1505 "Create QP failed - max exceeded! 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x", 1506 init_attr->cap.max_send_wr, dev_attr->max_qp_wqes, 1507 init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes, 1508 init_attr->cap.max_send_sge, dev_attr->max_qp_sges, 1509 init_attr->cap.max_recv_sge, dev_attr->max_qp_sges, 1510 init_attr->cap.max_inline_data, 1511 dev_attr->max_inline_data); 1512 rc = false; 1513 } 1514 return rc; 1515 } 1516 1517 int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, 1518 struct ib_udata *udata) 1519 { 1520 struct ib_pd *ib_pd = ib_qp->pd; 1521 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 1522 struct bnxt_re_dev *rdev = pd->rdev; 1523 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 1524 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 1525 u32 active_qps; 1526 int rc; 1527 1528 rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); 1529 if (!rc) { 1530 rc = -EINVAL; 1531 goto fail; 1532 } 1533 1534 qp->rdev = rdev; 1535 rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata); 1536 if (rc) 1537 goto fail; 1538 1539 if (qp_init_attr->qp_type == IB_QPT_GSI && 1540 !(bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))) { 1541 rc = bnxt_re_create_gsi_qp(qp, pd, qp_init_attr); 1542 if (rc == -ENODEV) 1543 goto qp_destroy; 1544 if (rc) 1545 goto fail; 1546 } else { 1547 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); 1548 if (rc) { 1549 ibdev_err(&rdev->ibdev, "Failed to create HW QP"); 1550 goto free_umem; 1551 } 1552 if (udata) { 1553 struct bnxt_re_qp_resp resp; 1554 1555 resp.qpid = qp->qplib_qp.id; 1556 resp.rsvd = 0; 1557 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 1558 if (rc) { 1559 ibdev_err(&rdev->ibdev, "Failed to copy QP udata"); 1560 goto qp_destroy; 1561 } 1562 } 1563 } 1564 1565 qp->ib_qp.qp_num = qp->qplib_qp.id; 1566 if (qp_init_attr->qp_type == IB_QPT_GSI) 1567 rdev->gsi_ctx.gsi_qp = qp; 1568 spin_lock_init(&qp->sq_lock); 1569 spin_lock_init(&qp->rq_lock); 1570 INIT_LIST_HEAD(&qp->list); 1571 mutex_lock(&rdev->qp_lock); 1572 list_add_tail(&qp->list, &rdev->qp_list); 1573 mutex_unlock(&rdev->qp_lock); 1574 active_qps = atomic_inc_return(&rdev->stats.res.qp_count); 1575 if (active_qps > rdev->stats.res.qp_watermark) 1576 rdev->stats.res.qp_watermark = active_qps; 1577 if (qp_init_attr->qp_type == IB_QPT_RC) { 1578 active_qps = atomic_inc_return(&rdev->stats.res.rc_qp_count); 1579 if (active_qps > rdev->stats.res.rc_qp_watermark) 1580 rdev->stats.res.rc_qp_watermark = active_qps; 1581 } else if (qp_init_attr->qp_type == IB_QPT_UD) { 1582 active_qps = atomic_inc_return(&rdev->stats.res.ud_qp_count); 1583 if (active_qps > rdev->stats.res.ud_qp_watermark) 1584 rdev->stats.res.ud_qp_watermark = active_qps; 1585 } 1586 1587 return 0; 1588 qp_destroy: 1589 bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 1590 free_umem: 1591 ib_umem_release(qp->rumem); 1592 ib_umem_release(qp->sumem); 1593 fail: 1594 return rc; 1595 } 1596 1597 static u8 __from_ib_qp_state(enum ib_qp_state state) 1598 { 1599 switch (state) { 1600 case IB_QPS_RESET: 1601 return CMDQ_MODIFY_QP_NEW_STATE_RESET; 1602 case IB_QPS_INIT: 1603 return CMDQ_MODIFY_QP_NEW_STATE_INIT; 1604 case IB_QPS_RTR: 1605 return CMDQ_MODIFY_QP_NEW_STATE_RTR; 1606 case IB_QPS_RTS: 1607 return CMDQ_MODIFY_QP_NEW_STATE_RTS; 1608 case IB_QPS_SQD: 1609 return CMDQ_MODIFY_QP_NEW_STATE_SQD; 1610 case IB_QPS_SQE: 1611 return CMDQ_MODIFY_QP_NEW_STATE_SQE; 1612 case IB_QPS_ERR: 1613 default: 1614 return CMDQ_MODIFY_QP_NEW_STATE_ERR; 1615 } 1616 } 1617 1618 static enum ib_qp_state __to_ib_qp_state(u8 state) 1619 { 1620 switch (state) { 1621 case CMDQ_MODIFY_QP_NEW_STATE_RESET: 1622 return IB_QPS_RESET; 1623 case CMDQ_MODIFY_QP_NEW_STATE_INIT: 1624 return IB_QPS_INIT; 1625 case CMDQ_MODIFY_QP_NEW_STATE_RTR: 1626 return IB_QPS_RTR; 1627 case CMDQ_MODIFY_QP_NEW_STATE_RTS: 1628 return IB_QPS_RTS; 1629 case CMDQ_MODIFY_QP_NEW_STATE_SQD: 1630 return IB_QPS_SQD; 1631 case CMDQ_MODIFY_QP_NEW_STATE_SQE: 1632 return IB_QPS_SQE; 1633 case CMDQ_MODIFY_QP_NEW_STATE_ERR: 1634 default: 1635 return IB_QPS_ERR; 1636 } 1637 } 1638 1639 static u32 __from_ib_mtu(enum ib_mtu mtu) 1640 { 1641 switch (mtu) { 1642 case IB_MTU_256: 1643 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256; 1644 case IB_MTU_512: 1645 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512; 1646 case IB_MTU_1024: 1647 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024; 1648 case IB_MTU_2048: 1649 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 1650 case IB_MTU_4096: 1651 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096; 1652 default: 1653 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 1654 } 1655 } 1656 1657 static enum ib_mtu __to_ib_mtu(u32 mtu) 1658 { 1659 switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) { 1660 case CMDQ_MODIFY_QP_PATH_MTU_MTU_256: 1661 return IB_MTU_256; 1662 case CMDQ_MODIFY_QP_PATH_MTU_MTU_512: 1663 return IB_MTU_512; 1664 case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024: 1665 return IB_MTU_1024; 1666 case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048: 1667 return IB_MTU_2048; 1668 case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096: 1669 return IB_MTU_4096; 1670 default: 1671 return IB_MTU_2048; 1672 } 1673 } 1674 1675 /* Shared Receive Queues */ 1676 int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) 1677 { 1678 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1679 ib_srq); 1680 struct bnxt_re_dev *rdev = srq->rdev; 1681 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; 1682 struct bnxt_qplib_nq *nq = NULL; 1683 1684 if (qplib_srq->cq) 1685 nq = qplib_srq->cq->nq; 1686 bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); 1687 ib_umem_release(srq->umem); 1688 atomic_dec(&rdev->stats.res.srq_count); 1689 if (nq) 1690 nq->budget--; 1691 return 0; 1692 } 1693 1694 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, 1695 struct bnxt_re_pd *pd, 1696 struct bnxt_re_srq *srq, 1697 struct ib_udata *udata) 1698 { 1699 struct bnxt_re_srq_req ureq; 1700 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; 1701 struct ib_umem *umem; 1702 int bytes = 0; 1703 struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context( 1704 udata, struct bnxt_re_ucontext, ib_uctx); 1705 1706 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 1707 return -EFAULT; 1708 1709 bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size); 1710 bytes = PAGE_ALIGN(bytes); 1711 umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes, 1712 IB_ACCESS_LOCAL_WRITE); 1713 if (IS_ERR(umem)) 1714 return PTR_ERR(umem); 1715 1716 srq->umem = umem; 1717 qplib_srq->sg_info.umem = umem; 1718 qplib_srq->sg_info.pgsize = PAGE_SIZE; 1719 qplib_srq->sg_info.pgshft = PAGE_SHIFT; 1720 qplib_srq->srq_handle = ureq.srq_handle; 1721 qplib_srq->dpi = &cntx->dpi; 1722 1723 return 0; 1724 } 1725 1726 int bnxt_re_create_srq(struct ib_srq *ib_srq, 1727 struct ib_srq_init_attr *srq_init_attr, 1728 struct ib_udata *udata) 1729 { 1730 struct bnxt_qplib_dev_attr *dev_attr; 1731 struct bnxt_qplib_nq *nq = NULL; 1732 struct bnxt_re_ucontext *uctx; 1733 struct bnxt_re_dev *rdev; 1734 struct bnxt_re_srq *srq; 1735 struct bnxt_re_pd *pd; 1736 struct ib_pd *ib_pd; 1737 u32 active_srqs; 1738 int rc, entries; 1739 1740 ib_pd = ib_srq->pd; 1741 pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 1742 rdev = pd->rdev; 1743 dev_attr = &rdev->dev_attr; 1744 srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq); 1745 1746 if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) { 1747 ibdev_err(&rdev->ibdev, "Create CQ failed - max exceeded"); 1748 rc = -EINVAL; 1749 goto exit; 1750 } 1751 1752 if (srq_init_attr->srq_type != IB_SRQT_BASIC) { 1753 rc = -EOPNOTSUPP; 1754 goto exit; 1755 } 1756 1757 uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx); 1758 srq->rdev = rdev; 1759 srq->qplib_srq.pd = &pd->qplib_pd; 1760 srq->qplib_srq.dpi = &rdev->dpi_privileged; 1761 /* Allocate 1 more than what's provided so posting max doesn't 1762 * mean empty 1763 */ 1764 entries = bnxt_re_init_depth(srq_init_attr->attr.max_wr + 1, uctx); 1765 if (entries > dev_attr->max_srq_wqes + 1) 1766 entries = dev_attr->max_srq_wqes + 1; 1767 srq->qplib_srq.max_wqe = entries; 1768 1769 srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge; 1770 /* 128 byte wqe size for SRQ . So use max sges */ 1771 srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr->max_srq_sges); 1772 srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit; 1773 srq->srq_limit = srq_init_attr->attr.srq_limit; 1774 srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id; 1775 nq = &rdev->nq[0]; 1776 1777 if (udata) { 1778 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata); 1779 if (rc) 1780 goto fail; 1781 } 1782 1783 rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq); 1784 if (rc) { 1785 ibdev_err(&rdev->ibdev, "Create HW SRQ failed!"); 1786 goto fail; 1787 } 1788 1789 if (udata) { 1790 struct bnxt_re_srq_resp resp; 1791 1792 resp.srqid = srq->qplib_srq.id; 1793 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 1794 if (rc) { 1795 ibdev_err(&rdev->ibdev, "SRQ copy to udata failed!"); 1796 bnxt_qplib_destroy_srq(&rdev->qplib_res, 1797 &srq->qplib_srq); 1798 goto fail; 1799 } 1800 } 1801 if (nq) 1802 nq->budget++; 1803 active_srqs = atomic_inc_return(&rdev->stats.res.srq_count); 1804 if (active_srqs > rdev->stats.res.srq_watermark) 1805 rdev->stats.res.srq_watermark = active_srqs; 1806 spin_lock_init(&srq->lock); 1807 1808 return 0; 1809 1810 fail: 1811 ib_umem_release(srq->umem); 1812 exit: 1813 return rc; 1814 } 1815 1816 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, 1817 enum ib_srq_attr_mask srq_attr_mask, 1818 struct ib_udata *udata) 1819 { 1820 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1821 ib_srq); 1822 struct bnxt_re_dev *rdev = srq->rdev; 1823 int rc; 1824 1825 switch (srq_attr_mask) { 1826 case IB_SRQ_MAX_WR: 1827 /* SRQ resize is not supported */ 1828 return -EINVAL; 1829 case IB_SRQ_LIMIT: 1830 /* Change the SRQ threshold */ 1831 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) 1832 return -EINVAL; 1833 1834 srq->qplib_srq.threshold = srq_attr->srq_limit; 1835 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq); 1836 if (rc) { 1837 ibdev_err(&rdev->ibdev, "Modify HW SRQ failed!"); 1838 return rc; 1839 } 1840 /* On success, update the shadow */ 1841 srq->srq_limit = srq_attr->srq_limit; 1842 /* No need to Build and send response back to udata */ 1843 return 0; 1844 default: 1845 ibdev_err(&rdev->ibdev, 1846 "Unsupported srq_attr_mask 0x%x", srq_attr_mask); 1847 return -EINVAL; 1848 } 1849 } 1850 1851 int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr) 1852 { 1853 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1854 ib_srq); 1855 struct bnxt_re_srq tsrq; 1856 struct bnxt_re_dev *rdev = srq->rdev; 1857 int rc; 1858 1859 /* Get live SRQ attr */ 1860 tsrq.qplib_srq.id = srq->qplib_srq.id; 1861 rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq); 1862 if (rc) { 1863 ibdev_err(&rdev->ibdev, "Query HW SRQ failed!"); 1864 return rc; 1865 } 1866 srq_attr->max_wr = srq->qplib_srq.max_wqe; 1867 srq_attr->max_sge = srq->qplib_srq.max_sge; 1868 srq_attr->srq_limit = tsrq.qplib_srq.threshold; 1869 1870 return 0; 1871 } 1872 1873 int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, 1874 const struct ib_recv_wr **bad_wr) 1875 { 1876 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1877 ib_srq); 1878 struct bnxt_qplib_swqe wqe; 1879 unsigned long flags; 1880 int rc = 0; 1881 1882 spin_lock_irqsave(&srq->lock, flags); 1883 while (wr) { 1884 /* Transcribe each ib_recv_wr to qplib_swqe */ 1885 wqe.num_sge = wr->num_sge; 1886 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); 1887 wqe.wr_id = wr->wr_id; 1888 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 1889 1890 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe); 1891 if (rc) { 1892 *bad_wr = wr; 1893 break; 1894 } 1895 wr = wr->next; 1896 } 1897 spin_unlock_irqrestore(&srq->lock, flags); 1898 1899 return rc; 1900 } 1901 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, 1902 struct bnxt_re_qp *qp1_qp, 1903 int qp_attr_mask) 1904 { 1905 struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp; 1906 int rc; 1907 1908 if (qp_attr_mask & IB_QP_STATE) { 1909 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 1910 qp->qplib_qp.state = qp1_qp->qplib_qp.state; 1911 } 1912 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 1913 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 1914 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; 1915 } 1916 1917 if (qp_attr_mask & IB_QP_QKEY) { 1918 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 1919 /* Using a Random QKEY */ 1920 qp->qplib_qp.qkey = 0x81818181; 1921 } 1922 if (qp_attr_mask & IB_QP_SQ_PSN) { 1923 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 1924 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; 1925 } 1926 1927 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 1928 if (rc) 1929 ibdev_err(&rdev->ibdev, "Failed to modify Shadow QP for QP1"); 1930 return rc; 1931 } 1932 1933 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 1934 int qp_attr_mask, struct ib_udata *udata) 1935 { 1936 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 1937 struct bnxt_re_dev *rdev = qp->rdev; 1938 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 1939 enum ib_qp_state curr_qp_state, new_qp_state; 1940 int rc, entries; 1941 unsigned int flags; 1942 u8 nw_type; 1943 1944 if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1945 return -EOPNOTSUPP; 1946 1947 qp->qplib_qp.modify_flags = 0; 1948 if (qp_attr_mask & IB_QP_STATE) { 1949 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); 1950 new_qp_state = qp_attr->qp_state; 1951 if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state, 1952 ib_qp->qp_type, qp_attr_mask)) { 1953 ibdev_err(&rdev->ibdev, 1954 "Invalid attribute mask: %#x specified ", 1955 qp_attr_mask); 1956 ibdev_err(&rdev->ibdev, 1957 "for qpn: %#x type: %#x", 1958 ib_qp->qp_num, ib_qp->qp_type); 1959 ibdev_err(&rdev->ibdev, 1960 "curr_qp_state=0x%x, new_qp_state=0x%x\n", 1961 curr_qp_state, new_qp_state); 1962 return -EINVAL; 1963 } 1964 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 1965 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); 1966 1967 if (!qp->sumem && 1968 qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { 1969 ibdev_dbg(&rdev->ibdev, 1970 "Move QP = %p to flush list\n", qp); 1971 flags = bnxt_re_lock_cqs(qp); 1972 bnxt_qplib_add_flush_qp(&qp->qplib_qp); 1973 bnxt_re_unlock_cqs(qp, flags); 1974 } 1975 if (!qp->sumem && 1976 qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { 1977 ibdev_dbg(&rdev->ibdev, 1978 "Move QP = %p out of flush list\n", qp); 1979 flags = bnxt_re_lock_cqs(qp); 1980 bnxt_qplib_clean_qp(&qp->qplib_qp); 1981 bnxt_re_unlock_cqs(qp, flags); 1982 } 1983 } 1984 if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { 1985 qp->qplib_qp.modify_flags |= 1986 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY; 1987 qp->qplib_qp.en_sqd_async_notify = true; 1988 } 1989 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) { 1990 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; 1991 qp->qplib_qp.access = 1992 __from_ib_access_flags(qp_attr->qp_access_flags); 1993 /* LOCAL_WRITE access must be set to allow RC receive */ 1994 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 1995 /* Temp: Set all params on QP as of now */ 1996 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE; 1997 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ; 1998 } 1999 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 2000 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 2001 qp->qplib_qp.pkey_index = qp_attr->pkey_index; 2002 } 2003 if (qp_attr_mask & IB_QP_QKEY) { 2004 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 2005 qp->qplib_qp.qkey = qp_attr->qkey; 2006 } 2007 if (qp_attr_mask & IB_QP_AV) { 2008 const struct ib_global_route *grh = 2009 rdma_ah_read_grh(&qp_attr->ah_attr); 2010 const struct ib_gid_attr *sgid_attr; 2011 struct bnxt_re_gid_ctx *ctx; 2012 2013 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | 2014 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | 2015 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | 2016 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | 2017 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | 2018 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | 2019 CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; 2020 memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw, 2021 sizeof(qp->qplib_qp.ah.dgid.data)); 2022 qp->qplib_qp.ah.flow_label = grh->flow_label; 2023 sgid_attr = grh->sgid_attr; 2024 /* Get the HW context of the GID. The reference 2025 * of GID table entry is already taken by the caller. 2026 */ 2027 ctx = rdma_read_gid_hw_context(sgid_attr); 2028 qp->qplib_qp.ah.sgid_index = ctx->idx; 2029 qp->qplib_qp.ah.host_sgid_index = grh->sgid_index; 2030 qp->qplib_qp.ah.hop_limit = grh->hop_limit; 2031 qp->qplib_qp.ah.traffic_class = grh->traffic_class; 2032 qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr); 2033 ether_addr_copy(qp->qplib_qp.ah.dmac, 2034 qp_attr->ah_attr.roce.dmac); 2035 2036 rc = rdma_read_gid_l2_fields(sgid_attr, NULL, 2037 &qp->qplib_qp.smac[0]); 2038 if (rc) 2039 return rc; 2040 2041 nw_type = rdma_gid_attr_network_type(sgid_attr); 2042 switch (nw_type) { 2043 case RDMA_NETWORK_IPV4: 2044 qp->qplib_qp.nw_type = 2045 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4; 2046 break; 2047 case RDMA_NETWORK_IPV6: 2048 qp->qplib_qp.nw_type = 2049 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6; 2050 break; 2051 default: 2052 qp->qplib_qp.nw_type = 2053 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1; 2054 break; 2055 } 2056 } 2057 2058 if (qp_attr_mask & IB_QP_PATH_MTU) { 2059 qp->qplib_qp.modify_flags |= 2060 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 2061 qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu); 2062 qp->qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr->path_mtu); 2063 } else if (qp_attr->qp_state == IB_QPS_RTR) { 2064 qp->qplib_qp.modify_flags |= 2065 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 2066 qp->qplib_qp.path_mtu = 2067 __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu)); 2068 qp->qplib_qp.mtu = 2069 ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); 2070 } 2071 2072 if (qp_attr_mask & IB_QP_TIMEOUT) { 2073 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; 2074 qp->qplib_qp.timeout = qp_attr->timeout; 2075 } 2076 if (qp_attr_mask & IB_QP_RETRY_CNT) { 2077 qp->qplib_qp.modify_flags |= 2078 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT; 2079 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; 2080 } 2081 if (qp_attr_mask & IB_QP_RNR_RETRY) { 2082 qp->qplib_qp.modify_flags |= 2083 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY; 2084 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; 2085 } 2086 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) { 2087 qp->qplib_qp.modify_flags |= 2088 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER; 2089 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; 2090 } 2091 if (qp_attr_mask & IB_QP_RQ_PSN) { 2092 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; 2093 qp->qplib_qp.rq.psn = qp_attr->rq_psn; 2094 } 2095 if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 2096 qp->qplib_qp.modify_flags |= 2097 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC; 2098 /* Cap the max_rd_atomic to device max */ 2099 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic, 2100 dev_attr->max_qp_rd_atom); 2101 } 2102 if (qp_attr_mask & IB_QP_SQ_PSN) { 2103 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 2104 qp->qplib_qp.sq.psn = qp_attr->sq_psn; 2105 } 2106 if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 2107 if (qp_attr->max_dest_rd_atomic > 2108 dev_attr->max_qp_init_rd_atom) { 2109 ibdev_err(&rdev->ibdev, 2110 "max_dest_rd_atomic requested%d is > dev_max%d", 2111 qp_attr->max_dest_rd_atomic, 2112 dev_attr->max_qp_init_rd_atom); 2113 return -EINVAL; 2114 } 2115 2116 qp->qplib_qp.modify_flags |= 2117 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC; 2118 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; 2119 } 2120 if (qp_attr_mask & IB_QP_CAP) { 2121 struct bnxt_re_ucontext *uctx = 2122 rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx); 2123 2124 qp->qplib_qp.modify_flags |= 2125 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE | 2126 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE | 2127 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE | 2128 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE | 2129 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA; 2130 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || 2131 (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || 2132 (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || 2133 (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || 2134 (qp_attr->cap.max_inline_data >= 2135 dev_attr->max_inline_data)) { 2136 ibdev_err(&rdev->ibdev, 2137 "Create QP failed - max exceeded"); 2138 return -EINVAL; 2139 } 2140 entries = bnxt_re_init_depth(qp_attr->cap.max_send_wr, uctx); 2141 qp->qplib_qp.sq.max_wqe = min_t(u32, entries, 2142 dev_attr->max_qp_wqes + 1); 2143 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - 2144 qp_attr->cap.max_send_wr; 2145 /* 2146 * Reserving one slot for Phantom WQE. Some application can 2147 * post one extra entry in this case. Allowing this to avoid 2148 * unexpected Queue full condition 2149 */ 2150 qp->qplib_qp.sq.q_full_delta -= 1; 2151 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; 2152 if (qp->qplib_qp.rq.max_wqe) { 2153 entries = bnxt_re_init_depth(qp_attr->cap.max_recv_wr, uctx); 2154 qp->qplib_qp.rq.max_wqe = 2155 min_t(u32, entries, dev_attr->max_qp_wqes + 1); 2156 qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - 2157 qp_attr->cap.max_recv_wr; 2158 qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; 2159 } else { 2160 /* SRQ was used prior, just ignore the RQ caps */ 2161 } 2162 } 2163 if (qp_attr_mask & IB_QP_DEST_QPN) { 2164 qp->qplib_qp.modify_flags |= 2165 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID; 2166 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; 2167 } 2168 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 2169 if (rc) { 2170 ibdev_err(&rdev->ibdev, "Failed to modify HW QP"); 2171 return rc; 2172 } 2173 if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) 2174 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); 2175 return rc; 2176 } 2177 2178 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 2179 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) 2180 { 2181 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 2182 struct bnxt_re_dev *rdev = qp->rdev; 2183 struct bnxt_qplib_qp *qplib_qp; 2184 int rc; 2185 2186 qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL); 2187 if (!qplib_qp) 2188 return -ENOMEM; 2189 2190 qplib_qp->id = qp->qplib_qp.id; 2191 qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; 2192 2193 rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp); 2194 if (rc) { 2195 ibdev_err(&rdev->ibdev, "Failed to query HW QP"); 2196 goto out; 2197 } 2198 qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state); 2199 qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state); 2200 qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0; 2201 qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access); 2202 qp_attr->pkey_index = qplib_qp->pkey_index; 2203 qp_attr->qkey = qplib_qp->qkey; 2204 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2205 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->ah.flow_label, 2206 qplib_qp->ah.host_sgid_index, 2207 qplib_qp->ah.hop_limit, 2208 qplib_qp->ah.traffic_class); 2209 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data); 2210 rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl); 2211 ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac); 2212 qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu); 2213 qp_attr->timeout = qplib_qp->timeout; 2214 qp_attr->retry_cnt = qplib_qp->retry_cnt; 2215 qp_attr->rnr_retry = qplib_qp->rnr_retry; 2216 qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer; 2217 qp_attr->rq_psn = qplib_qp->rq.psn; 2218 qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic; 2219 qp_attr->sq_psn = qplib_qp->sq.psn; 2220 qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic; 2221 qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR : 2222 IB_SIGNAL_REQ_WR; 2223 qp_attr->dest_qp_num = qplib_qp->dest_qpn; 2224 2225 qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; 2226 qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; 2227 qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; 2228 qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; 2229 qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; 2230 qp_init_attr->cap = qp_attr->cap; 2231 2232 out: 2233 kfree(qplib_qp); 2234 return rc; 2235 } 2236 2237 /* Routine for sending QP1 packets for RoCE V1 an V2 2238 */ 2239 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp, 2240 const struct ib_send_wr *wr, 2241 struct bnxt_qplib_swqe *wqe, 2242 int payload_size) 2243 { 2244 struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, 2245 ib_ah); 2246 struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; 2247 const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr; 2248 struct bnxt_qplib_sge sge; 2249 u8 nw_type; 2250 u16 ether_type; 2251 union ib_gid dgid; 2252 bool is_eth = false; 2253 bool is_vlan = false; 2254 bool is_grh = false; 2255 bool is_udp = false; 2256 u8 ip_version = 0; 2257 u16 vlan_id = 0xFFFF; 2258 void *buf; 2259 int i, rc; 2260 2261 memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); 2262 2263 rc = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, NULL); 2264 if (rc) 2265 return rc; 2266 2267 /* Get network header type for this GID */ 2268 nw_type = rdma_gid_attr_network_type(sgid_attr); 2269 switch (nw_type) { 2270 case RDMA_NETWORK_IPV4: 2271 nw_type = BNXT_RE_ROCEV2_IPV4_PACKET; 2272 break; 2273 case RDMA_NETWORK_IPV6: 2274 nw_type = BNXT_RE_ROCEV2_IPV6_PACKET; 2275 break; 2276 default: 2277 nw_type = BNXT_RE_ROCE_V1_PACKET; 2278 break; 2279 } 2280 memcpy(&dgid.raw, &qplib_ah->dgid, 16); 2281 is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP; 2282 if (is_udp) { 2283 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) { 2284 ip_version = 4; 2285 ether_type = ETH_P_IP; 2286 } else { 2287 ip_version = 6; 2288 ether_type = ETH_P_IPV6; 2289 } 2290 is_grh = false; 2291 } else { 2292 ether_type = ETH_P_IBOE; 2293 is_grh = true; 2294 } 2295 2296 is_eth = true; 2297 is_vlan = vlan_id && (vlan_id < 0x1000); 2298 2299 ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh, 2300 ip_version, is_udp, 0, &qp->qp1_hdr); 2301 2302 /* ETH */ 2303 ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac); 2304 ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac); 2305 2306 /* For vlan, check the sgid for vlan existence */ 2307 2308 if (!is_vlan) { 2309 qp->qp1_hdr.eth.type = cpu_to_be16(ether_type); 2310 } else { 2311 qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type); 2312 qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id); 2313 } 2314 2315 if (is_grh || (ip_version == 6)) { 2316 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw, 2317 sizeof(sgid_attr->gid)); 2318 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, 2319 sizeof(sgid_attr->gid)); 2320 qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit; 2321 } 2322 2323 if (ip_version == 4) { 2324 qp->qp1_hdr.ip4.tos = 0; 2325 qp->qp1_hdr.ip4.id = 0; 2326 qp->qp1_hdr.ip4.frag_off = htons(IP_DF); 2327 qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit; 2328 2329 memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4); 2330 memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4); 2331 qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr); 2332 } 2333 2334 if (is_udp) { 2335 qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT); 2336 qp->qp1_hdr.udp.sport = htons(0x8CD1); 2337 qp->qp1_hdr.udp.csum = 0; 2338 } 2339 2340 /* BTH */ 2341 if (wr->opcode == IB_WR_SEND_WITH_IMM) { 2342 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; 2343 qp->qp1_hdr.immediate_present = 1; 2344 } else { 2345 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; 2346 } 2347 if (wr->send_flags & IB_SEND_SOLICITED) 2348 qp->qp1_hdr.bth.solicited_event = 1; 2349 /* pad_count */ 2350 qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; 2351 2352 /* P_key for QP1 is for all members */ 2353 qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); 2354 qp->qp1_hdr.bth.destination_qpn = IB_QP1; 2355 qp->qp1_hdr.bth.ack_req = 0; 2356 qp->send_psn++; 2357 qp->send_psn &= BTH_PSN_MASK; 2358 qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); 2359 /* DETH */ 2360 /* Use the priviledged Q_Key for QP1 */ 2361 qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); 2362 qp->qp1_hdr.deth.source_qpn = IB_QP1; 2363 2364 /* Pack the QP1 to the transmit buffer */ 2365 buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); 2366 if (buf) { 2367 ib_ud_header_pack(&qp->qp1_hdr, buf); 2368 for (i = wqe->num_sge; i; i--) { 2369 wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; 2370 wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; 2371 wqe->sg_list[i].size = wqe->sg_list[i - 1].size; 2372 } 2373 2374 /* 2375 * Max Header buf size for IPV6 RoCE V2 is 86, 2376 * which is same as the QP1 SQ header buffer. 2377 * Header buf size for IPV4 RoCE V2 can be 66. 2378 * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20). 2379 * Subtract 20 bytes from QP1 SQ header buf size 2380 */ 2381 if (is_udp && ip_version == 4) 2382 sge.size -= 20; 2383 /* 2384 * Max Header buf size for RoCE V1 is 78. 2385 * ETH(14) + VLAN(4) + GRH(40) + BTH(20). 2386 * Subtract 8 bytes from QP1 SQ header buf size 2387 */ 2388 if (!is_udp) 2389 sge.size -= 8; 2390 2391 /* Subtract 4 bytes for non vlan packets */ 2392 if (!is_vlan) 2393 sge.size -= 4; 2394 2395 wqe->sg_list[0].addr = sge.addr; 2396 wqe->sg_list[0].lkey = sge.lkey; 2397 wqe->sg_list[0].size = sge.size; 2398 wqe->num_sge++; 2399 2400 } else { 2401 ibdev_err(&qp->rdev->ibdev, "QP1 buffer is empty!"); 2402 rc = -ENOMEM; 2403 } 2404 return rc; 2405 } 2406 2407 /* For the MAD layer, it only provides the recv SGE the size of 2408 * ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH, 2409 * nor RoCE iCRC. The Cu+ solution must provide buffer for the entire 2410 * receive packet (334 bytes) with no VLAN and then copy the GRH 2411 * and the MAD datagram out to the provided SGE. 2412 */ 2413 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, 2414 const struct ib_recv_wr *wr, 2415 struct bnxt_qplib_swqe *wqe, 2416 int payload_size) 2417 { 2418 struct bnxt_re_sqp_entries *sqp_entry; 2419 struct bnxt_qplib_sge ref, sge; 2420 struct bnxt_re_dev *rdev; 2421 u32 rq_prod_index; 2422 2423 rdev = qp->rdev; 2424 2425 rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); 2426 2427 if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) 2428 return -ENOMEM; 2429 2430 /* Create 1 SGE to receive the entire 2431 * ethernet packet 2432 */ 2433 /* Save the reference from ULP */ 2434 ref.addr = wqe->sg_list[0].addr; 2435 ref.lkey = wqe->sg_list[0].lkey; 2436 ref.size = wqe->sg_list[0].size; 2437 2438 sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index]; 2439 2440 /* SGE 1 */ 2441 wqe->sg_list[0].addr = sge.addr; 2442 wqe->sg_list[0].lkey = sge.lkey; 2443 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; 2444 sge.size -= wqe->sg_list[0].size; 2445 2446 sqp_entry->sge.addr = ref.addr; 2447 sqp_entry->sge.lkey = ref.lkey; 2448 sqp_entry->sge.size = ref.size; 2449 /* Store the wrid for reporting completion */ 2450 sqp_entry->wrid = wqe->wr_id; 2451 /* change the wqe->wrid to table index */ 2452 wqe->wr_id = rq_prod_index; 2453 return 0; 2454 } 2455 2456 static int is_ud_qp(struct bnxt_re_qp *qp) 2457 { 2458 return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD || 2459 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI); 2460 } 2461 2462 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, 2463 const struct ib_send_wr *wr, 2464 struct bnxt_qplib_swqe *wqe) 2465 { 2466 struct bnxt_re_ah *ah = NULL; 2467 2468 if (is_ud_qp(qp)) { 2469 ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah); 2470 wqe->send.q_key = ud_wr(wr)->remote_qkey; 2471 wqe->send.dst_qp = ud_wr(wr)->remote_qpn; 2472 wqe->send.avid = ah->qplib_ah.id; 2473 } 2474 switch (wr->opcode) { 2475 case IB_WR_SEND: 2476 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; 2477 break; 2478 case IB_WR_SEND_WITH_IMM: 2479 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; 2480 wqe->send.imm_data = be32_to_cpu(wr->ex.imm_data); 2481 break; 2482 case IB_WR_SEND_WITH_INV: 2483 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; 2484 wqe->send.inv_key = wr->ex.invalidate_rkey; 2485 break; 2486 default: 2487 return -EINVAL; 2488 } 2489 if (wr->send_flags & IB_SEND_SIGNALED) 2490 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2491 if (wr->send_flags & IB_SEND_FENCE) 2492 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2493 if (wr->send_flags & IB_SEND_SOLICITED) 2494 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2495 if (wr->send_flags & IB_SEND_INLINE) 2496 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 2497 2498 return 0; 2499 } 2500 2501 static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr, 2502 struct bnxt_qplib_swqe *wqe) 2503 { 2504 switch (wr->opcode) { 2505 case IB_WR_RDMA_WRITE: 2506 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; 2507 break; 2508 case IB_WR_RDMA_WRITE_WITH_IMM: 2509 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; 2510 wqe->rdma.imm_data = be32_to_cpu(wr->ex.imm_data); 2511 break; 2512 case IB_WR_RDMA_READ: 2513 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; 2514 wqe->rdma.inv_key = wr->ex.invalidate_rkey; 2515 break; 2516 default: 2517 return -EINVAL; 2518 } 2519 wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; 2520 wqe->rdma.r_key = rdma_wr(wr)->rkey; 2521 if (wr->send_flags & IB_SEND_SIGNALED) 2522 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2523 if (wr->send_flags & IB_SEND_FENCE) 2524 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2525 if (wr->send_flags & IB_SEND_SOLICITED) 2526 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2527 if (wr->send_flags & IB_SEND_INLINE) 2528 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 2529 2530 return 0; 2531 } 2532 2533 static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr, 2534 struct bnxt_qplib_swqe *wqe) 2535 { 2536 switch (wr->opcode) { 2537 case IB_WR_ATOMIC_CMP_AND_SWP: 2538 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; 2539 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 2540 wqe->atomic.swap_data = atomic_wr(wr)->swap; 2541 break; 2542 case IB_WR_ATOMIC_FETCH_AND_ADD: 2543 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; 2544 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 2545 break; 2546 default: 2547 return -EINVAL; 2548 } 2549 wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; 2550 wqe->atomic.r_key = atomic_wr(wr)->rkey; 2551 if (wr->send_flags & IB_SEND_SIGNALED) 2552 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2553 if (wr->send_flags & IB_SEND_FENCE) 2554 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2555 if (wr->send_flags & IB_SEND_SOLICITED) 2556 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2557 return 0; 2558 } 2559 2560 static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr, 2561 struct bnxt_qplib_swqe *wqe) 2562 { 2563 wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; 2564 wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; 2565 2566 if (wr->send_flags & IB_SEND_SIGNALED) 2567 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2568 if (wr->send_flags & IB_SEND_SOLICITED) 2569 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2570 2571 return 0; 2572 } 2573 2574 static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr, 2575 struct bnxt_qplib_swqe *wqe) 2576 { 2577 struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr); 2578 struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; 2579 int access = wr->access; 2580 2581 wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; 2582 wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; 2583 wqe->frmr.page_list = mr->pages; 2584 wqe->frmr.page_list_len = mr->npages; 2585 wqe->frmr.levels = qplib_frpl->hwq.level; 2586 wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; 2587 2588 if (wr->wr.send_flags & IB_SEND_SIGNALED) 2589 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2590 2591 if (access & IB_ACCESS_LOCAL_WRITE) 2592 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; 2593 if (access & IB_ACCESS_REMOTE_READ) 2594 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; 2595 if (access & IB_ACCESS_REMOTE_WRITE) 2596 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; 2597 if (access & IB_ACCESS_REMOTE_ATOMIC) 2598 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; 2599 if (access & IB_ACCESS_MW_BIND) 2600 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; 2601 2602 wqe->frmr.l_key = wr->key; 2603 wqe->frmr.length = wr->mr->length; 2604 wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K); 2605 wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K); 2606 wqe->frmr.va = wr->mr->iova; 2607 return 0; 2608 } 2609 2610 static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev, 2611 const struct ib_send_wr *wr, 2612 struct bnxt_qplib_swqe *wqe) 2613 { 2614 /* Copy the inline data to the data field */ 2615 u8 *in_data; 2616 u32 i, sge_len; 2617 void *sge_addr; 2618 2619 in_data = wqe->inline_data; 2620 for (i = 0; i < wr->num_sge; i++) { 2621 sge_addr = (void *)(unsigned long) 2622 wr->sg_list[i].addr; 2623 sge_len = wr->sg_list[i].length; 2624 2625 if ((sge_len + wqe->inline_len) > 2626 BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { 2627 ibdev_err(&rdev->ibdev, 2628 "Inline data size requested > supported value"); 2629 return -EINVAL; 2630 } 2631 sge_len = wr->sg_list[i].length; 2632 2633 memcpy(in_data, sge_addr, sge_len); 2634 in_data += wr->sg_list[i].length; 2635 wqe->inline_len += wr->sg_list[i].length; 2636 } 2637 return wqe->inline_len; 2638 } 2639 2640 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev, 2641 const struct ib_send_wr *wr, 2642 struct bnxt_qplib_swqe *wqe) 2643 { 2644 int payload_sz = 0; 2645 2646 if (wr->send_flags & IB_SEND_INLINE) 2647 payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe); 2648 else 2649 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list, 2650 wqe->num_sge); 2651 2652 return payload_sz; 2653 } 2654 2655 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp) 2656 { 2657 if ((qp->ib_qp.qp_type == IB_QPT_UD || 2658 qp->ib_qp.qp_type == IB_QPT_GSI || 2659 qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) && 2660 qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) { 2661 int qp_attr_mask; 2662 struct ib_qp_attr qp_attr; 2663 2664 qp_attr_mask = IB_QP_STATE; 2665 qp_attr.qp_state = IB_QPS_RTS; 2666 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL); 2667 qp->qplib_qp.wqe_cnt = 0; 2668 } 2669 } 2670 2671 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, 2672 struct bnxt_re_qp *qp, 2673 const struct ib_send_wr *wr) 2674 { 2675 int rc = 0, payload_sz = 0; 2676 unsigned long flags; 2677 2678 spin_lock_irqsave(&qp->sq_lock, flags); 2679 while (wr) { 2680 struct bnxt_qplib_swqe wqe = {}; 2681 2682 /* Common */ 2683 wqe.num_sge = wr->num_sge; 2684 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 2685 ibdev_err(&rdev->ibdev, 2686 "Limit exceeded for Send SGEs"); 2687 rc = -EINVAL; 2688 goto bad; 2689 } 2690 2691 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); 2692 if (payload_sz < 0) { 2693 rc = -EINVAL; 2694 goto bad; 2695 } 2696 wqe.wr_id = wr->wr_id; 2697 2698 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; 2699 2700 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 2701 if (!rc) 2702 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 2703 bad: 2704 if (rc) { 2705 ibdev_err(&rdev->ibdev, 2706 "Post send failed opcode = %#x rc = %d", 2707 wr->opcode, rc); 2708 break; 2709 } 2710 wr = wr->next; 2711 } 2712 bnxt_qplib_post_send_db(&qp->qplib_qp); 2713 bnxt_ud_qp_hw_stall_workaround(qp); 2714 spin_unlock_irqrestore(&qp->sq_lock, flags); 2715 return rc; 2716 } 2717 2718 static void bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe *wqe) 2719 { 2720 /* Need unconditional fence for non-wire memory opcode 2721 * to work as expected. 2722 */ 2723 if (wqe->type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV || 2724 wqe->type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR || 2725 wqe->type == BNXT_QPLIB_SWQE_TYPE_REG_MR || 2726 wqe->type == BNXT_QPLIB_SWQE_TYPE_BIND_MW) 2727 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2728 } 2729 2730 int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, 2731 const struct ib_send_wr **bad_wr) 2732 { 2733 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 2734 struct bnxt_qplib_swqe wqe; 2735 int rc = 0, payload_sz = 0; 2736 unsigned long flags; 2737 2738 spin_lock_irqsave(&qp->sq_lock, flags); 2739 while (wr) { 2740 /* House keeping */ 2741 memset(&wqe, 0, sizeof(wqe)); 2742 2743 /* Common */ 2744 wqe.num_sge = wr->num_sge; 2745 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 2746 ibdev_err(&qp->rdev->ibdev, 2747 "Limit exceeded for Send SGEs"); 2748 rc = -EINVAL; 2749 goto bad; 2750 } 2751 2752 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); 2753 if (payload_sz < 0) { 2754 rc = -EINVAL; 2755 goto bad; 2756 } 2757 wqe.wr_id = wr->wr_id; 2758 2759 switch (wr->opcode) { 2760 case IB_WR_SEND: 2761 case IB_WR_SEND_WITH_IMM: 2762 if (qp->qplib_qp.type == CMDQ_CREATE_QP1_TYPE_GSI) { 2763 rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe, 2764 payload_sz); 2765 if (rc) 2766 goto bad; 2767 wqe.rawqp1.lflags |= 2768 SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; 2769 } 2770 switch (wr->send_flags) { 2771 case IB_SEND_IP_CSUM: 2772 wqe.rawqp1.lflags |= 2773 SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; 2774 break; 2775 default: 2776 break; 2777 } 2778 fallthrough; 2779 case IB_WR_SEND_WITH_INV: 2780 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 2781 break; 2782 case IB_WR_RDMA_WRITE: 2783 case IB_WR_RDMA_WRITE_WITH_IMM: 2784 case IB_WR_RDMA_READ: 2785 rc = bnxt_re_build_rdma_wqe(wr, &wqe); 2786 break; 2787 case IB_WR_ATOMIC_CMP_AND_SWP: 2788 case IB_WR_ATOMIC_FETCH_AND_ADD: 2789 rc = bnxt_re_build_atomic_wqe(wr, &wqe); 2790 break; 2791 case IB_WR_RDMA_READ_WITH_INV: 2792 ibdev_err(&qp->rdev->ibdev, 2793 "RDMA Read with Invalidate is not supported"); 2794 rc = -EINVAL; 2795 goto bad; 2796 case IB_WR_LOCAL_INV: 2797 rc = bnxt_re_build_inv_wqe(wr, &wqe); 2798 break; 2799 case IB_WR_REG_MR: 2800 rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); 2801 break; 2802 default: 2803 /* Unsupported WRs */ 2804 ibdev_err(&qp->rdev->ibdev, 2805 "WR (%#x) is not supported", wr->opcode); 2806 rc = -EINVAL; 2807 goto bad; 2808 } 2809 if (!rc) { 2810 if (!bnxt_qplib_is_chip_gen_p5_p7(qp->rdev->chip_ctx)) 2811 bnxt_re_legacy_set_uc_fence(&wqe); 2812 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 2813 } 2814 bad: 2815 if (rc) { 2816 ibdev_err(&qp->rdev->ibdev, 2817 "post_send failed op:%#x qps = %#x rc = %d\n", 2818 wr->opcode, qp->qplib_qp.state, rc); 2819 *bad_wr = wr; 2820 break; 2821 } 2822 wr = wr->next; 2823 } 2824 bnxt_qplib_post_send_db(&qp->qplib_qp); 2825 bnxt_ud_qp_hw_stall_workaround(qp); 2826 spin_unlock_irqrestore(&qp->sq_lock, flags); 2827 2828 return rc; 2829 } 2830 2831 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, 2832 struct bnxt_re_qp *qp, 2833 const struct ib_recv_wr *wr) 2834 { 2835 struct bnxt_qplib_swqe wqe; 2836 int rc = 0; 2837 2838 while (wr) { 2839 /* House keeping */ 2840 memset(&wqe, 0, sizeof(wqe)); 2841 2842 /* Common */ 2843 wqe.num_sge = wr->num_sge; 2844 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 2845 ibdev_err(&rdev->ibdev, 2846 "Limit exceeded for Receive SGEs"); 2847 rc = -EINVAL; 2848 break; 2849 } 2850 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); 2851 wqe.wr_id = wr->wr_id; 2852 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 2853 2854 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 2855 if (rc) 2856 break; 2857 2858 wr = wr->next; 2859 } 2860 if (!rc) 2861 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2862 return rc; 2863 } 2864 2865 int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, 2866 const struct ib_recv_wr **bad_wr) 2867 { 2868 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 2869 struct bnxt_qplib_swqe wqe; 2870 int rc = 0, payload_sz = 0; 2871 unsigned long flags; 2872 u32 count = 0; 2873 2874 spin_lock_irqsave(&qp->rq_lock, flags); 2875 while (wr) { 2876 /* House keeping */ 2877 memset(&wqe, 0, sizeof(wqe)); 2878 2879 /* Common */ 2880 wqe.num_sge = wr->num_sge; 2881 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 2882 ibdev_err(&qp->rdev->ibdev, 2883 "Limit exceeded for Receive SGEs"); 2884 rc = -EINVAL; 2885 *bad_wr = wr; 2886 break; 2887 } 2888 2889 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, 2890 wr->num_sge); 2891 wqe.wr_id = wr->wr_id; 2892 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 2893 2894 if (ib_qp->qp_type == IB_QPT_GSI && 2895 qp->qplib_qp.type != CMDQ_CREATE_QP_TYPE_GSI) 2896 rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe, 2897 payload_sz); 2898 if (!rc) 2899 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 2900 if (rc) { 2901 *bad_wr = wr; 2902 break; 2903 } 2904 2905 /* Ring DB if the RQEs posted reaches a threshold value */ 2906 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) { 2907 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2908 count = 0; 2909 } 2910 2911 wr = wr->next; 2912 } 2913 2914 if (count) 2915 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2916 2917 spin_unlock_irqrestore(&qp->rq_lock, flags); 2918 2919 return rc; 2920 } 2921 2922 /* Completion Queues */ 2923 int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 2924 { 2925 struct bnxt_re_cq *cq; 2926 struct bnxt_qplib_nq *nq; 2927 struct bnxt_re_dev *rdev; 2928 2929 cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 2930 rdev = cq->rdev; 2931 nq = cq->qplib_cq.nq; 2932 2933 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); 2934 ib_umem_release(cq->umem); 2935 2936 atomic_dec(&rdev->stats.res.cq_count); 2937 nq->budget--; 2938 kfree(cq->cql); 2939 return 0; 2940 } 2941 2942 int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 2943 struct ib_udata *udata) 2944 { 2945 struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq); 2946 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev); 2947 struct bnxt_re_ucontext *uctx = 2948 rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx); 2949 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 2950 int rc, entries; 2951 int cqe = attr->cqe; 2952 struct bnxt_qplib_nq *nq = NULL; 2953 unsigned int nq_alloc_cnt; 2954 u32 active_cqs; 2955 2956 if (attr->flags) 2957 return -EOPNOTSUPP; 2958 2959 /* Validate CQ fields */ 2960 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { 2961 ibdev_err(&rdev->ibdev, "Failed to create CQ -max exceeded"); 2962 return -EINVAL; 2963 } 2964 2965 cq->rdev = rdev; 2966 cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq); 2967 2968 entries = bnxt_re_init_depth(cqe + 1, uctx); 2969 if (entries > dev_attr->max_cq_wqes + 1) 2970 entries = dev_attr->max_cq_wqes + 1; 2971 2972 cq->qplib_cq.sg_info.pgsize = PAGE_SIZE; 2973 cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT; 2974 if (udata) { 2975 struct bnxt_re_cq_req req; 2976 if (ib_copy_from_udata(&req, udata, sizeof(req))) { 2977 rc = -EFAULT; 2978 goto fail; 2979 } 2980 2981 cq->umem = ib_umem_get(&rdev->ibdev, req.cq_va, 2982 entries * sizeof(struct cq_base), 2983 IB_ACCESS_LOCAL_WRITE); 2984 if (IS_ERR(cq->umem)) { 2985 rc = PTR_ERR(cq->umem); 2986 goto fail; 2987 } 2988 cq->qplib_cq.sg_info.umem = cq->umem; 2989 cq->qplib_cq.dpi = &uctx->dpi; 2990 } else { 2991 cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL); 2992 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), 2993 GFP_KERNEL); 2994 if (!cq->cql) { 2995 rc = -ENOMEM; 2996 goto fail; 2997 } 2998 2999 cq->qplib_cq.dpi = &rdev->dpi_privileged; 3000 } 3001 /* 3002 * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a 3003 * used for getting the NQ index. 3004 */ 3005 nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt); 3006 nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)]; 3007 cq->qplib_cq.max_wqe = entries; 3008 cq->qplib_cq.cnq_hw_ring_id = nq->ring_id; 3009 cq->qplib_cq.nq = nq; 3010 3011 rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq); 3012 if (rc) { 3013 ibdev_err(&rdev->ibdev, "Failed to create HW CQ"); 3014 goto fail; 3015 } 3016 3017 cq->ib_cq.cqe = entries; 3018 cq->cq_period = cq->qplib_cq.period; 3019 nq->budget++; 3020 3021 active_cqs = atomic_inc_return(&rdev->stats.res.cq_count); 3022 if (active_cqs > rdev->stats.res.cq_watermark) 3023 rdev->stats.res.cq_watermark = active_cqs; 3024 spin_lock_init(&cq->cq_lock); 3025 3026 if (udata) { 3027 struct bnxt_re_cq_resp resp; 3028 3029 resp.cqid = cq->qplib_cq.id; 3030 resp.tail = cq->qplib_cq.hwq.cons; 3031 resp.phase = cq->qplib_cq.period; 3032 resp.rsvd = 0; 3033 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 3034 if (rc) { 3035 ibdev_err(&rdev->ibdev, "Failed to copy CQ udata"); 3036 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); 3037 goto c2fail; 3038 } 3039 } 3040 3041 return 0; 3042 3043 c2fail: 3044 ib_umem_release(cq->umem); 3045 fail: 3046 kfree(cq->cql); 3047 return rc; 3048 } 3049 3050 static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq) 3051 { 3052 struct bnxt_re_dev *rdev = cq->rdev; 3053 3054 bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq); 3055 3056 cq->qplib_cq.max_wqe = cq->resize_cqe; 3057 if (cq->resize_umem) { 3058 ib_umem_release(cq->umem); 3059 cq->umem = cq->resize_umem; 3060 cq->resize_umem = NULL; 3061 cq->resize_cqe = 0; 3062 } 3063 } 3064 3065 int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata) 3066 { 3067 struct bnxt_qplib_sg_info sg_info = {}; 3068 struct bnxt_qplib_dpi *orig_dpi = NULL; 3069 struct bnxt_qplib_dev_attr *dev_attr; 3070 struct bnxt_re_ucontext *uctx = NULL; 3071 struct bnxt_re_resize_cq_req req; 3072 struct bnxt_re_dev *rdev; 3073 struct bnxt_re_cq *cq; 3074 int rc, entries; 3075 3076 cq = container_of(ibcq, struct bnxt_re_cq, ib_cq); 3077 rdev = cq->rdev; 3078 dev_attr = &rdev->dev_attr; 3079 if (!ibcq->uobject) { 3080 ibdev_err(&rdev->ibdev, "Kernel CQ Resize not supported"); 3081 return -EOPNOTSUPP; 3082 } 3083 3084 if (cq->resize_umem) { 3085 ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - Busy", 3086 cq->qplib_cq.id); 3087 return -EBUSY; 3088 } 3089 3090 /* Check the requested cq depth out of supported depth */ 3091 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { 3092 ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - out of range cqe %d", 3093 cq->qplib_cq.id, cqe); 3094 return -EINVAL; 3095 } 3096 3097 uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx); 3098 entries = bnxt_re_init_depth(cqe + 1, uctx); 3099 if (entries > dev_attr->max_cq_wqes + 1) 3100 entries = dev_attr->max_cq_wqes + 1; 3101 3102 /* uverbs consumer */ 3103 if (ib_copy_from_udata(&req, udata, sizeof(req))) { 3104 rc = -EFAULT; 3105 goto fail; 3106 } 3107 3108 cq->resize_umem = ib_umem_get(&rdev->ibdev, req.cq_va, 3109 entries * sizeof(struct cq_base), 3110 IB_ACCESS_LOCAL_WRITE); 3111 if (IS_ERR(cq->resize_umem)) { 3112 rc = PTR_ERR(cq->resize_umem); 3113 cq->resize_umem = NULL; 3114 ibdev_err(&rdev->ibdev, "%s: ib_umem_get failed! rc = %d\n", 3115 __func__, rc); 3116 goto fail; 3117 } 3118 cq->resize_cqe = entries; 3119 memcpy(&sg_info, &cq->qplib_cq.sg_info, sizeof(sg_info)); 3120 orig_dpi = cq->qplib_cq.dpi; 3121 3122 cq->qplib_cq.sg_info.umem = cq->resize_umem; 3123 cq->qplib_cq.sg_info.pgsize = PAGE_SIZE; 3124 cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT; 3125 cq->qplib_cq.dpi = &uctx->dpi; 3126 3127 rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries); 3128 if (rc) { 3129 ibdev_err(&rdev->ibdev, "Resize HW CQ %#x failed!", 3130 cq->qplib_cq.id); 3131 goto fail; 3132 } 3133 3134 cq->ib_cq.cqe = cq->resize_cqe; 3135 atomic_inc(&rdev->stats.res.resize_count); 3136 3137 return 0; 3138 3139 fail: 3140 if (cq->resize_umem) { 3141 ib_umem_release(cq->resize_umem); 3142 cq->resize_umem = NULL; 3143 cq->resize_cqe = 0; 3144 memcpy(&cq->qplib_cq.sg_info, &sg_info, sizeof(sg_info)); 3145 cq->qplib_cq.dpi = orig_dpi; 3146 } 3147 return rc; 3148 } 3149 3150 static u8 __req_to_ib_wc_status(u8 qstatus) 3151 { 3152 switch (qstatus) { 3153 case CQ_REQ_STATUS_OK: 3154 return IB_WC_SUCCESS; 3155 case CQ_REQ_STATUS_BAD_RESPONSE_ERR: 3156 return IB_WC_BAD_RESP_ERR; 3157 case CQ_REQ_STATUS_LOCAL_LENGTH_ERR: 3158 return IB_WC_LOC_LEN_ERR; 3159 case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR: 3160 return IB_WC_LOC_QP_OP_ERR; 3161 case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR: 3162 return IB_WC_LOC_PROT_ERR; 3163 case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR: 3164 return IB_WC_GENERAL_ERR; 3165 case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR: 3166 return IB_WC_REM_INV_REQ_ERR; 3167 case CQ_REQ_STATUS_REMOTE_ACCESS_ERR: 3168 return IB_WC_REM_ACCESS_ERR; 3169 case CQ_REQ_STATUS_REMOTE_OPERATION_ERR: 3170 return IB_WC_REM_OP_ERR; 3171 case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR: 3172 return IB_WC_RNR_RETRY_EXC_ERR; 3173 case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR: 3174 return IB_WC_RETRY_EXC_ERR; 3175 case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR: 3176 return IB_WC_WR_FLUSH_ERR; 3177 default: 3178 return IB_WC_GENERAL_ERR; 3179 } 3180 return 0; 3181 } 3182 3183 static u8 __rawqp1_to_ib_wc_status(u8 qstatus) 3184 { 3185 switch (qstatus) { 3186 case CQ_RES_RAWETH_QP1_STATUS_OK: 3187 return IB_WC_SUCCESS; 3188 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR: 3189 return IB_WC_LOC_ACCESS_ERR; 3190 case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR: 3191 return IB_WC_LOC_LEN_ERR; 3192 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR: 3193 return IB_WC_LOC_PROT_ERR; 3194 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR: 3195 return IB_WC_LOC_QP_OP_ERR; 3196 case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR: 3197 return IB_WC_GENERAL_ERR; 3198 case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR: 3199 return IB_WC_WR_FLUSH_ERR; 3200 case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR: 3201 return IB_WC_WR_FLUSH_ERR; 3202 default: 3203 return IB_WC_GENERAL_ERR; 3204 } 3205 } 3206 3207 static u8 __rc_to_ib_wc_status(u8 qstatus) 3208 { 3209 switch (qstatus) { 3210 case CQ_RES_RC_STATUS_OK: 3211 return IB_WC_SUCCESS; 3212 case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR: 3213 return IB_WC_LOC_ACCESS_ERR; 3214 case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR: 3215 return IB_WC_LOC_LEN_ERR; 3216 case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR: 3217 return IB_WC_LOC_PROT_ERR; 3218 case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR: 3219 return IB_WC_LOC_QP_OP_ERR; 3220 case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR: 3221 return IB_WC_GENERAL_ERR; 3222 case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR: 3223 return IB_WC_REM_INV_REQ_ERR; 3224 case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR: 3225 return IB_WC_WR_FLUSH_ERR; 3226 case CQ_RES_RC_STATUS_HW_FLUSH_ERR: 3227 return IB_WC_WR_FLUSH_ERR; 3228 default: 3229 return IB_WC_GENERAL_ERR; 3230 } 3231 } 3232 3233 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) 3234 { 3235 switch (cqe->type) { 3236 case BNXT_QPLIB_SWQE_TYPE_SEND: 3237 wc->opcode = IB_WC_SEND; 3238 break; 3239 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: 3240 wc->opcode = IB_WC_SEND; 3241 wc->wc_flags |= IB_WC_WITH_IMM; 3242 break; 3243 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: 3244 wc->opcode = IB_WC_SEND; 3245 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3246 break; 3247 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: 3248 wc->opcode = IB_WC_RDMA_WRITE; 3249 break; 3250 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: 3251 wc->opcode = IB_WC_RDMA_WRITE; 3252 wc->wc_flags |= IB_WC_WITH_IMM; 3253 break; 3254 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: 3255 wc->opcode = IB_WC_RDMA_READ; 3256 break; 3257 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: 3258 wc->opcode = IB_WC_COMP_SWAP; 3259 break; 3260 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: 3261 wc->opcode = IB_WC_FETCH_ADD; 3262 break; 3263 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: 3264 wc->opcode = IB_WC_LOCAL_INV; 3265 break; 3266 case BNXT_QPLIB_SWQE_TYPE_REG_MR: 3267 wc->opcode = IB_WC_REG_MR; 3268 break; 3269 default: 3270 wc->opcode = IB_WC_SEND; 3271 break; 3272 } 3273 3274 wc->status = __req_to_ib_wc_status(cqe->status); 3275 } 3276 3277 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, 3278 u16 raweth_qp1_flags2) 3279 { 3280 bool is_ipv6 = false, is_ipv4 = false; 3281 3282 /* raweth_qp1_flags Bit 9-6 indicates itype */ 3283 if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 3284 != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 3285 return -1; 3286 3287 if (raweth_qp1_flags2 & 3288 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC && 3289 raweth_qp1_flags2 & 3290 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) { 3291 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ 3292 (raweth_qp1_flags2 & 3293 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ? 3294 (is_ipv6 = true) : (is_ipv4 = true); 3295 return ((is_ipv6) ? 3296 BNXT_RE_ROCEV2_IPV6_PACKET : 3297 BNXT_RE_ROCEV2_IPV4_PACKET); 3298 } else { 3299 return BNXT_RE_ROCE_V1_PACKET; 3300 } 3301 } 3302 3303 static int bnxt_re_to_ib_nw_type(int nw_type) 3304 { 3305 u8 nw_hdr_type = 0xFF; 3306 3307 switch (nw_type) { 3308 case BNXT_RE_ROCE_V1_PACKET: 3309 nw_hdr_type = RDMA_NETWORK_ROCE_V1; 3310 break; 3311 case BNXT_RE_ROCEV2_IPV4_PACKET: 3312 nw_hdr_type = RDMA_NETWORK_IPV4; 3313 break; 3314 case BNXT_RE_ROCEV2_IPV6_PACKET: 3315 nw_hdr_type = RDMA_NETWORK_IPV6; 3316 break; 3317 } 3318 return nw_hdr_type; 3319 } 3320 3321 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev, 3322 void *rq_hdr_buf) 3323 { 3324 u8 *tmp_buf = NULL; 3325 struct ethhdr *eth_hdr; 3326 u16 eth_type; 3327 bool rc = false; 3328 3329 tmp_buf = (u8 *)rq_hdr_buf; 3330 /* 3331 * If dest mac is not same as I/F mac, this could be a 3332 * loopback address or multicast address, check whether 3333 * it is a loopback packet 3334 */ 3335 if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) { 3336 tmp_buf += 4; 3337 /* Check the ether type */ 3338 eth_hdr = (struct ethhdr *)tmp_buf; 3339 eth_type = ntohs(eth_hdr->h_proto); 3340 switch (eth_type) { 3341 case ETH_P_IBOE: 3342 rc = true; 3343 break; 3344 case ETH_P_IP: 3345 case ETH_P_IPV6: { 3346 u32 len; 3347 struct udphdr *udp_hdr; 3348 3349 len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) : 3350 sizeof(struct ipv6hdr)); 3351 tmp_buf += sizeof(struct ethhdr) + len; 3352 udp_hdr = (struct udphdr *)tmp_buf; 3353 if (ntohs(udp_hdr->dest) == 3354 ROCE_V2_UDP_DPORT) 3355 rc = true; 3356 break; 3357 } 3358 default: 3359 break; 3360 } 3361 } 3362 3363 return rc; 3364 } 3365 3366 static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *gsi_qp, 3367 struct bnxt_qplib_cqe *cqe) 3368 { 3369 struct bnxt_re_dev *rdev = gsi_qp->rdev; 3370 struct bnxt_re_sqp_entries *sqp_entry = NULL; 3371 struct bnxt_re_qp *gsi_sqp = rdev->gsi_ctx.gsi_sqp; 3372 dma_addr_t shrq_hdr_buf_map; 3373 struct ib_sge s_sge[2] = {}; 3374 struct ib_sge r_sge[2] = {}; 3375 struct bnxt_re_ah *gsi_sah; 3376 struct ib_recv_wr rwr = {}; 3377 dma_addr_t rq_hdr_buf_map; 3378 struct ib_ud_wr udwr = {}; 3379 struct ib_send_wr *swr; 3380 u32 skip_bytes = 0; 3381 int pkt_type = 0; 3382 void *rq_hdr_buf; 3383 u32 offset = 0; 3384 u32 tbl_idx; 3385 int rc; 3386 3387 swr = &udwr.wr; 3388 tbl_idx = cqe->wr_id; 3389 3390 rq_hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf + 3391 (tbl_idx * gsi_qp->qplib_qp.rq_hdr_buf_size); 3392 rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp, 3393 tbl_idx); 3394 3395 /* Shadow QP header buffer */ 3396 shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp, 3397 tbl_idx); 3398 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; 3399 3400 /* Store this cqe */ 3401 memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); 3402 sqp_entry->qp1_qp = gsi_qp; 3403 3404 /* Find packet type from the cqe */ 3405 3406 pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, 3407 cqe->raweth_qp1_flags2); 3408 if (pkt_type < 0) { 3409 ibdev_err(&rdev->ibdev, "Invalid packet\n"); 3410 return -EINVAL; 3411 } 3412 3413 /* Adjust the offset for the user buffer and post in the rq */ 3414 3415 if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET) 3416 offset = 20; 3417 3418 /* 3419 * QP1 loopback packet has 4 bytes of internal header before 3420 * ether header. Skip these four bytes. 3421 */ 3422 if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf)) 3423 skip_bytes = 4; 3424 3425 /* First send SGE . Skip the ether header*/ 3426 s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 3427 + skip_bytes; 3428 s_sge[0].lkey = 0xFFFFFFFF; 3429 s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 : 3430 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; 3431 3432 /* Second Send SGE */ 3433 s_sge[1].addr = s_sge[0].addr + s_sge[0].length + 3434 BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE; 3435 if (pkt_type != BNXT_RE_ROCE_V1_PACKET) 3436 s_sge[1].addr += 8; 3437 s_sge[1].lkey = 0xFFFFFFFF; 3438 s_sge[1].length = 256; 3439 3440 /* First recv SGE */ 3441 3442 r_sge[0].addr = shrq_hdr_buf_map; 3443 r_sge[0].lkey = 0xFFFFFFFF; 3444 r_sge[0].length = 40; 3445 3446 r_sge[1].addr = sqp_entry->sge.addr + offset; 3447 r_sge[1].lkey = sqp_entry->sge.lkey; 3448 r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; 3449 3450 /* Create receive work request */ 3451 rwr.num_sge = 2; 3452 rwr.sg_list = r_sge; 3453 rwr.wr_id = tbl_idx; 3454 rwr.next = NULL; 3455 3456 rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr); 3457 if (rc) { 3458 ibdev_err(&rdev->ibdev, 3459 "Failed to post Rx buffers to shadow QP"); 3460 return -ENOMEM; 3461 } 3462 3463 swr->num_sge = 2; 3464 swr->sg_list = s_sge; 3465 swr->wr_id = tbl_idx; 3466 swr->opcode = IB_WR_SEND; 3467 swr->next = NULL; 3468 gsi_sah = rdev->gsi_ctx.gsi_sah; 3469 udwr.ah = &gsi_sah->ib_ah; 3470 udwr.remote_qpn = gsi_sqp->qplib_qp.id; 3471 udwr.remote_qkey = gsi_sqp->qplib_qp.qkey; 3472 3473 /* post data received in the send queue */ 3474 return bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr); 3475 } 3476 3477 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc, 3478 struct bnxt_qplib_cqe *cqe) 3479 { 3480 wc->opcode = IB_WC_RECV; 3481 wc->status = __rawqp1_to_ib_wc_status(cqe->status); 3482 wc->wc_flags |= IB_WC_GRH; 3483 } 3484 3485 static bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev, 3486 u16 vlan_id) 3487 { 3488 /* 3489 * Check if the vlan is configured in the host. If not configured, it 3490 * can be a transparent VLAN. So dont report the vlan id. 3491 */ 3492 if (!__vlan_find_dev_deep_rcu(rdev->netdev, 3493 htons(ETH_P_8021Q), vlan_id)) 3494 return false; 3495 return true; 3496 } 3497 3498 static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe, 3499 u16 *vid, u8 *sl) 3500 { 3501 bool ret = false; 3502 u32 metadata; 3503 u16 tpid; 3504 3505 metadata = orig_cqe->raweth_qp1_metadata; 3506 if (orig_cqe->raweth_qp1_flags2 & 3507 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) { 3508 tpid = ((metadata & 3509 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >> 3510 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); 3511 if (tpid == ETH_P_8021Q) { 3512 *vid = metadata & 3513 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK; 3514 *sl = (metadata & 3515 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >> 3516 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT; 3517 ret = true; 3518 } 3519 } 3520 3521 return ret; 3522 } 3523 3524 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc, 3525 struct bnxt_qplib_cqe *cqe) 3526 { 3527 wc->opcode = IB_WC_RECV; 3528 wc->status = __rc_to_ib_wc_status(cqe->status); 3529 3530 if (cqe->flags & CQ_RES_RC_FLAGS_IMM) 3531 wc->wc_flags |= IB_WC_WITH_IMM; 3532 if (cqe->flags & CQ_RES_RC_FLAGS_INV) 3533 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3534 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == 3535 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) 3536 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 3537 } 3538 3539 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp, 3540 struct ib_wc *wc, 3541 struct bnxt_qplib_cqe *cqe) 3542 { 3543 struct bnxt_re_dev *rdev = gsi_sqp->rdev; 3544 struct bnxt_re_qp *gsi_qp = NULL; 3545 struct bnxt_qplib_cqe *orig_cqe = NULL; 3546 struct bnxt_re_sqp_entries *sqp_entry = NULL; 3547 int nw_type; 3548 u32 tbl_idx; 3549 u16 vlan_id; 3550 u8 sl; 3551 3552 tbl_idx = cqe->wr_id; 3553 3554 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; 3555 gsi_qp = sqp_entry->qp1_qp; 3556 orig_cqe = &sqp_entry->cqe; 3557 3558 wc->wr_id = sqp_entry->wrid; 3559 wc->byte_len = orig_cqe->length; 3560 wc->qp = &gsi_qp->ib_qp; 3561 3562 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(orig_cqe->immdata)); 3563 wc->src_qp = orig_cqe->src_qp; 3564 memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); 3565 if (bnxt_re_is_vlan_pkt(orig_cqe, &vlan_id, &sl)) { 3566 if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { 3567 wc->vlan_id = vlan_id; 3568 wc->sl = sl; 3569 wc->wc_flags |= IB_WC_WITH_VLAN; 3570 } 3571 } 3572 wc->port_num = 1; 3573 wc->vendor_err = orig_cqe->status; 3574 3575 wc->opcode = IB_WC_RECV; 3576 wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); 3577 wc->wc_flags |= IB_WC_GRH; 3578 3579 nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, 3580 orig_cqe->raweth_qp1_flags2); 3581 if (nw_type >= 0) { 3582 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type); 3583 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 3584 } 3585 } 3586 3587 static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp, 3588 struct ib_wc *wc, 3589 struct bnxt_qplib_cqe *cqe) 3590 { 3591 struct bnxt_re_dev *rdev; 3592 u16 vlan_id = 0; 3593 u8 nw_type; 3594 3595 rdev = qp->rdev; 3596 wc->opcode = IB_WC_RECV; 3597 wc->status = __rc_to_ib_wc_status(cqe->status); 3598 3599 if (cqe->flags & CQ_RES_UD_FLAGS_IMM) 3600 wc->wc_flags |= IB_WC_WITH_IMM; 3601 /* report only on GSI QP for Thor */ 3602 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI) { 3603 wc->wc_flags |= IB_WC_GRH; 3604 memcpy(wc->smac, cqe->smac, ETH_ALEN); 3605 wc->wc_flags |= IB_WC_WITH_SMAC; 3606 if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) { 3607 vlan_id = (cqe->cfa_meta & 0xFFF); 3608 } 3609 /* Mark only if vlan_id is non zero */ 3610 if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { 3611 wc->vlan_id = vlan_id; 3612 wc->wc_flags |= IB_WC_WITH_VLAN; 3613 } 3614 nw_type = (cqe->flags & CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK) >> 3615 CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT; 3616 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type); 3617 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 3618 } 3619 3620 } 3621 3622 static int send_phantom_wqe(struct bnxt_re_qp *qp) 3623 { 3624 struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp; 3625 unsigned long flags; 3626 int rc; 3627 3628 spin_lock_irqsave(&qp->sq_lock, flags); 3629 3630 rc = bnxt_re_bind_fence_mw(lib_qp); 3631 if (!rc) { 3632 lib_qp->sq.phantom_wqe_cnt++; 3633 ibdev_dbg(&qp->rdev->ibdev, 3634 "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n", 3635 lib_qp->id, lib_qp->sq.hwq.prod, 3636 HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq), 3637 lib_qp->sq.phantom_wqe_cnt); 3638 } 3639 3640 spin_unlock_irqrestore(&qp->sq_lock, flags); 3641 return rc; 3642 } 3643 3644 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) 3645 { 3646 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 3647 struct bnxt_re_qp *qp, *sh_qp; 3648 struct bnxt_qplib_cqe *cqe; 3649 int i, ncqe, budget; 3650 struct bnxt_qplib_q *sq; 3651 struct bnxt_qplib_qp *lib_qp; 3652 u32 tbl_idx; 3653 struct bnxt_re_sqp_entries *sqp_entry = NULL; 3654 unsigned long flags; 3655 3656 /* User CQ; the only processing we do is to 3657 * complete any pending CQ resize operation. 3658 */ 3659 if (cq->umem) { 3660 if (cq->resize_umem) 3661 bnxt_re_resize_cq_complete(cq); 3662 return 0; 3663 } 3664 3665 spin_lock_irqsave(&cq->cq_lock, flags); 3666 budget = min_t(u32, num_entries, cq->max_cql); 3667 num_entries = budget; 3668 if (!cq->cql) { 3669 ibdev_err(&cq->rdev->ibdev, "POLL CQ : no CQL to use"); 3670 goto exit; 3671 } 3672 cqe = &cq->cql[0]; 3673 while (budget) { 3674 lib_qp = NULL; 3675 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp); 3676 if (lib_qp) { 3677 sq = &lib_qp->sq; 3678 if (sq->send_phantom) { 3679 qp = container_of(lib_qp, 3680 struct bnxt_re_qp, qplib_qp); 3681 if (send_phantom_wqe(qp) == -ENOMEM) 3682 ibdev_err(&cq->rdev->ibdev, 3683 "Phantom failed! Scheduled to send again\n"); 3684 else 3685 sq->send_phantom = false; 3686 } 3687 } 3688 if (ncqe < budget) 3689 ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq, 3690 cqe + ncqe, 3691 budget - ncqe); 3692 3693 if (!ncqe) 3694 break; 3695 3696 for (i = 0; i < ncqe; i++, cqe++) { 3697 /* Transcribe each qplib_wqe back to ib_wc */ 3698 memset(wc, 0, sizeof(*wc)); 3699 3700 wc->wr_id = cqe->wr_id; 3701 wc->byte_len = cqe->length; 3702 qp = container_of 3703 ((struct bnxt_qplib_qp *) 3704 (unsigned long)(cqe->qp_handle), 3705 struct bnxt_re_qp, qplib_qp); 3706 wc->qp = &qp->ib_qp; 3707 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(cqe->immdata)); 3708 wc->src_qp = cqe->src_qp; 3709 memcpy(wc->smac, cqe->smac, ETH_ALEN); 3710 wc->port_num = 1; 3711 wc->vendor_err = cqe->status; 3712 3713 switch (cqe->opcode) { 3714 case CQ_BASE_CQE_TYPE_REQ: 3715 sh_qp = qp->rdev->gsi_ctx.gsi_sqp; 3716 if (sh_qp && 3717 qp->qplib_qp.id == sh_qp->qplib_qp.id) { 3718 /* Handle this completion with 3719 * the stored completion 3720 */ 3721 memset(wc, 0, sizeof(*wc)); 3722 continue; 3723 } 3724 bnxt_re_process_req_wc(wc, cqe); 3725 break; 3726 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 3727 if (!cqe->status) { 3728 int rc = 0; 3729 3730 rc = bnxt_re_process_raw_qp_pkt_rx 3731 (qp, cqe); 3732 if (!rc) { 3733 memset(wc, 0, sizeof(*wc)); 3734 continue; 3735 } 3736 cqe->status = -1; 3737 } 3738 /* Errors need not be looped back. 3739 * But change the wr_id to the one 3740 * stored in the table 3741 */ 3742 tbl_idx = cqe->wr_id; 3743 sqp_entry = &cq->rdev->gsi_ctx.sqp_tbl[tbl_idx]; 3744 wc->wr_id = sqp_entry->wrid; 3745 bnxt_re_process_res_rawqp1_wc(wc, cqe); 3746 break; 3747 case CQ_BASE_CQE_TYPE_RES_RC: 3748 bnxt_re_process_res_rc_wc(wc, cqe); 3749 break; 3750 case CQ_BASE_CQE_TYPE_RES_UD: 3751 sh_qp = qp->rdev->gsi_ctx.gsi_sqp; 3752 if (sh_qp && 3753 qp->qplib_qp.id == sh_qp->qplib_qp.id) { 3754 /* Handle this completion with 3755 * the stored completion 3756 */ 3757 if (cqe->status) { 3758 continue; 3759 } else { 3760 bnxt_re_process_res_shadow_qp_wc 3761 (qp, wc, cqe); 3762 break; 3763 } 3764 } 3765 bnxt_re_process_res_ud_wc(qp, wc, cqe); 3766 break; 3767 default: 3768 ibdev_err(&cq->rdev->ibdev, 3769 "POLL CQ : type 0x%x not handled", 3770 cqe->opcode); 3771 continue; 3772 } 3773 wc++; 3774 budget--; 3775 } 3776 } 3777 exit: 3778 spin_unlock_irqrestore(&cq->cq_lock, flags); 3779 return num_entries - budget; 3780 } 3781 3782 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq, 3783 enum ib_cq_notify_flags ib_cqn_flags) 3784 { 3785 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 3786 int type = 0, rc = 0; 3787 unsigned long flags; 3788 3789 spin_lock_irqsave(&cq->cq_lock, flags); 3790 /* Trigger on the very next completion */ 3791 if (ib_cqn_flags & IB_CQ_NEXT_COMP) 3792 type = DBC_DBC_TYPE_CQ_ARMALL; 3793 /* Trigger on the next solicited completion */ 3794 else if (ib_cqn_flags & IB_CQ_SOLICITED) 3795 type = DBC_DBC_TYPE_CQ_ARMSE; 3796 3797 /* Poll to see if there are missed events */ 3798 if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) && 3799 !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) { 3800 rc = 1; 3801 goto exit; 3802 } 3803 bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); 3804 3805 exit: 3806 spin_unlock_irqrestore(&cq->cq_lock, flags); 3807 return rc; 3808 } 3809 3810 /* Memory Regions */ 3811 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) 3812 { 3813 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3814 struct bnxt_re_dev *rdev = pd->rdev; 3815 struct bnxt_re_mr *mr; 3816 u32 active_mrs; 3817 int rc; 3818 3819 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3820 if (!mr) 3821 return ERR_PTR(-ENOMEM); 3822 3823 mr->rdev = rdev; 3824 mr->qplib_mr.pd = &pd->qplib_pd; 3825 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 3826 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 3827 3828 /* Allocate and register 0 as the address */ 3829 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3830 if (rc) 3831 goto fail; 3832 3833 mr->qplib_mr.hwq.level = PBL_LVL_MAX; 3834 mr->qplib_mr.total_size = -1; /* Infinte length */ 3835 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 0, 3836 PAGE_SIZE); 3837 if (rc) 3838 goto fail_mr; 3839 3840 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3841 if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | 3842 IB_ACCESS_REMOTE_ATOMIC)) 3843 mr->ib_mr.rkey = mr->ib_mr.lkey; 3844 active_mrs = atomic_inc_return(&rdev->stats.res.mr_count); 3845 if (active_mrs > rdev->stats.res.mr_watermark) 3846 rdev->stats.res.mr_watermark = active_mrs; 3847 3848 return &mr->ib_mr; 3849 3850 fail_mr: 3851 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3852 fail: 3853 kfree(mr); 3854 return ERR_PTR(rc); 3855 } 3856 3857 int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 3858 { 3859 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3860 struct bnxt_re_dev *rdev = mr->rdev; 3861 int rc; 3862 3863 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3864 if (rc) { 3865 ibdev_err(&rdev->ibdev, "Dereg MR failed: %#x\n", rc); 3866 return rc; 3867 } 3868 3869 if (mr->pages) { 3870 rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, 3871 &mr->qplib_frpl); 3872 kfree(mr->pages); 3873 mr->npages = 0; 3874 mr->pages = NULL; 3875 } 3876 ib_umem_release(mr->ib_umem); 3877 3878 kfree(mr); 3879 atomic_dec(&rdev->stats.res.mr_count); 3880 return rc; 3881 } 3882 3883 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) 3884 { 3885 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3886 3887 if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) 3888 return -ENOMEM; 3889 3890 mr->pages[mr->npages++] = addr; 3891 return 0; 3892 } 3893 3894 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, 3895 unsigned int *sg_offset) 3896 { 3897 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3898 3899 mr->npages = 0; 3900 return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page); 3901 } 3902 3903 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, 3904 u32 max_num_sg) 3905 { 3906 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3907 struct bnxt_re_dev *rdev = pd->rdev; 3908 struct bnxt_re_mr *mr = NULL; 3909 u32 active_mrs; 3910 int rc; 3911 3912 if (type != IB_MR_TYPE_MEM_REG) { 3913 ibdev_dbg(&rdev->ibdev, "MR type 0x%x not supported", type); 3914 return ERR_PTR(-EINVAL); 3915 } 3916 if (max_num_sg > MAX_PBL_LVL_1_PGS) 3917 return ERR_PTR(-EINVAL); 3918 3919 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3920 if (!mr) 3921 return ERR_PTR(-ENOMEM); 3922 3923 mr->rdev = rdev; 3924 mr->qplib_mr.pd = &pd->qplib_pd; 3925 mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; 3926 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 3927 3928 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3929 if (rc) 3930 goto bail; 3931 3932 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3933 mr->ib_mr.rkey = mr->ib_mr.lkey; 3934 3935 mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); 3936 if (!mr->pages) { 3937 rc = -ENOMEM; 3938 goto fail; 3939 } 3940 rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, 3941 &mr->qplib_frpl, max_num_sg); 3942 if (rc) { 3943 ibdev_err(&rdev->ibdev, 3944 "Failed to allocate HW FR page list"); 3945 goto fail_mr; 3946 } 3947 3948 active_mrs = atomic_inc_return(&rdev->stats.res.mr_count); 3949 if (active_mrs > rdev->stats.res.mr_watermark) 3950 rdev->stats.res.mr_watermark = active_mrs; 3951 return &mr->ib_mr; 3952 3953 fail_mr: 3954 kfree(mr->pages); 3955 fail: 3956 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3957 bail: 3958 kfree(mr); 3959 return ERR_PTR(rc); 3960 } 3961 3962 struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, 3963 struct ib_udata *udata) 3964 { 3965 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3966 struct bnxt_re_dev *rdev = pd->rdev; 3967 struct bnxt_re_mw *mw; 3968 u32 active_mws; 3969 int rc; 3970 3971 mw = kzalloc(sizeof(*mw), GFP_KERNEL); 3972 if (!mw) 3973 return ERR_PTR(-ENOMEM); 3974 mw->rdev = rdev; 3975 mw->qplib_mw.pd = &pd->qplib_pd; 3976 3977 mw->qplib_mw.type = (type == IB_MW_TYPE_1 ? 3978 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 : 3979 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B); 3980 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw); 3981 if (rc) { 3982 ibdev_err(&rdev->ibdev, "Allocate MW failed!"); 3983 goto fail; 3984 } 3985 mw->ib_mw.rkey = mw->qplib_mw.rkey; 3986 3987 active_mws = atomic_inc_return(&rdev->stats.res.mw_count); 3988 if (active_mws > rdev->stats.res.mw_watermark) 3989 rdev->stats.res.mw_watermark = active_mws; 3990 return &mw->ib_mw; 3991 3992 fail: 3993 kfree(mw); 3994 return ERR_PTR(rc); 3995 } 3996 3997 int bnxt_re_dealloc_mw(struct ib_mw *ib_mw) 3998 { 3999 struct bnxt_re_mw *mw = container_of(ib_mw, struct bnxt_re_mw, ib_mw); 4000 struct bnxt_re_dev *rdev = mw->rdev; 4001 int rc; 4002 4003 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw); 4004 if (rc) { 4005 ibdev_err(&rdev->ibdev, "Free MW failed: %#x\n", rc); 4006 return rc; 4007 } 4008 4009 kfree(mw); 4010 atomic_dec(&rdev->stats.res.mw_count); 4011 return rc; 4012 } 4013 4014 static struct ib_mr *__bnxt_re_user_reg_mr(struct ib_pd *ib_pd, u64 length, u64 virt_addr, 4015 int mr_access_flags, struct ib_umem *umem) 4016 { 4017 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 4018 struct bnxt_re_dev *rdev = pd->rdev; 4019 unsigned long page_size; 4020 struct bnxt_re_mr *mr; 4021 int umem_pgs, rc; 4022 u32 active_mrs; 4023 4024 if (length > BNXT_RE_MAX_MR_SIZE) { 4025 ibdev_err(&rdev->ibdev, "MR Size: %lld > Max supported:%lld\n", 4026 length, BNXT_RE_MAX_MR_SIZE); 4027 return ERR_PTR(-ENOMEM); 4028 } 4029 4030 page_size = ib_umem_find_best_pgsz(umem, BNXT_RE_PAGE_SIZE_SUPPORTED, virt_addr); 4031 if (!page_size) { 4032 ibdev_err(&rdev->ibdev, "umem page size unsupported!"); 4033 return ERR_PTR(-EINVAL); 4034 } 4035 4036 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 4037 if (!mr) 4038 return ERR_PTR(-ENOMEM); 4039 4040 mr->rdev = rdev; 4041 mr->qplib_mr.pd = &pd->qplib_pd; 4042 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 4043 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; 4044 4045 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 4046 if (rc) { 4047 ibdev_err(&rdev->ibdev, "Failed to allocate MR rc = %d", rc); 4048 rc = -EIO; 4049 goto free_mr; 4050 } 4051 /* The fixed portion of the rkey is the same as the lkey */ 4052 mr->ib_mr.rkey = mr->qplib_mr.rkey; 4053 mr->ib_umem = umem; 4054 mr->qplib_mr.va = virt_addr; 4055 mr->qplib_mr.total_size = length; 4056 4057 umem_pgs = ib_umem_num_dma_blocks(umem, page_size); 4058 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, umem, 4059 umem_pgs, page_size); 4060 if (rc) { 4061 ibdev_err(&rdev->ibdev, "Failed to register user MR - rc = %d\n", rc); 4062 rc = -EIO; 4063 goto free_mrw; 4064 } 4065 4066 mr->ib_mr.lkey = mr->qplib_mr.lkey; 4067 mr->ib_mr.rkey = mr->qplib_mr.lkey; 4068 active_mrs = atomic_inc_return(&rdev->stats.res.mr_count); 4069 if (active_mrs > rdev->stats.res.mr_watermark) 4070 rdev->stats.res.mr_watermark = active_mrs; 4071 4072 return &mr->ib_mr; 4073 4074 free_mrw: 4075 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 4076 free_mr: 4077 kfree(mr); 4078 return ERR_PTR(rc); 4079 } 4080 4081 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, 4082 u64 virt_addr, int mr_access_flags, 4083 struct ib_udata *udata) 4084 { 4085 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 4086 struct bnxt_re_dev *rdev = pd->rdev; 4087 struct ib_umem *umem; 4088 struct ib_mr *ib_mr; 4089 4090 umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags); 4091 if (IS_ERR(umem)) 4092 return ERR_CAST(umem); 4093 4094 ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem); 4095 if (IS_ERR(ib_mr)) 4096 ib_umem_release(umem); 4097 return ib_mr; 4098 } 4099 4100 struct ib_mr *bnxt_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start, 4101 u64 length, u64 virt_addr, int fd, 4102 int mr_access_flags, struct ib_udata *udata) 4103 { 4104 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 4105 struct bnxt_re_dev *rdev = pd->rdev; 4106 struct ib_umem_dmabuf *umem_dmabuf; 4107 struct ib_umem *umem; 4108 struct ib_mr *ib_mr; 4109 4110 umem_dmabuf = ib_umem_dmabuf_get_pinned(&rdev->ibdev, start, length, 4111 fd, mr_access_flags); 4112 if (IS_ERR(umem_dmabuf)) 4113 return ERR_CAST(umem_dmabuf); 4114 4115 umem = &umem_dmabuf->umem; 4116 4117 ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem); 4118 if (IS_ERR(ib_mr)) 4119 ib_umem_release(umem); 4120 return ib_mr; 4121 } 4122 4123 int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata) 4124 { 4125 struct ib_device *ibdev = ctx->device; 4126 struct bnxt_re_ucontext *uctx = 4127 container_of(ctx, struct bnxt_re_ucontext, ib_uctx); 4128 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 4129 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 4130 struct bnxt_re_user_mmap_entry *entry; 4131 struct bnxt_re_uctx_resp resp = {}; 4132 struct bnxt_re_uctx_req ureq = {}; 4133 u32 chip_met_rev_num = 0; 4134 int rc; 4135 4136 ibdev_dbg(ibdev, "ABI version requested %u", ibdev->ops.uverbs_abi_ver); 4137 4138 if (ibdev->ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) { 4139 ibdev_dbg(ibdev, " is different from the device %d ", 4140 BNXT_RE_ABI_VERSION); 4141 return -EPERM; 4142 } 4143 4144 uctx->rdev = rdev; 4145 4146 uctx->shpg = (void *)__get_free_page(GFP_KERNEL); 4147 if (!uctx->shpg) { 4148 rc = -ENOMEM; 4149 goto fail; 4150 } 4151 spin_lock_init(&uctx->sh_lock); 4152 4153 resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX; 4154 chip_met_rev_num = rdev->chip_ctx->chip_num; 4155 chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_rev & 0xFF) << 4156 BNXT_RE_CHIP_ID0_CHIP_REV_SFT; 4157 chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_metal & 0xFF) << 4158 BNXT_RE_CHIP_ID0_CHIP_MET_SFT; 4159 resp.chip_id0 = chip_met_rev_num; 4160 /*Temp, Use xa_alloc instead */ 4161 resp.dev_id = rdev->en_dev->pdev->devfn; 4162 resp.max_qp = rdev->qplib_ctx.qpc_count; 4163 resp.pg_size = PAGE_SIZE; 4164 resp.cqe_sz = sizeof(struct cq_base); 4165 resp.max_cqd = dev_attr->max_cq_wqes; 4166 4167 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_HAVE_MODE; 4168 resp.mode = rdev->chip_ctx->modes.wqe_mode; 4169 4170 if (rdev->chip_ctx->modes.db_push) 4171 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_WC_DPI_ENABLED; 4172 4173 entry = bnxt_re_mmap_entry_insert(uctx, 0, BNXT_RE_MMAP_SH_PAGE, NULL); 4174 if (!entry) { 4175 rc = -ENOMEM; 4176 goto cfail; 4177 } 4178 uctx->shpage_mmap = &entry->rdma_entry; 4179 if (rdev->pacing.dbr_pacing) 4180 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_DBR_PACING_ENABLED; 4181 4182 if (udata->inlen >= sizeof(ureq)) { 4183 rc = ib_copy_from_udata(&ureq, udata, min(udata->inlen, sizeof(ureq))); 4184 if (rc) 4185 goto cfail; 4186 if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT) { 4187 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_POW2_DISABLED; 4188 uctx->cmask |= BNXT_RE_UCNTX_CMASK_POW2_DISABLED; 4189 } 4190 } 4191 4192 rc = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp))); 4193 if (rc) { 4194 ibdev_err(ibdev, "Failed to copy user context"); 4195 rc = -EFAULT; 4196 goto cfail; 4197 } 4198 4199 return 0; 4200 cfail: 4201 free_page((unsigned long)uctx->shpg); 4202 uctx->shpg = NULL; 4203 fail: 4204 return rc; 4205 } 4206 4207 void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx) 4208 { 4209 struct bnxt_re_ucontext *uctx = container_of(ib_uctx, 4210 struct bnxt_re_ucontext, 4211 ib_uctx); 4212 4213 struct bnxt_re_dev *rdev = uctx->rdev; 4214 4215 rdma_user_mmap_entry_remove(uctx->shpage_mmap); 4216 uctx->shpage_mmap = NULL; 4217 if (uctx->shpg) 4218 free_page((unsigned long)uctx->shpg); 4219 4220 if (uctx->dpi.dbr) { 4221 /* Free DPI only if this is the first PD allocated by the 4222 * application and mark the context dpi as NULL 4223 */ 4224 bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->dpi); 4225 uctx->dpi.dbr = NULL; 4226 } 4227 } 4228 4229 /* Helper function to mmap the virtual memory from user app */ 4230 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) 4231 { 4232 struct bnxt_re_ucontext *uctx = container_of(ib_uctx, 4233 struct bnxt_re_ucontext, 4234 ib_uctx); 4235 struct bnxt_re_user_mmap_entry *bnxt_entry; 4236 struct rdma_user_mmap_entry *rdma_entry; 4237 int ret = 0; 4238 u64 pfn; 4239 4240 rdma_entry = rdma_user_mmap_entry_get(&uctx->ib_uctx, vma); 4241 if (!rdma_entry) 4242 return -EINVAL; 4243 4244 bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry, 4245 rdma_entry); 4246 4247 switch (bnxt_entry->mmap_flag) { 4248 case BNXT_RE_MMAP_WC_DB: 4249 pfn = bnxt_entry->mem_offset >> PAGE_SHIFT; 4250 ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE, 4251 pgprot_writecombine(vma->vm_page_prot), 4252 rdma_entry); 4253 break; 4254 case BNXT_RE_MMAP_UC_DB: 4255 pfn = bnxt_entry->mem_offset >> PAGE_SHIFT; 4256 ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE, 4257 pgprot_noncached(vma->vm_page_prot), 4258 rdma_entry); 4259 break; 4260 case BNXT_RE_MMAP_SH_PAGE: 4261 ret = vm_insert_page(vma, vma->vm_start, virt_to_page(uctx->shpg)); 4262 break; 4263 case BNXT_RE_MMAP_DBR_BAR: 4264 pfn = bnxt_entry->mem_offset >> PAGE_SHIFT; 4265 ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE, 4266 pgprot_noncached(vma->vm_page_prot), 4267 rdma_entry); 4268 break; 4269 case BNXT_RE_MMAP_DBR_PAGE: 4270 /* Driver doesn't expect write access for user space */ 4271 if (vma->vm_flags & VM_WRITE) 4272 return -EFAULT; 4273 ret = vm_insert_page(vma, vma->vm_start, 4274 virt_to_page((void *)bnxt_entry->mem_offset)); 4275 break; 4276 default: 4277 ret = -EINVAL; 4278 break; 4279 } 4280 4281 rdma_user_mmap_entry_put(rdma_entry); 4282 return ret; 4283 } 4284 4285 void bnxt_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 4286 { 4287 struct bnxt_re_user_mmap_entry *bnxt_entry; 4288 4289 bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry, 4290 rdma_entry); 4291 4292 kfree(bnxt_entry); 4293 } 4294 4295 static int UVERBS_HANDLER(BNXT_RE_METHOD_NOTIFY_DRV)(struct uverbs_attr_bundle *attrs) 4296 { 4297 struct bnxt_re_ucontext *uctx; 4298 4299 uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx); 4300 bnxt_re_pacing_alert(uctx->rdev); 4301 return 0; 4302 } 4303 4304 static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle *attrs) 4305 { 4306 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_ALLOC_PAGE_HANDLE); 4307 enum bnxt_re_alloc_page_type alloc_type; 4308 struct bnxt_re_user_mmap_entry *entry; 4309 enum bnxt_re_mmap_flag mmap_flag; 4310 struct bnxt_qplib_chip_ctx *cctx; 4311 struct bnxt_re_ucontext *uctx; 4312 struct bnxt_re_dev *rdev; 4313 u64 mmap_offset; 4314 u32 length; 4315 u32 dpi; 4316 u64 addr; 4317 int err; 4318 4319 uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx); 4320 if (IS_ERR(uctx)) 4321 return PTR_ERR(uctx); 4322 4323 err = uverbs_get_const(&alloc_type, attrs, BNXT_RE_ALLOC_PAGE_TYPE); 4324 if (err) 4325 return err; 4326 4327 rdev = uctx->rdev; 4328 cctx = rdev->chip_ctx; 4329 4330 switch (alloc_type) { 4331 case BNXT_RE_ALLOC_WC_PAGE: 4332 if (cctx->modes.db_push) { 4333 if (bnxt_qplib_alloc_dpi(&rdev->qplib_res, &uctx->wcdpi, 4334 uctx, BNXT_QPLIB_DPI_TYPE_WC)) 4335 return -ENOMEM; 4336 length = PAGE_SIZE; 4337 dpi = uctx->wcdpi.dpi; 4338 addr = (u64)uctx->wcdpi.umdbr; 4339 mmap_flag = BNXT_RE_MMAP_WC_DB; 4340 } else { 4341 return -EINVAL; 4342 } 4343 4344 break; 4345 case BNXT_RE_ALLOC_DBR_BAR_PAGE: 4346 length = PAGE_SIZE; 4347 addr = (u64)rdev->pacing.dbr_bar_addr; 4348 mmap_flag = BNXT_RE_MMAP_DBR_BAR; 4349 break; 4350 4351 case BNXT_RE_ALLOC_DBR_PAGE: 4352 length = PAGE_SIZE; 4353 addr = (u64)rdev->pacing.dbr_page; 4354 mmap_flag = BNXT_RE_MMAP_DBR_PAGE; 4355 break; 4356 4357 default: 4358 return -EOPNOTSUPP; 4359 } 4360 4361 entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mmap_offset); 4362 if (!entry) 4363 return -ENOMEM; 4364 4365 uobj->object = entry; 4366 uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_PAGE_HANDLE); 4367 err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_OFFSET, 4368 &mmap_offset, sizeof(mmap_offset)); 4369 if (err) 4370 return err; 4371 4372 err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_LENGTH, 4373 &length, sizeof(length)); 4374 if (err) 4375 return err; 4376 4377 err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_DPI, 4378 &dpi, sizeof(length)); 4379 if (err) 4380 return err; 4381 4382 return 0; 4383 } 4384 4385 static int alloc_page_obj_cleanup(struct ib_uobject *uobject, 4386 enum rdma_remove_reason why, 4387 struct uverbs_attr_bundle *attrs) 4388 { 4389 struct bnxt_re_user_mmap_entry *entry = uobject->object; 4390 struct bnxt_re_ucontext *uctx = entry->uctx; 4391 4392 switch (entry->mmap_flag) { 4393 case BNXT_RE_MMAP_WC_DB: 4394 if (uctx && uctx->wcdpi.dbr) { 4395 struct bnxt_re_dev *rdev = uctx->rdev; 4396 4397 bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi); 4398 uctx->wcdpi.dbr = NULL; 4399 } 4400 break; 4401 case BNXT_RE_MMAP_DBR_BAR: 4402 case BNXT_RE_MMAP_DBR_PAGE: 4403 break; 4404 default: 4405 goto exit; 4406 } 4407 rdma_user_mmap_entry_remove(&entry->rdma_entry); 4408 exit: 4409 return 0; 4410 } 4411 4412 DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_ALLOC_PAGE, 4413 UVERBS_ATTR_IDR(BNXT_RE_ALLOC_PAGE_HANDLE, 4414 BNXT_RE_OBJECT_ALLOC_PAGE, 4415 UVERBS_ACCESS_NEW, 4416 UA_MANDATORY), 4417 UVERBS_ATTR_CONST_IN(BNXT_RE_ALLOC_PAGE_TYPE, 4418 enum bnxt_re_alloc_page_type, 4419 UA_MANDATORY), 4420 UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_OFFSET, 4421 UVERBS_ATTR_TYPE(u64), 4422 UA_MANDATORY), 4423 UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_LENGTH, 4424 UVERBS_ATTR_TYPE(u32), 4425 UA_MANDATORY), 4426 UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_DPI, 4427 UVERBS_ATTR_TYPE(u32), 4428 UA_MANDATORY)); 4429 4430 DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_DESTROY_PAGE, 4431 UVERBS_ATTR_IDR(BNXT_RE_DESTROY_PAGE_HANDLE, 4432 BNXT_RE_OBJECT_ALLOC_PAGE, 4433 UVERBS_ACCESS_DESTROY, 4434 UA_MANDATORY)); 4435 4436 DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_ALLOC_PAGE, 4437 UVERBS_TYPE_ALLOC_IDR(alloc_page_obj_cleanup), 4438 &UVERBS_METHOD(BNXT_RE_METHOD_ALLOC_PAGE), 4439 &UVERBS_METHOD(BNXT_RE_METHOD_DESTROY_PAGE)); 4440 4441 DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_NOTIFY_DRV); 4442 4443 DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_NOTIFY_DRV, 4444 &UVERBS_METHOD(BNXT_RE_METHOD_NOTIFY_DRV)); 4445 4446 const struct uapi_definition bnxt_re_uapi_defs[] = { 4447 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE), 4448 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV), 4449 {} 4450 }; 4451