1 /* 2 * Broadcom NetXtreme-E RoCE driver. 3 * 4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term 5 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Description: IB Verbs interpreter 37 */ 38 39 #include <linux/interrupt.h> 40 #include <linux/types.h> 41 #include <linux/pci.h> 42 #include <linux/netdevice.h> 43 #include <linux/if_ether.h> 44 45 #include <rdma/ib_verbs.h> 46 #include <rdma/ib_user_verbs.h> 47 #include <rdma/ib_umem.h> 48 #include <rdma/ib_addr.h> 49 #include <rdma/ib_mad.h> 50 #include <rdma/ib_cache.h> 51 52 #include "bnxt_ulp.h" 53 54 #include "roce_hsi.h" 55 #include "qplib_res.h" 56 #include "qplib_sp.h" 57 #include "qplib_fp.h" 58 #include "qplib_rcfw.h" 59 60 #include "bnxt_re.h" 61 #include "ib_verbs.h" 62 #include <rdma/bnxt_re-abi.h> 63 64 static int __from_ib_access_flags(int iflags) 65 { 66 int qflags = 0; 67 68 if (iflags & IB_ACCESS_LOCAL_WRITE) 69 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 70 if (iflags & IB_ACCESS_REMOTE_READ) 71 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ; 72 if (iflags & IB_ACCESS_REMOTE_WRITE) 73 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE; 74 if (iflags & IB_ACCESS_REMOTE_ATOMIC) 75 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC; 76 if (iflags & IB_ACCESS_MW_BIND) 77 qflags |= BNXT_QPLIB_ACCESS_MW_BIND; 78 if (iflags & IB_ZERO_BASED) 79 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED; 80 if (iflags & IB_ACCESS_ON_DEMAND) 81 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND; 82 return qflags; 83 }; 84 85 static enum ib_access_flags __to_ib_access_flags(int qflags) 86 { 87 enum ib_access_flags iflags = 0; 88 89 if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE) 90 iflags |= IB_ACCESS_LOCAL_WRITE; 91 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE) 92 iflags |= IB_ACCESS_REMOTE_WRITE; 93 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ) 94 iflags |= IB_ACCESS_REMOTE_READ; 95 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC) 96 iflags |= IB_ACCESS_REMOTE_ATOMIC; 97 if (qflags & BNXT_QPLIB_ACCESS_MW_BIND) 98 iflags |= IB_ACCESS_MW_BIND; 99 if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED) 100 iflags |= IB_ZERO_BASED; 101 if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND) 102 iflags |= IB_ACCESS_ON_DEMAND; 103 return iflags; 104 }; 105 106 static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list, 107 struct bnxt_qplib_sge *sg_list, int num) 108 { 109 int i, total = 0; 110 111 for (i = 0; i < num; i++) { 112 sg_list[i].addr = ib_sg_list[i].addr; 113 sg_list[i].lkey = ib_sg_list[i].lkey; 114 sg_list[i].size = ib_sg_list[i].length; 115 total += sg_list[i].size; 116 } 117 return total; 118 } 119 120 /* Device */ 121 struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num) 122 { 123 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 124 struct net_device *netdev = NULL; 125 126 rcu_read_lock(); 127 if (rdev) 128 netdev = rdev->netdev; 129 if (netdev) 130 dev_hold(netdev); 131 132 rcu_read_unlock(); 133 return netdev; 134 } 135 136 int bnxt_re_query_device(struct ib_device *ibdev, 137 struct ib_device_attr *ib_attr, 138 struct ib_udata *udata) 139 { 140 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 141 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 142 143 memset(ib_attr, 0, sizeof(*ib_attr)); 144 memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, 145 min(sizeof(dev_attr->fw_ver), 146 sizeof(ib_attr->fw_ver))); 147 bnxt_qplib_get_guid(rdev->netdev->dev_addr, 148 (u8 *)&ib_attr->sys_image_guid); 149 ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE; 150 ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M; 151 152 ib_attr->vendor_id = rdev->en_dev->pdev->vendor; 153 ib_attr->vendor_part_id = rdev->en_dev->pdev->device; 154 ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; 155 ib_attr->max_qp = dev_attr->max_qp; 156 ib_attr->max_qp_wr = dev_attr->max_qp_wqes; 157 ib_attr->device_cap_flags = 158 IB_DEVICE_CURR_QP_STATE_MOD 159 | IB_DEVICE_RC_RNR_NAK_GEN 160 | IB_DEVICE_SHUTDOWN_PORT 161 | IB_DEVICE_SYS_IMAGE_GUID 162 | IB_DEVICE_LOCAL_DMA_LKEY 163 | IB_DEVICE_RESIZE_MAX_WR 164 | IB_DEVICE_PORT_ACTIVE_EVENT 165 | IB_DEVICE_N_NOTIFY_CQ 166 | IB_DEVICE_MEM_WINDOW 167 | IB_DEVICE_MEM_WINDOW_TYPE_2B 168 | IB_DEVICE_MEM_MGT_EXTENSIONS; 169 ib_attr->max_send_sge = dev_attr->max_qp_sges; 170 ib_attr->max_recv_sge = dev_attr->max_qp_sges; 171 ib_attr->max_sge_rd = dev_attr->max_qp_sges; 172 ib_attr->max_cq = dev_attr->max_cq; 173 ib_attr->max_cqe = dev_attr->max_cq_wqes; 174 ib_attr->max_mr = dev_attr->max_mr; 175 ib_attr->max_pd = dev_attr->max_pd; 176 ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; 177 ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom; 178 ib_attr->atomic_cap = IB_ATOMIC_NONE; 179 ib_attr->masked_atomic_cap = IB_ATOMIC_NONE; 180 181 ib_attr->max_ee_rd_atom = 0; 182 ib_attr->max_res_rd_atom = 0; 183 ib_attr->max_ee_init_rd_atom = 0; 184 ib_attr->max_ee = 0; 185 ib_attr->max_rdd = 0; 186 ib_attr->max_mw = dev_attr->max_mw; 187 ib_attr->max_raw_ipv6_qp = 0; 188 ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; 189 ib_attr->max_mcast_grp = 0; 190 ib_attr->max_mcast_qp_attach = 0; 191 ib_attr->max_total_mcast_qp_attach = 0; 192 ib_attr->max_ah = dev_attr->max_ah; 193 194 ib_attr->max_fmr = 0; 195 ib_attr->max_map_per_fmr = 0; 196 197 ib_attr->max_srq = dev_attr->max_srq; 198 ib_attr->max_srq_wr = dev_attr->max_srq_wqes; 199 ib_attr->max_srq_sge = dev_attr->max_srq_sges; 200 201 ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; 202 203 ib_attr->max_pkeys = 1; 204 ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY; 205 return 0; 206 } 207 208 int bnxt_re_modify_device(struct ib_device *ibdev, 209 int device_modify_mask, 210 struct ib_device_modify *device_modify) 211 { 212 switch (device_modify_mask) { 213 case IB_DEVICE_MODIFY_SYS_IMAGE_GUID: 214 /* Modify the GUID requires the modification of the GID table */ 215 /* GUID should be made as READ-ONLY */ 216 break; 217 case IB_DEVICE_MODIFY_NODE_DESC: 218 /* Node Desc should be made as READ-ONLY */ 219 break; 220 default: 221 break; 222 } 223 return 0; 224 } 225 226 /* Port */ 227 int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, 228 struct ib_port_attr *port_attr) 229 { 230 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 231 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 232 233 memset(port_attr, 0, sizeof(*port_attr)); 234 235 if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) { 236 port_attr->state = IB_PORT_ACTIVE; 237 port_attr->phys_state = 5; 238 } else { 239 port_attr->state = IB_PORT_DOWN; 240 port_attr->phys_state = 3; 241 } 242 port_attr->max_mtu = IB_MTU_4096; 243 port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu); 244 port_attr->gid_tbl_len = dev_attr->max_sgid; 245 port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 246 IB_PORT_DEVICE_MGMT_SUP | 247 IB_PORT_VENDOR_CLASS_SUP; 248 port_attr->ip_gids = true; 249 250 port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW; 251 port_attr->bad_pkey_cntr = 0; 252 port_attr->qkey_viol_cntr = 0; 253 port_attr->pkey_tbl_len = dev_attr->max_pkey; 254 port_attr->lid = 0; 255 port_attr->sm_lid = 0; 256 port_attr->lmc = 0; 257 port_attr->max_vl_num = 4; 258 port_attr->sm_sl = 0; 259 port_attr->subnet_timeout = 0; 260 port_attr->init_type_reply = 0; 261 port_attr->active_speed = rdev->active_speed; 262 port_attr->active_width = rdev->active_width; 263 264 return 0; 265 } 266 267 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, 268 struct ib_port_immutable *immutable) 269 { 270 struct ib_port_attr port_attr; 271 272 if (bnxt_re_query_port(ibdev, port_num, &port_attr)) 273 return -EINVAL; 274 275 immutable->pkey_tbl_len = port_attr.pkey_tbl_len; 276 immutable->gid_tbl_len = port_attr.gid_tbl_len; 277 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; 278 immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP; 279 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 280 return 0; 281 } 282 283 void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str) 284 { 285 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 286 287 snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d", 288 rdev->dev_attr.fw_ver[0], rdev->dev_attr.fw_ver[1], 289 rdev->dev_attr.fw_ver[2], rdev->dev_attr.fw_ver[3]); 290 } 291 292 int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, 293 u16 index, u16 *pkey) 294 { 295 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 296 297 /* Ignore port_num */ 298 299 memset(pkey, 0, sizeof(*pkey)); 300 return bnxt_qplib_get_pkey(&rdev->qplib_res, 301 &rdev->qplib_res.pkey_tbl, index, pkey); 302 } 303 304 int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, 305 int index, union ib_gid *gid) 306 { 307 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 308 int rc = 0; 309 310 /* Ignore port_num */ 311 memset(gid, 0, sizeof(*gid)); 312 rc = bnxt_qplib_get_sgid(&rdev->qplib_res, 313 &rdev->qplib_res.sgid_tbl, index, 314 (struct bnxt_qplib_gid *)gid); 315 return rc; 316 } 317 318 int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context) 319 { 320 int rc = 0; 321 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 322 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); 323 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 324 struct bnxt_qplib_gid *gid_to_del; 325 326 /* Delete the entry from the hardware */ 327 ctx = *context; 328 if (!ctx) 329 return -EINVAL; 330 331 if (sgid_tbl && sgid_tbl->active) { 332 if (ctx->idx >= sgid_tbl->max) 333 return -EINVAL; 334 gid_to_del = &sgid_tbl->tbl[ctx->idx]; 335 /* DEL_GID is called in WQ context(netdevice_event_work_handler) 336 * or via the ib_unregister_device path. In the former case QP1 337 * may not be destroyed yet, in which case just return as FW 338 * needs that entry to be present and will fail it's deletion. 339 * We could get invoked again after QP1 is destroyed OR get an 340 * ADD_GID call with a different GID value for the same index 341 * where we issue MODIFY_GID cmd to update the GID entry -- TBD 342 */ 343 if (ctx->idx == 0 && 344 rdma_link_local_addr((struct in6_addr *)gid_to_del) && 345 ctx->refcnt == 1 && rdev->qp1_sqp) { 346 dev_dbg(rdev_to_dev(rdev), 347 "Trying to delete GID0 while QP1 is alive\n"); 348 return -EFAULT; 349 } 350 ctx->refcnt--; 351 if (!ctx->refcnt) { 352 rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, true); 353 if (rc) { 354 dev_err(rdev_to_dev(rdev), 355 "Failed to remove GID: %#x", rc); 356 } else { 357 ctx_tbl = sgid_tbl->ctx; 358 ctx_tbl[ctx->idx] = NULL; 359 kfree(ctx); 360 } 361 } 362 } else { 363 return -EINVAL; 364 } 365 return rc; 366 } 367 368 int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context) 369 { 370 int rc; 371 u32 tbl_idx = 0; 372 u16 vlan_id = 0xFFFF; 373 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 374 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); 375 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 376 377 if ((attr->ndev) && is_vlan_dev(attr->ndev)) 378 vlan_id = vlan_dev_vlan_id(attr->ndev); 379 380 rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid, 381 rdev->qplib_res.netdev->dev_addr, 382 vlan_id, true, &tbl_idx); 383 if (rc == -EALREADY) { 384 ctx_tbl = sgid_tbl->ctx; 385 ctx_tbl[tbl_idx]->refcnt++; 386 *context = ctx_tbl[tbl_idx]; 387 return 0; 388 } 389 390 if (rc < 0) { 391 dev_err(rdev_to_dev(rdev), "Failed to add GID: %#x", rc); 392 return rc; 393 } 394 395 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 396 if (!ctx) 397 return -ENOMEM; 398 ctx_tbl = sgid_tbl->ctx; 399 ctx->idx = tbl_idx; 400 ctx->refcnt = 1; 401 ctx_tbl[tbl_idx] = ctx; 402 *context = ctx; 403 404 return rc; 405 } 406 407 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, 408 u8 port_num) 409 { 410 return IB_LINK_LAYER_ETHERNET; 411 } 412 413 #define BNXT_RE_FENCE_PBL_SIZE DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE) 414 415 static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd) 416 { 417 struct bnxt_re_fence_data *fence = &pd->fence; 418 struct ib_mr *ib_mr = &fence->mr->ib_mr; 419 struct bnxt_qplib_swqe *wqe = &fence->bind_wqe; 420 421 memset(wqe, 0, sizeof(*wqe)); 422 wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW; 423 wqe->wr_id = BNXT_QPLIB_FENCE_WRID; 424 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 425 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 426 wqe->bind.zero_based = false; 427 wqe->bind.parent_l_key = ib_mr->lkey; 428 wqe->bind.va = (u64)(unsigned long)fence->va; 429 wqe->bind.length = fence->size; 430 wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ); 431 wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1; 432 433 /* Save the initial rkey in fence structure for now; 434 * wqe->bind.r_key will be set at (re)bind time. 435 */ 436 fence->bind_rkey = ib_inc_rkey(fence->mw->rkey); 437 } 438 439 static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp) 440 { 441 struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp, 442 qplib_qp); 443 struct ib_pd *ib_pd = qp->ib_qp.pd; 444 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 445 struct bnxt_re_fence_data *fence = &pd->fence; 446 struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe; 447 struct bnxt_qplib_swqe wqe; 448 int rc; 449 450 memcpy(&wqe, fence_wqe, sizeof(wqe)); 451 wqe.bind.r_key = fence->bind_rkey; 452 fence->bind_rkey = ib_inc_rkey(fence->bind_rkey); 453 454 dev_dbg(rdev_to_dev(qp->rdev), 455 "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n", 456 wqe.bind.r_key, qp->qplib_qp.id, pd); 457 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 458 if (rc) { 459 dev_err(rdev_to_dev(qp->rdev), "Failed to bind fence-WQE\n"); 460 return rc; 461 } 462 bnxt_qplib_post_send_db(&qp->qplib_qp); 463 464 return rc; 465 } 466 467 static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd) 468 { 469 struct bnxt_re_fence_data *fence = &pd->fence; 470 struct bnxt_re_dev *rdev = pd->rdev; 471 struct device *dev = &rdev->en_dev->pdev->dev; 472 struct bnxt_re_mr *mr = fence->mr; 473 474 if (fence->mw) { 475 bnxt_re_dealloc_mw(fence->mw); 476 fence->mw = NULL; 477 } 478 if (mr) { 479 if (mr->ib_mr.rkey) 480 bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr, 481 true); 482 if (mr->ib_mr.lkey) 483 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 484 kfree(mr); 485 fence->mr = NULL; 486 } 487 if (fence->dma_addr) { 488 dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES, 489 DMA_BIDIRECTIONAL); 490 fence->dma_addr = 0; 491 } 492 } 493 494 static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd) 495 { 496 int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND; 497 struct bnxt_re_fence_data *fence = &pd->fence; 498 struct bnxt_re_dev *rdev = pd->rdev; 499 struct device *dev = &rdev->en_dev->pdev->dev; 500 struct bnxt_re_mr *mr = NULL; 501 dma_addr_t dma_addr = 0; 502 struct ib_mw *mw; 503 u64 pbl_tbl; 504 int rc; 505 506 dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES, 507 DMA_BIDIRECTIONAL); 508 rc = dma_mapping_error(dev, dma_addr); 509 if (rc) { 510 dev_err(rdev_to_dev(rdev), "Failed to dma-map fence-MR-mem\n"); 511 rc = -EIO; 512 fence->dma_addr = 0; 513 goto fail; 514 } 515 fence->dma_addr = dma_addr; 516 517 /* Allocate a MR */ 518 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 519 if (!mr) { 520 rc = -ENOMEM; 521 goto fail; 522 } 523 fence->mr = mr; 524 mr->rdev = rdev; 525 mr->qplib_mr.pd = &pd->qplib_pd; 526 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 527 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 528 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 529 if (rc) { 530 dev_err(rdev_to_dev(rdev), "Failed to alloc fence-HW-MR\n"); 531 goto fail; 532 } 533 534 /* Register MR */ 535 mr->ib_mr.lkey = mr->qplib_mr.lkey; 536 mr->qplib_mr.va = (u64)(unsigned long)fence->va; 537 mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES; 538 pbl_tbl = dma_addr; 539 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl_tbl, 540 BNXT_RE_FENCE_PBL_SIZE, false, PAGE_SIZE); 541 if (rc) { 542 dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n"); 543 goto fail; 544 } 545 mr->ib_mr.rkey = mr->qplib_mr.rkey; 546 547 /* Create a fence MW only for kernel consumers */ 548 mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL); 549 if (IS_ERR(mw)) { 550 dev_err(rdev_to_dev(rdev), 551 "Failed to create fence-MW for PD: %p\n", pd); 552 rc = PTR_ERR(mw); 553 goto fail; 554 } 555 fence->mw = mw; 556 557 bnxt_re_create_fence_wqe(pd); 558 return 0; 559 560 fail: 561 bnxt_re_destroy_fence_mr(pd); 562 return rc; 563 } 564 565 /* Protection Domains */ 566 int bnxt_re_dealloc_pd(struct ib_pd *ib_pd) 567 { 568 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 569 struct bnxt_re_dev *rdev = pd->rdev; 570 int rc; 571 572 bnxt_re_destroy_fence_mr(pd); 573 574 if (pd->qplib_pd.id) { 575 rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res, 576 &rdev->qplib_res.pd_tbl, 577 &pd->qplib_pd); 578 if (rc) 579 dev_err(rdev_to_dev(rdev), "Failed to deallocate HW PD"); 580 } 581 582 kfree(pd); 583 return 0; 584 } 585 586 struct ib_pd *bnxt_re_alloc_pd(struct ib_device *ibdev, 587 struct ib_ucontext *ucontext, 588 struct ib_udata *udata) 589 { 590 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 591 struct bnxt_re_ucontext *ucntx = container_of(ucontext, 592 struct bnxt_re_ucontext, 593 ib_uctx); 594 struct bnxt_re_pd *pd; 595 int rc; 596 597 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 598 if (!pd) 599 return ERR_PTR(-ENOMEM); 600 601 pd->rdev = rdev; 602 if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) { 603 dev_err(rdev_to_dev(rdev), "Failed to allocate HW PD"); 604 rc = -ENOMEM; 605 goto fail; 606 } 607 608 if (udata) { 609 struct bnxt_re_pd_resp resp; 610 611 if (!ucntx->dpi.dbr) { 612 /* Allocate DPI in alloc_pd to avoid failing of 613 * ibv_devinfo and family of application when DPIs 614 * are depleted. 615 */ 616 if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl, 617 &ucntx->dpi, ucntx)) { 618 rc = -ENOMEM; 619 goto dbfail; 620 } 621 } 622 623 resp.pdid = pd->qplib_pd.id; 624 /* Still allow mapping this DBR to the new user PD. */ 625 resp.dpi = ucntx->dpi.dpi; 626 resp.dbr = (u64)ucntx->dpi.umdbr; 627 628 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 629 if (rc) { 630 dev_err(rdev_to_dev(rdev), 631 "Failed to copy user response\n"); 632 goto dbfail; 633 } 634 } 635 636 if (!udata) 637 if (bnxt_re_create_fence_mr(pd)) 638 dev_warn(rdev_to_dev(rdev), 639 "Failed to create Fence-MR\n"); 640 return &pd->ib_pd; 641 dbfail: 642 (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, 643 &pd->qplib_pd); 644 fail: 645 kfree(pd); 646 return ERR_PTR(rc); 647 } 648 649 /* Address Handles */ 650 int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags) 651 { 652 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 653 struct bnxt_re_dev *rdev = ah->rdev; 654 int rc; 655 656 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, 657 !(flags & RDMA_DESTROY_AH_SLEEPABLE)); 658 if (rc) { 659 dev_err(rdev_to_dev(rdev), "Failed to destroy HW AH"); 660 return rc; 661 } 662 kfree(ah); 663 return 0; 664 } 665 666 struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd, 667 struct rdma_ah_attr *ah_attr, 668 u32 flags, 669 struct ib_udata *udata) 670 { 671 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 672 struct bnxt_re_dev *rdev = pd->rdev; 673 struct bnxt_re_ah *ah; 674 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); 675 int rc; 676 u8 nw_type; 677 678 if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) { 679 dev_err(rdev_to_dev(rdev), "Failed to alloc AH: GRH not set"); 680 return ERR_PTR(-EINVAL); 681 } 682 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 683 if (!ah) 684 return ERR_PTR(-ENOMEM); 685 686 ah->rdev = rdev; 687 ah->qplib_ah.pd = &pd->qplib_pd; 688 689 /* Supply the configuration for the HW */ 690 memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw, 691 sizeof(union ib_gid)); 692 /* 693 * If RoCE V2 is enabled, stack will have two entries for 694 * each GID entry. Avoiding this duplicte entry in HW. Dividing 695 * the GID index by 2 for RoCE V2 696 */ 697 ah->qplib_ah.sgid_index = grh->sgid_index / 2; 698 ah->qplib_ah.host_sgid_index = grh->sgid_index; 699 ah->qplib_ah.traffic_class = grh->traffic_class; 700 ah->qplib_ah.flow_label = grh->flow_label; 701 ah->qplib_ah.hop_limit = grh->hop_limit; 702 ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr); 703 if (udata && 704 !rdma_is_multicast_addr((struct in6_addr *) 705 grh->dgid.raw) && 706 !rdma_link_local_addr((struct in6_addr *) 707 grh->dgid.raw)) { 708 const struct ib_gid_attr *sgid_attr; 709 710 sgid_attr = grh->sgid_attr; 711 /* Get network header type for this GID */ 712 nw_type = rdma_gid_attr_network_type(sgid_attr); 713 switch (nw_type) { 714 case RDMA_NETWORK_IPV4: 715 ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4; 716 break; 717 case RDMA_NETWORK_IPV6: 718 ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6; 719 break; 720 default: 721 ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V1; 722 break; 723 } 724 } 725 726 memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN); 727 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, 728 !(flags & RDMA_CREATE_AH_SLEEPABLE)); 729 if (rc) { 730 dev_err(rdev_to_dev(rdev), "Failed to allocate HW AH"); 731 goto fail; 732 } 733 734 /* Write AVID to shared page. */ 735 if (udata) { 736 struct ib_ucontext *ib_uctx = ib_pd->uobject->context; 737 struct bnxt_re_ucontext *uctx; 738 unsigned long flag; 739 u32 *wrptr; 740 741 uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx); 742 spin_lock_irqsave(&uctx->sh_lock, flag); 743 wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT); 744 *wrptr = ah->qplib_ah.id; 745 wmb(); /* make sure cache is updated. */ 746 spin_unlock_irqrestore(&uctx->sh_lock, flag); 747 } 748 749 return &ah->ib_ah; 750 751 fail: 752 kfree(ah); 753 return ERR_PTR(rc); 754 } 755 756 int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) 757 { 758 return 0; 759 } 760 761 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) 762 { 763 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 764 765 ah_attr->type = ib_ah->type; 766 rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl); 767 memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN); 768 rdma_ah_set_grh(ah_attr, NULL, 0, 769 ah->qplib_ah.host_sgid_index, 770 0, ah->qplib_ah.traffic_class); 771 rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data); 772 rdma_ah_set_port_num(ah_attr, 1); 773 rdma_ah_set_static_rate(ah_attr, 0); 774 return 0; 775 } 776 777 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp) 778 __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock) 779 { 780 unsigned long flags; 781 782 spin_lock_irqsave(&qp->scq->cq_lock, flags); 783 if (qp->rcq != qp->scq) 784 spin_lock(&qp->rcq->cq_lock); 785 else 786 __acquire(&qp->rcq->cq_lock); 787 788 return flags; 789 } 790 791 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, 792 unsigned long flags) 793 __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock) 794 { 795 if (qp->rcq != qp->scq) 796 spin_unlock(&qp->rcq->cq_lock); 797 else 798 __release(&qp->rcq->cq_lock); 799 spin_unlock_irqrestore(&qp->scq->cq_lock, flags); 800 } 801 802 /* Queue Pairs */ 803 int bnxt_re_destroy_qp(struct ib_qp *ib_qp) 804 { 805 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 806 struct bnxt_re_dev *rdev = qp->rdev; 807 int rc; 808 unsigned int flags; 809 810 bnxt_qplib_flush_cqn_wq(&qp->qplib_qp); 811 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 812 if (rc) { 813 dev_err(rdev_to_dev(rdev), "Failed to destroy HW QP"); 814 return rc; 815 } 816 817 flags = bnxt_re_lock_cqs(qp); 818 bnxt_qplib_clean_qp(&qp->qplib_qp); 819 bnxt_re_unlock_cqs(qp, flags); 820 bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); 821 822 if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) { 823 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, 824 &rdev->sqp_ah->qplib_ah, false); 825 if (rc) { 826 dev_err(rdev_to_dev(rdev), 827 "Failed to destroy HW AH for shadow QP"); 828 return rc; 829 } 830 831 bnxt_qplib_clean_qp(&qp->qplib_qp); 832 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, 833 &rdev->qp1_sqp->qplib_qp); 834 if (rc) { 835 dev_err(rdev_to_dev(rdev), 836 "Failed to destroy Shadow QP"); 837 return rc; 838 } 839 bnxt_qplib_free_qp_res(&rdev->qplib_res, 840 &rdev->qp1_sqp->qplib_qp); 841 mutex_lock(&rdev->qp_lock); 842 list_del(&rdev->qp1_sqp->list); 843 atomic_dec(&rdev->qp_count); 844 mutex_unlock(&rdev->qp_lock); 845 846 kfree(rdev->sqp_ah); 847 kfree(rdev->qp1_sqp); 848 rdev->qp1_sqp = NULL; 849 rdev->sqp_ah = NULL; 850 } 851 852 if (!IS_ERR_OR_NULL(qp->rumem)) 853 ib_umem_release(qp->rumem); 854 if (!IS_ERR_OR_NULL(qp->sumem)) 855 ib_umem_release(qp->sumem); 856 857 mutex_lock(&rdev->qp_lock); 858 list_del(&qp->list); 859 atomic_dec(&rdev->qp_count); 860 mutex_unlock(&rdev->qp_lock); 861 kfree(qp); 862 return 0; 863 } 864 865 static u8 __from_ib_qp_type(enum ib_qp_type type) 866 { 867 switch (type) { 868 case IB_QPT_GSI: 869 return CMDQ_CREATE_QP1_TYPE_GSI; 870 case IB_QPT_RC: 871 return CMDQ_CREATE_QP_TYPE_RC; 872 case IB_QPT_UD: 873 return CMDQ_CREATE_QP_TYPE_UD; 874 default: 875 return IB_QPT_MAX; 876 } 877 } 878 879 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, 880 struct bnxt_re_qp *qp, struct ib_udata *udata) 881 { 882 struct bnxt_re_qp_req ureq; 883 struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; 884 struct ib_umem *umem; 885 int bytes = 0; 886 struct ib_ucontext *context = pd->ib_pd.uobject->context; 887 struct bnxt_re_ucontext *cntx = container_of(context, 888 struct bnxt_re_ucontext, 889 ib_uctx); 890 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 891 return -EFAULT; 892 893 bytes = (qplib_qp->sq.max_wqe * BNXT_QPLIB_MAX_SQE_ENTRY_SIZE); 894 /* Consider mapping PSN search memory only for RC QPs. */ 895 if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) 896 bytes += (qplib_qp->sq.max_wqe * sizeof(struct sq_psn_search)); 897 bytes = PAGE_ALIGN(bytes); 898 umem = ib_umem_get(context, ureq.qpsva, bytes, 899 IB_ACCESS_LOCAL_WRITE, 1); 900 if (IS_ERR(umem)) 901 return PTR_ERR(umem); 902 903 qp->sumem = umem; 904 qplib_qp->sq.sglist = umem->sg_head.sgl; 905 qplib_qp->sq.nmap = umem->nmap; 906 qplib_qp->qp_handle = ureq.qp_handle; 907 908 if (!qp->qplib_qp.srq) { 909 bytes = (qplib_qp->rq.max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); 910 bytes = PAGE_ALIGN(bytes); 911 umem = ib_umem_get(context, ureq.qprva, bytes, 912 IB_ACCESS_LOCAL_WRITE, 1); 913 if (IS_ERR(umem)) 914 goto rqfail; 915 qp->rumem = umem; 916 qplib_qp->rq.sglist = umem->sg_head.sgl; 917 qplib_qp->rq.nmap = umem->nmap; 918 } 919 920 qplib_qp->dpi = &cntx->dpi; 921 return 0; 922 rqfail: 923 ib_umem_release(qp->sumem); 924 qp->sumem = NULL; 925 qplib_qp->sq.sglist = NULL; 926 qplib_qp->sq.nmap = 0; 927 928 return PTR_ERR(umem); 929 } 930 931 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah 932 (struct bnxt_re_pd *pd, 933 struct bnxt_qplib_res *qp1_res, 934 struct bnxt_qplib_qp *qp1_qp) 935 { 936 struct bnxt_re_dev *rdev = pd->rdev; 937 struct bnxt_re_ah *ah; 938 union ib_gid sgid; 939 int rc; 940 941 ah = kzalloc(sizeof(*ah), GFP_KERNEL); 942 if (!ah) 943 return NULL; 944 945 ah->rdev = rdev; 946 ah->qplib_ah.pd = &pd->qplib_pd; 947 948 rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); 949 if (rc) 950 goto fail; 951 952 /* supply the dgid data same as sgid */ 953 memcpy(ah->qplib_ah.dgid.data, &sgid.raw, 954 sizeof(union ib_gid)); 955 ah->qplib_ah.sgid_index = 0; 956 957 ah->qplib_ah.traffic_class = 0; 958 ah->qplib_ah.flow_label = 0; 959 ah->qplib_ah.hop_limit = 1; 960 ah->qplib_ah.sl = 0; 961 /* Have DMAC same as SMAC */ 962 ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr); 963 964 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, false); 965 if (rc) { 966 dev_err(rdev_to_dev(rdev), 967 "Failed to allocate HW AH for Shadow QP"); 968 goto fail; 969 } 970 971 return ah; 972 973 fail: 974 kfree(ah); 975 return NULL; 976 } 977 978 static struct bnxt_re_qp *bnxt_re_create_shadow_qp 979 (struct bnxt_re_pd *pd, 980 struct bnxt_qplib_res *qp1_res, 981 struct bnxt_qplib_qp *qp1_qp) 982 { 983 struct bnxt_re_dev *rdev = pd->rdev; 984 struct bnxt_re_qp *qp; 985 int rc; 986 987 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 988 if (!qp) 989 return NULL; 990 991 qp->rdev = rdev; 992 993 /* Initialize the shadow QP structure from the QP1 values */ 994 ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); 995 996 qp->qplib_qp.pd = &pd->qplib_pd; 997 qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); 998 qp->qplib_qp.type = IB_QPT_UD; 999 1000 qp->qplib_qp.max_inline_data = 0; 1001 qp->qplib_qp.sig_type = true; 1002 1003 /* Shadow QP SQ depth should be same as QP1 RQ depth */ 1004 qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; 1005 qp->qplib_qp.sq.max_sge = 2; 1006 /* Q full delta can be 1 since it is internal QP */ 1007 qp->qplib_qp.sq.q_full_delta = 1; 1008 1009 qp->qplib_qp.scq = qp1_qp->scq; 1010 qp->qplib_qp.rcq = qp1_qp->rcq; 1011 1012 qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; 1013 qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; 1014 /* Q full delta can be 1 since it is internal QP */ 1015 qp->qplib_qp.rq.q_full_delta = 1; 1016 1017 qp->qplib_qp.mtu = qp1_qp->mtu; 1018 1019 qp->qplib_qp.sq_hdr_buf_size = 0; 1020 qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; 1021 qp->qplib_qp.dpi = &rdev->dpi_privileged; 1022 1023 rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); 1024 if (rc) 1025 goto fail; 1026 1027 rdev->sqp_id = qp->qplib_qp.id; 1028 1029 spin_lock_init(&qp->sq_lock); 1030 INIT_LIST_HEAD(&qp->list); 1031 mutex_lock(&rdev->qp_lock); 1032 list_add_tail(&qp->list, &rdev->qp_list); 1033 atomic_inc(&rdev->qp_count); 1034 mutex_unlock(&rdev->qp_lock); 1035 return qp; 1036 fail: 1037 kfree(qp); 1038 return NULL; 1039 } 1040 1041 struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd, 1042 struct ib_qp_init_attr *qp_init_attr, 1043 struct ib_udata *udata) 1044 { 1045 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 1046 struct bnxt_re_dev *rdev = pd->rdev; 1047 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 1048 struct bnxt_re_qp *qp; 1049 struct bnxt_re_cq *cq; 1050 struct bnxt_re_srq *srq; 1051 int rc, entries; 1052 1053 if ((qp_init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) || 1054 (qp_init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) || 1055 (qp_init_attr->cap.max_send_sge > dev_attr->max_qp_sges) || 1056 (qp_init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) || 1057 (qp_init_attr->cap.max_inline_data > dev_attr->max_inline_data)) 1058 return ERR_PTR(-EINVAL); 1059 1060 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1061 if (!qp) 1062 return ERR_PTR(-ENOMEM); 1063 1064 qp->rdev = rdev; 1065 ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); 1066 qp->qplib_qp.pd = &pd->qplib_pd; 1067 qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); 1068 qp->qplib_qp.type = __from_ib_qp_type(qp_init_attr->qp_type); 1069 if (qp->qplib_qp.type == IB_QPT_MAX) { 1070 dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported", 1071 qp->qplib_qp.type); 1072 rc = -EINVAL; 1073 goto fail; 1074 } 1075 qp->qplib_qp.max_inline_data = qp_init_attr->cap.max_inline_data; 1076 qp->qplib_qp.sig_type = ((qp_init_attr->sq_sig_type == 1077 IB_SIGNAL_ALL_WR) ? true : false); 1078 1079 qp->qplib_qp.sq.max_sge = qp_init_attr->cap.max_send_sge; 1080 if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges) 1081 qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges; 1082 1083 if (qp_init_attr->send_cq) { 1084 cq = container_of(qp_init_attr->send_cq, struct bnxt_re_cq, 1085 ib_cq); 1086 if (!cq) { 1087 dev_err(rdev_to_dev(rdev), "Send CQ not found"); 1088 rc = -EINVAL; 1089 goto fail; 1090 } 1091 qp->qplib_qp.scq = &cq->qplib_cq; 1092 qp->scq = cq; 1093 } 1094 1095 if (qp_init_attr->recv_cq) { 1096 cq = container_of(qp_init_attr->recv_cq, struct bnxt_re_cq, 1097 ib_cq); 1098 if (!cq) { 1099 dev_err(rdev_to_dev(rdev), "Receive CQ not found"); 1100 rc = -EINVAL; 1101 goto fail; 1102 } 1103 qp->qplib_qp.rcq = &cq->qplib_cq; 1104 qp->rcq = cq; 1105 } 1106 1107 if (qp_init_attr->srq) { 1108 srq = container_of(qp_init_attr->srq, struct bnxt_re_srq, 1109 ib_srq); 1110 if (!srq) { 1111 dev_err(rdev_to_dev(rdev), "SRQ not found"); 1112 rc = -EINVAL; 1113 goto fail; 1114 } 1115 qp->qplib_qp.srq = &srq->qplib_srq; 1116 qp->qplib_qp.rq.max_wqe = 0; 1117 } else { 1118 /* Allocate 1 more than what's provided so posting max doesn't 1119 * mean empty 1120 */ 1121 entries = roundup_pow_of_two(qp_init_attr->cap.max_recv_wr + 1); 1122 qp->qplib_qp.rq.max_wqe = min_t(u32, entries, 1123 dev_attr->max_qp_wqes + 1); 1124 1125 qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - 1126 qp_init_attr->cap.max_recv_wr; 1127 1128 qp->qplib_qp.rq.max_sge = qp_init_attr->cap.max_recv_sge; 1129 if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges) 1130 qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; 1131 } 1132 1133 qp->qplib_qp.mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); 1134 1135 if (qp_init_attr->qp_type == IB_QPT_GSI) { 1136 /* Allocate 1 more than what's provided */ 1137 entries = roundup_pow_of_two(qp_init_attr->cap.max_send_wr + 1); 1138 qp->qplib_qp.sq.max_wqe = min_t(u32, entries, 1139 dev_attr->max_qp_wqes + 1); 1140 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - 1141 qp_init_attr->cap.max_send_wr; 1142 qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; 1143 if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges) 1144 qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; 1145 qp->qplib_qp.sq.max_sge++; 1146 if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges) 1147 qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges; 1148 1149 qp->qplib_qp.rq_hdr_buf_size = 1150 BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; 1151 1152 qp->qplib_qp.sq_hdr_buf_size = 1153 BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2; 1154 qp->qplib_qp.dpi = &rdev->dpi_privileged; 1155 rc = bnxt_qplib_create_qp1(&rdev->qplib_res, &qp->qplib_qp); 1156 if (rc) { 1157 dev_err(rdev_to_dev(rdev), "Failed to create HW QP1"); 1158 goto fail; 1159 } 1160 /* Create a shadow QP to handle the QP1 traffic */ 1161 rdev->qp1_sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, 1162 &qp->qplib_qp); 1163 if (!rdev->qp1_sqp) { 1164 rc = -EINVAL; 1165 dev_err(rdev_to_dev(rdev), 1166 "Failed to create Shadow QP for QP1"); 1167 goto qp_destroy; 1168 } 1169 rdev->sqp_ah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, 1170 &qp->qplib_qp); 1171 if (!rdev->sqp_ah) { 1172 bnxt_qplib_destroy_qp(&rdev->qplib_res, 1173 &rdev->qp1_sqp->qplib_qp); 1174 rc = -EINVAL; 1175 dev_err(rdev_to_dev(rdev), 1176 "Failed to create AH entry for ShadowQP"); 1177 goto qp_destroy; 1178 } 1179 1180 } else { 1181 /* Allocate 128 + 1 more than what's provided */ 1182 entries = roundup_pow_of_two(qp_init_attr->cap.max_send_wr + 1183 BNXT_QPLIB_RESERVED_QP_WRS + 1); 1184 qp->qplib_qp.sq.max_wqe = min_t(u32, entries, 1185 dev_attr->max_qp_wqes + 1186 BNXT_QPLIB_RESERVED_QP_WRS + 1); 1187 qp->qplib_qp.sq.q_full_delta = BNXT_QPLIB_RESERVED_QP_WRS + 1; 1188 1189 /* 1190 * Reserving one slot for Phantom WQE. Application can 1191 * post one extra entry in this case. But allowing this to avoid 1192 * unexpected Queue full condition 1193 */ 1194 1195 qp->qplib_qp.sq.q_full_delta -= 1; 1196 1197 qp->qplib_qp.max_rd_atomic = dev_attr->max_qp_rd_atom; 1198 qp->qplib_qp.max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; 1199 if (udata) { 1200 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); 1201 if (rc) 1202 goto fail; 1203 } else { 1204 qp->qplib_qp.dpi = &rdev->dpi_privileged; 1205 } 1206 1207 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); 1208 if (rc) { 1209 dev_err(rdev_to_dev(rdev), "Failed to create HW QP"); 1210 goto free_umem; 1211 } 1212 } 1213 1214 qp->ib_qp.qp_num = qp->qplib_qp.id; 1215 spin_lock_init(&qp->sq_lock); 1216 spin_lock_init(&qp->rq_lock); 1217 1218 if (udata) { 1219 struct bnxt_re_qp_resp resp; 1220 1221 resp.qpid = qp->ib_qp.qp_num; 1222 resp.rsvd = 0; 1223 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 1224 if (rc) { 1225 dev_err(rdev_to_dev(rdev), "Failed to copy QP udata"); 1226 goto qp_destroy; 1227 } 1228 } 1229 INIT_LIST_HEAD(&qp->list); 1230 mutex_lock(&rdev->qp_lock); 1231 list_add_tail(&qp->list, &rdev->qp_list); 1232 atomic_inc(&rdev->qp_count); 1233 mutex_unlock(&rdev->qp_lock); 1234 1235 return &qp->ib_qp; 1236 qp_destroy: 1237 bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 1238 free_umem: 1239 if (udata) { 1240 if (qp->rumem) 1241 ib_umem_release(qp->rumem); 1242 if (qp->sumem) 1243 ib_umem_release(qp->sumem); 1244 } 1245 fail: 1246 kfree(qp); 1247 return ERR_PTR(rc); 1248 } 1249 1250 static u8 __from_ib_qp_state(enum ib_qp_state state) 1251 { 1252 switch (state) { 1253 case IB_QPS_RESET: 1254 return CMDQ_MODIFY_QP_NEW_STATE_RESET; 1255 case IB_QPS_INIT: 1256 return CMDQ_MODIFY_QP_NEW_STATE_INIT; 1257 case IB_QPS_RTR: 1258 return CMDQ_MODIFY_QP_NEW_STATE_RTR; 1259 case IB_QPS_RTS: 1260 return CMDQ_MODIFY_QP_NEW_STATE_RTS; 1261 case IB_QPS_SQD: 1262 return CMDQ_MODIFY_QP_NEW_STATE_SQD; 1263 case IB_QPS_SQE: 1264 return CMDQ_MODIFY_QP_NEW_STATE_SQE; 1265 case IB_QPS_ERR: 1266 default: 1267 return CMDQ_MODIFY_QP_NEW_STATE_ERR; 1268 } 1269 } 1270 1271 static enum ib_qp_state __to_ib_qp_state(u8 state) 1272 { 1273 switch (state) { 1274 case CMDQ_MODIFY_QP_NEW_STATE_RESET: 1275 return IB_QPS_RESET; 1276 case CMDQ_MODIFY_QP_NEW_STATE_INIT: 1277 return IB_QPS_INIT; 1278 case CMDQ_MODIFY_QP_NEW_STATE_RTR: 1279 return IB_QPS_RTR; 1280 case CMDQ_MODIFY_QP_NEW_STATE_RTS: 1281 return IB_QPS_RTS; 1282 case CMDQ_MODIFY_QP_NEW_STATE_SQD: 1283 return IB_QPS_SQD; 1284 case CMDQ_MODIFY_QP_NEW_STATE_SQE: 1285 return IB_QPS_SQE; 1286 case CMDQ_MODIFY_QP_NEW_STATE_ERR: 1287 default: 1288 return IB_QPS_ERR; 1289 } 1290 } 1291 1292 static u32 __from_ib_mtu(enum ib_mtu mtu) 1293 { 1294 switch (mtu) { 1295 case IB_MTU_256: 1296 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256; 1297 case IB_MTU_512: 1298 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512; 1299 case IB_MTU_1024: 1300 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024; 1301 case IB_MTU_2048: 1302 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 1303 case IB_MTU_4096: 1304 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096; 1305 default: 1306 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 1307 } 1308 } 1309 1310 static enum ib_mtu __to_ib_mtu(u32 mtu) 1311 { 1312 switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) { 1313 case CMDQ_MODIFY_QP_PATH_MTU_MTU_256: 1314 return IB_MTU_256; 1315 case CMDQ_MODIFY_QP_PATH_MTU_MTU_512: 1316 return IB_MTU_512; 1317 case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024: 1318 return IB_MTU_1024; 1319 case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048: 1320 return IB_MTU_2048; 1321 case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096: 1322 return IB_MTU_4096; 1323 default: 1324 return IB_MTU_2048; 1325 } 1326 } 1327 1328 /* Shared Receive Queues */ 1329 int bnxt_re_destroy_srq(struct ib_srq *ib_srq) 1330 { 1331 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1332 ib_srq); 1333 struct bnxt_re_dev *rdev = srq->rdev; 1334 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; 1335 struct bnxt_qplib_nq *nq = NULL; 1336 int rc; 1337 1338 if (qplib_srq->cq) 1339 nq = qplib_srq->cq->nq; 1340 rc = bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); 1341 if (rc) { 1342 dev_err(rdev_to_dev(rdev), "Destroy HW SRQ failed!"); 1343 return rc; 1344 } 1345 1346 if (srq->umem) 1347 ib_umem_release(srq->umem); 1348 kfree(srq); 1349 atomic_dec(&rdev->srq_count); 1350 if (nq) 1351 nq->budget--; 1352 return 0; 1353 } 1354 1355 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, 1356 struct bnxt_re_pd *pd, 1357 struct bnxt_re_srq *srq, 1358 struct ib_udata *udata) 1359 { 1360 struct bnxt_re_srq_req ureq; 1361 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; 1362 struct ib_umem *umem; 1363 int bytes = 0; 1364 struct ib_ucontext *context = pd->ib_pd.uobject->context; 1365 struct bnxt_re_ucontext *cntx = container_of(context, 1366 struct bnxt_re_ucontext, 1367 ib_uctx); 1368 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 1369 return -EFAULT; 1370 1371 bytes = (qplib_srq->max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); 1372 bytes = PAGE_ALIGN(bytes); 1373 umem = ib_umem_get(context, ureq.srqva, bytes, 1374 IB_ACCESS_LOCAL_WRITE, 1); 1375 if (IS_ERR(umem)) 1376 return PTR_ERR(umem); 1377 1378 srq->umem = umem; 1379 qplib_srq->nmap = umem->nmap; 1380 qplib_srq->sglist = umem->sg_head.sgl; 1381 qplib_srq->srq_handle = ureq.srq_handle; 1382 qplib_srq->dpi = &cntx->dpi; 1383 1384 return 0; 1385 } 1386 1387 struct ib_srq *bnxt_re_create_srq(struct ib_pd *ib_pd, 1388 struct ib_srq_init_attr *srq_init_attr, 1389 struct ib_udata *udata) 1390 { 1391 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 1392 struct bnxt_re_dev *rdev = pd->rdev; 1393 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 1394 struct bnxt_re_srq *srq; 1395 struct bnxt_qplib_nq *nq = NULL; 1396 int rc, entries; 1397 1398 if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) { 1399 dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded"); 1400 rc = -EINVAL; 1401 goto exit; 1402 } 1403 1404 if (srq_init_attr->srq_type != IB_SRQT_BASIC) { 1405 rc = -EOPNOTSUPP; 1406 goto exit; 1407 } 1408 1409 srq = kzalloc(sizeof(*srq), GFP_KERNEL); 1410 if (!srq) { 1411 rc = -ENOMEM; 1412 goto exit; 1413 } 1414 srq->rdev = rdev; 1415 srq->qplib_srq.pd = &pd->qplib_pd; 1416 srq->qplib_srq.dpi = &rdev->dpi_privileged; 1417 /* Allocate 1 more than what's provided so posting max doesn't 1418 * mean empty 1419 */ 1420 entries = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1); 1421 if (entries > dev_attr->max_srq_wqes + 1) 1422 entries = dev_attr->max_srq_wqes + 1; 1423 1424 srq->qplib_srq.max_wqe = entries; 1425 srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge; 1426 srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit; 1427 srq->srq_limit = srq_init_attr->attr.srq_limit; 1428 srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id; 1429 nq = &rdev->nq[0]; 1430 1431 if (udata) { 1432 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata); 1433 if (rc) 1434 goto fail; 1435 } 1436 1437 rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq); 1438 if (rc) { 1439 dev_err(rdev_to_dev(rdev), "Create HW SRQ failed!"); 1440 goto fail; 1441 } 1442 1443 if (udata) { 1444 struct bnxt_re_srq_resp resp; 1445 1446 resp.srqid = srq->qplib_srq.id; 1447 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 1448 if (rc) { 1449 dev_err(rdev_to_dev(rdev), "SRQ copy to udata failed!"); 1450 bnxt_qplib_destroy_srq(&rdev->qplib_res, 1451 &srq->qplib_srq); 1452 goto exit; 1453 } 1454 } 1455 if (nq) 1456 nq->budget++; 1457 atomic_inc(&rdev->srq_count); 1458 1459 return &srq->ib_srq; 1460 1461 fail: 1462 if (srq->umem) 1463 ib_umem_release(srq->umem); 1464 kfree(srq); 1465 exit: 1466 return ERR_PTR(rc); 1467 } 1468 1469 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, 1470 enum ib_srq_attr_mask srq_attr_mask, 1471 struct ib_udata *udata) 1472 { 1473 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1474 ib_srq); 1475 struct bnxt_re_dev *rdev = srq->rdev; 1476 int rc; 1477 1478 switch (srq_attr_mask) { 1479 case IB_SRQ_MAX_WR: 1480 /* SRQ resize is not supported */ 1481 break; 1482 case IB_SRQ_LIMIT: 1483 /* Change the SRQ threshold */ 1484 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) 1485 return -EINVAL; 1486 1487 srq->qplib_srq.threshold = srq_attr->srq_limit; 1488 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq); 1489 if (rc) { 1490 dev_err(rdev_to_dev(rdev), "Modify HW SRQ failed!"); 1491 return rc; 1492 } 1493 /* On success, update the shadow */ 1494 srq->srq_limit = srq_attr->srq_limit; 1495 /* No need to Build and send response back to udata */ 1496 break; 1497 default: 1498 dev_err(rdev_to_dev(rdev), 1499 "Unsupported srq_attr_mask 0x%x", srq_attr_mask); 1500 return -EINVAL; 1501 } 1502 return 0; 1503 } 1504 1505 int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr) 1506 { 1507 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1508 ib_srq); 1509 struct bnxt_re_srq tsrq; 1510 struct bnxt_re_dev *rdev = srq->rdev; 1511 int rc; 1512 1513 /* Get live SRQ attr */ 1514 tsrq.qplib_srq.id = srq->qplib_srq.id; 1515 rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq); 1516 if (rc) { 1517 dev_err(rdev_to_dev(rdev), "Query HW SRQ failed!"); 1518 return rc; 1519 } 1520 srq_attr->max_wr = srq->qplib_srq.max_wqe; 1521 srq_attr->max_sge = srq->qplib_srq.max_sge; 1522 srq_attr->srq_limit = tsrq.qplib_srq.threshold; 1523 1524 return 0; 1525 } 1526 1527 int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, 1528 const struct ib_recv_wr **bad_wr) 1529 { 1530 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1531 ib_srq); 1532 struct bnxt_qplib_swqe wqe; 1533 unsigned long flags; 1534 int rc = 0; 1535 1536 spin_lock_irqsave(&srq->lock, flags); 1537 while (wr) { 1538 /* Transcribe each ib_recv_wr to qplib_swqe */ 1539 wqe.num_sge = wr->num_sge; 1540 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); 1541 wqe.wr_id = wr->wr_id; 1542 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 1543 1544 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe); 1545 if (rc) { 1546 *bad_wr = wr; 1547 break; 1548 } 1549 wr = wr->next; 1550 } 1551 spin_unlock_irqrestore(&srq->lock, flags); 1552 1553 return rc; 1554 } 1555 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, 1556 struct bnxt_re_qp *qp1_qp, 1557 int qp_attr_mask) 1558 { 1559 struct bnxt_re_qp *qp = rdev->qp1_sqp; 1560 int rc = 0; 1561 1562 if (qp_attr_mask & IB_QP_STATE) { 1563 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 1564 qp->qplib_qp.state = qp1_qp->qplib_qp.state; 1565 } 1566 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 1567 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 1568 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; 1569 } 1570 1571 if (qp_attr_mask & IB_QP_QKEY) { 1572 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 1573 /* Using a Random QKEY */ 1574 qp->qplib_qp.qkey = 0x81818181; 1575 } 1576 if (qp_attr_mask & IB_QP_SQ_PSN) { 1577 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 1578 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; 1579 } 1580 1581 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 1582 if (rc) 1583 dev_err(rdev_to_dev(rdev), 1584 "Failed to modify Shadow QP for QP1"); 1585 return rc; 1586 } 1587 1588 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 1589 int qp_attr_mask, struct ib_udata *udata) 1590 { 1591 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 1592 struct bnxt_re_dev *rdev = qp->rdev; 1593 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 1594 enum ib_qp_state curr_qp_state, new_qp_state; 1595 int rc, entries; 1596 unsigned int flags; 1597 u8 nw_type; 1598 1599 qp->qplib_qp.modify_flags = 0; 1600 if (qp_attr_mask & IB_QP_STATE) { 1601 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); 1602 new_qp_state = qp_attr->qp_state; 1603 if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state, 1604 ib_qp->qp_type, qp_attr_mask)) { 1605 dev_err(rdev_to_dev(rdev), 1606 "Invalid attribute mask: %#x specified ", 1607 qp_attr_mask); 1608 dev_err(rdev_to_dev(rdev), 1609 "for qpn: %#x type: %#x", 1610 ib_qp->qp_num, ib_qp->qp_type); 1611 dev_err(rdev_to_dev(rdev), 1612 "curr_qp_state=0x%x, new_qp_state=0x%x\n", 1613 curr_qp_state, new_qp_state); 1614 return -EINVAL; 1615 } 1616 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 1617 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); 1618 1619 if (!qp->sumem && 1620 qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { 1621 dev_dbg(rdev_to_dev(rdev), 1622 "Move QP = %p to flush list\n", 1623 qp); 1624 flags = bnxt_re_lock_cqs(qp); 1625 bnxt_qplib_add_flush_qp(&qp->qplib_qp); 1626 bnxt_re_unlock_cqs(qp, flags); 1627 } 1628 if (!qp->sumem && 1629 qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { 1630 dev_dbg(rdev_to_dev(rdev), 1631 "Move QP = %p out of flush list\n", 1632 qp); 1633 flags = bnxt_re_lock_cqs(qp); 1634 bnxt_qplib_clean_qp(&qp->qplib_qp); 1635 bnxt_re_unlock_cqs(qp, flags); 1636 } 1637 } 1638 if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { 1639 qp->qplib_qp.modify_flags |= 1640 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY; 1641 qp->qplib_qp.en_sqd_async_notify = true; 1642 } 1643 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) { 1644 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; 1645 qp->qplib_qp.access = 1646 __from_ib_access_flags(qp_attr->qp_access_flags); 1647 /* LOCAL_WRITE access must be set to allow RC receive */ 1648 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 1649 } 1650 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 1651 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 1652 qp->qplib_qp.pkey_index = qp_attr->pkey_index; 1653 } 1654 if (qp_attr_mask & IB_QP_QKEY) { 1655 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 1656 qp->qplib_qp.qkey = qp_attr->qkey; 1657 } 1658 if (qp_attr_mask & IB_QP_AV) { 1659 const struct ib_global_route *grh = 1660 rdma_ah_read_grh(&qp_attr->ah_attr); 1661 const struct ib_gid_attr *sgid_attr; 1662 1663 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | 1664 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | 1665 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | 1666 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | 1667 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | 1668 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | 1669 CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; 1670 memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw, 1671 sizeof(qp->qplib_qp.ah.dgid.data)); 1672 qp->qplib_qp.ah.flow_label = grh->flow_label; 1673 /* If RoCE V2 is enabled, stack will have two entries for 1674 * each GID entry. Avoiding this duplicte entry in HW. Dividing 1675 * the GID index by 2 for RoCE V2 1676 */ 1677 qp->qplib_qp.ah.sgid_index = grh->sgid_index / 2; 1678 qp->qplib_qp.ah.host_sgid_index = grh->sgid_index; 1679 qp->qplib_qp.ah.hop_limit = grh->hop_limit; 1680 qp->qplib_qp.ah.traffic_class = grh->traffic_class; 1681 qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr); 1682 ether_addr_copy(qp->qplib_qp.ah.dmac, 1683 qp_attr->ah_attr.roce.dmac); 1684 1685 sgid_attr = qp_attr->ah_attr.grh.sgid_attr; 1686 memcpy(qp->qplib_qp.smac, sgid_attr->ndev->dev_addr, 1687 ETH_ALEN); 1688 nw_type = rdma_gid_attr_network_type(sgid_attr); 1689 switch (nw_type) { 1690 case RDMA_NETWORK_IPV4: 1691 qp->qplib_qp.nw_type = 1692 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4; 1693 break; 1694 case RDMA_NETWORK_IPV6: 1695 qp->qplib_qp.nw_type = 1696 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6; 1697 break; 1698 default: 1699 qp->qplib_qp.nw_type = 1700 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1; 1701 break; 1702 } 1703 } 1704 1705 if (qp_attr_mask & IB_QP_PATH_MTU) { 1706 qp->qplib_qp.modify_flags |= 1707 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 1708 qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu); 1709 qp->qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr->path_mtu); 1710 } else if (qp_attr->qp_state == IB_QPS_RTR) { 1711 qp->qplib_qp.modify_flags |= 1712 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 1713 qp->qplib_qp.path_mtu = 1714 __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu)); 1715 qp->qplib_qp.mtu = 1716 ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); 1717 } 1718 1719 if (qp_attr_mask & IB_QP_TIMEOUT) { 1720 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; 1721 qp->qplib_qp.timeout = qp_attr->timeout; 1722 } 1723 if (qp_attr_mask & IB_QP_RETRY_CNT) { 1724 qp->qplib_qp.modify_flags |= 1725 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT; 1726 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; 1727 } 1728 if (qp_attr_mask & IB_QP_RNR_RETRY) { 1729 qp->qplib_qp.modify_flags |= 1730 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY; 1731 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; 1732 } 1733 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) { 1734 qp->qplib_qp.modify_flags |= 1735 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER; 1736 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; 1737 } 1738 if (qp_attr_mask & IB_QP_RQ_PSN) { 1739 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; 1740 qp->qplib_qp.rq.psn = qp_attr->rq_psn; 1741 } 1742 if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1743 qp->qplib_qp.modify_flags |= 1744 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC; 1745 /* Cap the max_rd_atomic to device max */ 1746 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic, 1747 dev_attr->max_qp_rd_atom); 1748 } 1749 if (qp_attr_mask & IB_QP_SQ_PSN) { 1750 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 1751 qp->qplib_qp.sq.psn = qp_attr->sq_psn; 1752 } 1753 if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1754 if (qp_attr->max_dest_rd_atomic > 1755 dev_attr->max_qp_init_rd_atom) { 1756 dev_err(rdev_to_dev(rdev), 1757 "max_dest_rd_atomic requested%d is > dev_max%d", 1758 qp_attr->max_dest_rd_atomic, 1759 dev_attr->max_qp_init_rd_atom); 1760 return -EINVAL; 1761 } 1762 1763 qp->qplib_qp.modify_flags |= 1764 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC; 1765 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; 1766 } 1767 if (qp_attr_mask & IB_QP_CAP) { 1768 qp->qplib_qp.modify_flags |= 1769 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE | 1770 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE | 1771 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE | 1772 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE | 1773 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA; 1774 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || 1775 (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || 1776 (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || 1777 (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || 1778 (qp_attr->cap.max_inline_data >= 1779 dev_attr->max_inline_data)) { 1780 dev_err(rdev_to_dev(rdev), 1781 "Create QP failed - max exceeded"); 1782 return -EINVAL; 1783 } 1784 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr); 1785 qp->qplib_qp.sq.max_wqe = min_t(u32, entries, 1786 dev_attr->max_qp_wqes + 1); 1787 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - 1788 qp_attr->cap.max_send_wr; 1789 /* 1790 * Reserving one slot for Phantom WQE. Some application can 1791 * post one extra entry in this case. Allowing this to avoid 1792 * unexpected Queue full condition 1793 */ 1794 qp->qplib_qp.sq.q_full_delta -= 1; 1795 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; 1796 if (qp->qplib_qp.rq.max_wqe) { 1797 entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr); 1798 qp->qplib_qp.rq.max_wqe = 1799 min_t(u32, entries, dev_attr->max_qp_wqes + 1); 1800 qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - 1801 qp_attr->cap.max_recv_wr; 1802 qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; 1803 } else { 1804 /* SRQ was used prior, just ignore the RQ caps */ 1805 } 1806 } 1807 if (qp_attr_mask & IB_QP_DEST_QPN) { 1808 qp->qplib_qp.modify_flags |= 1809 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID; 1810 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; 1811 } 1812 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 1813 if (rc) { 1814 dev_err(rdev_to_dev(rdev), "Failed to modify HW QP"); 1815 return rc; 1816 } 1817 if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) 1818 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); 1819 return rc; 1820 } 1821 1822 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 1823 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) 1824 { 1825 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 1826 struct bnxt_re_dev *rdev = qp->rdev; 1827 struct bnxt_qplib_qp *qplib_qp; 1828 int rc; 1829 1830 qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL); 1831 if (!qplib_qp) 1832 return -ENOMEM; 1833 1834 qplib_qp->id = qp->qplib_qp.id; 1835 qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; 1836 1837 rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp); 1838 if (rc) { 1839 dev_err(rdev_to_dev(rdev), "Failed to query HW QP"); 1840 goto out; 1841 } 1842 qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state); 1843 qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0; 1844 qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access); 1845 qp_attr->pkey_index = qplib_qp->pkey_index; 1846 qp_attr->qkey = qplib_qp->qkey; 1847 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 1848 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->ah.flow_label, 1849 qplib_qp->ah.host_sgid_index, 1850 qplib_qp->ah.hop_limit, 1851 qplib_qp->ah.traffic_class); 1852 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data); 1853 rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl); 1854 ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac); 1855 qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu); 1856 qp_attr->timeout = qplib_qp->timeout; 1857 qp_attr->retry_cnt = qplib_qp->retry_cnt; 1858 qp_attr->rnr_retry = qplib_qp->rnr_retry; 1859 qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer; 1860 qp_attr->rq_psn = qplib_qp->rq.psn; 1861 qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic; 1862 qp_attr->sq_psn = qplib_qp->sq.psn; 1863 qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic; 1864 qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR : 1865 IB_SIGNAL_REQ_WR; 1866 qp_attr->dest_qp_num = qplib_qp->dest_qpn; 1867 1868 qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; 1869 qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; 1870 qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; 1871 qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; 1872 qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; 1873 qp_init_attr->cap = qp_attr->cap; 1874 1875 out: 1876 kfree(qplib_qp); 1877 return rc; 1878 } 1879 1880 /* Routine for sending QP1 packets for RoCE V1 an V2 1881 */ 1882 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp, 1883 const struct ib_send_wr *wr, 1884 struct bnxt_qplib_swqe *wqe, 1885 int payload_size) 1886 { 1887 struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, 1888 ib_ah); 1889 struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; 1890 const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr; 1891 struct bnxt_qplib_sge sge; 1892 u8 nw_type; 1893 u16 ether_type; 1894 union ib_gid dgid; 1895 bool is_eth = false; 1896 bool is_vlan = false; 1897 bool is_grh = false; 1898 bool is_udp = false; 1899 u8 ip_version = 0; 1900 u16 vlan_id = 0xFFFF; 1901 void *buf; 1902 int i, rc = 0; 1903 1904 memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); 1905 1906 if (is_vlan_dev(sgid_attr->ndev)) 1907 vlan_id = vlan_dev_vlan_id(sgid_attr->ndev); 1908 /* Get network header type for this GID */ 1909 nw_type = rdma_gid_attr_network_type(sgid_attr); 1910 switch (nw_type) { 1911 case RDMA_NETWORK_IPV4: 1912 nw_type = BNXT_RE_ROCEV2_IPV4_PACKET; 1913 break; 1914 case RDMA_NETWORK_IPV6: 1915 nw_type = BNXT_RE_ROCEV2_IPV6_PACKET; 1916 break; 1917 default: 1918 nw_type = BNXT_RE_ROCE_V1_PACKET; 1919 break; 1920 } 1921 memcpy(&dgid.raw, &qplib_ah->dgid, 16); 1922 is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP; 1923 if (is_udp) { 1924 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) { 1925 ip_version = 4; 1926 ether_type = ETH_P_IP; 1927 } else { 1928 ip_version = 6; 1929 ether_type = ETH_P_IPV6; 1930 } 1931 is_grh = false; 1932 } else { 1933 ether_type = ETH_P_IBOE; 1934 is_grh = true; 1935 } 1936 1937 is_eth = true; 1938 is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false; 1939 1940 ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh, 1941 ip_version, is_udp, 0, &qp->qp1_hdr); 1942 1943 /* ETH */ 1944 ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac); 1945 ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac); 1946 1947 /* For vlan, check the sgid for vlan existence */ 1948 1949 if (!is_vlan) { 1950 qp->qp1_hdr.eth.type = cpu_to_be16(ether_type); 1951 } else { 1952 qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type); 1953 qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id); 1954 } 1955 1956 if (is_grh || (ip_version == 6)) { 1957 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw, 1958 sizeof(sgid_attr->gid)); 1959 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, 1960 sizeof(sgid_attr->gid)); 1961 qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit; 1962 } 1963 1964 if (ip_version == 4) { 1965 qp->qp1_hdr.ip4.tos = 0; 1966 qp->qp1_hdr.ip4.id = 0; 1967 qp->qp1_hdr.ip4.frag_off = htons(IP_DF); 1968 qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit; 1969 1970 memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4); 1971 memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4); 1972 qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr); 1973 } 1974 1975 if (is_udp) { 1976 qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT); 1977 qp->qp1_hdr.udp.sport = htons(0x8CD1); 1978 qp->qp1_hdr.udp.csum = 0; 1979 } 1980 1981 /* BTH */ 1982 if (wr->opcode == IB_WR_SEND_WITH_IMM) { 1983 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; 1984 qp->qp1_hdr.immediate_present = 1; 1985 } else { 1986 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; 1987 } 1988 if (wr->send_flags & IB_SEND_SOLICITED) 1989 qp->qp1_hdr.bth.solicited_event = 1; 1990 /* pad_count */ 1991 qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; 1992 1993 /* P_key for QP1 is for all members */ 1994 qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); 1995 qp->qp1_hdr.bth.destination_qpn = IB_QP1; 1996 qp->qp1_hdr.bth.ack_req = 0; 1997 qp->send_psn++; 1998 qp->send_psn &= BTH_PSN_MASK; 1999 qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); 2000 /* DETH */ 2001 /* Use the priviledged Q_Key for QP1 */ 2002 qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); 2003 qp->qp1_hdr.deth.source_qpn = IB_QP1; 2004 2005 /* Pack the QP1 to the transmit buffer */ 2006 buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); 2007 if (buf) { 2008 ib_ud_header_pack(&qp->qp1_hdr, buf); 2009 for (i = wqe->num_sge; i; i--) { 2010 wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; 2011 wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; 2012 wqe->sg_list[i].size = wqe->sg_list[i - 1].size; 2013 } 2014 2015 /* 2016 * Max Header buf size for IPV6 RoCE V2 is 86, 2017 * which is same as the QP1 SQ header buffer. 2018 * Header buf size for IPV4 RoCE V2 can be 66. 2019 * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20). 2020 * Subtract 20 bytes from QP1 SQ header buf size 2021 */ 2022 if (is_udp && ip_version == 4) 2023 sge.size -= 20; 2024 /* 2025 * Max Header buf size for RoCE V1 is 78. 2026 * ETH(14) + VLAN(4) + GRH(40) + BTH(20). 2027 * Subtract 8 bytes from QP1 SQ header buf size 2028 */ 2029 if (!is_udp) 2030 sge.size -= 8; 2031 2032 /* Subtract 4 bytes for non vlan packets */ 2033 if (!is_vlan) 2034 sge.size -= 4; 2035 2036 wqe->sg_list[0].addr = sge.addr; 2037 wqe->sg_list[0].lkey = sge.lkey; 2038 wqe->sg_list[0].size = sge.size; 2039 wqe->num_sge++; 2040 2041 } else { 2042 dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!"); 2043 rc = -ENOMEM; 2044 } 2045 return rc; 2046 } 2047 2048 /* For the MAD layer, it only provides the recv SGE the size of 2049 * ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH, 2050 * nor RoCE iCRC. The Cu+ solution must provide buffer for the entire 2051 * receive packet (334 bytes) with no VLAN and then copy the GRH 2052 * and the MAD datagram out to the provided SGE. 2053 */ 2054 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, 2055 const struct ib_recv_wr *wr, 2056 struct bnxt_qplib_swqe *wqe, 2057 int payload_size) 2058 { 2059 struct bnxt_qplib_sge ref, sge; 2060 u32 rq_prod_index; 2061 struct bnxt_re_sqp_entries *sqp_entry; 2062 2063 rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); 2064 2065 if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) 2066 return -ENOMEM; 2067 2068 /* Create 1 SGE to receive the entire 2069 * ethernet packet 2070 */ 2071 /* Save the reference from ULP */ 2072 ref.addr = wqe->sg_list[0].addr; 2073 ref.lkey = wqe->sg_list[0].lkey; 2074 ref.size = wqe->sg_list[0].size; 2075 2076 sqp_entry = &qp->rdev->sqp_tbl[rq_prod_index]; 2077 2078 /* SGE 1 */ 2079 wqe->sg_list[0].addr = sge.addr; 2080 wqe->sg_list[0].lkey = sge.lkey; 2081 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; 2082 sge.size -= wqe->sg_list[0].size; 2083 2084 sqp_entry->sge.addr = ref.addr; 2085 sqp_entry->sge.lkey = ref.lkey; 2086 sqp_entry->sge.size = ref.size; 2087 /* Store the wrid for reporting completion */ 2088 sqp_entry->wrid = wqe->wr_id; 2089 /* change the wqe->wrid to table index */ 2090 wqe->wr_id = rq_prod_index; 2091 return 0; 2092 } 2093 2094 static int is_ud_qp(struct bnxt_re_qp *qp) 2095 { 2096 return qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD; 2097 } 2098 2099 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, 2100 const struct ib_send_wr *wr, 2101 struct bnxt_qplib_swqe *wqe) 2102 { 2103 struct bnxt_re_ah *ah = NULL; 2104 2105 if (is_ud_qp(qp)) { 2106 ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah); 2107 wqe->send.q_key = ud_wr(wr)->remote_qkey; 2108 wqe->send.dst_qp = ud_wr(wr)->remote_qpn; 2109 wqe->send.avid = ah->qplib_ah.id; 2110 } 2111 switch (wr->opcode) { 2112 case IB_WR_SEND: 2113 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; 2114 break; 2115 case IB_WR_SEND_WITH_IMM: 2116 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; 2117 wqe->send.imm_data = wr->ex.imm_data; 2118 break; 2119 case IB_WR_SEND_WITH_INV: 2120 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; 2121 wqe->send.inv_key = wr->ex.invalidate_rkey; 2122 break; 2123 default: 2124 return -EINVAL; 2125 } 2126 if (wr->send_flags & IB_SEND_SIGNALED) 2127 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2128 if (wr->send_flags & IB_SEND_FENCE) 2129 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2130 if (wr->send_flags & IB_SEND_SOLICITED) 2131 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2132 if (wr->send_flags & IB_SEND_INLINE) 2133 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 2134 2135 return 0; 2136 } 2137 2138 static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr, 2139 struct bnxt_qplib_swqe *wqe) 2140 { 2141 switch (wr->opcode) { 2142 case IB_WR_RDMA_WRITE: 2143 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; 2144 break; 2145 case IB_WR_RDMA_WRITE_WITH_IMM: 2146 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; 2147 wqe->rdma.imm_data = wr->ex.imm_data; 2148 break; 2149 case IB_WR_RDMA_READ: 2150 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; 2151 wqe->rdma.inv_key = wr->ex.invalidate_rkey; 2152 break; 2153 default: 2154 return -EINVAL; 2155 } 2156 wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; 2157 wqe->rdma.r_key = rdma_wr(wr)->rkey; 2158 if (wr->send_flags & IB_SEND_SIGNALED) 2159 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2160 if (wr->send_flags & IB_SEND_FENCE) 2161 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2162 if (wr->send_flags & IB_SEND_SOLICITED) 2163 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2164 if (wr->send_flags & IB_SEND_INLINE) 2165 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 2166 2167 return 0; 2168 } 2169 2170 static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr, 2171 struct bnxt_qplib_swqe *wqe) 2172 { 2173 switch (wr->opcode) { 2174 case IB_WR_ATOMIC_CMP_AND_SWP: 2175 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; 2176 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 2177 wqe->atomic.swap_data = atomic_wr(wr)->swap; 2178 break; 2179 case IB_WR_ATOMIC_FETCH_AND_ADD: 2180 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; 2181 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 2182 break; 2183 default: 2184 return -EINVAL; 2185 } 2186 wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; 2187 wqe->atomic.r_key = atomic_wr(wr)->rkey; 2188 if (wr->send_flags & IB_SEND_SIGNALED) 2189 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2190 if (wr->send_flags & IB_SEND_FENCE) 2191 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2192 if (wr->send_flags & IB_SEND_SOLICITED) 2193 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2194 return 0; 2195 } 2196 2197 static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr, 2198 struct bnxt_qplib_swqe *wqe) 2199 { 2200 wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; 2201 wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; 2202 2203 /* Need unconditional fence for local invalidate 2204 * opcode to work as expected. 2205 */ 2206 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2207 2208 if (wr->send_flags & IB_SEND_SIGNALED) 2209 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2210 if (wr->send_flags & IB_SEND_SOLICITED) 2211 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2212 2213 return 0; 2214 } 2215 2216 static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr, 2217 struct bnxt_qplib_swqe *wqe) 2218 { 2219 struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr); 2220 struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; 2221 int access = wr->access; 2222 2223 wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; 2224 wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; 2225 wqe->frmr.page_list = mr->pages; 2226 wqe->frmr.page_list_len = mr->npages; 2227 wqe->frmr.levels = qplib_frpl->hwq.level + 1; 2228 wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; 2229 2230 /* Need unconditional fence for reg_mr 2231 * opcode to function as expected. 2232 */ 2233 2234 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2235 2236 if (wr->wr.send_flags & IB_SEND_SIGNALED) 2237 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2238 2239 if (access & IB_ACCESS_LOCAL_WRITE) 2240 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; 2241 if (access & IB_ACCESS_REMOTE_READ) 2242 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; 2243 if (access & IB_ACCESS_REMOTE_WRITE) 2244 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; 2245 if (access & IB_ACCESS_REMOTE_ATOMIC) 2246 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; 2247 if (access & IB_ACCESS_MW_BIND) 2248 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; 2249 2250 wqe->frmr.l_key = wr->key; 2251 wqe->frmr.length = wr->mr->length; 2252 wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1; 2253 wqe->frmr.va = wr->mr->iova; 2254 return 0; 2255 } 2256 2257 static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev, 2258 const struct ib_send_wr *wr, 2259 struct bnxt_qplib_swqe *wqe) 2260 { 2261 /* Copy the inline data to the data field */ 2262 u8 *in_data; 2263 u32 i, sge_len; 2264 void *sge_addr; 2265 2266 in_data = wqe->inline_data; 2267 for (i = 0; i < wr->num_sge; i++) { 2268 sge_addr = (void *)(unsigned long) 2269 wr->sg_list[i].addr; 2270 sge_len = wr->sg_list[i].length; 2271 2272 if ((sge_len + wqe->inline_len) > 2273 BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { 2274 dev_err(rdev_to_dev(rdev), 2275 "Inline data size requested > supported value"); 2276 return -EINVAL; 2277 } 2278 sge_len = wr->sg_list[i].length; 2279 2280 memcpy(in_data, sge_addr, sge_len); 2281 in_data += wr->sg_list[i].length; 2282 wqe->inline_len += wr->sg_list[i].length; 2283 } 2284 return wqe->inline_len; 2285 } 2286 2287 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev, 2288 const struct ib_send_wr *wr, 2289 struct bnxt_qplib_swqe *wqe) 2290 { 2291 int payload_sz = 0; 2292 2293 if (wr->send_flags & IB_SEND_INLINE) 2294 payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe); 2295 else 2296 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list, 2297 wqe->num_sge); 2298 2299 return payload_sz; 2300 } 2301 2302 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp) 2303 { 2304 if ((qp->ib_qp.qp_type == IB_QPT_UD || 2305 qp->ib_qp.qp_type == IB_QPT_GSI || 2306 qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) && 2307 qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) { 2308 int qp_attr_mask; 2309 struct ib_qp_attr qp_attr; 2310 2311 qp_attr_mask = IB_QP_STATE; 2312 qp_attr.qp_state = IB_QPS_RTS; 2313 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL); 2314 qp->qplib_qp.wqe_cnt = 0; 2315 } 2316 } 2317 2318 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, 2319 struct bnxt_re_qp *qp, 2320 const struct ib_send_wr *wr) 2321 { 2322 struct bnxt_qplib_swqe wqe; 2323 int rc = 0, payload_sz = 0; 2324 unsigned long flags; 2325 2326 spin_lock_irqsave(&qp->sq_lock, flags); 2327 memset(&wqe, 0, sizeof(wqe)); 2328 while (wr) { 2329 /* House keeping */ 2330 memset(&wqe, 0, sizeof(wqe)); 2331 2332 /* Common */ 2333 wqe.num_sge = wr->num_sge; 2334 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 2335 dev_err(rdev_to_dev(rdev), 2336 "Limit exceeded for Send SGEs"); 2337 rc = -EINVAL; 2338 goto bad; 2339 } 2340 2341 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); 2342 if (payload_sz < 0) { 2343 rc = -EINVAL; 2344 goto bad; 2345 } 2346 wqe.wr_id = wr->wr_id; 2347 2348 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; 2349 2350 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 2351 if (!rc) 2352 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 2353 bad: 2354 if (rc) { 2355 dev_err(rdev_to_dev(rdev), 2356 "Post send failed opcode = %#x rc = %d", 2357 wr->opcode, rc); 2358 break; 2359 } 2360 wr = wr->next; 2361 } 2362 bnxt_qplib_post_send_db(&qp->qplib_qp); 2363 bnxt_ud_qp_hw_stall_workaround(qp); 2364 spin_unlock_irqrestore(&qp->sq_lock, flags); 2365 return rc; 2366 } 2367 2368 int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, 2369 const struct ib_send_wr **bad_wr) 2370 { 2371 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 2372 struct bnxt_qplib_swqe wqe; 2373 int rc = 0, payload_sz = 0; 2374 unsigned long flags; 2375 2376 spin_lock_irqsave(&qp->sq_lock, flags); 2377 while (wr) { 2378 /* House keeping */ 2379 memset(&wqe, 0, sizeof(wqe)); 2380 2381 /* Common */ 2382 wqe.num_sge = wr->num_sge; 2383 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 2384 dev_err(rdev_to_dev(qp->rdev), 2385 "Limit exceeded for Send SGEs"); 2386 rc = -EINVAL; 2387 goto bad; 2388 } 2389 2390 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); 2391 if (payload_sz < 0) { 2392 rc = -EINVAL; 2393 goto bad; 2394 } 2395 wqe.wr_id = wr->wr_id; 2396 2397 switch (wr->opcode) { 2398 case IB_WR_SEND: 2399 case IB_WR_SEND_WITH_IMM: 2400 if (ib_qp->qp_type == IB_QPT_GSI) { 2401 rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe, 2402 payload_sz); 2403 if (rc) 2404 goto bad; 2405 wqe.rawqp1.lflags |= 2406 SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; 2407 } 2408 switch (wr->send_flags) { 2409 case IB_SEND_IP_CSUM: 2410 wqe.rawqp1.lflags |= 2411 SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; 2412 break; 2413 default: 2414 break; 2415 } 2416 /* fall through */ 2417 case IB_WR_SEND_WITH_INV: 2418 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 2419 break; 2420 case IB_WR_RDMA_WRITE: 2421 case IB_WR_RDMA_WRITE_WITH_IMM: 2422 case IB_WR_RDMA_READ: 2423 rc = bnxt_re_build_rdma_wqe(wr, &wqe); 2424 break; 2425 case IB_WR_ATOMIC_CMP_AND_SWP: 2426 case IB_WR_ATOMIC_FETCH_AND_ADD: 2427 rc = bnxt_re_build_atomic_wqe(wr, &wqe); 2428 break; 2429 case IB_WR_RDMA_READ_WITH_INV: 2430 dev_err(rdev_to_dev(qp->rdev), 2431 "RDMA Read with Invalidate is not supported"); 2432 rc = -EINVAL; 2433 goto bad; 2434 case IB_WR_LOCAL_INV: 2435 rc = bnxt_re_build_inv_wqe(wr, &wqe); 2436 break; 2437 case IB_WR_REG_MR: 2438 rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); 2439 break; 2440 default: 2441 /* Unsupported WRs */ 2442 dev_err(rdev_to_dev(qp->rdev), 2443 "WR (%#x) is not supported", wr->opcode); 2444 rc = -EINVAL; 2445 goto bad; 2446 } 2447 if (!rc) 2448 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 2449 bad: 2450 if (rc) { 2451 dev_err(rdev_to_dev(qp->rdev), 2452 "post_send failed op:%#x qps = %#x rc = %d\n", 2453 wr->opcode, qp->qplib_qp.state, rc); 2454 *bad_wr = wr; 2455 break; 2456 } 2457 wr = wr->next; 2458 } 2459 bnxt_qplib_post_send_db(&qp->qplib_qp); 2460 bnxt_ud_qp_hw_stall_workaround(qp); 2461 spin_unlock_irqrestore(&qp->sq_lock, flags); 2462 2463 return rc; 2464 } 2465 2466 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, 2467 struct bnxt_re_qp *qp, 2468 const struct ib_recv_wr *wr) 2469 { 2470 struct bnxt_qplib_swqe wqe; 2471 int rc = 0; 2472 2473 memset(&wqe, 0, sizeof(wqe)); 2474 while (wr) { 2475 /* House keeping */ 2476 memset(&wqe, 0, sizeof(wqe)); 2477 2478 /* Common */ 2479 wqe.num_sge = wr->num_sge; 2480 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 2481 dev_err(rdev_to_dev(rdev), 2482 "Limit exceeded for Receive SGEs"); 2483 rc = -EINVAL; 2484 break; 2485 } 2486 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); 2487 wqe.wr_id = wr->wr_id; 2488 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 2489 2490 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 2491 if (rc) 2492 break; 2493 2494 wr = wr->next; 2495 } 2496 if (!rc) 2497 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2498 return rc; 2499 } 2500 2501 int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, 2502 const struct ib_recv_wr **bad_wr) 2503 { 2504 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 2505 struct bnxt_qplib_swqe wqe; 2506 int rc = 0, payload_sz = 0; 2507 unsigned long flags; 2508 u32 count = 0; 2509 2510 spin_lock_irqsave(&qp->rq_lock, flags); 2511 while (wr) { 2512 /* House keeping */ 2513 memset(&wqe, 0, sizeof(wqe)); 2514 2515 /* Common */ 2516 wqe.num_sge = wr->num_sge; 2517 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 2518 dev_err(rdev_to_dev(qp->rdev), 2519 "Limit exceeded for Receive SGEs"); 2520 rc = -EINVAL; 2521 *bad_wr = wr; 2522 break; 2523 } 2524 2525 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, 2526 wr->num_sge); 2527 wqe.wr_id = wr->wr_id; 2528 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 2529 2530 if (ib_qp->qp_type == IB_QPT_GSI) 2531 rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe, 2532 payload_sz); 2533 if (!rc) 2534 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 2535 if (rc) { 2536 *bad_wr = wr; 2537 break; 2538 } 2539 2540 /* Ring DB if the RQEs posted reaches a threshold value */ 2541 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) { 2542 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2543 count = 0; 2544 } 2545 2546 wr = wr->next; 2547 } 2548 2549 if (count) 2550 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2551 2552 spin_unlock_irqrestore(&qp->rq_lock, flags); 2553 2554 return rc; 2555 } 2556 2557 /* Completion Queues */ 2558 int bnxt_re_destroy_cq(struct ib_cq *ib_cq) 2559 { 2560 int rc; 2561 struct bnxt_re_cq *cq; 2562 struct bnxt_qplib_nq *nq; 2563 struct bnxt_re_dev *rdev; 2564 2565 cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 2566 rdev = cq->rdev; 2567 nq = cq->qplib_cq.nq; 2568 2569 rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); 2570 if (rc) { 2571 dev_err(rdev_to_dev(rdev), "Failed to destroy HW CQ"); 2572 return rc; 2573 } 2574 if (!IS_ERR_OR_NULL(cq->umem)) 2575 ib_umem_release(cq->umem); 2576 2577 atomic_dec(&rdev->cq_count); 2578 nq->budget--; 2579 kfree(cq->cql); 2580 kfree(cq); 2581 2582 return 0; 2583 } 2584 2585 struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, 2586 const struct ib_cq_init_attr *attr, 2587 struct ib_ucontext *context, 2588 struct ib_udata *udata) 2589 { 2590 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 2591 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 2592 struct bnxt_re_cq *cq = NULL; 2593 int rc, entries; 2594 int cqe = attr->cqe; 2595 struct bnxt_qplib_nq *nq = NULL; 2596 unsigned int nq_alloc_cnt; 2597 2598 /* Validate CQ fields */ 2599 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { 2600 dev_err(rdev_to_dev(rdev), "Failed to create CQ -max exceeded"); 2601 return ERR_PTR(-EINVAL); 2602 } 2603 cq = kzalloc(sizeof(*cq), GFP_KERNEL); 2604 if (!cq) 2605 return ERR_PTR(-ENOMEM); 2606 2607 cq->rdev = rdev; 2608 cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq); 2609 2610 entries = roundup_pow_of_two(cqe + 1); 2611 if (entries > dev_attr->max_cq_wqes + 1) 2612 entries = dev_attr->max_cq_wqes + 1; 2613 2614 if (context) { 2615 struct bnxt_re_cq_req req; 2616 struct bnxt_re_ucontext *uctx = container_of 2617 (context, 2618 struct bnxt_re_ucontext, 2619 ib_uctx); 2620 if (ib_copy_from_udata(&req, udata, sizeof(req))) { 2621 rc = -EFAULT; 2622 goto fail; 2623 } 2624 2625 cq->umem = ib_umem_get(context, req.cq_va, 2626 entries * sizeof(struct cq_base), 2627 IB_ACCESS_LOCAL_WRITE, 1); 2628 if (IS_ERR(cq->umem)) { 2629 rc = PTR_ERR(cq->umem); 2630 goto fail; 2631 } 2632 cq->qplib_cq.sghead = cq->umem->sg_head.sgl; 2633 cq->qplib_cq.nmap = cq->umem->nmap; 2634 cq->qplib_cq.dpi = &uctx->dpi; 2635 } else { 2636 cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL); 2637 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), 2638 GFP_KERNEL); 2639 if (!cq->cql) { 2640 rc = -ENOMEM; 2641 goto fail; 2642 } 2643 2644 cq->qplib_cq.dpi = &rdev->dpi_privileged; 2645 cq->qplib_cq.sghead = NULL; 2646 cq->qplib_cq.nmap = 0; 2647 } 2648 /* 2649 * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a 2650 * used for getting the NQ index. 2651 */ 2652 nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt); 2653 nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)]; 2654 cq->qplib_cq.max_wqe = entries; 2655 cq->qplib_cq.cnq_hw_ring_id = nq->ring_id; 2656 cq->qplib_cq.nq = nq; 2657 2658 rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq); 2659 if (rc) { 2660 dev_err(rdev_to_dev(rdev), "Failed to create HW CQ"); 2661 goto fail; 2662 } 2663 2664 cq->ib_cq.cqe = entries; 2665 cq->cq_period = cq->qplib_cq.period; 2666 nq->budget++; 2667 2668 atomic_inc(&rdev->cq_count); 2669 spin_lock_init(&cq->cq_lock); 2670 2671 if (context) { 2672 struct bnxt_re_cq_resp resp; 2673 2674 resp.cqid = cq->qplib_cq.id; 2675 resp.tail = cq->qplib_cq.hwq.cons; 2676 resp.phase = cq->qplib_cq.period; 2677 resp.rsvd = 0; 2678 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 2679 if (rc) { 2680 dev_err(rdev_to_dev(rdev), "Failed to copy CQ udata"); 2681 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); 2682 goto c2fail; 2683 } 2684 } 2685 2686 return &cq->ib_cq; 2687 2688 c2fail: 2689 if (context) 2690 ib_umem_release(cq->umem); 2691 fail: 2692 kfree(cq->cql); 2693 kfree(cq); 2694 return ERR_PTR(rc); 2695 } 2696 2697 static u8 __req_to_ib_wc_status(u8 qstatus) 2698 { 2699 switch (qstatus) { 2700 case CQ_REQ_STATUS_OK: 2701 return IB_WC_SUCCESS; 2702 case CQ_REQ_STATUS_BAD_RESPONSE_ERR: 2703 return IB_WC_BAD_RESP_ERR; 2704 case CQ_REQ_STATUS_LOCAL_LENGTH_ERR: 2705 return IB_WC_LOC_LEN_ERR; 2706 case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR: 2707 return IB_WC_LOC_QP_OP_ERR; 2708 case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR: 2709 return IB_WC_LOC_PROT_ERR; 2710 case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR: 2711 return IB_WC_GENERAL_ERR; 2712 case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR: 2713 return IB_WC_REM_INV_REQ_ERR; 2714 case CQ_REQ_STATUS_REMOTE_ACCESS_ERR: 2715 return IB_WC_REM_ACCESS_ERR; 2716 case CQ_REQ_STATUS_REMOTE_OPERATION_ERR: 2717 return IB_WC_REM_OP_ERR; 2718 case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR: 2719 return IB_WC_RNR_RETRY_EXC_ERR; 2720 case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR: 2721 return IB_WC_RETRY_EXC_ERR; 2722 case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR: 2723 return IB_WC_WR_FLUSH_ERR; 2724 default: 2725 return IB_WC_GENERAL_ERR; 2726 } 2727 return 0; 2728 } 2729 2730 static u8 __rawqp1_to_ib_wc_status(u8 qstatus) 2731 { 2732 switch (qstatus) { 2733 case CQ_RES_RAWETH_QP1_STATUS_OK: 2734 return IB_WC_SUCCESS; 2735 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR: 2736 return IB_WC_LOC_ACCESS_ERR; 2737 case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR: 2738 return IB_WC_LOC_LEN_ERR; 2739 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR: 2740 return IB_WC_LOC_PROT_ERR; 2741 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR: 2742 return IB_WC_LOC_QP_OP_ERR; 2743 case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR: 2744 return IB_WC_GENERAL_ERR; 2745 case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR: 2746 return IB_WC_WR_FLUSH_ERR; 2747 case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR: 2748 return IB_WC_WR_FLUSH_ERR; 2749 default: 2750 return IB_WC_GENERAL_ERR; 2751 } 2752 } 2753 2754 static u8 __rc_to_ib_wc_status(u8 qstatus) 2755 { 2756 switch (qstatus) { 2757 case CQ_RES_RC_STATUS_OK: 2758 return IB_WC_SUCCESS; 2759 case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR: 2760 return IB_WC_LOC_ACCESS_ERR; 2761 case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR: 2762 return IB_WC_LOC_LEN_ERR; 2763 case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR: 2764 return IB_WC_LOC_PROT_ERR; 2765 case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR: 2766 return IB_WC_LOC_QP_OP_ERR; 2767 case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR: 2768 return IB_WC_GENERAL_ERR; 2769 case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR: 2770 return IB_WC_REM_INV_REQ_ERR; 2771 case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR: 2772 return IB_WC_WR_FLUSH_ERR; 2773 case CQ_RES_RC_STATUS_HW_FLUSH_ERR: 2774 return IB_WC_WR_FLUSH_ERR; 2775 default: 2776 return IB_WC_GENERAL_ERR; 2777 } 2778 } 2779 2780 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) 2781 { 2782 switch (cqe->type) { 2783 case BNXT_QPLIB_SWQE_TYPE_SEND: 2784 wc->opcode = IB_WC_SEND; 2785 break; 2786 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: 2787 wc->opcode = IB_WC_SEND; 2788 wc->wc_flags |= IB_WC_WITH_IMM; 2789 break; 2790 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: 2791 wc->opcode = IB_WC_SEND; 2792 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 2793 break; 2794 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: 2795 wc->opcode = IB_WC_RDMA_WRITE; 2796 break; 2797 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: 2798 wc->opcode = IB_WC_RDMA_WRITE; 2799 wc->wc_flags |= IB_WC_WITH_IMM; 2800 break; 2801 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: 2802 wc->opcode = IB_WC_RDMA_READ; 2803 break; 2804 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: 2805 wc->opcode = IB_WC_COMP_SWAP; 2806 break; 2807 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: 2808 wc->opcode = IB_WC_FETCH_ADD; 2809 break; 2810 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: 2811 wc->opcode = IB_WC_LOCAL_INV; 2812 break; 2813 case BNXT_QPLIB_SWQE_TYPE_REG_MR: 2814 wc->opcode = IB_WC_REG_MR; 2815 break; 2816 default: 2817 wc->opcode = IB_WC_SEND; 2818 break; 2819 } 2820 2821 wc->status = __req_to_ib_wc_status(cqe->status); 2822 } 2823 2824 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, 2825 u16 raweth_qp1_flags2) 2826 { 2827 bool is_ipv6 = false, is_ipv4 = false; 2828 2829 /* raweth_qp1_flags Bit 9-6 indicates itype */ 2830 if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 2831 != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 2832 return -1; 2833 2834 if (raweth_qp1_flags2 & 2835 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC && 2836 raweth_qp1_flags2 & 2837 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) { 2838 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ 2839 (raweth_qp1_flags2 & 2840 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ? 2841 (is_ipv6 = true) : (is_ipv4 = true); 2842 return ((is_ipv6) ? 2843 BNXT_RE_ROCEV2_IPV6_PACKET : 2844 BNXT_RE_ROCEV2_IPV4_PACKET); 2845 } else { 2846 return BNXT_RE_ROCE_V1_PACKET; 2847 } 2848 } 2849 2850 static int bnxt_re_to_ib_nw_type(int nw_type) 2851 { 2852 u8 nw_hdr_type = 0xFF; 2853 2854 switch (nw_type) { 2855 case BNXT_RE_ROCE_V1_PACKET: 2856 nw_hdr_type = RDMA_NETWORK_ROCE_V1; 2857 break; 2858 case BNXT_RE_ROCEV2_IPV4_PACKET: 2859 nw_hdr_type = RDMA_NETWORK_IPV4; 2860 break; 2861 case BNXT_RE_ROCEV2_IPV6_PACKET: 2862 nw_hdr_type = RDMA_NETWORK_IPV6; 2863 break; 2864 } 2865 return nw_hdr_type; 2866 } 2867 2868 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev, 2869 void *rq_hdr_buf) 2870 { 2871 u8 *tmp_buf = NULL; 2872 struct ethhdr *eth_hdr; 2873 u16 eth_type; 2874 bool rc = false; 2875 2876 tmp_buf = (u8 *)rq_hdr_buf; 2877 /* 2878 * If dest mac is not same as I/F mac, this could be a 2879 * loopback address or multicast address, check whether 2880 * it is a loopback packet 2881 */ 2882 if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) { 2883 tmp_buf += 4; 2884 /* Check the ether type */ 2885 eth_hdr = (struct ethhdr *)tmp_buf; 2886 eth_type = ntohs(eth_hdr->h_proto); 2887 switch (eth_type) { 2888 case ETH_P_IBOE: 2889 rc = true; 2890 break; 2891 case ETH_P_IP: 2892 case ETH_P_IPV6: { 2893 u32 len; 2894 struct udphdr *udp_hdr; 2895 2896 len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) : 2897 sizeof(struct ipv6hdr)); 2898 tmp_buf += sizeof(struct ethhdr) + len; 2899 udp_hdr = (struct udphdr *)tmp_buf; 2900 if (ntohs(udp_hdr->dest) == 2901 ROCE_V2_UDP_DPORT) 2902 rc = true; 2903 break; 2904 } 2905 default: 2906 break; 2907 } 2908 } 2909 2910 return rc; 2911 } 2912 2913 static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *qp1_qp, 2914 struct bnxt_qplib_cqe *cqe) 2915 { 2916 struct bnxt_re_dev *rdev = qp1_qp->rdev; 2917 struct bnxt_re_sqp_entries *sqp_entry = NULL; 2918 struct bnxt_re_qp *qp = rdev->qp1_sqp; 2919 struct ib_send_wr *swr; 2920 struct ib_ud_wr udwr; 2921 struct ib_recv_wr rwr; 2922 int pkt_type = 0; 2923 u32 tbl_idx; 2924 void *rq_hdr_buf; 2925 dma_addr_t rq_hdr_buf_map; 2926 dma_addr_t shrq_hdr_buf_map; 2927 u32 offset = 0; 2928 u32 skip_bytes = 0; 2929 struct ib_sge s_sge[2]; 2930 struct ib_sge r_sge[2]; 2931 int rc; 2932 2933 memset(&udwr, 0, sizeof(udwr)); 2934 memset(&rwr, 0, sizeof(rwr)); 2935 memset(&s_sge, 0, sizeof(s_sge)); 2936 memset(&r_sge, 0, sizeof(r_sge)); 2937 2938 swr = &udwr.wr; 2939 tbl_idx = cqe->wr_id; 2940 2941 rq_hdr_buf = qp1_qp->qplib_qp.rq_hdr_buf + 2942 (tbl_idx * qp1_qp->qplib_qp.rq_hdr_buf_size); 2943 rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp1_qp->qplib_qp, 2944 tbl_idx); 2945 2946 /* Shadow QP header buffer */ 2947 shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp->qplib_qp, 2948 tbl_idx); 2949 sqp_entry = &rdev->sqp_tbl[tbl_idx]; 2950 2951 /* Store this cqe */ 2952 memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); 2953 sqp_entry->qp1_qp = qp1_qp; 2954 2955 /* Find packet type from the cqe */ 2956 2957 pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, 2958 cqe->raweth_qp1_flags2); 2959 if (pkt_type < 0) { 2960 dev_err(rdev_to_dev(rdev), "Invalid packet\n"); 2961 return -EINVAL; 2962 } 2963 2964 /* Adjust the offset for the user buffer and post in the rq */ 2965 2966 if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET) 2967 offset = 20; 2968 2969 /* 2970 * QP1 loopback packet has 4 bytes of internal header before 2971 * ether header. Skip these four bytes. 2972 */ 2973 if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf)) 2974 skip_bytes = 4; 2975 2976 /* First send SGE . Skip the ether header*/ 2977 s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 2978 + skip_bytes; 2979 s_sge[0].lkey = 0xFFFFFFFF; 2980 s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 : 2981 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; 2982 2983 /* Second Send SGE */ 2984 s_sge[1].addr = s_sge[0].addr + s_sge[0].length + 2985 BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE; 2986 if (pkt_type != BNXT_RE_ROCE_V1_PACKET) 2987 s_sge[1].addr += 8; 2988 s_sge[1].lkey = 0xFFFFFFFF; 2989 s_sge[1].length = 256; 2990 2991 /* First recv SGE */ 2992 2993 r_sge[0].addr = shrq_hdr_buf_map; 2994 r_sge[0].lkey = 0xFFFFFFFF; 2995 r_sge[0].length = 40; 2996 2997 r_sge[1].addr = sqp_entry->sge.addr + offset; 2998 r_sge[1].lkey = sqp_entry->sge.lkey; 2999 r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; 3000 3001 /* Create receive work request */ 3002 rwr.num_sge = 2; 3003 rwr.sg_list = r_sge; 3004 rwr.wr_id = tbl_idx; 3005 rwr.next = NULL; 3006 3007 rc = bnxt_re_post_recv_shadow_qp(rdev, qp, &rwr); 3008 if (rc) { 3009 dev_err(rdev_to_dev(rdev), 3010 "Failed to post Rx buffers to shadow QP"); 3011 return -ENOMEM; 3012 } 3013 3014 swr->num_sge = 2; 3015 swr->sg_list = s_sge; 3016 swr->wr_id = tbl_idx; 3017 swr->opcode = IB_WR_SEND; 3018 swr->next = NULL; 3019 3020 udwr.ah = &rdev->sqp_ah->ib_ah; 3021 udwr.remote_qpn = rdev->qp1_sqp->qplib_qp.id; 3022 udwr.remote_qkey = rdev->qp1_sqp->qplib_qp.qkey; 3023 3024 /* post data received in the send queue */ 3025 rc = bnxt_re_post_send_shadow_qp(rdev, qp, swr); 3026 3027 return 0; 3028 } 3029 3030 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc, 3031 struct bnxt_qplib_cqe *cqe) 3032 { 3033 wc->opcode = IB_WC_RECV; 3034 wc->status = __rawqp1_to_ib_wc_status(cqe->status); 3035 wc->wc_flags |= IB_WC_GRH; 3036 } 3037 3038 static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe, 3039 u16 *vid, u8 *sl) 3040 { 3041 bool ret = false; 3042 u32 metadata; 3043 u16 tpid; 3044 3045 metadata = orig_cqe->raweth_qp1_metadata; 3046 if (orig_cqe->raweth_qp1_flags2 & 3047 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) { 3048 tpid = ((metadata & 3049 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >> 3050 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); 3051 if (tpid == ETH_P_8021Q) { 3052 *vid = metadata & 3053 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK; 3054 *sl = (metadata & 3055 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >> 3056 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT; 3057 ret = true; 3058 } 3059 } 3060 3061 return ret; 3062 } 3063 3064 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc, 3065 struct bnxt_qplib_cqe *cqe) 3066 { 3067 wc->opcode = IB_WC_RECV; 3068 wc->status = __rc_to_ib_wc_status(cqe->status); 3069 3070 if (cqe->flags & CQ_RES_RC_FLAGS_IMM) 3071 wc->wc_flags |= IB_WC_WITH_IMM; 3072 if (cqe->flags & CQ_RES_RC_FLAGS_INV) 3073 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3074 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == 3075 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) 3076 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 3077 } 3078 3079 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *qp, 3080 struct ib_wc *wc, 3081 struct bnxt_qplib_cqe *cqe) 3082 { 3083 struct bnxt_re_dev *rdev = qp->rdev; 3084 struct bnxt_re_qp *qp1_qp = NULL; 3085 struct bnxt_qplib_cqe *orig_cqe = NULL; 3086 struct bnxt_re_sqp_entries *sqp_entry = NULL; 3087 int nw_type; 3088 u32 tbl_idx; 3089 u16 vlan_id; 3090 u8 sl; 3091 3092 tbl_idx = cqe->wr_id; 3093 3094 sqp_entry = &rdev->sqp_tbl[tbl_idx]; 3095 qp1_qp = sqp_entry->qp1_qp; 3096 orig_cqe = &sqp_entry->cqe; 3097 3098 wc->wr_id = sqp_entry->wrid; 3099 wc->byte_len = orig_cqe->length; 3100 wc->qp = &qp1_qp->ib_qp; 3101 3102 wc->ex.imm_data = orig_cqe->immdata; 3103 wc->src_qp = orig_cqe->src_qp; 3104 memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); 3105 if (bnxt_re_is_vlan_pkt(orig_cqe, &vlan_id, &sl)) { 3106 wc->vlan_id = vlan_id; 3107 wc->sl = sl; 3108 wc->wc_flags |= IB_WC_WITH_VLAN; 3109 } 3110 wc->port_num = 1; 3111 wc->vendor_err = orig_cqe->status; 3112 3113 wc->opcode = IB_WC_RECV; 3114 wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); 3115 wc->wc_flags |= IB_WC_GRH; 3116 3117 nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, 3118 orig_cqe->raweth_qp1_flags2); 3119 if (nw_type >= 0) { 3120 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type); 3121 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 3122 } 3123 } 3124 3125 static void bnxt_re_process_res_ud_wc(struct ib_wc *wc, 3126 struct bnxt_qplib_cqe *cqe) 3127 { 3128 wc->opcode = IB_WC_RECV; 3129 wc->status = __rc_to_ib_wc_status(cqe->status); 3130 3131 if (cqe->flags & CQ_RES_RC_FLAGS_IMM) 3132 wc->wc_flags |= IB_WC_WITH_IMM; 3133 if (cqe->flags & CQ_RES_RC_FLAGS_INV) 3134 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3135 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == 3136 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) 3137 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 3138 } 3139 3140 static int send_phantom_wqe(struct bnxt_re_qp *qp) 3141 { 3142 struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp; 3143 unsigned long flags; 3144 int rc = 0; 3145 3146 spin_lock_irqsave(&qp->sq_lock, flags); 3147 3148 rc = bnxt_re_bind_fence_mw(lib_qp); 3149 if (!rc) { 3150 lib_qp->sq.phantom_wqe_cnt++; 3151 dev_dbg(&lib_qp->sq.hwq.pdev->dev, 3152 "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n", 3153 lib_qp->id, lib_qp->sq.hwq.prod, 3154 HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq), 3155 lib_qp->sq.phantom_wqe_cnt); 3156 } 3157 3158 spin_unlock_irqrestore(&qp->sq_lock, flags); 3159 return rc; 3160 } 3161 3162 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) 3163 { 3164 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 3165 struct bnxt_re_qp *qp; 3166 struct bnxt_qplib_cqe *cqe; 3167 int i, ncqe, budget; 3168 struct bnxt_qplib_q *sq; 3169 struct bnxt_qplib_qp *lib_qp; 3170 u32 tbl_idx; 3171 struct bnxt_re_sqp_entries *sqp_entry = NULL; 3172 unsigned long flags; 3173 3174 spin_lock_irqsave(&cq->cq_lock, flags); 3175 budget = min_t(u32, num_entries, cq->max_cql); 3176 num_entries = budget; 3177 if (!cq->cql) { 3178 dev_err(rdev_to_dev(cq->rdev), "POLL CQ : no CQL to use"); 3179 goto exit; 3180 } 3181 cqe = &cq->cql[0]; 3182 while (budget) { 3183 lib_qp = NULL; 3184 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp); 3185 if (lib_qp) { 3186 sq = &lib_qp->sq; 3187 if (sq->send_phantom) { 3188 qp = container_of(lib_qp, 3189 struct bnxt_re_qp, qplib_qp); 3190 if (send_phantom_wqe(qp) == -ENOMEM) 3191 dev_err(rdev_to_dev(cq->rdev), 3192 "Phantom failed! Scheduled to send again\n"); 3193 else 3194 sq->send_phantom = false; 3195 } 3196 } 3197 if (ncqe < budget) 3198 ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq, 3199 cqe + ncqe, 3200 budget - ncqe); 3201 3202 if (!ncqe) 3203 break; 3204 3205 for (i = 0; i < ncqe; i++, cqe++) { 3206 /* Transcribe each qplib_wqe back to ib_wc */ 3207 memset(wc, 0, sizeof(*wc)); 3208 3209 wc->wr_id = cqe->wr_id; 3210 wc->byte_len = cqe->length; 3211 qp = container_of 3212 ((struct bnxt_qplib_qp *) 3213 (unsigned long)(cqe->qp_handle), 3214 struct bnxt_re_qp, qplib_qp); 3215 if (!qp) { 3216 dev_err(rdev_to_dev(cq->rdev), 3217 "POLL CQ : bad QP handle"); 3218 continue; 3219 } 3220 wc->qp = &qp->ib_qp; 3221 wc->ex.imm_data = cqe->immdata; 3222 wc->src_qp = cqe->src_qp; 3223 memcpy(wc->smac, cqe->smac, ETH_ALEN); 3224 wc->port_num = 1; 3225 wc->vendor_err = cqe->status; 3226 3227 switch (cqe->opcode) { 3228 case CQ_BASE_CQE_TYPE_REQ: 3229 if (qp->qplib_qp.id == 3230 qp->rdev->qp1_sqp->qplib_qp.id) { 3231 /* Handle this completion with 3232 * the stored completion 3233 */ 3234 memset(wc, 0, sizeof(*wc)); 3235 continue; 3236 } 3237 bnxt_re_process_req_wc(wc, cqe); 3238 break; 3239 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 3240 if (!cqe->status) { 3241 int rc = 0; 3242 3243 rc = bnxt_re_process_raw_qp_pkt_rx 3244 (qp, cqe); 3245 if (!rc) { 3246 memset(wc, 0, sizeof(*wc)); 3247 continue; 3248 } 3249 cqe->status = -1; 3250 } 3251 /* Errors need not be looped back. 3252 * But change the wr_id to the one 3253 * stored in the table 3254 */ 3255 tbl_idx = cqe->wr_id; 3256 sqp_entry = &cq->rdev->sqp_tbl[tbl_idx]; 3257 wc->wr_id = sqp_entry->wrid; 3258 bnxt_re_process_res_rawqp1_wc(wc, cqe); 3259 break; 3260 case CQ_BASE_CQE_TYPE_RES_RC: 3261 bnxt_re_process_res_rc_wc(wc, cqe); 3262 break; 3263 case CQ_BASE_CQE_TYPE_RES_UD: 3264 if (qp->qplib_qp.id == 3265 qp->rdev->qp1_sqp->qplib_qp.id) { 3266 /* Handle this completion with 3267 * the stored completion 3268 */ 3269 if (cqe->status) { 3270 continue; 3271 } else { 3272 bnxt_re_process_res_shadow_qp_wc 3273 (qp, wc, cqe); 3274 break; 3275 } 3276 } 3277 bnxt_re_process_res_ud_wc(wc, cqe); 3278 break; 3279 default: 3280 dev_err(rdev_to_dev(cq->rdev), 3281 "POLL CQ : type 0x%x not handled", 3282 cqe->opcode); 3283 continue; 3284 } 3285 wc++; 3286 budget--; 3287 } 3288 } 3289 exit: 3290 spin_unlock_irqrestore(&cq->cq_lock, flags); 3291 return num_entries - budget; 3292 } 3293 3294 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq, 3295 enum ib_cq_notify_flags ib_cqn_flags) 3296 { 3297 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 3298 int type = 0, rc = 0; 3299 unsigned long flags; 3300 3301 spin_lock_irqsave(&cq->cq_lock, flags); 3302 /* Trigger on the very next completion */ 3303 if (ib_cqn_flags & IB_CQ_NEXT_COMP) 3304 type = DBR_DBR_TYPE_CQ_ARMALL; 3305 /* Trigger on the next solicited completion */ 3306 else if (ib_cqn_flags & IB_CQ_SOLICITED) 3307 type = DBR_DBR_TYPE_CQ_ARMSE; 3308 3309 /* Poll to see if there are missed events */ 3310 if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) && 3311 !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) { 3312 rc = 1; 3313 goto exit; 3314 } 3315 bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); 3316 3317 exit: 3318 spin_unlock_irqrestore(&cq->cq_lock, flags); 3319 return rc; 3320 } 3321 3322 /* Memory Regions */ 3323 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) 3324 { 3325 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3326 struct bnxt_re_dev *rdev = pd->rdev; 3327 struct bnxt_re_mr *mr; 3328 u64 pbl = 0; 3329 int rc; 3330 3331 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3332 if (!mr) 3333 return ERR_PTR(-ENOMEM); 3334 3335 mr->rdev = rdev; 3336 mr->qplib_mr.pd = &pd->qplib_pd; 3337 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 3338 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 3339 3340 /* Allocate and register 0 as the address */ 3341 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3342 if (rc) 3343 goto fail; 3344 3345 mr->qplib_mr.hwq.level = PBL_LVL_MAX; 3346 mr->qplib_mr.total_size = -1; /* Infinte length */ 3347 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl, 0, false, 3348 PAGE_SIZE); 3349 if (rc) 3350 goto fail_mr; 3351 3352 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3353 if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | 3354 IB_ACCESS_REMOTE_ATOMIC)) 3355 mr->ib_mr.rkey = mr->ib_mr.lkey; 3356 atomic_inc(&rdev->mr_count); 3357 3358 return &mr->ib_mr; 3359 3360 fail_mr: 3361 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3362 fail: 3363 kfree(mr); 3364 return ERR_PTR(rc); 3365 } 3366 3367 int bnxt_re_dereg_mr(struct ib_mr *ib_mr) 3368 { 3369 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3370 struct bnxt_re_dev *rdev = mr->rdev; 3371 int rc; 3372 3373 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3374 if (rc) 3375 dev_err(rdev_to_dev(rdev), "Dereg MR failed: %#x\n", rc); 3376 3377 if (mr->pages) { 3378 rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, 3379 &mr->qplib_frpl); 3380 kfree(mr->pages); 3381 mr->npages = 0; 3382 mr->pages = NULL; 3383 } 3384 if (!IS_ERR_OR_NULL(mr->ib_umem)) 3385 ib_umem_release(mr->ib_umem); 3386 3387 kfree(mr); 3388 atomic_dec(&rdev->mr_count); 3389 return rc; 3390 } 3391 3392 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) 3393 { 3394 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3395 3396 if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) 3397 return -ENOMEM; 3398 3399 mr->pages[mr->npages++] = addr; 3400 return 0; 3401 } 3402 3403 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, 3404 unsigned int *sg_offset) 3405 { 3406 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3407 3408 mr->npages = 0; 3409 return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page); 3410 } 3411 3412 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, 3413 u32 max_num_sg) 3414 { 3415 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3416 struct bnxt_re_dev *rdev = pd->rdev; 3417 struct bnxt_re_mr *mr = NULL; 3418 int rc; 3419 3420 if (type != IB_MR_TYPE_MEM_REG) { 3421 dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported", type); 3422 return ERR_PTR(-EINVAL); 3423 } 3424 if (max_num_sg > MAX_PBL_LVL_1_PGS) 3425 return ERR_PTR(-EINVAL); 3426 3427 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3428 if (!mr) 3429 return ERR_PTR(-ENOMEM); 3430 3431 mr->rdev = rdev; 3432 mr->qplib_mr.pd = &pd->qplib_pd; 3433 mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; 3434 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 3435 3436 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3437 if (rc) 3438 goto bail; 3439 3440 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3441 mr->ib_mr.rkey = mr->ib_mr.lkey; 3442 3443 mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); 3444 if (!mr->pages) { 3445 rc = -ENOMEM; 3446 goto fail; 3447 } 3448 rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, 3449 &mr->qplib_frpl, max_num_sg); 3450 if (rc) { 3451 dev_err(rdev_to_dev(rdev), 3452 "Failed to allocate HW FR page list"); 3453 goto fail_mr; 3454 } 3455 3456 atomic_inc(&rdev->mr_count); 3457 return &mr->ib_mr; 3458 3459 fail_mr: 3460 kfree(mr->pages); 3461 fail: 3462 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3463 bail: 3464 kfree(mr); 3465 return ERR_PTR(rc); 3466 } 3467 3468 struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, 3469 struct ib_udata *udata) 3470 { 3471 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3472 struct bnxt_re_dev *rdev = pd->rdev; 3473 struct bnxt_re_mw *mw; 3474 int rc; 3475 3476 mw = kzalloc(sizeof(*mw), GFP_KERNEL); 3477 if (!mw) 3478 return ERR_PTR(-ENOMEM); 3479 mw->rdev = rdev; 3480 mw->qplib_mw.pd = &pd->qplib_pd; 3481 3482 mw->qplib_mw.type = (type == IB_MW_TYPE_1 ? 3483 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 : 3484 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B); 3485 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw); 3486 if (rc) { 3487 dev_err(rdev_to_dev(rdev), "Allocate MW failed!"); 3488 goto fail; 3489 } 3490 mw->ib_mw.rkey = mw->qplib_mw.rkey; 3491 3492 atomic_inc(&rdev->mw_count); 3493 return &mw->ib_mw; 3494 3495 fail: 3496 kfree(mw); 3497 return ERR_PTR(rc); 3498 } 3499 3500 int bnxt_re_dealloc_mw(struct ib_mw *ib_mw) 3501 { 3502 struct bnxt_re_mw *mw = container_of(ib_mw, struct bnxt_re_mw, ib_mw); 3503 struct bnxt_re_dev *rdev = mw->rdev; 3504 int rc; 3505 3506 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw); 3507 if (rc) { 3508 dev_err(rdev_to_dev(rdev), "Free MW failed: %#x\n", rc); 3509 return rc; 3510 } 3511 3512 kfree(mw); 3513 atomic_dec(&rdev->mw_count); 3514 return rc; 3515 } 3516 3517 static int bnxt_re_page_size_ok(int page_shift) 3518 { 3519 switch (page_shift) { 3520 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K: 3521 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K: 3522 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K: 3523 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M: 3524 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K: 3525 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M: 3526 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M: 3527 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G: 3528 return 1; 3529 default: 3530 return 0; 3531 } 3532 } 3533 3534 static int fill_umem_pbl_tbl(struct ib_umem *umem, u64 *pbl_tbl_orig, 3535 int page_shift) 3536 { 3537 u64 *pbl_tbl = pbl_tbl_orig; 3538 u64 paddr; 3539 u64 page_mask = (1ULL << page_shift) - 1; 3540 int i, pages; 3541 struct scatterlist *sg; 3542 int entry; 3543 3544 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 3545 pages = sg_dma_len(sg) >> PAGE_SHIFT; 3546 for (i = 0; i < pages; i++) { 3547 paddr = sg_dma_address(sg) + (i << PAGE_SHIFT); 3548 if (pbl_tbl == pbl_tbl_orig) 3549 *pbl_tbl++ = paddr & ~page_mask; 3550 else if ((paddr & page_mask) == 0) 3551 *pbl_tbl++ = paddr; 3552 } 3553 } 3554 return pbl_tbl - pbl_tbl_orig; 3555 } 3556 3557 /* uverbs */ 3558 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, 3559 u64 virt_addr, int mr_access_flags, 3560 struct ib_udata *udata) 3561 { 3562 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3563 struct bnxt_re_dev *rdev = pd->rdev; 3564 struct bnxt_re_mr *mr; 3565 struct ib_umem *umem; 3566 u64 *pbl_tbl = NULL; 3567 int umem_pgs, page_shift, rc; 3568 3569 if (length > BNXT_RE_MAX_MR_SIZE) { 3570 dev_err(rdev_to_dev(rdev), "MR Size: %lld > Max supported:%lld\n", 3571 length, BNXT_RE_MAX_MR_SIZE); 3572 return ERR_PTR(-ENOMEM); 3573 } 3574 3575 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3576 if (!mr) 3577 return ERR_PTR(-ENOMEM); 3578 3579 mr->rdev = rdev; 3580 mr->qplib_mr.pd = &pd->qplib_pd; 3581 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 3582 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; 3583 3584 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3585 if (rc) { 3586 dev_err(rdev_to_dev(rdev), "Failed to allocate MR"); 3587 goto free_mr; 3588 } 3589 /* The fixed portion of the rkey is the same as the lkey */ 3590 mr->ib_mr.rkey = mr->qplib_mr.rkey; 3591 3592 umem = ib_umem_get(ib_pd->uobject->context, start, length, 3593 mr_access_flags, 0); 3594 if (IS_ERR(umem)) { 3595 dev_err(rdev_to_dev(rdev), "Failed to get umem"); 3596 rc = -EFAULT; 3597 goto free_mrw; 3598 } 3599 mr->ib_umem = umem; 3600 3601 mr->qplib_mr.va = virt_addr; 3602 umem_pgs = ib_umem_page_count(umem); 3603 if (!umem_pgs) { 3604 dev_err(rdev_to_dev(rdev), "umem is invalid!"); 3605 rc = -EINVAL; 3606 goto free_umem; 3607 } 3608 mr->qplib_mr.total_size = length; 3609 3610 pbl_tbl = kcalloc(umem_pgs, sizeof(u64 *), GFP_KERNEL); 3611 if (!pbl_tbl) { 3612 rc = -ENOMEM; 3613 goto free_umem; 3614 } 3615 3616 page_shift = umem->page_shift; 3617 3618 if (!bnxt_re_page_size_ok(page_shift)) { 3619 dev_err(rdev_to_dev(rdev), "umem page size unsupported!"); 3620 rc = -EFAULT; 3621 goto fail; 3622 } 3623 3624 if (!umem->hugetlb && length > BNXT_RE_MAX_MR_SIZE_LOW) { 3625 dev_err(rdev_to_dev(rdev), "Requested MR Sz:%llu Max sup:%llu", 3626 length, (u64)BNXT_RE_MAX_MR_SIZE_LOW); 3627 rc = -EINVAL; 3628 goto fail; 3629 } 3630 if (umem->hugetlb && length > BNXT_RE_PAGE_SIZE_2M) { 3631 page_shift = BNXT_RE_PAGE_SHIFT_2M; 3632 dev_warn(rdev_to_dev(rdev), "umem hugetlb set page_size %x", 3633 1 << page_shift); 3634 } 3635 3636 /* Map umem buf ptrs to the PBL */ 3637 umem_pgs = fill_umem_pbl_tbl(umem, pbl_tbl, page_shift); 3638 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl, 3639 umem_pgs, false, 1 << page_shift); 3640 if (rc) { 3641 dev_err(rdev_to_dev(rdev), "Failed to register user MR"); 3642 goto fail; 3643 } 3644 3645 kfree(pbl_tbl); 3646 3647 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3648 mr->ib_mr.rkey = mr->qplib_mr.lkey; 3649 atomic_inc(&rdev->mr_count); 3650 3651 return &mr->ib_mr; 3652 fail: 3653 kfree(pbl_tbl); 3654 free_umem: 3655 ib_umem_release(umem); 3656 free_mrw: 3657 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3658 free_mr: 3659 kfree(mr); 3660 return ERR_PTR(rc); 3661 } 3662 3663 struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev, 3664 struct ib_udata *udata) 3665 { 3666 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 3667 struct bnxt_re_uctx_resp resp; 3668 struct bnxt_re_ucontext *uctx; 3669 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 3670 int rc; 3671 3672 dev_dbg(rdev_to_dev(rdev), "ABI version requested %d", 3673 ibdev->uverbs_abi_ver); 3674 3675 if (ibdev->uverbs_abi_ver != BNXT_RE_ABI_VERSION) { 3676 dev_dbg(rdev_to_dev(rdev), " is different from the device %d ", 3677 BNXT_RE_ABI_VERSION); 3678 return ERR_PTR(-EPERM); 3679 } 3680 3681 uctx = kzalloc(sizeof(*uctx), GFP_KERNEL); 3682 if (!uctx) 3683 return ERR_PTR(-ENOMEM); 3684 3685 uctx->rdev = rdev; 3686 3687 uctx->shpg = (void *)__get_free_page(GFP_KERNEL); 3688 if (!uctx->shpg) { 3689 rc = -ENOMEM; 3690 goto fail; 3691 } 3692 spin_lock_init(&uctx->sh_lock); 3693 3694 resp.dev_id = rdev->en_dev->pdev->devfn; /*Temp, Use idr_alloc instead*/ 3695 resp.max_qp = rdev->qplib_ctx.qpc_count; 3696 resp.pg_size = PAGE_SIZE; 3697 resp.cqe_sz = sizeof(struct cq_base); 3698 resp.max_cqd = dev_attr->max_cq_wqes; 3699 resp.rsvd = 0; 3700 3701 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 3702 if (rc) { 3703 dev_err(rdev_to_dev(rdev), "Failed to copy user context"); 3704 rc = -EFAULT; 3705 goto cfail; 3706 } 3707 3708 return &uctx->ib_uctx; 3709 cfail: 3710 free_page((unsigned long)uctx->shpg); 3711 uctx->shpg = NULL; 3712 fail: 3713 kfree(uctx); 3714 return ERR_PTR(rc); 3715 } 3716 3717 int bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx) 3718 { 3719 struct bnxt_re_ucontext *uctx = container_of(ib_uctx, 3720 struct bnxt_re_ucontext, 3721 ib_uctx); 3722 3723 struct bnxt_re_dev *rdev = uctx->rdev; 3724 int rc = 0; 3725 3726 if (uctx->shpg) 3727 free_page((unsigned long)uctx->shpg); 3728 3729 if (uctx->dpi.dbr) { 3730 /* Free DPI only if this is the first PD allocated by the 3731 * application and mark the context dpi as NULL 3732 */ 3733 rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, 3734 &rdev->qplib_res.dpi_tbl, 3735 &uctx->dpi); 3736 if (rc) 3737 dev_err(rdev_to_dev(rdev), "Deallocate HW DPI failed!"); 3738 /* Don't fail, continue*/ 3739 uctx->dpi.dbr = NULL; 3740 } 3741 3742 kfree(uctx); 3743 return 0; 3744 } 3745 3746 /* Helper function to mmap the virtual memory from user app */ 3747 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) 3748 { 3749 struct bnxt_re_ucontext *uctx = container_of(ib_uctx, 3750 struct bnxt_re_ucontext, 3751 ib_uctx); 3752 struct bnxt_re_dev *rdev = uctx->rdev; 3753 u64 pfn; 3754 3755 if (vma->vm_end - vma->vm_start != PAGE_SIZE) 3756 return -EINVAL; 3757 3758 if (vma->vm_pgoff) { 3759 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 3760 if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 3761 PAGE_SIZE, vma->vm_page_prot)) { 3762 dev_err(rdev_to_dev(rdev), "Failed to map DPI"); 3763 return -EAGAIN; 3764 } 3765 } else { 3766 pfn = virt_to_phys(uctx->shpg) >> PAGE_SHIFT; 3767 if (remap_pfn_range(vma, vma->vm_start, 3768 pfn, PAGE_SIZE, vma->vm_page_prot)) { 3769 dev_err(rdev_to_dev(rdev), 3770 "Failed to map shared page"); 3771 return -EAGAIN; 3772 } 3773 } 3774 3775 return 0; 3776 } 3777