1 /* 2 * Broadcom NetXtreme-E RoCE driver. 3 * 4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term 5 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Description: IB Verbs interpreter 37 */ 38 39 #include <linux/interrupt.h> 40 #include <linux/types.h> 41 #include <linux/pci.h> 42 #include <linux/netdevice.h> 43 #include <linux/if_ether.h> 44 45 #include <rdma/ib_verbs.h> 46 #include <rdma/ib_user_verbs.h> 47 #include <rdma/ib_umem.h> 48 #include <rdma/ib_addr.h> 49 #include <rdma/ib_mad.h> 50 #include <rdma/ib_cache.h> 51 #include <rdma/uverbs_ioctl.h> 52 53 #include "bnxt_ulp.h" 54 55 #include "roce_hsi.h" 56 #include "qplib_res.h" 57 #include "qplib_sp.h" 58 #include "qplib_fp.h" 59 #include "qplib_rcfw.h" 60 61 #include "bnxt_re.h" 62 #include "ib_verbs.h" 63 #include <rdma/bnxt_re-abi.h> 64 65 static int __from_ib_access_flags(int iflags) 66 { 67 int qflags = 0; 68 69 if (iflags & IB_ACCESS_LOCAL_WRITE) 70 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 71 if (iflags & IB_ACCESS_REMOTE_READ) 72 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ; 73 if (iflags & IB_ACCESS_REMOTE_WRITE) 74 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE; 75 if (iflags & IB_ACCESS_REMOTE_ATOMIC) 76 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC; 77 if (iflags & IB_ACCESS_MW_BIND) 78 qflags |= BNXT_QPLIB_ACCESS_MW_BIND; 79 if (iflags & IB_ZERO_BASED) 80 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED; 81 if (iflags & IB_ACCESS_ON_DEMAND) 82 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND; 83 return qflags; 84 }; 85 86 static enum ib_access_flags __to_ib_access_flags(int qflags) 87 { 88 enum ib_access_flags iflags = 0; 89 90 if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE) 91 iflags |= IB_ACCESS_LOCAL_WRITE; 92 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE) 93 iflags |= IB_ACCESS_REMOTE_WRITE; 94 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ) 95 iflags |= IB_ACCESS_REMOTE_READ; 96 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC) 97 iflags |= IB_ACCESS_REMOTE_ATOMIC; 98 if (qflags & BNXT_QPLIB_ACCESS_MW_BIND) 99 iflags |= IB_ACCESS_MW_BIND; 100 if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED) 101 iflags |= IB_ZERO_BASED; 102 if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND) 103 iflags |= IB_ACCESS_ON_DEMAND; 104 return iflags; 105 }; 106 107 static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list, 108 struct bnxt_qplib_sge *sg_list, int num) 109 { 110 int i, total = 0; 111 112 for (i = 0; i < num; i++) { 113 sg_list[i].addr = ib_sg_list[i].addr; 114 sg_list[i].lkey = ib_sg_list[i].lkey; 115 sg_list[i].size = ib_sg_list[i].length; 116 total += sg_list[i].size; 117 } 118 return total; 119 } 120 121 /* Device */ 122 int bnxt_re_query_device(struct ib_device *ibdev, 123 struct ib_device_attr *ib_attr, 124 struct ib_udata *udata) 125 { 126 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 127 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 128 129 memset(ib_attr, 0, sizeof(*ib_attr)); 130 memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, 131 min(sizeof(dev_attr->fw_ver), 132 sizeof(ib_attr->fw_ver))); 133 bnxt_qplib_get_guid(rdev->netdev->dev_addr, 134 (u8 *)&ib_attr->sys_image_guid); 135 ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE; 136 ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M; 137 138 ib_attr->vendor_id = rdev->en_dev->pdev->vendor; 139 ib_attr->vendor_part_id = rdev->en_dev->pdev->device; 140 ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; 141 ib_attr->max_qp = dev_attr->max_qp; 142 ib_attr->max_qp_wr = dev_attr->max_qp_wqes; 143 ib_attr->device_cap_flags = 144 IB_DEVICE_CURR_QP_STATE_MOD 145 | IB_DEVICE_RC_RNR_NAK_GEN 146 | IB_DEVICE_SHUTDOWN_PORT 147 | IB_DEVICE_SYS_IMAGE_GUID 148 | IB_DEVICE_LOCAL_DMA_LKEY 149 | IB_DEVICE_RESIZE_MAX_WR 150 | IB_DEVICE_PORT_ACTIVE_EVENT 151 | IB_DEVICE_N_NOTIFY_CQ 152 | IB_DEVICE_MEM_WINDOW 153 | IB_DEVICE_MEM_WINDOW_TYPE_2B 154 | IB_DEVICE_MEM_MGT_EXTENSIONS; 155 ib_attr->max_send_sge = dev_attr->max_qp_sges; 156 ib_attr->max_recv_sge = dev_attr->max_qp_sges; 157 ib_attr->max_sge_rd = dev_attr->max_qp_sges; 158 ib_attr->max_cq = dev_attr->max_cq; 159 ib_attr->max_cqe = dev_attr->max_cq_wqes; 160 ib_attr->max_mr = dev_attr->max_mr; 161 ib_attr->max_pd = dev_attr->max_pd; 162 ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; 163 ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom; 164 ib_attr->atomic_cap = IB_ATOMIC_NONE; 165 ib_attr->masked_atomic_cap = IB_ATOMIC_NONE; 166 167 ib_attr->max_ee_rd_atom = 0; 168 ib_attr->max_res_rd_atom = 0; 169 ib_attr->max_ee_init_rd_atom = 0; 170 ib_attr->max_ee = 0; 171 ib_attr->max_rdd = 0; 172 ib_attr->max_mw = dev_attr->max_mw; 173 ib_attr->max_raw_ipv6_qp = 0; 174 ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; 175 ib_attr->max_mcast_grp = 0; 176 ib_attr->max_mcast_qp_attach = 0; 177 ib_attr->max_total_mcast_qp_attach = 0; 178 ib_attr->max_ah = dev_attr->max_ah; 179 180 ib_attr->max_fmr = 0; 181 ib_attr->max_map_per_fmr = 0; 182 183 ib_attr->max_srq = dev_attr->max_srq; 184 ib_attr->max_srq_wr = dev_attr->max_srq_wqes; 185 ib_attr->max_srq_sge = dev_attr->max_srq_sges; 186 187 ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; 188 189 ib_attr->max_pkeys = 1; 190 ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY; 191 return 0; 192 } 193 194 int bnxt_re_modify_device(struct ib_device *ibdev, 195 int device_modify_mask, 196 struct ib_device_modify *device_modify) 197 { 198 switch (device_modify_mask) { 199 case IB_DEVICE_MODIFY_SYS_IMAGE_GUID: 200 /* Modify the GUID requires the modification of the GID table */ 201 /* GUID should be made as READ-ONLY */ 202 break; 203 case IB_DEVICE_MODIFY_NODE_DESC: 204 /* Node Desc should be made as READ-ONLY */ 205 break; 206 default: 207 break; 208 } 209 return 0; 210 } 211 212 /* Port */ 213 int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, 214 struct ib_port_attr *port_attr) 215 { 216 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 217 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 218 219 memset(port_attr, 0, sizeof(*port_attr)); 220 221 if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) { 222 port_attr->state = IB_PORT_ACTIVE; 223 port_attr->phys_state = 5; 224 } else { 225 port_attr->state = IB_PORT_DOWN; 226 port_attr->phys_state = 3; 227 } 228 port_attr->max_mtu = IB_MTU_4096; 229 port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu); 230 port_attr->gid_tbl_len = dev_attr->max_sgid; 231 port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 232 IB_PORT_DEVICE_MGMT_SUP | 233 IB_PORT_VENDOR_CLASS_SUP; 234 port_attr->ip_gids = true; 235 236 port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW; 237 port_attr->bad_pkey_cntr = 0; 238 port_attr->qkey_viol_cntr = 0; 239 port_attr->pkey_tbl_len = dev_attr->max_pkey; 240 port_attr->lid = 0; 241 port_attr->sm_lid = 0; 242 port_attr->lmc = 0; 243 port_attr->max_vl_num = 4; 244 port_attr->sm_sl = 0; 245 port_attr->subnet_timeout = 0; 246 port_attr->init_type_reply = 0; 247 port_attr->active_speed = rdev->active_speed; 248 port_attr->active_width = rdev->active_width; 249 250 return 0; 251 } 252 253 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, 254 struct ib_port_immutable *immutable) 255 { 256 struct ib_port_attr port_attr; 257 258 if (bnxt_re_query_port(ibdev, port_num, &port_attr)) 259 return -EINVAL; 260 261 immutable->pkey_tbl_len = port_attr.pkey_tbl_len; 262 immutable->gid_tbl_len = port_attr.gid_tbl_len; 263 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; 264 immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP; 265 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 266 return 0; 267 } 268 269 void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str) 270 { 271 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 272 273 snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d", 274 rdev->dev_attr.fw_ver[0], rdev->dev_attr.fw_ver[1], 275 rdev->dev_attr.fw_ver[2], rdev->dev_attr.fw_ver[3]); 276 } 277 278 int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, 279 u16 index, u16 *pkey) 280 { 281 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 282 283 /* Ignore port_num */ 284 285 memset(pkey, 0, sizeof(*pkey)); 286 return bnxt_qplib_get_pkey(&rdev->qplib_res, 287 &rdev->qplib_res.pkey_tbl, index, pkey); 288 } 289 290 int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, 291 int index, union ib_gid *gid) 292 { 293 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 294 int rc = 0; 295 296 /* Ignore port_num */ 297 memset(gid, 0, sizeof(*gid)); 298 rc = bnxt_qplib_get_sgid(&rdev->qplib_res, 299 &rdev->qplib_res.sgid_tbl, index, 300 (struct bnxt_qplib_gid *)gid); 301 return rc; 302 } 303 304 int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context) 305 { 306 int rc = 0; 307 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 308 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); 309 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 310 struct bnxt_qplib_gid *gid_to_del; 311 312 /* Delete the entry from the hardware */ 313 ctx = *context; 314 if (!ctx) 315 return -EINVAL; 316 317 if (sgid_tbl && sgid_tbl->active) { 318 if (ctx->idx >= sgid_tbl->max) 319 return -EINVAL; 320 gid_to_del = &sgid_tbl->tbl[ctx->idx]; 321 /* DEL_GID is called in WQ context(netdevice_event_work_handler) 322 * or via the ib_unregister_device path. In the former case QP1 323 * may not be destroyed yet, in which case just return as FW 324 * needs that entry to be present and will fail it's deletion. 325 * We could get invoked again after QP1 is destroyed OR get an 326 * ADD_GID call with a different GID value for the same index 327 * where we issue MODIFY_GID cmd to update the GID entry -- TBD 328 */ 329 if (ctx->idx == 0 && 330 rdma_link_local_addr((struct in6_addr *)gid_to_del) && 331 ctx->refcnt == 1 && rdev->qp1_sqp) { 332 dev_dbg(rdev_to_dev(rdev), 333 "Trying to delete GID0 while QP1 is alive\n"); 334 return -EFAULT; 335 } 336 ctx->refcnt--; 337 if (!ctx->refcnt) { 338 rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, true); 339 if (rc) { 340 dev_err(rdev_to_dev(rdev), 341 "Failed to remove GID: %#x", rc); 342 } else { 343 ctx_tbl = sgid_tbl->ctx; 344 ctx_tbl[ctx->idx] = NULL; 345 kfree(ctx); 346 } 347 } 348 } else { 349 return -EINVAL; 350 } 351 return rc; 352 } 353 354 int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context) 355 { 356 int rc; 357 u32 tbl_idx = 0; 358 u16 vlan_id = 0xFFFF; 359 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 360 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); 361 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 362 363 rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL); 364 if (rc) 365 return rc; 366 367 rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid, 368 rdev->qplib_res.netdev->dev_addr, 369 vlan_id, true, &tbl_idx); 370 if (rc == -EALREADY) { 371 ctx_tbl = sgid_tbl->ctx; 372 ctx_tbl[tbl_idx]->refcnt++; 373 *context = ctx_tbl[tbl_idx]; 374 return 0; 375 } 376 377 if (rc < 0) { 378 dev_err(rdev_to_dev(rdev), "Failed to add GID: %#x", rc); 379 return rc; 380 } 381 382 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 383 if (!ctx) 384 return -ENOMEM; 385 ctx_tbl = sgid_tbl->ctx; 386 ctx->idx = tbl_idx; 387 ctx->refcnt = 1; 388 ctx_tbl[tbl_idx] = ctx; 389 *context = ctx; 390 391 return rc; 392 } 393 394 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, 395 u8 port_num) 396 { 397 return IB_LINK_LAYER_ETHERNET; 398 } 399 400 #define BNXT_RE_FENCE_PBL_SIZE DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE) 401 402 static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd) 403 { 404 struct bnxt_re_fence_data *fence = &pd->fence; 405 struct ib_mr *ib_mr = &fence->mr->ib_mr; 406 struct bnxt_qplib_swqe *wqe = &fence->bind_wqe; 407 408 memset(wqe, 0, sizeof(*wqe)); 409 wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW; 410 wqe->wr_id = BNXT_QPLIB_FENCE_WRID; 411 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 412 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 413 wqe->bind.zero_based = false; 414 wqe->bind.parent_l_key = ib_mr->lkey; 415 wqe->bind.va = (u64)(unsigned long)fence->va; 416 wqe->bind.length = fence->size; 417 wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ); 418 wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1; 419 420 /* Save the initial rkey in fence structure for now; 421 * wqe->bind.r_key will be set at (re)bind time. 422 */ 423 fence->bind_rkey = ib_inc_rkey(fence->mw->rkey); 424 } 425 426 static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp) 427 { 428 struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp, 429 qplib_qp); 430 struct ib_pd *ib_pd = qp->ib_qp.pd; 431 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 432 struct bnxt_re_fence_data *fence = &pd->fence; 433 struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe; 434 struct bnxt_qplib_swqe wqe; 435 int rc; 436 437 memcpy(&wqe, fence_wqe, sizeof(wqe)); 438 wqe.bind.r_key = fence->bind_rkey; 439 fence->bind_rkey = ib_inc_rkey(fence->bind_rkey); 440 441 dev_dbg(rdev_to_dev(qp->rdev), 442 "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n", 443 wqe.bind.r_key, qp->qplib_qp.id, pd); 444 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 445 if (rc) { 446 dev_err(rdev_to_dev(qp->rdev), "Failed to bind fence-WQE\n"); 447 return rc; 448 } 449 bnxt_qplib_post_send_db(&qp->qplib_qp); 450 451 return rc; 452 } 453 454 static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd) 455 { 456 struct bnxt_re_fence_data *fence = &pd->fence; 457 struct bnxt_re_dev *rdev = pd->rdev; 458 struct device *dev = &rdev->en_dev->pdev->dev; 459 struct bnxt_re_mr *mr = fence->mr; 460 461 if (fence->mw) { 462 bnxt_re_dealloc_mw(fence->mw); 463 fence->mw = NULL; 464 } 465 if (mr) { 466 if (mr->ib_mr.rkey) 467 bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr, 468 true); 469 if (mr->ib_mr.lkey) 470 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 471 kfree(mr); 472 fence->mr = NULL; 473 } 474 if (fence->dma_addr) { 475 dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES, 476 DMA_BIDIRECTIONAL); 477 fence->dma_addr = 0; 478 } 479 } 480 481 static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd) 482 { 483 int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND; 484 struct bnxt_re_fence_data *fence = &pd->fence; 485 struct bnxt_re_dev *rdev = pd->rdev; 486 struct device *dev = &rdev->en_dev->pdev->dev; 487 struct bnxt_re_mr *mr = NULL; 488 dma_addr_t dma_addr = 0; 489 struct ib_mw *mw; 490 u64 pbl_tbl; 491 int rc; 492 493 dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES, 494 DMA_BIDIRECTIONAL); 495 rc = dma_mapping_error(dev, dma_addr); 496 if (rc) { 497 dev_err(rdev_to_dev(rdev), "Failed to dma-map fence-MR-mem\n"); 498 rc = -EIO; 499 fence->dma_addr = 0; 500 goto fail; 501 } 502 fence->dma_addr = dma_addr; 503 504 /* Allocate a MR */ 505 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 506 if (!mr) { 507 rc = -ENOMEM; 508 goto fail; 509 } 510 fence->mr = mr; 511 mr->rdev = rdev; 512 mr->qplib_mr.pd = &pd->qplib_pd; 513 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 514 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 515 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 516 if (rc) { 517 dev_err(rdev_to_dev(rdev), "Failed to alloc fence-HW-MR\n"); 518 goto fail; 519 } 520 521 /* Register MR */ 522 mr->ib_mr.lkey = mr->qplib_mr.lkey; 523 mr->qplib_mr.va = (u64)(unsigned long)fence->va; 524 mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES; 525 pbl_tbl = dma_addr; 526 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl_tbl, 527 BNXT_RE_FENCE_PBL_SIZE, false, PAGE_SIZE); 528 if (rc) { 529 dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n"); 530 goto fail; 531 } 532 mr->ib_mr.rkey = mr->qplib_mr.rkey; 533 534 /* Create a fence MW only for kernel consumers */ 535 mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL); 536 if (IS_ERR(mw)) { 537 dev_err(rdev_to_dev(rdev), 538 "Failed to create fence-MW for PD: %p\n", pd); 539 rc = PTR_ERR(mw); 540 goto fail; 541 } 542 fence->mw = mw; 543 544 bnxt_re_create_fence_wqe(pd); 545 return 0; 546 547 fail: 548 bnxt_re_destroy_fence_mr(pd); 549 return rc; 550 } 551 552 /* Protection Domains */ 553 void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) 554 { 555 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 556 struct bnxt_re_dev *rdev = pd->rdev; 557 558 bnxt_re_destroy_fence_mr(pd); 559 560 if (pd->qplib_pd.id) 561 bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, 562 &pd->qplib_pd); 563 } 564 565 int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 566 { 567 struct ib_device *ibdev = ibpd->device; 568 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 569 struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context( 570 udata, struct bnxt_re_ucontext, ib_uctx); 571 struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd); 572 int rc; 573 574 pd->rdev = rdev; 575 if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) { 576 dev_err(rdev_to_dev(rdev), "Failed to allocate HW PD"); 577 rc = -ENOMEM; 578 goto fail; 579 } 580 581 if (udata) { 582 struct bnxt_re_pd_resp resp; 583 584 if (!ucntx->dpi.dbr) { 585 /* Allocate DPI in alloc_pd to avoid failing of 586 * ibv_devinfo and family of application when DPIs 587 * are depleted. 588 */ 589 if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl, 590 &ucntx->dpi, ucntx)) { 591 rc = -ENOMEM; 592 goto dbfail; 593 } 594 } 595 596 resp.pdid = pd->qplib_pd.id; 597 /* Still allow mapping this DBR to the new user PD. */ 598 resp.dpi = ucntx->dpi.dpi; 599 resp.dbr = (u64)ucntx->dpi.umdbr; 600 601 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 602 if (rc) { 603 dev_err(rdev_to_dev(rdev), 604 "Failed to copy user response\n"); 605 goto dbfail; 606 } 607 } 608 609 if (!udata) 610 if (bnxt_re_create_fence_mr(pd)) 611 dev_warn(rdev_to_dev(rdev), 612 "Failed to create Fence-MR\n"); 613 return 0; 614 dbfail: 615 bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, 616 &pd->qplib_pd); 617 fail: 618 return rc; 619 } 620 621 /* Address Handles */ 622 void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags) 623 { 624 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 625 struct bnxt_re_dev *rdev = ah->rdev; 626 627 bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, 628 !(flags & RDMA_DESTROY_AH_SLEEPABLE)); 629 } 630 631 static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype) 632 { 633 u8 nw_type; 634 635 switch (ntype) { 636 case RDMA_NETWORK_IPV4: 637 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4; 638 break; 639 case RDMA_NETWORK_IPV6: 640 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6; 641 break; 642 default: 643 nw_type = CMDQ_CREATE_AH_TYPE_V1; 644 break; 645 } 646 return nw_type; 647 } 648 649 int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr, 650 u32 flags, struct ib_udata *udata) 651 { 652 struct ib_pd *ib_pd = ib_ah->pd; 653 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 654 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); 655 struct bnxt_re_dev *rdev = pd->rdev; 656 const struct ib_gid_attr *sgid_attr; 657 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 658 u8 nw_type; 659 int rc; 660 661 if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) { 662 dev_err(rdev_to_dev(rdev), "Failed to alloc AH: GRH not set"); 663 return -EINVAL; 664 } 665 666 ah->rdev = rdev; 667 ah->qplib_ah.pd = &pd->qplib_pd; 668 669 /* Supply the configuration for the HW */ 670 memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw, 671 sizeof(union ib_gid)); 672 /* 673 * If RoCE V2 is enabled, stack will have two entries for 674 * each GID entry. Avoiding this duplicte entry in HW. Dividing 675 * the GID index by 2 for RoCE V2 676 */ 677 ah->qplib_ah.sgid_index = grh->sgid_index / 2; 678 ah->qplib_ah.host_sgid_index = grh->sgid_index; 679 ah->qplib_ah.traffic_class = grh->traffic_class; 680 ah->qplib_ah.flow_label = grh->flow_label; 681 ah->qplib_ah.hop_limit = grh->hop_limit; 682 ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr); 683 684 sgid_attr = grh->sgid_attr; 685 /* Get network header type for this GID */ 686 nw_type = rdma_gid_attr_network_type(sgid_attr); 687 ah->qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(nw_type); 688 689 memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN); 690 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, 691 !(flags & RDMA_CREATE_AH_SLEEPABLE)); 692 if (rc) { 693 dev_err(rdev_to_dev(rdev), "Failed to allocate HW AH"); 694 return rc; 695 } 696 697 /* Write AVID to shared page. */ 698 if (udata) { 699 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context( 700 udata, struct bnxt_re_ucontext, ib_uctx); 701 unsigned long flag; 702 u32 *wrptr; 703 704 spin_lock_irqsave(&uctx->sh_lock, flag); 705 wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT); 706 *wrptr = ah->qplib_ah.id; 707 wmb(); /* make sure cache is updated. */ 708 spin_unlock_irqrestore(&uctx->sh_lock, flag); 709 } 710 711 return 0; 712 } 713 714 int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) 715 { 716 return 0; 717 } 718 719 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) 720 { 721 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 722 723 ah_attr->type = ib_ah->type; 724 rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl); 725 memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN); 726 rdma_ah_set_grh(ah_attr, NULL, 0, 727 ah->qplib_ah.host_sgid_index, 728 0, ah->qplib_ah.traffic_class); 729 rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data); 730 rdma_ah_set_port_num(ah_attr, 1); 731 rdma_ah_set_static_rate(ah_attr, 0); 732 return 0; 733 } 734 735 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp) 736 __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock) 737 { 738 unsigned long flags; 739 740 spin_lock_irqsave(&qp->scq->cq_lock, flags); 741 if (qp->rcq != qp->scq) 742 spin_lock(&qp->rcq->cq_lock); 743 else 744 __acquire(&qp->rcq->cq_lock); 745 746 return flags; 747 } 748 749 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, 750 unsigned long flags) 751 __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock) 752 { 753 if (qp->rcq != qp->scq) 754 spin_unlock(&qp->rcq->cq_lock); 755 else 756 __release(&qp->rcq->cq_lock); 757 spin_unlock_irqrestore(&qp->scq->cq_lock, flags); 758 } 759 760 /* Queue Pairs */ 761 int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) 762 { 763 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 764 struct bnxt_re_dev *rdev = qp->rdev; 765 unsigned int flags; 766 int rc; 767 768 bnxt_qplib_flush_cqn_wq(&qp->qplib_qp); 769 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 770 if (rc) { 771 dev_err(rdev_to_dev(rdev), "Failed to destroy HW QP"); 772 return rc; 773 } 774 775 if (rdma_is_kernel_res(&qp->ib_qp.res)) { 776 flags = bnxt_re_lock_cqs(qp); 777 bnxt_qplib_clean_qp(&qp->qplib_qp); 778 bnxt_re_unlock_cqs(qp, flags); 779 } 780 781 bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); 782 783 if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) { 784 bnxt_qplib_destroy_ah(&rdev->qplib_res, &rdev->sqp_ah->qplib_ah, 785 false); 786 787 bnxt_qplib_clean_qp(&qp->qplib_qp); 788 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, 789 &rdev->qp1_sqp->qplib_qp); 790 if (rc) { 791 dev_err(rdev_to_dev(rdev), 792 "Failed to destroy Shadow QP"); 793 return rc; 794 } 795 bnxt_qplib_free_qp_res(&rdev->qplib_res, 796 &rdev->qp1_sqp->qplib_qp); 797 mutex_lock(&rdev->qp_lock); 798 list_del(&rdev->qp1_sqp->list); 799 atomic_dec(&rdev->qp_count); 800 mutex_unlock(&rdev->qp_lock); 801 802 kfree(rdev->sqp_ah); 803 kfree(rdev->qp1_sqp); 804 rdev->qp1_sqp = NULL; 805 rdev->sqp_ah = NULL; 806 } 807 808 ib_umem_release(qp->rumem); 809 ib_umem_release(qp->sumem); 810 811 mutex_lock(&rdev->qp_lock); 812 list_del(&qp->list); 813 atomic_dec(&rdev->qp_count); 814 mutex_unlock(&rdev->qp_lock); 815 kfree(qp); 816 return 0; 817 } 818 819 static u8 __from_ib_qp_type(enum ib_qp_type type) 820 { 821 switch (type) { 822 case IB_QPT_GSI: 823 return CMDQ_CREATE_QP1_TYPE_GSI; 824 case IB_QPT_RC: 825 return CMDQ_CREATE_QP_TYPE_RC; 826 case IB_QPT_UD: 827 return CMDQ_CREATE_QP_TYPE_UD; 828 default: 829 return IB_QPT_MAX; 830 } 831 } 832 833 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, 834 struct bnxt_re_qp *qp, struct ib_udata *udata) 835 { 836 struct bnxt_re_qp_req ureq; 837 struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; 838 struct ib_umem *umem; 839 int bytes = 0, psn_sz; 840 struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context( 841 udata, struct bnxt_re_ucontext, ib_uctx); 842 843 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 844 return -EFAULT; 845 846 bytes = (qplib_qp->sq.max_wqe * BNXT_QPLIB_MAX_SQE_ENTRY_SIZE); 847 /* Consider mapping PSN search memory only for RC QPs. */ 848 if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { 849 psn_sz = bnxt_qplib_is_chip_gen_p5(&rdev->chip_ctx) ? 850 sizeof(struct sq_psn_search_ext) : 851 sizeof(struct sq_psn_search); 852 bytes += (qplib_qp->sq.max_wqe * psn_sz); 853 } 854 bytes = PAGE_ALIGN(bytes); 855 umem = ib_umem_get(udata, ureq.qpsva, bytes, IB_ACCESS_LOCAL_WRITE, 1); 856 if (IS_ERR(umem)) 857 return PTR_ERR(umem); 858 859 qp->sumem = umem; 860 qplib_qp->sq.sg_info.sglist = umem->sg_head.sgl; 861 qplib_qp->sq.sg_info.npages = ib_umem_num_pages(umem); 862 qplib_qp->sq.sg_info.nmap = umem->nmap; 863 qplib_qp->qp_handle = ureq.qp_handle; 864 865 if (!qp->qplib_qp.srq) { 866 bytes = (qplib_qp->rq.max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); 867 bytes = PAGE_ALIGN(bytes); 868 umem = ib_umem_get(udata, ureq.qprva, bytes, 869 IB_ACCESS_LOCAL_WRITE, 1); 870 if (IS_ERR(umem)) 871 goto rqfail; 872 qp->rumem = umem; 873 qplib_qp->rq.sg_info.sglist = umem->sg_head.sgl; 874 qplib_qp->rq.sg_info.npages = ib_umem_num_pages(umem); 875 qplib_qp->rq.sg_info.nmap = umem->nmap; 876 } 877 878 qplib_qp->dpi = &cntx->dpi; 879 return 0; 880 rqfail: 881 ib_umem_release(qp->sumem); 882 qp->sumem = NULL; 883 memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info)); 884 885 return PTR_ERR(umem); 886 } 887 888 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah 889 (struct bnxt_re_pd *pd, 890 struct bnxt_qplib_res *qp1_res, 891 struct bnxt_qplib_qp *qp1_qp) 892 { 893 struct bnxt_re_dev *rdev = pd->rdev; 894 struct bnxt_re_ah *ah; 895 union ib_gid sgid; 896 int rc; 897 898 ah = kzalloc(sizeof(*ah), GFP_KERNEL); 899 if (!ah) 900 return NULL; 901 902 ah->rdev = rdev; 903 ah->qplib_ah.pd = &pd->qplib_pd; 904 905 rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); 906 if (rc) 907 goto fail; 908 909 /* supply the dgid data same as sgid */ 910 memcpy(ah->qplib_ah.dgid.data, &sgid.raw, 911 sizeof(union ib_gid)); 912 ah->qplib_ah.sgid_index = 0; 913 914 ah->qplib_ah.traffic_class = 0; 915 ah->qplib_ah.flow_label = 0; 916 ah->qplib_ah.hop_limit = 1; 917 ah->qplib_ah.sl = 0; 918 /* Have DMAC same as SMAC */ 919 ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr); 920 921 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, false); 922 if (rc) { 923 dev_err(rdev_to_dev(rdev), 924 "Failed to allocate HW AH for Shadow QP"); 925 goto fail; 926 } 927 928 return ah; 929 930 fail: 931 kfree(ah); 932 return NULL; 933 } 934 935 static struct bnxt_re_qp *bnxt_re_create_shadow_qp 936 (struct bnxt_re_pd *pd, 937 struct bnxt_qplib_res *qp1_res, 938 struct bnxt_qplib_qp *qp1_qp) 939 { 940 struct bnxt_re_dev *rdev = pd->rdev; 941 struct bnxt_re_qp *qp; 942 int rc; 943 944 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 945 if (!qp) 946 return NULL; 947 948 qp->rdev = rdev; 949 950 /* Initialize the shadow QP structure from the QP1 values */ 951 ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); 952 953 qp->qplib_qp.pd = &pd->qplib_pd; 954 qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); 955 qp->qplib_qp.type = IB_QPT_UD; 956 957 qp->qplib_qp.max_inline_data = 0; 958 qp->qplib_qp.sig_type = true; 959 960 /* Shadow QP SQ depth should be same as QP1 RQ depth */ 961 qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; 962 qp->qplib_qp.sq.max_sge = 2; 963 /* Q full delta can be 1 since it is internal QP */ 964 qp->qplib_qp.sq.q_full_delta = 1; 965 966 qp->qplib_qp.scq = qp1_qp->scq; 967 qp->qplib_qp.rcq = qp1_qp->rcq; 968 969 qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; 970 qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; 971 /* Q full delta can be 1 since it is internal QP */ 972 qp->qplib_qp.rq.q_full_delta = 1; 973 974 qp->qplib_qp.mtu = qp1_qp->mtu; 975 976 qp->qplib_qp.sq_hdr_buf_size = 0; 977 qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; 978 qp->qplib_qp.dpi = &rdev->dpi_privileged; 979 980 rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); 981 if (rc) 982 goto fail; 983 984 rdev->sqp_id = qp->qplib_qp.id; 985 986 spin_lock_init(&qp->sq_lock); 987 INIT_LIST_HEAD(&qp->list); 988 mutex_lock(&rdev->qp_lock); 989 list_add_tail(&qp->list, &rdev->qp_list); 990 atomic_inc(&rdev->qp_count); 991 mutex_unlock(&rdev->qp_lock); 992 return qp; 993 fail: 994 kfree(qp); 995 return NULL; 996 } 997 998 struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd, 999 struct ib_qp_init_attr *qp_init_attr, 1000 struct ib_udata *udata) 1001 { 1002 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 1003 struct bnxt_re_dev *rdev = pd->rdev; 1004 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 1005 struct bnxt_re_qp *qp; 1006 struct bnxt_re_cq *cq; 1007 struct bnxt_re_srq *srq; 1008 int rc, entries; 1009 1010 if ((qp_init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) || 1011 (qp_init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) || 1012 (qp_init_attr->cap.max_send_sge > dev_attr->max_qp_sges) || 1013 (qp_init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) || 1014 (qp_init_attr->cap.max_inline_data > dev_attr->max_inline_data)) 1015 return ERR_PTR(-EINVAL); 1016 1017 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1018 if (!qp) 1019 return ERR_PTR(-ENOMEM); 1020 1021 qp->rdev = rdev; 1022 ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); 1023 qp->qplib_qp.pd = &pd->qplib_pd; 1024 qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); 1025 qp->qplib_qp.type = __from_ib_qp_type(qp_init_attr->qp_type); 1026 1027 if (qp_init_attr->qp_type == IB_QPT_GSI && 1028 bnxt_qplib_is_chip_gen_p5(&rdev->chip_ctx)) 1029 qp->qplib_qp.type = CMDQ_CREATE_QP_TYPE_GSI; 1030 if (qp->qplib_qp.type == IB_QPT_MAX) { 1031 dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported", 1032 qp->qplib_qp.type); 1033 rc = -EINVAL; 1034 goto fail; 1035 } 1036 1037 qp->qplib_qp.max_inline_data = qp_init_attr->cap.max_inline_data; 1038 qp->qplib_qp.sig_type = ((qp_init_attr->sq_sig_type == 1039 IB_SIGNAL_ALL_WR) ? true : false); 1040 1041 qp->qplib_qp.sq.max_sge = qp_init_attr->cap.max_send_sge; 1042 if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges) 1043 qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges; 1044 1045 if (qp_init_attr->send_cq) { 1046 cq = container_of(qp_init_attr->send_cq, struct bnxt_re_cq, 1047 ib_cq); 1048 if (!cq) { 1049 dev_err(rdev_to_dev(rdev), "Send CQ not found"); 1050 rc = -EINVAL; 1051 goto fail; 1052 } 1053 qp->qplib_qp.scq = &cq->qplib_cq; 1054 qp->scq = cq; 1055 } 1056 1057 if (qp_init_attr->recv_cq) { 1058 cq = container_of(qp_init_attr->recv_cq, struct bnxt_re_cq, 1059 ib_cq); 1060 if (!cq) { 1061 dev_err(rdev_to_dev(rdev), "Receive CQ not found"); 1062 rc = -EINVAL; 1063 goto fail; 1064 } 1065 qp->qplib_qp.rcq = &cq->qplib_cq; 1066 qp->rcq = cq; 1067 } 1068 1069 if (qp_init_attr->srq) { 1070 srq = container_of(qp_init_attr->srq, struct bnxt_re_srq, 1071 ib_srq); 1072 if (!srq) { 1073 dev_err(rdev_to_dev(rdev), "SRQ not found"); 1074 rc = -EINVAL; 1075 goto fail; 1076 } 1077 qp->qplib_qp.srq = &srq->qplib_srq; 1078 qp->qplib_qp.rq.max_wqe = 0; 1079 } else { 1080 /* Allocate 1 more than what's provided so posting max doesn't 1081 * mean empty 1082 */ 1083 entries = roundup_pow_of_two(qp_init_attr->cap.max_recv_wr + 1); 1084 qp->qplib_qp.rq.max_wqe = min_t(u32, entries, 1085 dev_attr->max_qp_wqes + 1); 1086 1087 qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - 1088 qp_init_attr->cap.max_recv_wr; 1089 1090 qp->qplib_qp.rq.max_sge = qp_init_attr->cap.max_recv_sge; 1091 if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges) 1092 qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; 1093 } 1094 1095 qp->qplib_qp.mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); 1096 1097 if (qp_init_attr->qp_type == IB_QPT_GSI && 1098 !(bnxt_qplib_is_chip_gen_p5(&rdev->chip_ctx))) { 1099 /* Allocate 1 more than what's provided */ 1100 entries = roundup_pow_of_two(qp_init_attr->cap.max_send_wr + 1); 1101 qp->qplib_qp.sq.max_wqe = min_t(u32, entries, 1102 dev_attr->max_qp_wqes + 1); 1103 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - 1104 qp_init_attr->cap.max_send_wr; 1105 qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; 1106 if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges) 1107 qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; 1108 qp->qplib_qp.sq.max_sge++; 1109 if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges) 1110 qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges; 1111 1112 qp->qplib_qp.rq_hdr_buf_size = 1113 BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; 1114 1115 qp->qplib_qp.sq_hdr_buf_size = 1116 BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2; 1117 qp->qplib_qp.dpi = &rdev->dpi_privileged; 1118 rc = bnxt_qplib_create_qp1(&rdev->qplib_res, &qp->qplib_qp); 1119 if (rc) { 1120 dev_err(rdev_to_dev(rdev), "Failed to create HW QP1"); 1121 goto fail; 1122 } 1123 /* Create a shadow QP to handle the QP1 traffic */ 1124 rdev->qp1_sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, 1125 &qp->qplib_qp); 1126 if (!rdev->qp1_sqp) { 1127 rc = -EINVAL; 1128 dev_err(rdev_to_dev(rdev), 1129 "Failed to create Shadow QP for QP1"); 1130 goto qp_destroy; 1131 } 1132 rdev->sqp_ah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, 1133 &qp->qplib_qp); 1134 if (!rdev->sqp_ah) { 1135 bnxt_qplib_destroy_qp(&rdev->qplib_res, 1136 &rdev->qp1_sqp->qplib_qp); 1137 rc = -EINVAL; 1138 dev_err(rdev_to_dev(rdev), 1139 "Failed to create AH entry for ShadowQP"); 1140 goto qp_destroy; 1141 } 1142 1143 } else { 1144 /* Allocate 128 + 1 more than what's provided */ 1145 entries = roundup_pow_of_two(qp_init_attr->cap.max_send_wr + 1146 BNXT_QPLIB_RESERVED_QP_WRS + 1); 1147 qp->qplib_qp.sq.max_wqe = min_t(u32, entries, 1148 dev_attr->max_qp_wqes + 1149 BNXT_QPLIB_RESERVED_QP_WRS + 1); 1150 qp->qplib_qp.sq.q_full_delta = BNXT_QPLIB_RESERVED_QP_WRS + 1; 1151 1152 /* 1153 * Reserving one slot for Phantom WQE. Application can 1154 * post one extra entry in this case. But allowing this to avoid 1155 * unexpected Queue full condition 1156 */ 1157 1158 qp->qplib_qp.sq.q_full_delta -= 1; 1159 1160 qp->qplib_qp.max_rd_atomic = dev_attr->max_qp_rd_atom; 1161 qp->qplib_qp.max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; 1162 if (udata) { 1163 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); 1164 if (rc) 1165 goto fail; 1166 } else { 1167 qp->qplib_qp.dpi = &rdev->dpi_privileged; 1168 } 1169 1170 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); 1171 if (rc) { 1172 dev_err(rdev_to_dev(rdev), "Failed to create HW QP"); 1173 goto free_umem; 1174 } 1175 } 1176 1177 qp->ib_qp.qp_num = qp->qplib_qp.id; 1178 spin_lock_init(&qp->sq_lock); 1179 spin_lock_init(&qp->rq_lock); 1180 1181 if (udata) { 1182 struct bnxt_re_qp_resp resp; 1183 1184 resp.qpid = qp->ib_qp.qp_num; 1185 resp.rsvd = 0; 1186 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 1187 if (rc) { 1188 dev_err(rdev_to_dev(rdev), "Failed to copy QP udata"); 1189 goto qp_destroy; 1190 } 1191 } 1192 INIT_LIST_HEAD(&qp->list); 1193 mutex_lock(&rdev->qp_lock); 1194 list_add_tail(&qp->list, &rdev->qp_list); 1195 atomic_inc(&rdev->qp_count); 1196 mutex_unlock(&rdev->qp_lock); 1197 1198 return &qp->ib_qp; 1199 qp_destroy: 1200 bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 1201 free_umem: 1202 ib_umem_release(qp->rumem); 1203 ib_umem_release(qp->sumem); 1204 fail: 1205 kfree(qp); 1206 return ERR_PTR(rc); 1207 } 1208 1209 static u8 __from_ib_qp_state(enum ib_qp_state state) 1210 { 1211 switch (state) { 1212 case IB_QPS_RESET: 1213 return CMDQ_MODIFY_QP_NEW_STATE_RESET; 1214 case IB_QPS_INIT: 1215 return CMDQ_MODIFY_QP_NEW_STATE_INIT; 1216 case IB_QPS_RTR: 1217 return CMDQ_MODIFY_QP_NEW_STATE_RTR; 1218 case IB_QPS_RTS: 1219 return CMDQ_MODIFY_QP_NEW_STATE_RTS; 1220 case IB_QPS_SQD: 1221 return CMDQ_MODIFY_QP_NEW_STATE_SQD; 1222 case IB_QPS_SQE: 1223 return CMDQ_MODIFY_QP_NEW_STATE_SQE; 1224 case IB_QPS_ERR: 1225 default: 1226 return CMDQ_MODIFY_QP_NEW_STATE_ERR; 1227 } 1228 } 1229 1230 static enum ib_qp_state __to_ib_qp_state(u8 state) 1231 { 1232 switch (state) { 1233 case CMDQ_MODIFY_QP_NEW_STATE_RESET: 1234 return IB_QPS_RESET; 1235 case CMDQ_MODIFY_QP_NEW_STATE_INIT: 1236 return IB_QPS_INIT; 1237 case CMDQ_MODIFY_QP_NEW_STATE_RTR: 1238 return IB_QPS_RTR; 1239 case CMDQ_MODIFY_QP_NEW_STATE_RTS: 1240 return IB_QPS_RTS; 1241 case CMDQ_MODIFY_QP_NEW_STATE_SQD: 1242 return IB_QPS_SQD; 1243 case CMDQ_MODIFY_QP_NEW_STATE_SQE: 1244 return IB_QPS_SQE; 1245 case CMDQ_MODIFY_QP_NEW_STATE_ERR: 1246 default: 1247 return IB_QPS_ERR; 1248 } 1249 } 1250 1251 static u32 __from_ib_mtu(enum ib_mtu mtu) 1252 { 1253 switch (mtu) { 1254 case IB_MTU_256: 1255 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256; 1256 case IB_MTU_512: 1257 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512; 1258 case IB_MTU_1024: 1259 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024; 1260 case IB_MTU_2048: 1261 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 1262 case IB_MTU_4096: 1263 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096; 1264 default: 1265 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 1266 } 1267 } 1268 1269 static enum ib_mtu __to_ib_mtu(u32 mtu) 1270 { 1271 switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) { 1272 case CMDQ_MODIFY_QP_PATH_MTU_MTU_256: 1273 return IB_MTU_256; 1274 case CMDQ_MODIFY_QP_PATH_MTU_MTU_512: 1275 return IB_MTU_512; 1276 case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024: 1277 return IB_MTU_1024; 1278 case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048: 1279 return IB_MTU_2048; 1280 case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096: 1281 return IB_MTU_4096; 1282 default: 1283 return IB_MTU_2048; 1284 } 1285 } 1286 1287 /* Shared Receive Queues */ 1288 void bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) 1289 { 1290 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1291 ib_srq); 1292 struct bnxt_re_dev *rdev = srq->rdev; 1293 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; 1294 struct bnxt_qplib_nq *nq = NULL; 1295 1296 if (qplib_srq->cq) 1297 nq = qplib_srq->cq->nq; 1298 bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); 1299 ib_umem_release(srq->umem); 1300 atomic_dec(&rdev->srq_count); 1301 if (nq) 1302 nq->budget--; 1303 } 1304 1305 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, 1306 struct bnxt_re_pd *pd, 1307 struct bnxt_re_srq *srq, 1308 struct ib_udata *udata) 1309 { 1310 struct bnxt_re_srq_req ureq; 1311 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; 1312 struct ib_umem *umem; 1313 int bytes = 0; 1314 struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context( 1315 udata, struct bnxt_re_ucontext, ib_uctx); 1316 1317 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 1318 return -EFAULT; 1319 1320 bytes = (qplib_srq->max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); 1321 bytes = PAGE_ALIGN(bytes); 1322 umem = ib_umem_get(udata, ureq.srqva, bytes, IB_ACCESS_LOCAL_WRITE, 1); 1323 if (IS_ERR(umem)) 1324 return PTR_ERR(umem); 1325 1326 srq->umem = umem; 1327 qplib_srq->sg_info.sglist = umem->sg_head.sgl; 1328 qplib_srq->sg_info.npages = ib_umem_num_pages(umem); 1329 qplib_srq->sg_info.nmap = umem->nmap; 1330 qplib_srq->srq_handle = ureq.srq_handle; 1331 qplib_srq->dpi = &cntx->dpi; 1332 1333 return 0; 1334 } 1335 1336 int bnxt_re_create_srq(struct ib_srq *ib_srq, 1337 struct ib_srq_init_attr *srq_init_attr, 1338 struct ib_udata *udata) 1339 { 1340 struct ib_pd *ib_pd = ib_srq->pd; 1341 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 1342 struct bnxt_re_dev *rdev = pd->rdev; 1343 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 1344 struct bnxt_re_srq *srq = 1345 container_of(ib_srq, struct bnxt_re_srq, ib_srq); 1346 struct bnxt_qplib_nq *nq = NULL; 1347 int rc, entries; 1348 1349 if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) { 1350 dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded"); 1351 rc = -EINVAL; 1352 goto exit; 1353 } 1354 1355 if (srq_init_attr->srq_type != IB_SRQT_BASIC) { 1356 rc = -EOPNOTSUPP; 1357 goto exit; 1358 } 1359 1360 srq->rdev = rdev; 1361 srq->qplib_srq.pd = &pd->qplib_pd; 1362 srq->qplib_srq.dpi = &rdev->dpi_privileged; 1363 /* Allocate 1 more than what's provided so posting max doesn't 1364 * mean empty 1365 */ 1366 entries = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1); 1367 if (entries > dev_attr->max_srq_wqes + 1) 1368 entries = dev_attr->max_srq_wqes + 1; 1369 1370 srq->qplib_srq.max_wqe = entries; 1371 srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge; 1372 srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit; 1373 srq->srq_limit = srq_init_attr->attr.srq_limit; 1374 srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id; 1375 nq = &rdev->nq[0]; 1376 1377 if (udata) { 1378 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata); 1379 if (rc) 1380 goto fail; 1381 } 1382 1383 rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq); 1384 if (rc) { 1385 dev_err(rdev_to_dev(rdev), "Create HW SRQ failed!"); 1386 goto fail; 1387 } 1388 1389 if (udata) { 1390 struct bnxt_re_srq_resp resp; 1391 1392 resp.srqid = srq->qplib_srq.id; 1393 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 1394 if (rc) { 1395 dev_err(rdev_to_dev(rdev), "SRQ copy to udata failed!"); 1396 bnxt_qplib_destroy_srq(&rdev->qplib_res, 1397 &srq->qplib_srq); 1398 goto exit; 1399 } 1400 } 1401 if (nq) 1402 nq->budget++; 1403 atomic_inc(&rdev->srq_count); 1404 1405 return 0; 1406 1407 fail: 1408 ib_umem_release(srq->umem); 1409 exit: 1410 return rc; 1411 } 1412 1413 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, 1414 enum ib_srq_attr_mask srq_attr_mask, 1415 struct ib_udata *udata) 1416 { 1417 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1418 ib_srq); 1419 struct bnxt_re_dev *rdev = srq->rdev; 1420 int rc; 1421 1422 switch (srq_attr_mask) { 1423 case IB_SRQ_MAX_WR: 1424 /* SRQ resize is not supported */ 1425 break; 1426 case IB_SRQ_LIMIT: 1427 /* Change the SRQ threshold */ 1428 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) 1429 return -EINVAL; 1430 1431 srq->qplib_srq.threshold = srq_attr->srq_limit; 1432 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq); 1433 if (rc) { 1434 dev_err(rdev_to_dev(rdev), "Modify HW SRQ failed!"); 1435 return rc; 1436 } 1437 /* On success, update the shadow */ 1438 srq->srq_limit = srq_attr->srq_limit; 1439 /* No need to Build and send response back to udata */ 1440 break; 1441 default: 1442 dev_err(rdev_to_dev(rdev), 1443 "Unsupported srq_attr_mask 0x%x", srq_attr_mask); 1444 return -EINVAL; 1445 } 1446 return 0; 1447 } 1448 1449 int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr) 1450 { 1451 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1452 ib_srq); 1453 struct bnxt_re_srq tsrq; 1454 struct bnxt_re_dev *rdev = srq->rdev; 1455 int rc; 1456 1457 /* Get live SRQ attr */ 1458 tsrq.qplib_srq.id = srq->qplib_srq.id; 1459 rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq); 1460 if (rc) { 1461 dev_err(rdev_to_dev(rdev), "Query HW SRQ failed!"); 1462 return rc; 1463 } 1464 srq_attr->max_wr = srq->qplib_srq.max_wqe; 1465 srq_attr->max_sge = srq->qplib_srq.max_sge; 1466 srq_attr->srq_limit = tsrq.qplib_srq.threshold; 1467 1468 return 0; 1469 } 1470 1471 int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, 1472 const struct ib_recv_wr **bad_wr) 1473 { 1474 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1475 ib_srq); 1476 struct bnxt_qplib_swqe wqe; 1477 unsigned long flags; 1478 int rc = 0; 1479 1480 spin_lock_irqsave(&srq->lock, flags); 1481 while (wr) { 1482 /* Transcribe each ib_recv_wr to qplib_swqe */ 1483 wqe.num_sge = wr->num_sge; 1484 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); 1485 wqe.wr_id = wr->wr_id; 1486 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 1487 1488 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe); 1489 if (rc) { 1490 *bad_wr = wr; 1491 break; 1492 } 1493 wr = wr->next; 1494 } 1495 spin_unlock_irqrestore(&srq->lock, flags); 1496 1497 return rc; 1498 } 1499 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, 1500 struct bnxt_re_qp *qp1_qp, 1501 int qp_attr_mask) 1502 { 1503 struct bnxt_re_qp *qp = rdev->qp1_sqp; 1504 int rc = 0; 1505 1506 if (qp_attr_mask & IB_QP_STATE) { 1507 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 1508 qp->qplib_qp.state = qp1_qp->qplib_qp.state; 1509 } 1510 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 1511 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 1512 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; 1513 } 1514 1515 if (qp_attr_mask & IB_QP_QKEY) { 1516 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 1517 /* Using a Random QKEY */ 1518 qp->qplib_qp.qkey = 0x81818181; 1519 } 1520 if (qp_attr_mask & IB_QP_SQ_PSN) { 1521 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 1522 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; 1523 } 1524 1525 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 1526 if (rc) 1527 dev_err(rdev_to_dev(rdev), 1528 "Failed to modify Shadow QP for QP1"); 1529 return rc; 1530 } 1531 1532 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 1533 int qp_attr_mask, struct ib_udata *udata) 1534 { 1535 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 1536 struct bnxt_re_dev *rdev = qp->rdev; 1537 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 1538 enum ib_qp_state curr_qp_state, new_qp_state; 1539 int rc, entries; 1540 unsigned int flags; 1541 u8 nw_type; 1542 1543 qp->qplib_qp.modify_flags = 0; 1544 if (qp_attr_mask & IB_QP_STATE) { 1545 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); 1546 new_qp_state = qp_attr->qp_state; 1547 if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state, 1548 ib_qp->qp_type, qp_attr_mask)) { 1549 dev_err(rdev_to_dev(rdev), 1550 "Invalid attribute mask: %#x specified ", 1551 qp_attr_mask); 1552 dev_err(rdev_to_dev(rdev), 1553 "for qpn: %#x type: %#x", 1554 ib_qp->qp_num, ib_qp->qp_type); 1555 dev_err(rdev_to_dev(rdev), 1556 "curr_qp_state=0x%x, new_qp_state=0x%x\n", 1557 curr_qp_state, new_qp_state); 1558 return -EINVAL; 1559 } 1560 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 1561 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); 1562 1563 if (!qp->sumem && 1564 qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { 1565 dev_dbg(rdev_to_dev(rdev), 1566 "Move QP = %p to flush list\n", 1567 qp); 1568 flags = bnxt_re_lock_cqs(qp); 1569 bnxt_qplib_add_flush_qp(&qp->qplib_qp); 1570 bnxt_re_unlock_cqs(qp, flags); 1571 } 1572 if (!qp->sumem && 1573 qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { 1574 dev_dbg(rdev_to_dev(rdev), 1575 "Move QP = %p out of flush list\n", 1576 qp); 1577 flags = bnxt_re_lock_cqs(qp); 1578 bnxt_qplib_clean_qp(&qp->qplib_qp); 1579 bnxt_re_unlock_cqs(qp, flags); 1580 } 1581 } 1582 if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { 1583 qp->qplib_qp.modify_flags |= 1584 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY; 1585 qp->qplib_qp.en_sqd_async_notify = true; 1586 } 1587 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) { 1588 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; 1589 qp->qplib_qp.access = 1590 __from_ib_access_flags(qp_attr->qp_access_flags); 1591 /* LOCAL_WRITE access must be set to allow RC receive */ 1592 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 1593 /* Temp: Set all params on QP as of now */ 1594 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE; 1595 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ; 1596 } 1597 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 1598 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 1599 qp->qplib_qp.pkey_index = qp_attr->pkey_index; 1600 } 1601 if (qp_attr_mask & IB_QP_QKEY) { 1602 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 1603 qp->qplib_qp.qkey = qp_attr->qkey; 1604 } 1605 if (qp_attr_mask & IB_QP_AV) { 1606 const struct ib_global_route *grh = 1607 rdma_ah_read_grh(&qp_attr->ah_attr); 1608 const struct ib_gid_attr *sgid_attr; 1609 1610 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | 1611 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | 1612 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | 1613 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | 1614 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | 1615 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | 1616 CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; 1617 memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw, 1618 sizeof(qp->qplib_qp.ah.dgid.data)); 1619 qp->qplib_qp.ah.flow_label = grh->flow_label; 1620 /* If RoCE V2 is enabled, stack will have two entries for 1621 * each GID entry. Avoiding this duplicte entry in HW. Dividing 1622 * the GID index by 2 for RoCE V2 1623 */ 1624 qp->qplib_qp.ah.sgid_index = grh->sgid_index / 2; 1625 qp->qplib_qp.ah.host_sgid_index = grh->sgid_index; 1626 qp->qplib_qp.ah.hop_limit = grh->hop_limit; 1627 qp->qplib_qp.ah.traffic_class = grh->traffic_class; 1628 qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr); 1629 ether_addr_copy(qp->qplib_qp.ah.dmac, 1630 qp_attr->ah_attr.roce.dmac); 1631 1632 sgid_attr = qp_attr->ah_attr.grh.sgid_attr; 1633 rc = rdma_read_gid_l2_fields(sgid_attr, NULL, 1634 &qp->qplib_qp.smac[0]); 1635 if (rc) 1636 return rc; 1637 1638 nw_type = rdma_gid_attr_network_type(sgid_attr); 1639 switch (nw_type) { 1640 case RDMA_NETWORK_IPV4: 1641 qp->qplib_qp.nw_type = 1642 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4; 1643 break; 1644 case RDMA_NETWORK_IPV6: 1645 qp->qplib_qp.nw_type = 1646 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6; 1647 break; 1648 default: 1649 qp->qplib_qp.nw_type = 1650 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1; 1651 break; 1652 } 1653 } 1654 1655 if (qp_attr_mask & IB_QP_PATH_MTU) { 1656 qp->qplib_qp.modify_flags |= 1657 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 1658 qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu); 1659 qp->qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr->path_mtu); 1660 } else if (qp_attr->qp_state == IB_QPS_RTR) { 1661 qp->qplib_qp.modify_flags |= 1662 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 1663 qp->qplib_qp.path_mtu = 1664 __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu)); 1665 qp->qplib_qp.mtu = 1666 ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); 1667 } 1668 1669 if (qp_attr_mask & IB_QP_TIMEOUT) { 1670 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; 1671 qp->qplib_qp.timeout = qp_attr->timeout; 1672 } 1673 if (qp_attr_mask & IB_QP_RETRY_CNT) { 1674 qp->qplib_qp.modify_flags |= 1675 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT; 1676 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; 1677 } 1678 if (qp_attr_mask & IB_QP_RNR_RETRY) { 1679 qp->qplib_qp.modify_flags |= 1680 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY; 1681 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; 1682 } 1683 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) { 1684 qp->qplib_qp.modify_flags |= 1685 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER; 1686 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; 1687 } 1688 if (qp_attr_mask & IB_QP_RQ_PSN) { 1689 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; 1690 qp->qplib_qp.rq.psn = qp_attr->rq_psn; 1691 } 1692 if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1693 qp->qplib_qp.modify_flags |= 1694 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC; 1695 /* Cap the max_rd_atomic to device max */ 1696 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic, 1697 dev_attr->max_qp_rd_atom); 1698 } 1699 if (qp_attr_mask & IB_QP_SQ_PSN) { 1700 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 1701 qp->qplib_qp.sq.psn = qp_attr->sq_psn; 1702 } 1703 if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1704 if (qp_attr->max_dest_rd_atomic > 1705 dev_attr->max_qp_init_rd_atom) { 1706 dev_err(rdev_to_dev(rdev), 1707 "max_dest_rd_atomic requested%d is > dev_max%d", 1708 qp_attr->max_dest_rd_atomic, 1709 dev_attr->max_qp_init_rd_atom); 1710 return -EINVAL; 1711 } 1712 1713 qp->qplib_qp.modify_flags |= 1714 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC; 1715 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; 1716 } 1717 if (qp_attr_mask & IB_QP_CAP) { 1718 qp->qplib_qp.modify_flags |= 1719 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE | 1720 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE | 1721 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE | 1722 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE | 1723 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA; 1724 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || 1725 (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || 1726 (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || 1727 (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || 1728 (qp_attr->cap.max_inline_data >= 1729 dev_attr->max_inline_data)) { 1730 dev_err(rdev_to_dev(rdev), 1731 "Create QP failed - max exceeded"); 1732 return -EINVAL; 1733 } 1734 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr); 1735 qp->qplib_qp.sq.max_wqe = min_t(u32, entries, 1736 dev_attr->max_qp_wqes + 1); 1737 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - 1738 qp_attr->cap.max_send_wr; 1739 /* 1740 * Reserving one slot for Phantom WQE. Some application can 1741 * post one extra entry in this case. Allowing this to avoid 1742 * unexpected Queue full condition 1743 */ 1744 qp->qplib_qp.sq.q_full_delta -= 1; 1745 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; 1746 if (qp->qplib_qp.rq.max_wqe) { 1747 entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr); 1748 qp->qplib_qp.rq.max_wqe = 1749 min_t(u32, entries, dev_attr->max_qp_wqes + 1); 1750 qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - 1751 qp_attr->cap.max_recv_wr; 1752 qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; 1753 } else { 1754 /* SRQ was used prior, just ignore the RQ caps */ 1755 } 1756 } 1757 if (qp_attr_mask & IB_QP_DEST_QPN) { 1758 qp->qplib_qp.modify_flags |= 1759 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID; 1760 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; 1761 } 1762 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 1763 if (rc) { 1764 dev_err(rdev_to_dev(rdev), "Failed to modify HW QP"); 1765 return rc; 1766 } 1767 if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) 1768 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); 1769 return rc; 1770 } 1771 1772 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 1773 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) 1774 { 1775 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 1776 struct bnxt_re_dev *rdev = qp->rdev; 1777 struct bnxt_qplib_qp *qplib_qp; 1778 int rc; 1779 1780 qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL); 1781 if (!qplib_qp) 1782 return -ENOMEM; 1783 1784 qplib_qp->id = qp->qplib_qp.id; 1785 qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; 1786 1787 rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp); 1788 if (rc) { 1789 dev_err(rdev_to_dev(rdev), "Failed to query HW QP"); 1790 goto out; 1791 } 1792 qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state); 1793 qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0; 1794 qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access); 1795 qp_attr->pkey_index = qplib_qp->pkey_index; 1796 qp_attr->qkey = qplib_qp->qkey; 1797 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 1798 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->ah.flow_label, 1799 qplib_qp->ah.host_sgid_index, 1800 qplib_qp->ah.hop_limit, 1801 qplib_qp->ah.traffic_class); 1802 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data); 1803 rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl); 1804 ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac); 1805 qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu); 1806 qp_attr->timeout = qplib_qp->timeout; 1807 qp_attr->retry_cnt = qplib_qp->retry_cnt; 1808 qp_attr->rnr_retry = qplib_qp->rnr_retry; 1809 qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer; 1810 qp_attr->rq_psn = qplib_qp->rq.psn; 1811 qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic; 1812 qp_attr->sq_psn = qplib_qp->sq.psn; 1813 qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic; 1814 qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR : 1815 IB_SIGNAL_REQ_WR; 1816 qp_attr->dest_qp_num = qplib_qp->dest_qpn; 1817 1818 qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; 1819 qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; 1820 qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; 1821 qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; 1822 qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; 1823 qp_init_attr->cap = qp_attr->cap; 1824 1825 out: 1826 kfree(qplib_qp); 1827 return rc; 1828 } 1829 1830 /* Routine for sending QP1 packets for RoCE V1 an V2 1831 */ 1832 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp, 1833 const struct ib_send_wr *wr, 1834 struct bnxt_qplib_swqe *wqe, 1835 int payload_size) 1836 { 1837 struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, 1838 ib_ah); 1839 struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; 1840 const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr; 1841 struct bnxt_qplib_sge sge; 1842 u8 nw_type; 1843 u16 ether_type; 1844 union ib_gid dgid; 1845 bool is_eth = false; 1846 bool is_vlan = false; 1847 bool is_grh = false; 1848 bool is_udp = false; 1849 u8 ip_version = 0; 1850 u16 vlan_id = 0xFFFF; 1851 void *buf; 1852 int i, rc = 0; 1853 1854 memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); 1855 1856 rc = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, NULL); 1857 if (rc) 1858 return rc; 1859 1860 /* Get network header type for this GID */ 1861 nw_type = rdma_gid_attr_network_type(sgid_attr); 1862 switch (nw_type) { 1863 case RDMA_NETWORK_IPV4: 1864 nw_type = BNXT_RE_ROCEV2_IPV4_PACKET; 1865 break; 1866 case RDMA_NETWORK_IPV6: 1867 nw_type = BNXT_RE_ROCEV2_IPV6_PACKET; 1868 break; 1869 default: 1870 nw_type = BNXT_RE_ROCE_V1_PACKET; 1871 break; 1872 } 1873 memcpy(&dgid.raw, &qplib_ah->dgid, 16); 1874 is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP; 1875 if (is_udp) { 1876 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) { 1877 ip_version = 4; 1878 ether_type = ETH_P_IP; 1879 } else { 1880 ip_version = 6; 1881 ether_type = ETH_P_IPV6; 1882 } 1883 is_grh = false; 1884 } else { 1885 ether_type = ETH_P_IBOE; 1886 is_grh = true; 1887 } 1888 1889 is_eth = true; 1890 is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false; 1891 1892 ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh, 1893 ip_version, is_udp, 0, &qp->qp1_hdr); 1894 1895 /* ETH */ 1896 ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac); 1897 ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac); 1898 1899 /* For vlan, check the sgid for vlan existence */ 1900 1901 if (!is_vlan) { 1902 qp->qp1_hdr.eth.type = cpu_to_be16(ether_type); 1903 } else { 1904 qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type); 1905 qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id); 1906 } 1907 1908 if (is_grh || (ip_version == 6)) { 1909 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw, 1910 sizeof(sgid_attr->gid)); 1911 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, 1912 sizeof(sgid_attr->gid)); 1913 qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit; 1914 } 1915 1916 if (ip_version == 4) { 1917 qp->qp1_hdr.ip4.tos = 0; 1918 qp->qp1_hdr.ip4.id = 0; 1919 qp->qp1_hdr.ip4.frag_off = htons(IP_DF); 1920 qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit; 1921 1922 memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4); 1923 memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4); 1924 qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr); 1925 } 1926 1927 if (is_udp) { 1928 qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT); 1929 qp->qp1_hdr.udp.sport = htons(0x8CD1); 1930 qp->qp1_hdr.udp.csum = 0; 1931 } 1932 1933 /* BTH */ 1934 if (wr->opcode == IB_WR_SEND_WITH_IMM) { 1935 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; 1936 qp->qp1_hdr.immediate_present = 1; 1937 } else { 1938 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; 1939 } 1940 if (wr->send_flags & IB_SEND_SOLICITED) 1941 qp->qp1_hdr.bth.solicited_event = 1; 1942 /* pad_count */ 1943 qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; 1944 1945 /* P_key for QP1 is for all members */ 1946 qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); 1947 qp->qp1_hdr.bth.destination_qpn = IB_QP1; 1948 qp->qp1_hdr.bth.ack_req = 0; 1949 qp->send_psn++; 1950 qp->send_psn &= BTH_PSN_MASK; 1951 qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); 1952 /* DETH */ 1953 /* Use the priviledged Q_Key for QP1 */ 1954 qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); 1955 qp->qp1_hdr.deth.source_qpn = IB_QP1; 1956 1957 /* Pack the QP1 to the transmit buffer */ 1958 buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); 1959 if (buf) { 1960 ib_ud_header_pack(&qp->qp1_hdr, buf); 1961 for (i = wqe->num_sge; i; i--) { 1962 wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; 1963 wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; 1964 wqe->sg_list[i].size = wqe->sg_list[i - 1].size; 1965 } 1966 1967 /* 1968 * Max Header buf size for IPV6 RoCE V2 is 86, 1969 * which is same as the QP1 SQ header buffer. 1970 * Header buf size for IPV4 RoCE V2 can be 66. 1971 * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20). 1972 * Subtract 20 bytes from QP1 SQ header buf size 1973 */ 1974 if (is_udp && ip_version == 4) 1975 sge.size -= 20; 1976 /* 1977 * Max Header buf size for RoCE V1 is 78. 1978 * ETH(14) + VLAN(4) + GRH(40) + BTH(20). 1979 * Subtract 8 bytes from QP1 SQ header buf size 1980 */ 1981 if (!is_udp) 1982 sge.size -= 8; 1983 1984 /* Subtract 4 bytes for non vlan packets */ 1985 if (!is_vlan) 1986 sge.size -= 4; 1987 1988 wqe->sg_list[0].addr = sge.addr; 1989 wqe->sg_list[0].lkey = sge.lkey; 1990 wqe->sg_list[0].size = sge.size; 1991 wqe->num_sge++; 1992 1993 } else { 1994 dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!"); 1995 rc = -ENOMEM; 1996 } 1997 return rc; 1998 } 1999 2000 /* For the MAD layer, it only provides the recv SGE the size of 2001 * ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH, 2002 * nor RoCE iCRC. The Cu+ solution must provide buffer for the entire 2003 * receive packet (334 bytes) with no VLAN and then copy the GRH 2004 * and the MAD datagram out to the provided SGE. 2005 */ 2006 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, 2007 const struct ib_recv_wr *wr, 2008 struct bnxt_qplib_swqe *wqe, 2009 int payload_size) 2010 { 2011 struct bnxt_qplib_sge ref, sge; 2012 u32 rq_prod_index; 2013 struct bnxt_re_sqp_entries *sqp_entry; 2014 2015 rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); 2016 2017 if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) 2018 return -ENOMEM; 2019 2020 /* Create 1 SGE to receive the entire 2021 * ethernet packet 2022 */ 2023 /* Save the reference from ULP */ 2024 ref.addr = wqe->sg_list[0].addr; 2025 ref.lkey = wqe->sg_list[0].lkey; 2026 ref.size = wqe->sg_list[0].size; 2027 2028 sqp_entry = &qp->rdev->sqp_tbl[rq_prod_index]; 2029 2030 /* SGE 1 */ 2031 wqe->sg_list[0].addr = sge.addr; 2032 wqe->sg_list[0].lkey = sge.lkey; 2033 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; 2034 sge.size -= wqe->sg_list[0].size; 2035 2036 sqp_entry->sge.addr = ref.addr; 2037 sqp_entry->sge.lkey = ref.lkey; 2038 sqp_entry->sge.size = ref.size; 2039 /* Store the wrid for reporting completion */ 2040 sqp_entry->wrid = wqe->wr_id; 2041 /* change the wqe->wrid to table index */ 2042 wqe->wr_id = rq_prod_index; 2043 return 0; 2044 } 2045 2046 static int is_ud_qp(struct bnxt_re_qp *qp) 2047 { 2048 return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD || 2049 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI); 2050 } 2051 2052 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, 2053 const struct ib_send_wr *wr, 2054 struct bnxt_qplib_swqe *wqe) 2055 { 2056 struct bnxt_re_ah *ah = NULL; 2057 2058 if (is_ud_qp(qp)) { 2059 ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah); 2060 wqe->send.q_key = ud_wr(wr)->remote_qkey; 2061 wqe->send.dst_qp = ud_wr(wr)->remote_qpn; 2062 wqe->send.avid = ah->qplib_ah.id; 2063 } 2064 switch (wr->opcode) { 2065 case IB_WR_SEND: 2066 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; 2067 break; 2068 case IB_WR_SEND_WITH_IMM: 2069 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; 2070 wqe->send.imm_data = wr->ex.imm_data; 2071 break; 2072 case IB_WR_SEND_WITH_INV: 2073 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; 2074 wqe->send.inv_key = wr->ex.invalidate_rkey; 2075 break; 2076 default: 2077 return -EINVAL; 2078 } 2079 if (wr->send_flags & IB_SEND_SIGNALED) 2080 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2081 if (wr->send_flags & IB_SEND_FENCE) 2082 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2083 if (wr->send_flags & IB_SEND_SOLICITED) 2084 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2085 if (wr->send_flags & IB_SEND_INLINE) 2086 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 2087 2088 return 0; 2089 } 2090 2091 static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr, 2092 struct bnxt_qplib_swqe *wqe) 2093 { 2094 switch (wr->opcode) { 2095 case IB_WR_RDMA_WRITE: 2096 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; 2097 break; 2098 case IB_WR_RDMA_WRITE_WITH_IMM: 2099 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; 2100 wqe->rdma.imm_data = wr->ex.imm_data; 2101 break; 2102 case IB_WR_RDMA_READ: 2103 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; 2104 wqe->rdma.inv_key = wr->ex.invalidate_rkey; 2105 break; 2106 default: 2107 return -EINVAL; 2108 } 2109 wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; 2110 wqe->rdma.r_key = rdma_wr(wr)->rkey; 2111 if (wr->send_flags & IB_SEND_SIGNALED) 2112 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2113 if (wr->send_flags & IB_SEND_FENCE) 2114 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2115 if (wr->send_flags & IB_SEND_SOLICITED) 2116 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2117 if (wr->send_flags & IB_SEND_INLINE) 2118 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 2119 2120 return 0; 2121 } 2122 2123 static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr, 2124 struct bnxt_qplib_swqe *wqe) 2125 { 2126 switch (wr->opcode) { 2127 case IB_WR_ATOMIC_CMP_AND_SWP: 2128 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; 2129 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 2130 wqe->atomic.swap_data = atomic_wr(wr)->swap; 2131 break; 2132 case IB_WR_ATOMIC_FETCH_AND_ADD: 2133 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; 2134 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 2135 break; 2136 default: 2137 return -EINVAL; 2138 } 2139 wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; 2140 wqe->atomic.r_key = atomic_wr(wr)->rkey; 2141 if (wr->send_flags & IB_SEND_SIGNALED) 2142 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2143 if (wr->send_flags & IB_SEND_FENCE) 2144 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2145 if (wr->send_flags & IB_SEND_SOLICITED) 2146 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2147 return 0; 2148 } 2149 2150 static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr, 2151 struct bnxt_qplib_swqe *wqe) 2152 { 2153 wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; 2154 wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; 2155 2156 /* Need unconditional fence for local invalidate 2157 * opcode to work as expected. 2158 */ 2159 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2160 2161 if (wr->send_flags & IB_SEND_SIGNALED) 2162 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2163 if (wr->send_flags & IB_SEND_SOLICITED) 2164 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2165 2166 return 0; 2167 } 2168 2169 static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr, 2170 struct bnxt_qplib_swqe *wqe) 2171 { 2172 struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr); 2173 struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; 2174 int access = wr->access; 2175 2176 wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; 2177 wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; 2178 wqe->frmr.page_list = mr->pages; 2179 wqe->frmr.page_list_len = mr->npages; 2180 wqe->frmr.levels = qplib_frpl->hwq.level + 1; 2181 wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; 2182 2183 /* Need unconditional fence for reg_mr 2184 * opcode to function as expected. 2185 */ 2186 2187 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2188 2189 if (wr->wr.send_flags & IB_SEND_SIGNALED) 2190 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2191 2192 if (access & IB_ACCESS_LOCAL_WRITE) 2193 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; 2194 if (access & IB_ACCESS_REMOTE_READ) 2195 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; 2196 if (access & IB_ACCESS_REMOTE_WRITE) 2197 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; 2198 if (access & IB_ACCESS_REMOTE_ATOMIC) 2199 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; 2200 if (access & IB_ACCESS_MW_BIND) 2201 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; 2202 2203 wqe->frmr.l_key = wr->key; 2204 wqe->frmr.length = wr->mr->length; 2205 wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1; 2206 wqe->frmr.va = wr->mr->iova; 2207 return 0; 2208 } 2209 2210 static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev, 2211 const struct ib_send_wr *wr, 2212 struct bnxt_qplib_swqe *wqe) 2213 { 2214 /* Copy the inline data to the data field */ 2215 u8 *in_data; 2216 u32 i, sge_len; 2217 void *sge_addr; 2218 2219 in_data = wqe->inline_data; 2220 for (i = 0; i < wr->num_sge; i++) { 2221 sge_addr = (void *)(unsigned long) 2222 wr->sg_list[i].addr; 2223 sge_len = wr->sg_list[i].length; 2224 2225 if ((sge_len + wqe->inline_len) > 2226 BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { 2227 dev_err(rdev_to_dev(rdev), 2228 "Inline data size requested > supported value"); 2229 return -EINVAL; 2230 } 2231 sge_len = wr->sg_list[i].length; 2232 2233 memcpy(in_data, sge_addr, sge_len); 2234 in_data += wr->sg_list[i].length; 2235 wqe->inline_len += wr->sg_list[i].length; 2236 } 2237 return wqe->inline_len; 2238 } 2239 2240 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev, 2241 const struct ib_send_wr *wr, 2242 struct bnxt_qplib_swqe *wqe) 2243 { 2244 int payload_sz = 0; 2245 2246 if (wr->send_flags & IB_SEND_INLINE) 2247 payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe); 2248 else 2249 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list, 2250 wqe->num_sge); 2251 2252 return payload_sz; 2253 } 2254 2255 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp) 2256 { 2257 if ((qp->ib_qp.qp_type == IB_QPT_UD || 2258 qp->ib_qp.qp_type == IB_QPT_GSI || 2259 qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) && 2260 qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) { 2261 int qp_attr_mask; 2262 struct ib_qp_attr qp_attr; 2263 2264 qp_attr_mask = IB_QP_STATE; 2265 qp_attr.qp_state = IB_QPS_RTS; 2266 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL); 2267 qp->qplib_qp.wqe_cnt = 0; 2268 } 2269 } 2270 2271 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, 2272 struct bnxt_re_qp *qp, 2273 const struct ib_send_wr *wr) 2274 { 2275 struct bnxt_qplib_swqe wqe; 2276 int rc = 0, payload_sz = 0; 2277 unsigned long flags; 2278 2279 spin_lock_irqsave(&qp->sq_lock, flags); 2280 memset(&wqe, 0, sizeof(wqe)); 2281 while (wr) { 2282 /* House keeping */ 2283 memset(&wqe, 0, sizeof(wqe)); 2284 2285 /* Common */ 2286 wqe.num_sge = wr->num_sge; 2287 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 2288 dev_err(rdev_to_dev(rdev), 2289 "Limit exceeded for Send SGEs"); 2290 rc = -EINVAL; 2291 goto bad; 2292 } 2293 2294 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); 2295 if (payload_sz < 0) { 2296 rc = -EINVAL; 2297 goto bad; 2298 } 2299 wqe.wr_id = wr->wr_id; 2300 2301 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; 2302 2303 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 2304 if (!rc) 2305 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 2306 bad: 2307 if (rc) { 2308 dev_err(rdev_to_dev(rdev), 2309 "Post send failed opcode = %#x rc = %d", 2310 wr->opcode, rc); 2311 break; 2312 } 2313 wr = wr->next; 2314 } 2315 bnxt_qplib_post_send_db(&qp->qplib_qp); 2316 bnxt_ud_qp_hw_stall_workaround(qp); 2317 spin_unlock_irqrestore(&qp->sq_lock, flags); 2318 return rc; 2319 } 2320 2321 int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, 2322 const struct ib_send_wr **bad_wr) 2323 { 2324 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 2325 struct bnxt_qplib_swqe wqe; 2326 int rc = 0, payload_sz = 0; 2327 unsigned long flags; 2328 2329 spin_lock_irqsave(&qp->sq_lock, flags); 2330 while (wr) { 2331 /* House keeping */ 2332 memset(&wqe, 0, sizeof(wqe)); 2333 2334 /* Common */ 2335 wqe.num_sge = wr->num_sge; 2336 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 2337 dev_err(rdev_to_dev(qp->rdev), 2338 "Limit exceeded for Send SGEs"); 2339 rc = -EINVAL; 2340 goto bad; 2341 } 2342 2343 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); 2344 if (payload_sz < 0) { 2345 rc = -EINVAL; 2346 goto bad; 2347 } 2348 wqe.wr_id = wr->wr_id; 2349 2350 switch (wr->opcode) { 2351 case IB_WR_SEND: 2352 case IB_WR_SEND_WITH_IMM: 2353 if (qp->qplib_qp.type == CMDQ_CREATE_QP1_TYPE_GSI) { 2354 rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe, 2355 payload_sz); 2356 if (rc) 2357 goto bad; 2358 wqe.rawqp1.lflags |= 2359 SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; 2360 } 2361 switch (wr->send_flags) { 2362 case IB_SEND_IP_CSUM: 2363 wqe.rawqp1.lflags |= 2364 SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; 2365 break; 2366 default: 2367 break; 2368 } 2369 /* fall through */ 2370 case IB_WR_SEND_WITH_INV: 2371 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 2372 break; 2373 case IB_WR_RDMA_WRITE: 2374 case IB_WR_RDMA_WRITE_WITH_IMM: 2375 case IB_WR_RDMA_READ: 2376 rc = bnxt_re_build_rdma_wqe(wr, &wqe); 2377 break; 2378 case IB_WR_ATOMIC_CMP_AND_SWP: 2379 case IB_WR_ATOMIC_FETCH_AND_ADD: 2380 rc = bnxt_re_build_atomic_wqe(wr, &wqe); 2381 break; 2382 case IB_WR_RDMA_READ_WITH_INV: 2383 dev_err(rdev_to_dev(qp->rdev), 2384 "RDMA Read with Invalidate is not supported"); 2385 rc = -EINVAL; 2386 goto bad; 2387 case IB_WR_LOCAL_INV: 2388 rc = bnxt_re_build_inv_wqe(wr, &wqe); 2389 break; 2390 case IB_WR_REG_MR: 2391 rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); 2392 break; 2393 default: 2394 /* Unsupported WRs */ 2395 dev_err(rdev_to_dev(qp->rdev), 2396 "WR (%#x) is not supported", wr->opcode); 2397 rc = -EINVAL; 2398 goto bad; 2399 } 2400 if (!rc) 2401 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 2402 bad: 2403 if (rc) { 2404 dev_err(rdev_to_dev(qp->rdev), 2405 "post_send failed op:%#x qps = %#x rc = %d\n", 2406 wr->opcode, qp->qplib_qp.state, rc); 2407 *bad_wr = wr; 2408 break; 2409 } 2410 wr = wr->next; 2411 } 2412 bnxt_qplib_post_send_db(&qp->qplib_qp); 2413 bnxt_ud_qp_hw_stall_workaround(qp); 2414 spin_unlock_irqrestore(&qp->sq_lock, flags); 2415 2416 return rc; 2417 } 2418 2419 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, 2420 struct bnxt_re_qp *qp, 2421 const struct ib_recv_wr *wr) 2422 { 2423 struct bnxt_qplib_swqe wqe; 2424 int rc = 0; 2425 2426 memset(&wqe, 0, sizeof(wqe)); 2427 while (wr) { 2428 /* House keeping */ 2429 memset(&wqe, 0, sizeof(wqe)); 2430 2431 /* Common */ 2432 wqe.num_sge = wr->num_sge; 2433 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 2434 dev_err(rdev_to_dev(rdev), 2435 "Limit exceeded for Receive SGEs"); 2436 rc = -EINVAL; 2437 break; 2438 } 2439 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); 2440 wqe.wr_id = wr->wr_id; 2441 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 2442 2443 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 2444 if (rc) 2445 break; 2446 2447 wr = wr->next; 2448 } 2449 if (!rc) 2450 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2451 return rc; 2452 } 2453 2454 int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, 2455 const struct ib_recv_wr **bad_wr) 2456 { 2457 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 2458 struct bnxt_qplib_swqe wqe; 2459 int rc = 0, payload_sz = 0; 2460 unsigned long flags; 2461 u32 count = 0; 2462 2463 spin_lock_irqsave(&qp->rq_lock, flags); 2464 while (wr) { 2465 /* House keeping */ 2466 memset(&wqe, 0, sizeof(wqe)); 2467 2468 /* Common */ 2469 wqe.num_sge = wr->num_sge; 2470 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 2471 dev_err(rdev_to_dev(qp->rdev), 2472 "Limit exceeded for Receive SGEs"); 2473 rc = -EINVAL; 2474 *bad_wr = wr; 2475 break; 2476 } 2477 2478 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, 2479 wr->num_sge); 2480 wqe.wr_id = wr->wr_id; 2481 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 2482 2483 if (ib_qp->qp_type == IB_QPT_GSI && 2484 qp->qplib_qp.type != CMDQ_CREATE_QP_TYPE_GSI) 2485 rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe, 2486 payload_sz); 2487 if (!rc) 2488 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 2489 if (rc) { 2490 *bad_wr = wr; 2491 break; 2492 } 2493 2494 /* Ring DB if the RQEs posted reaches a threshold value */ 2495 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) { 2496 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2497 count = 0; 2498 } 2499 2500 wr = wr->next; 2501 } 2502 2503 if (count) 2504 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2505 2506 spin_unlock_irqrestore(&qp->rq_lock, flags); 2507 2508 return rc; 2509 } 2510 2511 /* Completion Queues */ 2512 void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 2513 { 2514 struct bnxt_re_cq *cq; 2515 struct bnxt_qplib_nq *nq; 2516 struct bnxt_re_dev *rdev; 2517 2518 cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 2519 rdev = cq->rdev; 2520 nq = cq->qplib_cq.nq; 2521 2522 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); 2523 ib_umem_release(cq->umem); 2524 2525 atomic_dec(&rdev->cq_count); 2526 nq->budget--; 2527 kfree(cq->cql); 2528 } 2529 2530 int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 2531 struct ib_udata *udata) 2532 { 2533 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev); 2534 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 2535 struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq); 2536 int rc, entries; 2537 int cqe = attr->cqe; 2538 struct bnxt_qplib_nq *nq = NULL; 2539 unsigned int nq_alloc_cnt; 2540 2541 /* Validate CQ fields */ 2542 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { 2543 dev_err(rdev_to_dev(rdev), "Failed to create CQ -max exceeded"); 2544 return -EINVAL; 2545 } 2546 2547 cq->rdev = rdev; 2548 cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq); 2549 2550 entries = roundup_pow_of_two(cqe + 1); 2551 if (entries > dev_attr->max_cq_wqes + 1) 2552 entries = dev_attr->max_cq_wqes + 1; 2553 2554 if (udata) { 2555 struct bnxt_re_cq_req req; 2556 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context( 2557 udata, struct bnxt_re_ucontext, ib_uctx); 2558 if (ib_copy_from_udata(&req, udata, sizeof(req))) { 2559 rc = -EFAULT; 2560 goto fail; 2561 } 2562 2563 cq->umem = ib_umem_get(udata, req.cq_va, 2564 entries * sizeof(struct cq_base), 2565 IB_ACCESS_LOCAL_WRITE, 1); 2566 if (IS_ERR(cq->umem)) { 2567 rc = PTR_ERR(cq->umem); 2568 goto fail; 2569 } 2570 cq->qplib_cq.sg_info.sglist = cq->umem->sg_head.sgl; 2571 cq->qplib_cq.sg_info.npages = ib_umem_num_pages(cq->umem); 2572 cq->qplib_cq.sg_info.nmap = cq->umem->nmap; 2573 cq->qplib_cq.dpi = &uctx->dpi; 2574 } else { 2575 cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL); 2576 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), 2577 GFP_KERNEL); 2578 if (!cq->cql) { 2579 rc = -ENOMEM; 2580 goto fail; 2581 } 2582 2583 cq->qplib_cq.dpi = &rdev->dpi_privileged; 2584 } 2585 /* 2586 * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a 2587 * used for getting the NQ index. 2588 */ 2589 nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt); 2590 nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)]; 2591 cq->qplib_cq.max_wqe = entries; 2592 cq->qplib_cq.cnq_hw_ring_id = nq->ring_id; 2593 cq->qplib_cq.nq = nq; 2594 2595 rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq); 2596 if (rc) { 2597 dev_err(rdev_to_dev(rdev), "Failed to create HW CQ"); 2598 goto fail; 2599 } 2600 2601 cq->ib_cq.cqe = entries; 2602 cq->cq_period = cq->qplib_cq.period; 2603 nq->budget++; 2604 2605 atomic_inc(&rdev->cq_count); 2606 spin_lock_init(&cq->cq_lock); 2607 2608 if (udata) { 2609 struct bnxt_re_cq_resp resp; 2610 2611 resp.cqid = cq->qplib_cq.id; 2612 resp.tail = cq->qplib_cq.hwq.cons; 2613 resp.phase = cq->qplib_cq.period; 2614 resp.rsvd = 0; 2615 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 2616 if (rc) { 2617 dev_err(rdev_to_dev(rdev), "Failed to copy CQ udata"); 2618 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); 2619 goto c2fail; 2620 } 2621 } 2622 2623 return 0; 2624 2625 c2fail: 2626 ib_umem_release(cq->umem); 2627 fail: 2628 kfree(cq->cql); 2629 return rc; 2630 } 2631 2632 static u8 __req_to_ib_wc_status(u8 qstatus) 2633 { 2634 switch (qstatus) { 2635 case CQ_REQ_STATUS_OK: 2636 return IB_WC_SUCCESS; 2637 case CQ_REQ_STATUS_BAD_RESPONSE_ERR: 2638 return IB_WC_BAD_RESP_ERR; 2639 case CQ_REQ_STATUS_LOCAL_LENGTH_ERR: 2640 return IB_WC_LOC_LEN_ERR; 2641 case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR: 2642 return IB_WC_LOC_QP_OP_ERR; 2643 case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR: 2644 return IB_WC_LOC_PROT_ERR; 2645 case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR: 2646 return IB_WC_GENERAL_ERR; 2647 case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR: 2648 return IB_WC_REM_INV_REQ_ERR; 2649 case CQ_REQ_STATUS_REMOTE_ACCESS_ERR: 2650 return IB_WC_REM_ACCESS_ERR; 2651 case CQ_REQ_STATUS_REMOTE_OPERATION_ERR: 2652 return IB_WC_REM_OP_ERR; 2653 case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR: 2654 return IB_WC_RNR_RETRY_EXC_ERR; 2655 case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR: 2656 return IB_WC_RETRY_EXC_ERR; 2657 case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR: 2658 return IB_WC_WR_FLUSH_ERR; 2659 default: 2660 return IB_WC_GENERAL_ERR; 2661 } 2662 return 0; 2663 } 2664 2665 static u8 __rawqp1_to_ib_wc_status(u8 qstatus) 2666 { 2667 switch (qstatus) { 2668 case CQ_RES_RAWETH_QP1_STATUS_OK: 2669 return IB_WC_SUCCESS; 2670 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR: 2671 return IB_WC_LOC_ACCESS_ERR; 2672 case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR: 2673 return IB_WC_LOC_LEN_ERR; 2674 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR: 2675 return IB_WC_LOC_PROT_ERR; 2676 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR: 2677 return IB_WC_LOC_QP_OP_ERR; 2678 case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR: 2679 return IB_WC_GENERAL_ERR; 2680 case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR: 2681 return IB_WC_WR_FLUSH_ERR; 2682 case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR: 2683 return IB_WC_WR_FLUSH_ERR; 2684 default: 2685 return IB_WC_GENERAL_ERR; 2686 } 2687 } 2688 2689 static u8 __rc_to_ib_wc_status(u8 qstatus) 2690 { 2691 switch (qstatus) { 2692 case CQ_RES_RC_STATUS_OK: 2693 return IB_WC_SUCCESS; 2694 case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR: 2695 return IB_WC_LOC_ACCESS_ERR; 2696 case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR: 2697 return IB_WC_LOC_LEN_ERR; 2698 case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR: 2699 return IB_WC_LOC_PROT_ERR; 2700 case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR: 2701 return IB_WC_LOC_QP_OP_ERR; 2702 case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR: 2703 return IB_WC_GENERAL_ERR; 2704 case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR: 2705 return IB_WC_REM_INV_REQ_ERR; 2706 case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR: 2707 return IB_WC_WR_FLUSH_ERR; 2708 case CQ_RES_RC_STATUS_HW_FLUSH_ERR: 2709 return IB_WC_WR_FLUSH_ERR; 2710 default: 2711 return IB_WC_GENERAL_ERR; 2712 } 2713 } 2714 2715 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) 2716 { 2717 switch (cqe->type) { 2718 case BNXT_QPLIB_SWQE_TYPE_SEND: 2719 wc->opcode = IB_WC_SEND; 2720 break; 2721 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: 2722 wc->opcode = IB_WC_SEND; 2723 wc->wc_flags |= IB_WC_WITH_IMM; 2724 break; 2725 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: 2726 wc->opcode = IB_WC_SEND; 2727 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 2728 break; 2729 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: 2730 wc->opcode = IB_WC_RDMA_WRITE; 2731 break; 2732 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: 2733 wc->opcode = IB_WC_RDMA_WRITE; 2734 wc->wc_flags |= IB_WC_WITH_IMM; 2735 break; 2736 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: 2737 wc->opcode = IB_WC_RDMA_READ; 2738 break; 2739 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: 2740 wc->opcode = IB_WC_COMP_SWAP; 2741 break; 2742 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: 2743 wc->opcode = IB_WC_FETCH_ADD; 2744 break; 2745 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: 2746 wc->opcode = IB_WC_LOCAL_INV; 2747 break; 2748 case BNXT_QPLIB_SWQE_TYPE_REG_MR: 2749 wc->opcode = IB_WC_REG_MR; 2750 break; 2751 default: 2752 wc->opcode = IB_WC_SEND; 2753 break; 2754 } 2755 2756 wc->status = __req_to_ib_wc_status(cqe->status); 2757 } 2758 2759 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, 2760 u16 raweth_qp1_flags2) 2761 { 2762 bool is_ipv6 = false, is_ipv4 = false; 2763 2764 /* raweth_qp1_flags Bit 9-6 indicates itype */ 2765 if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 2766 != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 2767 return -1; 2768 2769 if (raweth_qp1_flags2 & 2770 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC && 2771 raweth_qp1_flags2 & 2772 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) { 2773 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ 2774 (raweth_qp1_flags2 & 2775 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ? 2776 (is_ipv6 = true) : (is_ipv4 = true); 2777 return ((is_ipv6) ? 2778 BNXT_RE_ROCEV2_IPV6_PACKET : 2779 BNXT_RE_ROCEV2_IPV4_PACKET); 2780 } else { 2781 return BNXT_RE_ROCE_V1_PACKET; 2782 } 2783 } 2784 2785 static int bnxt_re_to_ib_nw_type(int nw_type) 2786 { 2787 u8 nw_hdr_type = 0xFF; 2788 2789 switch (nw_type) { 2790 case BNXT_RE_ROCE_V1_PACKET: 2791 nw_hdr_type = RDMA_NETWORK_ROCE_V1; 2792 break; 2793 case BNXT_RE_ROCEV2_IPV4_PACKET: 2794 nw_hdr_type = RDMA_NETWORK_IPV4; 2795 break; 2796 case BNXT_RE_ROCEV2_IPV6_PACKET: 2797 nw_hdr_type = RDMA_NETWORK_IPV6; 2798 break; 2799 } 2800 return nw_hdr_type; 2801 } 2802 2803 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev, 2804 void *rq_hdr_buf) 2805 { 2806 u8 *tmp_buf = NULL; 2807 struct ethhdr *eth_hdr; 2808 u16 eth_type; 2809 bool rc = false; 2810 2811 tmp_buf = (u8 *)rq_hdr_buf; 2812 /* 2813 * If dest mac is not same as I/F mac, this could be a 2814 * loopback address or multicast address, check whether 2815 * it is a loopback packet 2816 */ 2817 if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) { 2818 tmp_buf += 4; 2819 /* Check the ether type */ 2820 eth_hdr = (struct ethhdr *)tmp_buf; 2821 eth_type = ntohs(eth_hdr->h_proto); 2822 switch (eth_type) { 2823 case ETH_P_IBOE: 2824 rc = true; 2825 break; 2826 case ETH_P_IP: 2827 case ETH_P_IPV6: { 2828 u32 len; 2829 struct udphdr *udp_hdr; 2830 2831 len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) : 2832 sizeof(struct ipv6hdr)); 2833 tmp_buf += sizeof(struct ethhdr) + len; 2834 udp_hdr = (struct udphdr *)tmp_buf; 2835 if (ntohs(udp_hdr->dest) == 2836 ROCE_V2_UDP_DPORT) 2837 rc = true; 2838 break; 2839 } 2840 default: 2841 break; 2842 } 2843 } 2844 2845 return rc; 2846 } 2847 2848 static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *qp1_qp, 2849 struct bnxt_qplib_cqe *cqe) 2850 { 2851 struct bnxt_re_dev *rdev = qp1_qp->rdev; 2852 struct bnxt_re_sqp_entries *sqp_entry = NULL; 2853 struct bnxt_re_qp *qp = rdev->qp1_sqp; 2854 struct ib_send_wr *swr; 2855 struct ib_ud_wr udwr; 2856 struct ib_recv_wr rwr; 2857 int pkt_type = 0; 2858 u32 tbl_idx; 2859 void *rq_hdr_buf; 2860 dma_addr_t rq_hdr_buf_map; 2861 dma_addr_t shrq_hdr_buf_map; 2862 u32 offset = 0; 2863 u32 skip_bytes = 0; 2864 struct ib_sge s_sge[2]; 2865 struct ib_sge r_sge[2]; 2866 int rc; 2867 2868 memset(&udwr, 0, sizeof(udwr)); 2869 memset(&rwr, 0, sizeof(rwr)); 2870 memset(&s_sge, 0, sizeof(s_sge)); 2871 memset(&r_sge, 0, sizeof(r_sge)); 2872 2873 swr = &udwr.wr; 2874 tbl_idx = cqe->wr_id; 2875 2876 rq_hdr_buf = qp1_qp->qplib_qp.rq_hdr_buf + 2877 (tbl_idx * qp1_qp->qplib_qp.rq_hdr_buf_size); 2878 rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp1_qp->qplib_qp, 2879 tbl_idx); 2880 2881 /* Shadow QP header buffer */ 2882 shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp->qplib_qp, 2883 tbl_idx); 2884 sqp_entry = &rdev->sqp_tbl[tbl_idx]; 2885 2886 /* Store this cqe */ 2887 memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); 2888 sqp_entry->qp1_qp = qp1_qp; 2889 2890 /* Find packet type from the cqe */ 2891 2892 pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, 2893 cqe->raweth_qp1_flags2); 2894 if (pkt_type < 0) { 2895 dev_err(rdev_to_dev(rdev), "Invalid packet\n"); 2896 return -EINVAL; 2897 } 2898 2899 /* Adjust the offset for the user buffer and post in the rq */ 2900 2901 if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET) 2902 offset = 20; 2903 2904 /* 2905 * QP1 loopback packet has 4 bytes of internal header before 2906 * ether header. Skip these four bytes. 2907 */ 2908 if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf)) 2909 skip_bytes = 4; 2910 2911 /* First send SGE . Skip the ether header*/ 2912 s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 2913 + skip_bytes; 2914 s_sge[0].lkey = 0xFFFFFFFF; 2915 s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 : 2916 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; 2917 2918 /* Second Send SGE */ 2919 s_sge[1].addr = s_sge[0].addr + s_sge[0].length + 2920 BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE; 2921 if (pkt_type != BNXT_RE_ROCE_V1_PACKET) 2922 s_sge[1].addr += 8; 2923 s_sge[1].lkey = 0xFFFFFFFF; 2924 s_sge[1].length = 256; 2925 2926 /* First recv SGE */ 2927 2928 r_sge[0].addr = shrq_hdr_buf_map; 2929 r_sge[0].lkey = 0xFFFFFFFF; 2930 r_sge[0].length = 40; 2931 2932 r_sge[1].addr = sqp_entry->sge.addr + offset; 2933 r_sge[1].lkey = sqp_entry->sge.lkey; 2934 r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; 2935 2936 /* Create receive work request */ 2937 rwr.num_sge = 2; 2938 rwr.sg_list = r_sge; 2939 rwr.wr_id = tbl_idx; 2940 rwr.next = NULL; 2941 2942 rc = bnxt_re_post_recv_shadow_qp(rdev, qp, &rwr); 2943 if (rc) { 2944 dev_err(rdev_to_dev(rdev), 2945 "Failed to post Rx buffers to shadow QP"); 2946 return -ENOMEM; 2947 } 2948 2949 swr->num_sge = 2; 2950 swr->sg_list = s_sge; 2951 swr->wr_id = tbl_idx; 2952 swr->opcode = IB_WR_SEND; 2953 swr->next = NULL; 2954 2955 udwr.ah = &rdev->sqp_ah->ib_ah; 2956 udwr.remote_qpn = rdev->qp1_sqp->qplib_qp.id; 2957 udwr.remote_qkey = rdev->qp1_sqp->qplib_qp.qkey; 2958 2959 /* post data received in the send queue */ 2960 rc = bnxt_re_post_send_shadow_qp(rdev, qp, swr); 2961 2962 return 0; 2963 } 2964 2965 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc, 2966 struct bnxt_qplib_cqe *cqe) 2967 { 2968 wc->opcode = IB_WC_RECV; 2969 wc->status = __rawqp1_to_ib_wc_status(cqe->status); 2970 wc->wc_flags |= IB_WC_GRH; 2971 } 2972 2973 static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe, 2974 u16 *vid, u8 *sl) 2975 { 2976 bool ret = false; 2977 u32 metadata; 2978 u16 tpid; 2979 2980 metadata = orig_cqe->raweth_qp1_metadata; 2981 if (orig_cqe->raweth_qp1_flags2 & 2982 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) { 2983 tpid = ((metadata & 2984 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >> 2985 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); 2986 if (tpid == ETH_P_8021Q) { 2987 *vid = metadata & 2988 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK; 2989 *sl = (metadata & 2990 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >> 2991 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT; 2992 ret = true; 2993 } 2994 } 2995 2996 return ret; 2997 } 2998 2999 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc, 3000 struct bnxt_qplib_cqe *cqe) 3001 { 3002 wc->opcode = IB_WC_RECV; 3003 wc->status = __rc_to_ib_wc_status(cqe->status); 3004 3005 if (cqe->flags & CQ_RES_RC_FLAGS_IMM) 3006 wc->wc_flags |= IB_WC_WITH_IMM; 3007 if (cqe->flags & CQ_RES_RC_FLAGS_INV) 3008 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3009 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == 3010 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) 3011 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 3012 } 3013 3014 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *qp, 3015 struct ib_wc *wc, 3016 struct bnxt_qplib_cqe *cqe) 3017 { 3018 struct bnxt_re_dev *rdev = qp->rdev; 3019 struct bnxt_re_qp *qp1_qp = NULL; 3020 struct bnxt_qplib_cqe *orig_cqe = NULL; 3021 struct bnxt_re_sqp_entries *sqp_entry = NULL; 3022 int nw_type; 3023 u32 tbl_idx; 3024 u16 vlan_id; 3025 u8 sl; 3026 3027 tbl_idx = cqe->wr_id; 3028 3029 sqp_entry = &rdev->sqp_tbl[tbl_idx]; 3030 qp1_qp = sqp_entry->qp1_qp; 3031 orig_cqe = &sqp_entry->cqe; 3032 3033 wc->wr_id = sqp_entry->wrid; 3034 wc->byte_len = orig_cqe->length; 3035 wc->qp = &qp1_qp->ib_qp; 3036 3037 wc->ex.imm_data = orig_cqe->immdata; 3038 wc->src_qp = orig_cqe->src_qp; 3039 memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); 3040 if (bnxt_re_is_vlan_pkt(orig_cqe, &vlan_id, &sl)) { 3041 wc->vlan_id = vlan_id; 3042 wc->sl = sl; 3043 wc->wc_flags |= IB_WC_WITH_VLAN; 3044 } 3045 wc->port_num = 1; 3046 wc->vendor_err = orig_cqe->status; 3047 3048 wc->opcode = IB_WC_RECV; 3049 wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); 3050 wc->wc_flags |= IB_WC_GRH; 3051 3052 nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, 3053 orig_cqe->raweth_qp1_flags2); 3054 if (nw_type >= 0) { 3055 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type); 3056 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 3057 } 3058 } 3059 3060 static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp, 3061 struct ib_wc *wc, 3062 struct bnxt_qplib_cqe *cqe) 3063 { 3064 u8 nw_type; 3065 3066 wc->opcode = IB_WC_RECV; 3067 wc->status = __rc_to_ib_wc_status(cqe->status); 3068 3069 if (cqe->flags & CQ_RES_UD_FLAGS_IMM) 3070 wc->wc_flags |= IB_WC_WITH_IMM; 3071 /* report only on GSI QP for Thor */ 3072 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI) { 3073 wc->wc_flags |= IB_WC_GRH; 3074 memcpy(wc->smac, cqe->smac, ETH_ALEN); 3075 wc->wc_flags |= IB_WC_WITH_SMAC; 3076 if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) { 3077 wc->vlan_id = (cqe->cfa_meta & 0xFFF); 3078 if (wc->vlan_id < 0x1000) 3079 wc->wc_flags |= IB_WC_WITH_VLAN; 3080 } 3081 nw_type = (cqe->flags & CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK) >> 3082 CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT; 3083 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type); 3084 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 3085 } 3086 3087 } 3088 3089 static int send_phantom_wqe(struct bnxt_re_qp *qp) 3090 { 3091 struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp; 3092 unsigned long flags; 3093 int rc = 0; 3094 3095 spin_lock_irqsave(&qp->sq_lock, flags); 3096 3097 rc = bnxt_re_bind_fence_mw(lib_qp); 3098 if (!rc) { 3099 lib_qp->sq.phantom_wqe_cnt++; 3100 dev_dbg(&lib_qp->sq.hwq.pdev->dev, 3101 "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n", 3102 lib_qp->id, lib_qp->sq.hwq.prod, 3103 HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq), 3104 lib_qp->sq.phantom_wqe_cnt); 3105 } 3106 3107 spin_unlock_irqrestore(&qp->sq_lock, flags); 3108 return rc; 3109 } 3110 3111 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) 3112 { 3113 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 3114 struct bnxt_re_qp *qp; 3115 struct bnxt_qplib_cqe *cqe; 3116 int i, ncqe, budget; 3117 struct bnxt_qplib_q *sq; 3118 struct bnxt_qplib_qp *lib_qp; 3119 u32 tbl_idx; 3120 struct bnxt_re_sqp_entries *sqp_entry = NULL; 3121 unsigned long flags; 3122 3123 spin_lock_irqsave(&cq->cq_lock, flags); 3124 budget = min_t(u32, num_entries, cq->max_cql); 3125 num_entries = budget; 3126 if (!cq->cql) { 3127 dev_err(rdev_to_dev(cq->rdev), "POLL CQ : no CQL to use"); 3128 goto exit; 3129 } 3130 cqe = &cq->cql[0]; 3131 while (budget) { 3132 lib_qp = NULL; 3133 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp); 3134 if (lib_qp) { 3135 sq = &lib_qp->sq; 3136 if (sq->send_phantom) { 3137 qp = container_of(lib_qp, 3138 struct bnxt_re_qp, qplib_qp); 3139 if (send_phantom_wqe(qp) == -ENOMEM) 3140 dev_err(rdev_to_dev(cq->rdev), 3141 "Phantom failed! Scheduled to send again\n"); 3142 else 3143 sq->send_phantom = false; 3144 } 3145 } 3146 if (ncqe < budget) 3147 ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq, 3148 cqe + ncqe, 3149 budget - ncqe); 3150 3151 if (!ncqe) 3152 break; 3153 3154 for (i = 0; i < ncqe; i++, cqe++) { 3155 /* Transcribe each qplib_wqe back to ib_wc */ 3156 memset(wc, 0, sizeof(*wc)); 3157 3158 wc->wr_id = cqe->wr_id; 3159 wc->byte_len = cqe->length; 3160 qp = container_of 3161 ((struct bnxt_qplib_qp *) 3162 (unsigned long)(cqe->qp_handle), 3163 struct bnxt_re_qp, qplib_qp); 3164 if (!qp) { 3165 dev_err(rdev_to_dev(cq->rdev), 3166 "POLL CQ : bad QP handle"); 3167 continue; 3168 } 3169 wc->qp = &qp->ib_qp; 3170 wc->ex.imm_data = cqe->immdata; 3171 wc->src_qp = cqe->src_qp; 3172 memcpy(wc->smac, cqe->smac, ETH_ALEN); 3173 wc->port_num = 1; 3174 wc->vendor_err = cqe->status; 3175 3176 switch (cqe->opcode) { 3177 case CQ_BASE_CQE_TYPE_REQ: 3178 if (qp->rdev->qp1_sqp && qp->qplib_qp.id == 3179 qp->rdev->qp1_sqp->qplib_qp.id) { 3180 /* Handle this completion with 3181 * the stored completion 3182 */ 3183 memset(wc, 0, sizeof(*wc)); 3184 continue; 3185 } 3186 bnxt_re_process_req_wc(wc, cqe); 3187 break; 3188 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 3189 if (!cqe->status) { 3190 int rc = 0; 3191 3192 rc = bnxt_re_process_raw_qp_pkt_rx 3193 (qp, cqe); 3194 if (!rc) { 3195 memset(wc, 0, sizeof(*wc)); 3196 continue; 3197 } 3198 cqe->status = -1; 3199 } 3200 /* Errors need not be looped back. 3201 * But change the wr_id to the one 3202 * stored in the table 3203 */ 3204 tbl_idx = cqe->wr_id; 3205 sqp_entry = &cq->rdev->sqp_tbl[tbl_idx]; 3206 wc->wr_id = sqp_entry->wrid; 3207 bnxt_re_process_res_rawqp1_wc(wc, cqe); 3208 break; 3209 case CQ_BASE_CQE_TYPE_RES_RC: 3210 bnxt_re_process_res_rc_wc(wc, cqe); 3211 break; 3212 case CQ_BASE_CQE_TYPE_RES_UD: 3213 if (qp->rdev->qp1_sqp && qp->qplib_qp.id == 3214 qp->rdev->qp1_sqp->qplib_qp.id) { 3215 /* Handle this completion with 3216 * the stored completion 3217 */ 3218 if (cqe->status) { 3219 continue; 3220 } else { 3221 bnxt_re_process_res_shadow_qp_wc 3222 (qp, wc, cqe); 3223 break; 3224 } 3225 } 3226 bnxt_re_process_res_ud_wc(qp, wc, cqe); 3227 break; 3228 default: 3229 dev_err(rdev_to_dev(cq->rdev), 3230 "POLL CQ : type 0x%x not handled", 3231 cqe->opcode); 3232 continue; 3233 } 3234 wc++; 3235 budget--; 3236 } 3237 } 3238 exit: 3239 spin_unlock_irqrestore(&cq->cq_lock, flags); 3240 return num_entries - budget; 3241 } 3242 3243 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq, 3244 enum ib_cq_notify_flags ib_cqn_flags) 3245 { 3246 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 3247 int type = 0, rc = 0; 3248 unsigned long flags; 3249 3250 spin_lock_irqsave(&cq->cq_lock, flags); 3251 /* Trigger on the very next completion */ 3252 if (ib_cqn_flags & IB_CQ_NEXT_COMP) 3253 type = DBC_DBC_TYPE_CQ_ARMALL; 3254 /* Trigger on the next solicited completion */ 3255 else if (ib_cqn_flags & IB_CQ_SOLICITED) 3256 type = DBC_DBC_TYPE_CQ_ARMSE; 3257 3258 /* Poll to see if there are missed events */ 3259 if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) && 3260 !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) { 3261 rc = 1; 3262 goto exit; 3263 } 3264 bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); 3265 3266 exit: 3267 spin_unlock_irqrestore(&cq->cq_lock, flags); 3268 return rc; 3269 } 3270 3271 /* Memory Regions */ 3272 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) 3273 { 3274 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3275 struct bnxt_re_dev *rdev = pd->rdev; 3276 struct bnxt_re_mr *mr; 3277 u64 pbl = 0; 3278 int rc; 3279 3280 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3281 if (!mr) 3282 return ERR_PTR(-ENOMEM); 3283 3284 mr->rdev = rdev; 3285 mr->qplib_mr.pd = &pd->qplib_pd; 3286 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 3287 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 3288 3289 /* Allocate and register 0 as the address */ 3290 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3291 if (rc) 3292 goto fail; 3293 3294 mr->qplib_mr.hwq.level = PBL_LVL_MAX; 3295 mr->qplib_mr.total_size = -1; /* Infinte length */ 3296 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl, 0, false, 3297 PAGE_SIZE); 3298 if (rc) 3299 goto fail_mr; 3300 3301 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3302 if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | 3303 IB_ACCESS_REMOTE_ATOMIC)) 3304 mr->ib_mr.rkey = mr->ib_mr.lkey; 3305 atomic_inc(&rdev->mr_count); 3306 3307 return &mr->ib_mr; 3308 3309 fail_mr: 3310 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3311 fail: 3312 kfree(mr); 3313 return ERR_PTR(rc); 3314 } 3315 3316 int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 3317 { 3318 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3319 struct bnxt_re_dev *rdev = mr->rdev; 3320 int rc; 3321 3322 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3323 if (rc) 3324 dev_err(rdev_to_dev(rdev), "Dereg MR failed: %#x\n", rc); 3325 3326 if (mr->pages) { 3327 rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, 3328 &mr->qplib_frpl); 3329 kfree(mr->pages); 3330 mr->npages = 0; 3331 mr->pages = NULL; 3332 } 3333 ib_umem_release(mr->ib_umem); 3334 3335 kfree(mr); 3336 atomic_dec(&rdev->mr_count); 3337 return rc; 3338 } 3339 3340 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) 3341 { 3342 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3343 3344 if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) 3345 return -ENOMEM; 3346 3347 mr->pages[mr->npages++] = addr; 3348 return 0; 3349 } 3350 3351 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, 3352 unsigned int *sg_offset) 3353 { 3354 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3355 3356 mr->npages = 0; 3357 return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page); 3358 } 3359 3360 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, 3361 u32 max_num_sg, struct ib_udata *udata) 3362 { 3363 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3364 struct bnxt_re_dev *rdev = pd->rdev; 3365 struct bnxt_re_mr *mr = NULL; 3366 int rc; 3367 3368 if (type != IB_MR_TYPE_MEM_REG) { 3369 dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported", type); 3370 return ERR_PTR(-EINVAL); 3371 } 3372 if (max_num_sg > MAX_PBL_LVL_1_PGS) 3373 return ERR_PTR(-EINVAL); 3374 3375 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3376 if (!mr) 3377 return ERR_PTR(-ENOMEM); 3378 3379 mr->rdev = rdev; 3380 mr->qplib_mr.pd = &pd->qplib_pd; 3381 mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; 3382 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 3383 3384 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3385 if (rc) 3386 goto bail; 3387 3388 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3389 mr->ib_mr.rkey = mr->ib_mr.lkey; 3390 3391 mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); 3392 if (!mr->pages) { 3393 rc = -ENOMEM; 3394 goto fail; 3395 } 3396 rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, 3397 &mr->qplib_frpl, max_num_sg); 3398 if (rc) { 3399 dev_err(rdev_to_dev(rdev), 3400 "Failed to allocate HW FR page list"); 3401 goto fail_mr; 3402 } 3403 3404 atomic_inc(&rdev->mr_count); 3405 return &mr->ib_mr; 3406 3407 fail_mr: 3408 kfree(mr->pages); 3409 fail: 3410 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3411 bail: 3412 kfree(mr); 3413 return ERR_PTR(rc); 3414 } 3415 3416 struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, 3417 struct ib_udata *udata) 3418 { 3419 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3420 struct bnxt_re_dev *rdev = pd->rdev; 3421 struct bnxt_re_mw *mw; 3422 int rc; 3423 3424 mw = kzalloc(sizeof(*mw), GFP_KERNEL); 3425 if (!mw) 3426 return ERR_PTR(-ENOMEM); 3427 mw->rdev = rdev; 3428 mw->qplib_mw.pd = &pd->qplib_pd; 3429 3430 mw->qplib_mw.type = (type == IB_MW_TYPE_1 ? 3431 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 : 3432 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B); 3433 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw); 3434 if (rc) { 3435 dev_err(rdev_to_dev(rdev), "Allocate MW failed!"); 3436 goto fail; 3437 } 3438 mw->ib_mw.rkey = mw->qplib_mw.rkey; 3439 3440 atomic_inc(&rdev->mw_count); 3441 return &mw->ib_mw; 3442 3443 fail: 3444 kfree(mw); 3445 return ERR_PTR(rc); 3446 } 3447 3448 int bnxt_re_dealloc_mw(struct ib_mw *ib_mw) 3449 { 3450 struct bnxt_re_mw *mw = container_of(ib_mw, struct bnxt_re_mw, ib_mw); 3451 struct bnxt_re_dev *rdev = mw->rdev; 3452 int rc; 3453 3454 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw); 3455 if (rc) { 3456 dev_err(rdev_to_dev(rdev), "Free MW failed: %#x\n", rc); 3457 return rc; 3458 } 3459 3460 kfree(mw); 3461 atomic_dec(&rdev->mw_count); 3462 return rc; 3463 } 3464 3465 static int bnxt_re_page_size_ok(int page_shift) 3466 { 3467 switch (page_shift) { 3468 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K: 3469 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K: 3470 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K: 3471 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M: 3472 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K: 3473 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M: 3474 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M: 3475 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G: 3476 return 1; 3477 default: 3478 return 0; 3479 } 3480 } 3481 3482 static int fill_umem_pbl_tbl(struct ib_umem *umem, u64 *pbl_tbl_orig, 3483 int page_shift) 3484 { 3485 u64 *pbl_tbl = pbl_tbl_orig; 3486 u64 page_size = BIT_ULL(page_shift); 3487 struct ib_block_iter biter; 3488 3489 rdma_for_each_block(umem->sg_head.sgl, &biter, umem->nmap, page_size) 3490 *pbl_tbl++ = rdma_block_iter_dma_address(&biter); 3491 3492 return pbl_tbl - pbl_tbl_orig; 3493 } 3494 3495 /* uverbs */ 3496 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, 3497 u64 virt_addr, int mr_access_flags, 3498 struct ib_udata *udata) 3499 { 3500 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3501 struct bnxt_re_dev *rdev = pd->rdev; 3502 struct bnxt_re_mr *mr; 3503 struct ib_umem *umem; 3504 u64 *pbl_tbl = NULL; 3505 int umem_pgs, page_shift, rc; 3506 3507 if (length > BNXT_RE_MAX_MR_SIZE) { 3508 dev_err(rdev_to_dev(rdev), "MR Size: %lld > Max supported:%lld\n", 3509 length, BNXT_RE_MAX_MR_SIZE); 3510 return ERR_PTR(-ENOMEM); 3511 } 3512 3513 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3514 if (!mr) 3515 return ERR_PTR(-ENOMEM); 3516 3517 mr->rdev = rdev; 3518 mr->qplib_mr.pd = &pd->qplib_pd; 3519 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 3520 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; 3521 3522 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3523 if (rc) { 3524 dev_err(rdev_to_dev(rdev), "Failed to allocate MR"); 3525 goto free_mr; 3526 } 3527 /* The fixed portion of the rkey is the same as the lkey */ 3528 mr->ib_mr.rkey = mr->qplib_mr.rkey; 3529 3530 umem = ib_umem_get(udata, start, length, mr_access_flags, 0); 3531 if (IS_ERR(umem)) { 3532 dev_err(rdev_to_dev(rdev), "Failed to get umem"); 3533 rc = -EFAULT; 3534 goto free_mrw; 3535 } 3536 mr->ib_umem = umem; 3537 3538 mr->qplib_mr.va = virt_addr; 3539 umem_pgs = ib_umem_page_count(umem); 3540 if (!umem_pgs) { 3541 dev_err(rdev_to_dev(rdev), "umem is invalid!"); 3542 rc = -EINVAL; 3543 goto free_umem; 3544 } 3545 mr->qplib_mr.total_size = length; 3546 3547 pbl_tbl = kcalloc(umem_pgs, sizeof(u64 *), GFP_KERNEL); 3548 if (!pbl_tbl) { 3549 rc = -ENOMEM; 3550 goto free_umem; 3551 } 3552 3553 page_shift = __ffs(ib_umem_find_best_pgsz(umem, 3554 BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M, 3555 virt_addr)); 3556 3557 if (!bnxt_re_page_size_ok(page_shift)) { 3558 dev_err(rdev_to_dev(rdev), "umem page size unsupported!"); 3559 rc = -EFAULT; 3560 goto fail; 3561 } 3562 3563 if (page_shift == BNXT_RE_PAGE_SHIFT_4K && 3564 length > BNXT_RE_MAX_MR_SIZE_LOW) { 3565 dev_err(rdev_to_dev(rdev), "Requested MR Sz:%llu Max sup:%llu", 3566 length, (u64)BNXT_RE_MAX_MR_SIZE_LOW); 3567 rc = -EINVAL; 3568 goto fail; 3569 } 3570 3571 /* Map umem buf ptrs to the PBL */ 3572 umem_pgs = fill_umem_pbl_tbl(umem, pbl_tbl, page_shift); 3573 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl, 3574 umem_pgs, false, 1 << page_shift); 3575 if (rc) { 3576 dev_err(rdev_to_dev(rdev), "Failed to register user MR"); 3577 goto fail; 3578 } 3579 3580 kfree(pbl_tbl); 3581 3582 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3583 mr->ib_mr.rkey = mr->qplib_mr.lkey; 3584 atomic_inc(&rdev->mr_count); 3585 3586 return &mr->ib_mr; 3587 fail: 3588 kfree(pbl_tbl); 3589 free_umem: 3590 ib_umem_release(umem); 3591 free_mrw: 3592 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3593 free_mr: 3594 kfree(mr); 3595 return ERR_PTR(rc); 3596 } 3597 3598 int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata) 3599 { 3600 struct ib_device *ibdev = ctx->device; 3601 struct bnxt_re_ucontext *uctx = 3602 container_of(ctx, struct bnxt_re_ucontext, ib_uctx); 3603 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 3604 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 3605 struct bnxt_re_uctx_resp resp; 3606 u32 chip_met_rev_num = 0; 3607 int rc; 3608 3609 dev_dbg(rdev_to_dev(rdev), "ABI version requested %u", 3610 ibdev->ops.uverbs_abi_ver); 3611 3612 if (ibdev->ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) { 3613 dev_dbg(rdev_to_dev(rdev), " is different from the device %d ", 3614 BNXT_RE_ABI_VERSION); 3615 return -EPERM; 3616 } 3617 3618 uctx->rdev = rdev; 3619 3620 uctx->shpg = (void *)__get_free_page(GFP_KERNEL); 3621 if (!uctx->shpg) { 3622 rc = -ENOMEM; 3623 goto fail; 3624 } 3625 spin_lock_init(&uctx->sh_lock); 3626 3627 resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX; 3628 chip_met_rev_num = rdev->chip_ctx.chip_num; 3629 chip_met_rev_num |= ((u32)rdev->chip_ctx.chip_rev & 0xFF) << 3630 BNXT_RE_CHIP_ID0_CHIP_REV_SFT; 3631 chip_met_rev_num |= ((u32)rdev->chip_ctx.chip_metal & 0xFF) << 3632 BNXT_RE_CHIP_ID0_CHIP_MET_SFT; 3633 resp.chip_id0 = chip_met_rev_num; 3634 /* Future extension of chip info */ 3635 resp.chip_id1 = 0; 3636 /*Temp, Use xa_alloc instead */ 3637 resp.dev_id = rdev->en_dev->pdev->devfn; 3638 resp.max_qp = rdev->qplib_ctx.qpc_count; 3639 resp.pg_size = PAGE_SIZE; 3640 resp.cqe_sz = sizeof(struct cq_base); 3641 resp.max_cqd = dev_attr->max_cq_wqes; 3642 resp.rsvd = 0; 3643 3644 rc = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp))); 3645 if (rc) { 3646 dev_err(rdev_to_dev(rdev), "Failed to copy user context"); 3647 rc = -EFAULT; 3648 goto cfail; 3649 } 3650 3651 return 0; 3652 cfail: 3653 free_page((unsigned long)uctx->shpg); 3654 uctx->shpg = NULL; 3655 fail: 3656 return rc; 3657 } 3658 3659 void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx) 3660 { 3661 struct bnxt_re_ucontext *uctx = container_of(ib_uctx, 3662 struct bnxt_re_ucontext, 3663 ib_uctx); 3664 3665 struct bnxt_re_dev *rdev = uctx->rdev; 3666 3667 if (uctx->shpg) 3668 free_page((unsigned long)uctx->shpg); 3669 3670 if (uctx->dpi.dbr) { 3671 /* Free DPI only if this is the first PD allocated by the 3672 * application and mark the context dpi as NULL 3673 */ 3674 bnxt_qplib_dealloc_dpi(&rdev->qplib_res, 3675 &rdev->qplib_res.dpi_tbl, &uctx->dpi); 3676 uctx->dpi.dbr = NULL; 3677 } 3678 } 3679 3680 /* Helper function to mmap the virtual memory from user app */ 3681 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) 3682 { 3683 struct bnxt_re_ucontext *uctx = container_of(ib_uctx, 3684 struct bnxt_re_ucontext, 3685 ib_uctx); 3686 struct bnxt_re_dev *rdev = uctx->rdev; 3687 u64 pfn; 3688 3689 if (vma->vm_end - vma->vm_start != PAGE_SIZE) 3690 return -EINVAL; 3691 3692 if (vma->vm_pgoff) { 3693 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 3694 if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 3695 PAGE_SIZE, vma->vm_page_prot)) { 3696 dev_err(rdev_to_dev(rdev), "Failed to map DPI"); 3697 return -EAGAIN; 3698 } 3699 } else { 3700 pfn = virt_to_phys(uctx->shpg) >> PAGE_SHIFT; 3701 if (remap_pfn_range(vma, vma->vm_start, 3702 pfn, PAGE_SIZE, vma->vm_page_prot)) { 3703 dev_err(rdev_to_dev(rdev), 3704 "Failed to map shared page"); 3705 return -EAGAIN; 3706 } 3707 } 3708 3709 return 0; 3710 } 3711