1 /* 2 * Broadcom NetXtreme-E RoCE driver. 3 * 4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term 5 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Description: IB Verbs interpreter 37 */ 38 39 #include <linux/interrupt.h> 40 #include <linux/types.h> 41 #include <linux/pci.h> 42 #include <linux/netdevice.h> 43 #include <linux/if_ether.h> 44 45 #include <rdma/ib_verbs.h> 46 #include <rdma/ib_user_verbs.h> 47 #include <rdma/ib_umem.h> 48 #include <rdma/ib_addr.h> 49 #include <rdma/ib_mad.h> 50 #include <rdma/ib_cache.h> 51 52 #include "bnxt_ulp.h" 53 54 #include "roce_hsi.h" 55 #include "qplib_res.h" 56 #include "qplib_sp.h" 57 #include "qplib_fp.h" 58 #include "qplib_rcfw.h" 59 60 #include "bnxt_re.h" 61 #include "ib_verbs.h" 62 #include <rdma/bnxt_re-abi.h> 63 64 static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list, 65 struct bnxt_qplib_sge *sg_list, int num) 66 { 67 int i, total = 0; 68 69 for (i = 0; i < num; i++) { 70 sg_list[i].addr = ib_sg_list[i].addr; 71 sg_list[i].lkey = ib_sg_list[i].lkey; 72 sg_list[i].size = ib_sg_list[i].length; 73 total += sg_list[i].size; 74 } 75 return total; 76 } 77 78 /* Device */ 79 struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num) 80 { 81 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 82 struct net_device *netdev = NULL; 83 84 rcu_read_lock(); 85 if (rdev) 86 netdev = rdev->netdev; 87 if (netdev) 88 dev_hold(netdev); 89 90 rcu_read_unlock(); 91 return netdev; 92 } 93 94 int bnxt_re_query_device(struct ib_device *ibdev, 95 struct ib_device_attr *ib_attr, 96 struct ib_udata *udata) 97 { 98 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 99 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 100 101 memset(ib_attr, 0, sizeof(*ib_attr)); 102 103 ib_attr->fw_ver = (u64)(unsigned long)(dev_attr->fw_ver); 104 bnxt_qplib_get_guid(rdev->netdev->dev_addr, 105 (u8 *)&ib_attr->sys_image_guid); 106 ib_attr->max_mr_size = ~0ull; 107 ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_8K | 108 BNXT_RE_PAGE_SIZE_64K | BNXT_RE_PAGE_SIZE_2M | 109 BNXT_RE_PAGE_SIZE_8M | BNXT_RE_PAGE_SIZE_1G; 110 111 ib_attr->vendor_id = rdev->en_dev->pdev->vendor; 112 ib_attr->vendor_part_id = rdev->en_dev->pdev->device; 113 ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; 114 ib_attr->max_qp = dev_attr->max_qp; 115 ib_attr->max_qp_wr = dev_attr->max_qp_wqes; 116 ib_attr->device_cap_flags = 117 IB_DEVICE_CURR_QP_STATE_MOD 118 | IB_DEVICE_RC_RNR_NAK_GEN 119 | IB_DEVICE_SHUTDOWN_PORT 120 | IB_DEVICE_SYS_IMAGE_GUID 121 | IB_DEVICE_LOCAL_DMA_LKEY 122 | IB_DEVICE_RESIZE_MAX_WR 123 | IB_DEVICE_PORT_ACTIVE_EVENT 124 | IB_DEVICE_N_NOTIFY_CQ 125 | IB_DEVICE_MEM_WINDOW 126 | IB_DEVICE_MEM_WINDOW_TYPE_2B 127 | IB_DEVICE_MEM_MGT_EXTENSIONS; 128 ib_attr->max_sge = dev_attr->max_qp_sges; 129 ib_attr->max_sge_rd = dev_attr->max_qp_sges; 130 ib_attr->max_cq = dev_attr->max_cq; 131 ib_attr->max_cqe = dev_attr->max_cq_wqes; 132 ib_attr->max_mr = dev_attr->max_mr; 133 ib_attr->max_pd = dev_attr->max_pd; 134 ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; 135 ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_rd_atom; 136 ib_attr->atomic_cap = IB_ATOMIC_HCA; 137 ib_attr->masked_atomic_cap = IB_ATOMIC_HCA; 138 139 ib_attr->max_ee_rd_atom = 0; 140 ib_attr->max_res_rd_atom = 0; 141 ib_attr->max_ee_init_rd_atom = 0; 142 ib_attr->max_ee = 0; 143 ib_attr->max_rdd = 0; 144 ib_attr->max_mw = dev_attr->max_mw; 145 ib_attr->max_raw_ipv6_qp = 0; 146 ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; 147 ib_attr->max_mcast_grp = 0; 148 ib_attr->max_mcast_qp_attach = 0; 149 ib_attr->max_total_mcast_qp_attach = 0; 150 ib_attr->max_ah = dev_attr->max_ah; 151 152 ib_attr->max_fmr = dev_attr->max_fmr; 153 ib_attr->max_map_per_fmr = 1; /* ? */ 154 155 ib_attr->max_srq = dev_attr->max_srq; 156 ib_attr->max_srq_wr = dev_attr->max_srq_wqes; 157 ib_attr->max_srq_sge = dev_attr->max_srq_sges; 158 159 ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; 160 161 ib_attr->max_pkeys = 1; 162 ib_attr->local_ca_ack_delay = 0; 163 return 0; 164 } 165 166 int bnxt_re_modify_device(struct ib_device *ibdev, 167 int device_modify_mask, 168 struct ib_device_modify *device_modify) 169 { 170 switch (device_modify_mask) { 171 case IB_DEVICE_MODIFY_SYS_IMAGE_GUID: 172 /* Modify the GUID requires the modification of the GID table */ 173 /* GUID should be made as READ-ONLY */ 174 break; 175 case IB_DEVICE_MODIFY_NODE_DESC: 176 /* Node Desc should be made as READ-ONLY */ 177 break; 178 default: 179 break; 180 } 181 return 0; 182 } 183 184 static void __to_ib_speed_width(struct net_device *netdev, u8 *speed, u8 *width) 185 { 186 struct ethtool_link_ksettings lksettings; 187 u32 espeed; 188 189 if (netdev->ethtool_ops && netdev->ethtool_ops->get_link_ksettings) { 190 memset(&lksettings, 0, sizeof(lksettings)); 191 rtnl_lock(); 192 netdev->ethtool_ops->get_link_ksettings(netdev, &lksettings); 193 rtnl_unlock(); 194 espeed = lksettings.base.speed; 195 } else { 196 espeed = SPEED_UNKNOWN; 197 } 198 switch (espeed) { 199 case SPEED_1000: 200 *speed = IB_SPEED_SDR; 201 *width = IB_WIDTH_1X; 202 break; 203 case SPEED_10000: 204 *speed = IB_SPEED_QDR; 205 *width = IB_WIDTH_1X; 206 break; 207 case SPEED_20000: 208 *speed = IB_SPEED_DDR; 209 *width = IB_WIDTH_4X; 210 break; 211 case SPEED_25000: 212 *speed = IB_SPEED_EDR; 213 *width = IB_WIDTH_1X; 214 break; 215 case SPEED_40000: 216 *speed = IB_SPEED_QDR; 217 *width = IB_WIDTH_4X; 218 break; 219 case SPEED_50000: 220 break; 221 default: 222 *speed = IB_SPEED_SDR; 223 *width = IB_WIDTH_1X; 224 break; 225 } 226 } 227 228 /* Port */ 229 int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, 230 struct ib_port_attr *port_attr) 231 { 232 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 233 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 234 235 memset(port_attr, 0, sizeof(*port_attr)); 236 237 if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) { 238 port_attr->state = IB_PORT_ACTIVE; 239 port_attr->phys_state = 5; 240 } else { 241 port_attr->state = IB_PORT_DOWN; 242 port_attr->phys_state = 3; 243 } 244 port_attr->max_mtu = IB_MTU_4096; 245 port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu); 246 port_attr->gid_tbl_len = dev_attr->max_sgid; 247 port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 248 IB_PORT_DEVICE_MGMT_SUP | 249 IB_PORT_VENDOR_CLASS_SUP | 250 IB_PORT_IP_BASED_GIDS; 251 252 /* Max MSG size set to 2G for now */ 253 port_attr->max_msg_sz = 0x80000000; 254 port_attr->bad_pkey_cntr = 0; 255 port_attr->qkey_viol_cntr = 0; 256 port_attr->pkey_tbl_len = dev_attr->max_pkey; 257 port_attr->lid = 0; 258 port_attr->sm_lid = 0; 259 port_attr->lmc = 0; 260 port_attr->max_vl_num = 4; 261 port_attr->sm_sl = 0; 262 port_attr->subnet_timeout = 0; 263 port_attr->init_type_reply = 0; 264 /* call the underlying netdev's ethtool hooks to query speed settings 265 * for which we acquire rtnl_lock _only_ if it's registered with 266 * IB stack to avoid race in the NETDEV_UNREG path 267 */ 268 if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) 269 __to_ib_speed_width(rdev->netdev, &port_attr->active_speed, 270 &port_attr->active_width); 271 return 0; 272 } 273 274 int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num, 275 int port_modify_mask, 276 struct ib_port_modify *port_modify) 277 { 278 switch (port_modify_mask) { 279 case IB_PORT_SHUTDOWN: 280 break; 281 case IB_PORT_INIT_TYPE: 282 break; 283 case IB_PORT_RESET_QKEY_CNTR: 284 break; 285 default: 286 break; 287 } 288 return 0; 289 } 290 291 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, 292 struct ib_port_immutable *immutable) 293 { 294 struct ib_port_attr port_attr; 295 296 if (bnxt_re_query_port(ibdev, port_num, &port_attr)) 297 return -EINVAL; 298 299 immutable->pkey_tbl_len = port_attr.pkey_tbl_len; 300 immutable->gid_tbl_len = port_attr.gid_tbl_len; 301 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; 302 immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP; 303 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 304 return 0; 305 } 306 307 int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, 308 u16 index, u16 *pkey) 309 { 310 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 311 312 /* Ignore port_num */ 313 314 memset(pkey, 0, sizeof(*pkey)); 315 return bnxt_qplib_get_pkey(&rdev->qplib_res, 316 &rdev->qplib_res.pkey_tbl, index, pkey); 317 } 318 319 int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, 320 int index, union ib_gid *gid) 321 { 322 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 323 int rc = 0; 324 325 /* Ignore port_num */ 326 memset(gid, 0, sizeof(*gid)); 327 rc = bnxt_qplib_get_sgid(&rdev->qplib_res, 328 &rdev->qplib_res.sgid_tbl, index, 329 (struct bnxt_qplib_gid *)gid); 330 return rc; 331 } 332 333 int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num, 334 unsigned int index, void **context) 335 { 336 int rc = 0; 337 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 338 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 339 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 340 341 /* Delete the entry from the hardware */ 342 ctx = *context; 343 if (!ctx) 344 return -EINVAL; 345 346 if (sgid_tbl && sgid_tbl->active) { 347 if (ctx->idx >= sgid_tbl->max) 348 return -EINVAL; 349 ctx->refcnt--; 350 if (!ctx->refcnt) { 351 rc = bnxt_qplib_del_sgid 352 (sgid_tbl, 353 &sgid_tbl->tbl[ctx->idx], true); 354 if (rc) 355 dev_err(rdev_to_dev(rdev), 356 "Failed to remove GID: %#x", rc); 357 ctx_tbl = sgid_tbl->ctx; 358 ctx_tbl[ctx->idx] = NULL; 359 kfree(ctx); 360 } 361 } else { 362 return -EINVAL; 363 } 364 return rc; 365 } 366 367 int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num, 368 unsigned int index, const union ib_gid *gid, 369 const struct ib_gid_attr *attr, void **context) 370 { 371 int rc; 372 u32 tbl_idx = 0; 373 u16 vlan_id = 0xFFFF; 374 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 375 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 376 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 377 378 if ((attr->ndev) && is_vlan_dev(attr->ndev)) 379 vlan_id = vlan_dev_vlan_id(attr->ndev); 380 381 rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)gid, 382 rdev->qplib_res.netdev->dev_addr, 383 vlan_id, true, &tbl_idx); 384 if (rc == -EALREADY) { 385 ctx_tbl = sgid_tbl->ctx; 386 ctx_tbl[tbl_idx]->refcnt++; 387 *context = ctx_tbl[tbl_idx]; 388 return 0; 389 } 390 391 if (rc < 0) { 392 dev_err(rdev_to_dev(rdev), "Failed to add GID: %#x", rc); 393 return rc; 394 } 395 396 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 397 if (!ctx) 398 return -ENOMEM; 399 ctx_tbl = sgid_tbl->ctx; 400 ctx->idx = tbl_idx; 401 ctx->refcnt = 1; 402 ctx_tbl[tbl_idx] = ctx; 403 404 return rc; 405 } 406 407 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, 408 u8 port_num) 409 { 410 return IB_LINK_LAYER_ETHERNET; 411 } 412 413 /* Protection Domains */ 414 int bnxt_re_dealloc_pd(struct ib_pd *ib_pd) 415 { 416 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 417 struct bnxt_re_dev *rdev = pd->rdev; 418 int rc; 419 420 if (ib_pd->uobject && pd->dpi.dbr) { 421 struct ib_ucontext *ib_uctx = ib_pd->uobject->context; 422 struct bnxt_re_ucontext *ucntx; 423 424 /* Free DPI only if this is the first PD allocated by the 425 * application and mark the context dpi as NULL 426 */ 427 ucntx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx); 428 429 rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, 430 &rdev->qplib_res.dpi_tbl, 431 &pd->dpi); 432 if (rc) 433 dev_err(rdev_to_dev(rdev), "Failed to deallocate HW DPI"); 434 /* Don't fail, continue*/ 435 ucntx->dpi = NULL; 436 } 437 438 rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res, 439 &rdev->qplib_res.pd_tbl, 440 &pd->qplib_pd); 441 if (rc) { 442 dev_err(rdev_to_dev(rdev), "Failed to deallocate HW PD"); 443 return rc; 444 } 445 446 kfree(pd); 447 return 0; 448 } 449 450 struct ib_pd *bnxt_re_alloc_pd(struct ib_device *ibdev, 451 struct ib_ucontext *ucontext, 452 struct ib_udata *udata) 453 { 454 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 455 struct bnxt_re_ucontext *ucntx = container_of(ucontext, 456 struct bnxt_re_ucontext, 457 ib_uctx); 458 struct bnxt_re_pd *pd; 459 int rc; 460 461 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 462 if (!pd) 463 return ERR_PTR(-ENOMEM); 464 465 pd->rdev = rdev; 466 if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) { 467 dev_err(rdev_to_dev(rdev), "Failed to allocate HW PD"); 468 rc = -ENOMEM; 469 goto fail; 470 } 471 472 if (udata) { 473 struct bnxt_re_pd_resp resp; 474 475 if (!ucntx->dpi) { 476 /* Allocate DPI in alloc_pd to avoid failing of 477 * ibv_devinfo and family of application when DPIs 478 * are depleted. 479 */ 480 if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl, 481 &pd->dpi, ucntx)) { 482 rc = -ENOMEM; 483 goto dbfail; 484 } 485 ucntx->dpi = &pd->dpi; 486 } 487 488 resp.pdid = pd->qplib_pd.id; 489 /* Still allow mapping this DBR to the new user PD. */ 490 resp.dpi = ucntx->dpi->dpi; 491 resp.dbr = (u64)ucntx->dpi->umdbr; 492 493 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 494 if (rc) { 495 dev_err(rdev_to_dev(rdev), 496 "Failed to copy user response\n"); 497 goto dbfail; 498 } 499 } 500 501 return &pd->ib_pd; 502 dbfail: 503 (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, 504 &pd->qplib_pd); 505 fail: 506 kfree(pd); 507 return ERR_PTR(rc); 508 } 509 510 /* Address Handles */ 511 int bnxt_re_destroy_ah(struct ib_ah *ib_ah) 512 { 513 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 514 struct bnxt_re_dev *rdev = ah->rdev; 515 int rc; 516 517 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah); 518 if (rc) { 519 dev_err(rdev_to_dev(rdev), "Failed to destroy HW AH"); 520 return rc; 521 } 522 kfree(ah); 523 return 0; 524 } 525 526 struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd, 527 struct rdma_ah_attr *ah_attr, 528 struct ib_udata *udata) 529 { 530 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 531 struct bnxt_re_dev *rdev = pd->rdev; 532 struct bnxt_re_ah *ah; 533 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); 534 int rc; 535 u16 vlan_tag; 536 u8 nw_type; 537 538 struct ib_gid_attr sgid_attr; 539 540 if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) { 541 dev_err(rdev_to_dev(rdev), "Failed to alloc AH: GRH not set"); 542 return ERR_PTR(-EINVAL); 543 } 544 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 545 if (!ah) 546 return ERR_PTR(-ENOMEM); 547 548 ah->rdev = rdev; 549 ah->qplib_ah.pd = &pd->qplib_pd; 550 551 /* Supply the configuration for the HW */ 552 memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw, 553 sizeof(union ib_gid)); 554 /* 555 * If RoCE V2 is enabled, stack will have two entries for 556 * each GID entry. Avoiding this duplicte entry in HW. Dividing 557 * the GID index by 2 for RoCE V2 558 */ 559 ah->qplib_ah.sgid_index = grh->sgid_index / 2; 560 ah->qplib_ah.host_sgid_index = grh->sgid_index; 561 ah->qplib_ah.traffic_class = grh->traffic_class; 562 ah->qplib_ah.flow_label = grh->flow_label; 563 ah->qplib_ah.hop_limit = grh->hop_limit; 564 ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr); 565 if (ib_pd->uobject && 566 !rdma_is_multicast_addr((struct in6_addr *) 567 grh->dgid.raw) && 568 !rdma_link_local_addr((struct in6_addr *) 569 grh->dgid.raw)) { 570 union ib_gid sgid; 571 572 rc = ib_get_cached_gid(&rdev->ibdev, 1, 573 grh->sgid_index, &sgid, 574 &sgid_attr); 575 if (rc) { 576 dev_err(rdev_to_dev(rdev), 577 "Failed to query gid at index %d", 578 grh->sgid_index); 579 goto fail; 580 } 581 if (sgid_attr.ndev) { 582 if (is_vlan_dev(sgid_attr.ndev)) 583 vlan_tag = vlan_dev_vlan_id(sgid_attr.ndev); 584 dev_put(sgid_attr.ndev); 585 } 586 /* Get network header type for this GID */ 587 nw_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid); 588 switch (nw_type) { 589 case RDMA_NETWORK_IPV4: 590 ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4; 591 break; 592 case RDMA_NETWORK_IPV6: 593 ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6; 594 break; 595 default: 596 ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V1; 597 break; 598 } 599 rc = rdma_addr_find_l2_eth_by_grh(&sgid, &grh->dgid, 600 ah_attr->roce.dmac, &vlan_tag, 601 &sgid_attr.ndev->ifindex, 602 NULL); 603 if (rc) { 604 dev_err(rdev_to_dev(rdev), "Failed to get dmac\n"); 605 goto fail; 606 } 607 } 608 609 memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN); 610 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah); 611 if (rc) { 612 dev_err(rdev_to_dev(rdev), "Failed to allocate HW AH"); 613 goto fail; 614 } 615 616 /* Write AVID to shared page. */ 617 if (ib_pd->uobject) { 618 struct ib_ucontext *ib_uctx = ib_pd->uobject->context; 619 struct bnxt_re_ucontext *uctx; 620 unsigned long flag; 621 u32 *wrptr; 622 623 uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx); 624 spin_lock_irqsave(&uctx->sh_lock, flag); 625 wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT); 626 *wrptr = ah->qplib_ah.id; 627 wmb(); /* make sure cache is updated. */ 628 spin_unlock_irqrestore(&uctx->sh_lock, flag); 629 } 630 631 return &ah->ib_ah; 632 633 fail: 634 kfree(ah); 635 return ERR_PTR(rc); 636 } 637 638 int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) 639 { 640 return 0; 641 } 642 643 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) 644 { 645 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 646 647 ah_attr->type = ib_ah->type; 648 rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl); 649 memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN); 650 rdma_ah_set_grh(ah_attr, NULL, 0, 651 ah->qplib_ah.host_sgid_index, 652 0, ah->qplib_ah.traffic_class); 653 rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data); 654 rdma_ah_set_port_num(ah_attr, 1); 655 rdma_ah_set_static_rate(ah_attr, 0); 656 return 0; 657 } 658 659 /* Queue Pairs */ 660 int bnxt_re_destroy_qp(struct ib_qp *ib_qp) 661 { 662 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 663 struct bnxt_re_dev *rdev = qp->rdev; 664 int rc; 665 666 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 667 if (rc) { 668 dev_err(rdev_to_dev(rdev), "Failed to destroy HW QP"); 669 return rc; 670 } 671 if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) { 672 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, 673 &rdev->sqp_ah->qplib_ah); 674 if (rc) { 675 dev_err(rdev_to_dev(rdev), 676 "Failed to destroy HW AH for shadow QP"); 677 return rc; 678 } 679 680 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, 681 &rdev->qp1_sqp->qplib_qp); 682 if (rc) { 683 dev_err(rdev_to_dev(rdev), 684 "Failed to destroy Shadow QP"); 685 return rc; 686 } 687 mutex_lock(&rdev->qp_lock); 688 list_del(&rdev->qp1_sqp->list); 689 atomic_dec(&rdev->qp_count); 690 mutex_unlock(&rdev->qp_lock); 691 692 kfree(rdev->sqp_ah); 693 kfree(rdev->qp1_sqp); 694 } 695 696 if (!IS_ERR_OR_NULL(qp->rumem)) 697 ib_umem_release(qp->rumem); 698 if (!IS_ERR_OR_NULL(qp->sumem)) 699 ib_umem_release(qp->sumem); 700 701 mutex_lock(&rdev->qp_lock); 702 list_del(&qp->list); 703 atomic_dec(&rdev->qp_count); 704 mutex_unlock(&rdev->qp_lock); 705 kfree(qp); 706 return 0; 707 } 708 709 static u8 __from_ib_qp_type(enum ib_qp_type type) 710 { 711 switch (type) { 712 case IB_QPT_GSI: 713 return CMDQ_CREATE_QP1_TYPE_GSI; 714 case IB_QPT_RC: 715 return CMDQ_CREATE_QP_TYPE_RC; 716 case IB_QPT_UD: 717 return CMDQ_CREATE_QP_TYPE_UD; 718 default: 719 return IB_QPT_MAX; 720 } 721 } 722 723 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, 724 struct bnxt_re_qp *qp, struct ib_udata *udata) 725 { 726 struct bnxt_re_qp_req ureq; 727 struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; 728 struct ib_umem *umem; 729 int bytes = 0; 730 struct ib_ucontext *context = pd->ib_pd.uobject->context; 731 struct bnxt_re_ucontext *cntx = container_of(context, 732 struct bnxt_re_ucontext, 733 ib_uctx); 734 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 735 return -EFAULT; 736 737 bytes = (qplib_qp->sq.max_wqe * BNXT_QPLIB_MAX_SQE_ENTRY_SIZE); 738 /* Consider mapping PSN search memory only for RC QPs. */ 739 if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) 740 bytes += (qplib_qp->sq.max_wqe * sizeof(struct sq_psn_search)); 741 bytes = PAGE_ALIGN(bytes); 742 umem = ib_umem_get(context, ureq.qpsva, bytes, 743 IB_ACCESS_LOCAL_WRITE, 1); 744 if (IS_ERR(umem)) 745 return PTR_ERR(umem); 746 747 qp->sumem = umem; 748 qplib_qp->sq.sglist = umem->sg_head.sgl; 749 qplib_qp->sq.nmap = umem->nmap; 750 qplib_qp->qp_handle = ureq.qp_handle; 751 752 if (!qp->qplib_qp.srq) { 753 bytes = (qplib_qp->rq.max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); 754 bytes = PAGE_ALIGN(bytes); 755 umem = ib_umem_get(context, ureq.qprva, bytes, 756 IB_ACCESS_LOCAL_WRITE, 1); 757 if (IS_ERR(umem)) 758 goto rqfail; 759 qp->rumem = umem; 760 qplib_qp->rq.sglist = umem->sg_head.sgl; 761 qplib_qp->rq.nmap = umem->nmap; 762 } 763 764 qplib_qp->dpi = cntx->dpi; 765 return 0; 766 rqfail: 767 ib_umem_release(qp->sumem); 768 qp->sumem = NULL; 769 qplib_qp->sq.sglist = NULL; 770 qplib_qp->sq.nmap = 0; 771 772 return PTR_ERR(umem); 773 } 774 775 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah 776 (struct bnxt_re_pd *pd, 777 struct bnxt_qplib_res *qp1_res, 778 struct bnxt_qplib_qp *qp1_qp) 779 { 780 struct bnxt_re_dev *rdev = pd->rdev; 781 struct bnxt_re_ah *ah; 782 union ib_gid sgid; 783 int rc; 784 785 ah = kzalloc(sizeof(*ah), GFP_KERNEL); 786 if (!ah) 787 return NULL; 788 789 memset(ah, 0, sizeof(*ah)); 790 ah->rdev = rdev; 791 ah->qplib_ah.pd = &pd->qplib_pd; 792 793 rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); 794 if (rc) 795 goto fail; 796 797 /* supply the dgid data same as sgid */ 798 memcpy(ah->qplib_ah.dgid.data, &sgid.raw, 799 sizeof(union ib_gid)); 800 ah->qplib_ah.sgid_index = 0; 801 802 ah->qplib_ah.traffic_class = 0; 803 ah->qplib_ah.flow_label = 0; 804 ah->qplib_ah.hop_limit = 1; 805 ah->qplib_ah.sl = 0; 806 /* Have DMAC same as SMAC */ 807 ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr); 808 809 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah); 810 if (rc) { 811 dev_err(rdev_to_dev(rdev), 812 "Failed to allocate HW AH for Shadow QP"); 813 goto fail; 814 } 815 816 return ah; 817 818 fail: 819 kfree(ah); 820 return NULL; 821 } 822 823 static struct bnxt_re_qp *bnxt_re_create_shadow_qp 824 (struct bnxt_re_pd *pd, 825 struct bnxt_qplib_res *qp1_res, 826 struct bnxt_qplib_qp *qp1_qp) 827 { 828 struct bnxt_re_dev *rdev = pd->rdev; 829 struct bnxt_re_qp *qp; 830 int rc; 831 832 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 833 if (!qp) 834 return NULL; 835 836 memset(qp, 0, sizeof(*qp)); 837 qp->rdev = rdev; 838 839 /* Initialize the shadow QP structure from the QP1 values */ 840 ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); 841 842 qp->qplib_qp.pd = &pd->qplib_pd; 843 qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); 844 qp->qplib_qp.type = IB_QPT_UD; 845 846 qp->qplib_qp.max_inline_data = 0; 847 qp->qplib_qp.sig_type = true; 848 849 /* Shadow QP SQ depth should be same as QP1 RQ depth */ 850 qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; 851 qp->qplib_qp.sq.max_sge = 2; 852 853 qp->qplib_qp.scq = qp1_qp->scq; 854 qp->qplib_qp.rcq = qp1_qp->rcq; 855 856 qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; 857 qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; 858 859 qp->qplib_qp.mtu = qp1_qp->mtu; 860 861 qp->qplib_qp.sq_hdr_buf_size = 0; 862 qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; 863 qp->qplib_qp.dpi = &rdev->dpi_privileged; 864 865 rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); 866 if (rc) 867 goto fail; 868 869 rdev->sqp_id = qp->qplib_qp.id; 870 871 spin_lock_init(&qp->sq_lock); 872 INIT_LIST_HEAD(&qp->list); 873 mutex_lock(&rdev->qp_lock); 874 list_add_tail(&qp->list, &rdev->qp_list); 875 atomic_inc(&rdev->qp_count); 876 mutex_unlock(&rdev->qp_lock); 877 return qp; 878 fail: 879 kfree(qp); 880 return NULL; 881 } 882 883 struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd, 884 struct ib_qp_init_attr *qp_init_attr, 885 struct ib_udata *udata) 886 { 887 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 888 struct bnxt_re_dev *rdev = pd->rdev; 889 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 890 struct bnxt_re_qp *qp; 891 struct bnxt_re_cq *cq; 892 int rc, entries; 893 894 if ((qp_init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) || 895 (qp_init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) || 896 (qp_init_attr->cap.max_send_sge > dev_attr->max_qp_sges) || 897 (qp_init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) || 898 (qp_init_attr->cap.max_inline_data > dev_attr->max_inline_data)) 899 return ERR_PTR(-EINVAL); 900 901 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 902 if (!qp) 903 return ERR_PTR(-ENOMEM); 904 905 qp->rdev = rdev; 906 ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); 907 qp->qplib_qp.pd = &pd->qplib_pd; 908 qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); 909 qp->qplib_qp.type = __from_ib_qp_type(qp_init_attr->qp_type); 910 if (qp->qplib_qp.type == IB_QPT_MAX) { 911 dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported", 912 qp->qplib_qp.type); 913 rc = -EINVAL; 914 goto fail; 915 } 916 qp->qplib_qp.max_inline_data = qp_init_attr->cap.max_inline_data; 917 qp->qplib_qp.sig_type = ((qp_init_attr->sq_sig_type == 918 IB_SIGNAL_ALL_WR) ? true : false); 919 920 entries = roundup_pow_of_two(qp_init_attr->cap.max_send_wr + 1); 921 qp->qplib_qp.sq.max_wqe = min_t(u32, entries, 922 dev_attr->max_qp_wqes + 1); 923 924 qp->qplib_qp.sq.max_sge = qp_init_attr->cap.max_send_sge; 925 if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges) 926 qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges; 927 928 if (qp_init_attr->send_cq) { 929 cq = container_of(qp_init_attr->send_cq, struct bnxt_re_cq, 930 ib_cq); 931 if (!cq) { 932 dev_err(rdev_to_dev(rdev), "Send CQ not found"); 933 rc = -EINVAL; 934 goto fail; 935 } 936 qp->qplib_qp.scq = &cq->qplib_cq; 937 } 938 939 if (qp_init_attr->recv_cq) { 940 cq = container_of(qp_init_attr->recv_cq, struct bnxt_re_cq, 941 ib_cq); 942 if (!cq) { 943 dev_err(rdev_to_dev(rdev), "Receive CQ not found"); 944 rc = -EINVAL; 945 goto fail; 946 } 947 qp->qplib_qp.rcq = &cq->qplib_cq; 948 } 949 950 if (qp_init_attr->srq) { 951 dev_err(rdev_to_dev(rdev), "SRQ not supported"); 952 rc = -ENOTSUPP; 953 goto fail; 954 } else { 955 /* Allocate 1 more than what's provided so posting max doesn't 956 * mean empty 957 */ 958 entries = roundup_pow_of_two(qp_init_attr->cap.max_recv_wr + 1); 959 qp->qplib_qp.rq.max_wqe = min_t(u32, entries, 960 dev_attr->max_qp_wqes + 1); 961 962 qp->qplib_qp.rq.max_sge = qp_init_attr->cap.max_recv_sge; 963 if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges) 964 qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; 965 } 966 967 qp->qplib_qp.mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); 968 969 if (qp_init_attr->qp_type == IB_QPT_GSI) { 970 qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; 971 if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges) 972 qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; 973 qp->qplib_qp.sq.max_sge++; 974 if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges) 975 qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges; 976 977 qp->qplib_qp.rq_hdr_buf_size = 978 BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; 979 980 qp->qplib_qp.sq_hdr_buf_size = 981 BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2; 982 qp->qplib_qp.dpi = &rdev->dpi_privileged; 983 rc = bnxt_qplib_create_qp1(&rdev->qplib_res, &qp->qplib_qp); 984 if (rc) { 985 dev_err(rdev_to_dev(rdev), "Failed to create HW QP1"); 986 goto fail; 987 } 988 /* Create a shadow QP to handle the QP1 traffic */ 989 rdev->qp1_sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, 990 &qp->qplib_qp); 991 if (!rdev->qp1_sqp) { 992 rc = -EINVAL; 993 dev_err(rdev_to_dev(rdev), 994 "Failed to create Shadow QP for QP1"); 995 goto qp_destroy; 996 } 997 rdev->sqp_ah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, 998 &qp->qplib_qp); 999 if (!rdev->sqp_ah) { 1000 bnxt_qplib_destroy_qp(&rdev->qplib_res, 1001 &rdev->qp1_sqp->qplib_qp); 1002 rc = -EINVAL; 1003 dev_err(rdev_to_dev(rdev), 1004 "Failed to create AH entry for ShadowQP"); 1005 goto qp_destroy; 1006 } 1007 1008 } else { 1009 qp->qplib_qp.max_rd_atomic = dev_attr->max_qp_rd_atom; 1010 qp->qplib_qp.max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; 1011 if (udata) { 1012 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); 1013 if (rc) 1014 goto fail; 1015 } else { 1016 qp->qplib_qp.dpi = &rdev->dpi_privileged; 1017 } 1018 1019 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); 1020 if (rc) { 1021 dev_err(rdev_to_dev(rdev), "Failed to create HW QP"); 1022 goto fail; 1023 } 1024 } 1025 1026 qp->ib_qp.qp_num = qp->qplib_qp.id; 1027 spin_lock_init(&qp->sq_lock); 1028 1029 if (udata) { 1030 struct bnxt_re_qp_resp resp; 1031 1032 resp.qpid = qp->ib_qp.qp_num; 1033 resp.rsvd = 0; 1034 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 1035 if (rc) { 1036 dev_err(rdev_to_dev(rdev), "Failed to copy QP udata"); 1037 goto qp_destroy; 1038 } 1039 } 1040 INIT_LIST_HEAD(&qp->list); 1041 mutex_lock(&rdev->qp_lock); 1042 list_add_tail(&qp->list, &rdev->qp_list); 1043 atomic_inc(&rdev->qp_count); 1044 mutex_unlock(&rdev->qp_lock); 1045 1046 return &qp->ib_qp; 1047 qp_destroy: 1048 bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 1049 fail: 1050 kfree(qp); 1051 return ERR_PTR(rc); 1052 } 1053 1054 static u8 __from_ib_qp_state(enum ib_qp_state state) 1055 { 1056 switch (state) { 1057 case IB_QPS_RESET: 1058 return CMDQ_MODIFY_QP_NEW_STATE_RESET; 1059 case IB_QPS_INIT: 1060 return CMDQ_MODIFY_QP_NEW_STATE_INIT; 1061 case IB_QPS_RTR: 1062 return CMDQ_MODIFY_QP_NEW_STATE_RTR; 1063 case IB_QPS_RTS: 1064 return CMDQ_MODIFY_QP_NEW_STATE_RTS; 1065 case IB_QPS_SQD: 1066 return CMDQ_MODIFY_QP_NEW_STATE_SQD; 1067 case IB_QPS_SQE: 1068 return CMDQ_MODIFY_QP_NEW_STATE_SQE; 1069 case IB_QPS_ERR: 1070 default: 1071 return CMDQ_MODIFY_QP_NEW_STATE_ERR; 1072 } 1073 } 1074 1075 static enum ib_qp_state __to_ib_qp_state(u8 state) 1076 { 1077 switch (state) { 1078 case CMDQ_MODIFY_QP_NEW_STATE_RESET: 1079 return IB_QPS_RESET; 1080 case CMDQ_MODIFY_QP_NEW_STATE_INIT: 1081 return IB_QPS_INIT; 1082 case CMDQ_MODIFY_QP_NEW_STATE_RTR: 1083 return IB_QPS_RTR; 1084 case CMDQ_MODIFY_QP_NEW_STATE_RTS: 1085 return IB_QPS_RTS; 1086 case CMDQ_MODIFY_QP_NEW_STATE_SQD: 1087 return IB_QPS_SQD; 1088 case CMDQ_MODIFY_QP_NEW_STATE_SQE: 1089 return IB_QPS_SQE; 1090 case CMDQ_MODIFY_QP_NEW_STATE_ERR: 1091 default: 1092 return IB_QPS_ERR; 1093 } 1094 } 1095 1096 static u32 __from_ib_mtu(enum ib_mtu mtu) 1097 { 1098 switch (mtu) { 1099 case IB_MTU_256: 1100 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256; 1101 case IB_MTU_512: 1102 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512; 1103 case IB_MTU_1024: 1104 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024; 1105 case IB_MTU_2048: 1106 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 1107 case IB_MTU_4096: 1108 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096; 1109 default: 1110 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 1111 } 1112 } 1113 1114 static enum ib_mtu __to_ib_mtu(u32 mtu) 1115 { 1116 switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) { 1117 case CMDQ_MODIFY_QP_PATH_MTU_MTU_256: 1118 return IB_MTU_256; 1119 case CMDQ_MODIFY_QP_PATH_MTU_MTU_512: 1120 return IB_MTU_512; 1121 case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024: 1122 return IB_MTU_1024; 1123 case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048: 1124 return IB_MTU_2048; 1125 case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096: 1126 return IB_MTU_4096; 1127 default: 1128 return IB_MTU_2048; 1129 } 1130 } 1131 1132 static int __from_ib_access_flags(int iflags) 1133 { 1134 int qflags = 0; 1135 1136 if (iflags & IB_ACCESS_LOCAL_WRITE) 1137 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 1138 if (iflags & IB_ACCESS_REMOTE_READ) 1139 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ; 1140 if (iflags & IB_ACCESS_REMOTE_WRITE) 1141 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE; 1142 if (iflags & IB_ACCESS_REMOTE_ATOMIC) 1143 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC; 1144 if (iflags & IB_ACCESS_MW_BIND) 1145 qflags |= BNXT_QPLIB_ACCESS_MW_BIND; 1146 if (iflags & IB_ZERO_BASED) 1147 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED; 1148 if (iflags & IB_ACCESS_ON_DEMAND) 1149 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND; 1150 return qflags; 1151 }; 1152 1153 static enum ib_access_flags __to_ib_access_flags(int qflags) 1154 { 1155 enum ib_access_flags iflags = 0; 1156 1157 if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE) 1158 iflags |= IB_ACCESS_LOCAL_WRITE; 1159 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE) 1160 iflags |= IB_ACCESS_REMOTE_WRITE; 1161 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ) 1162 iflags |= IB_ACCESS_REMOTE_READ; 1163 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC) 1164 iflags |= IB_ACCESS_REMOTE_ATOMIC; 1165 if (qflags & BNXT_QPLIB_ACCESS_MW_BIND) 1166 iflags |= IB_ACCESS_MW_BIND; 1167 if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED) 1168 iflags |= IB_ZERO_BASED; 1169 if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND) 1170 iflags |= IB_ACCESS_ON_DEMAND; 1171 return iflags; 1172 }; 1173 1174 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, 1175 struct bnxt_re_qp *qp1_qp, 1176 int qp_attr_mask) 1177 { 1178 struct bnxt_re_qp *qp = rdev->qp1_sqp; 1179 int rc = 0; 1180 1181 if (qp_attr_mask & IB_QP_STATE) { 1182 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 1183 qp->qplib_qp.state = qp1_qp->qplib_qp.state; 1184 } 1185 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 1186 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 1187 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; 1188 } 1189 1190 if (qp_attr_mask & IB_QP_QKEY) { 1191 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 1192 /* Using a Random QKEY */ 1193 qp->qplib_qp.qkey = 0x81818181; 1194 } 1195 if (qp_attr_mask & IB_QP_SQ_PSN) { 1196 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 1197 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; 1198 } 1199 1200 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 1201 if (rc) 1202 dev_err(rdev_to_dev(rdev), 1203 "Failed to modify Shadow QP for QP1"); 1204 return rc; 1205 } 1206 1207 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 1208 int qp_attr_mask, struct ib_udata *udata) 1209 { 1210 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 1211 struct bnxt_re_dev *rdev = qp->rdev; 1212 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 1213 enum ib_qp_state curr_qp_state, new_qp_state; 1214 int rc, entries; 1215 int status; 1216 union ib_gid sgid; 1217 struct ib_gid_attr sgid_attr; 1218 u8 nw_type; 1219 1220 qp->qplib_qp.modify_flags = 0; 1221 if (qp_attr_mask & IB_QP_STATE) { 1222 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); 1223 new_qp_state = qp_attr->qp_state; 1224 if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state, 1225 ib_qp->qp_type, qp_attr_mask, 1226 IB_LINK_LAYER_ETHERNET)) { 1227 dev_err(rdev_to_dev(rdev), 1228 "Invalid attribute mask: %#x specified ", 1229 qp_attr_mask); 1230 dev_err(rdev_to_dev(rdev), 1231 "for qpn: %#x type: %#x", 1232 ib_qp->qp_num, ib_qp->qp_type); 1233 dev_err(rdev_to_dev(rdev), 1234 "curr_qp_state=0x%x, new_qp_state=0x%x\n", 1235 curr_qp_state, new_qp_state); 1236 return -EINVAL; 1237 } 1238 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 1239 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); 1240 } 1241 if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { 1242 qp->qplib_qp.modify_flags |= 1243 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY; 1244 qp->qplib_qp.en_sqd_async_notify = true; 1245 } 1246 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) { 1247 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; 1248 qp->qplib_qp.access = 1249 __from_ib_access_flags(qp_attr->qp_access_flags); 1250 /* LOCAL_WRITE access must be set to allow RC receive */ 1251 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 1252 } 1253 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 1254 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 1255 qp->qplib_qp.pkey_index = qp_attr->pkey_index; 1256 } 1257 if (qp_attr_mask & IB_QP_QKEY) { 1258 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 1259 qp->qplib_qp.qkey = qp_attr->qkey; 1260 } 1261 if (qp_attr_mask & IB_QP_AV) { 1262 const struct ib_global_route *grh = 1263 rdma_ah_read_grh(&qp_attr->ah_attr); 1264 1265 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | 1266 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | 1267 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | 1268 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | 1269 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | 1270 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | 1271 CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; 1272 memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw, 1273 sizeof(qp->qplib_qp.ah.dgid.data)); 1274 qp->qplib_qp.ah.flow_label = grh->flow_label; 1275 /* If RoCE V2 is enabled, stack will have two entries for 1276 * each GID entry. Avoiding this duplicte entry in HW. Dividing 1277 * the GID index by 2 for RoCE V2 1278 */ 1279 qp->qplib_qp.ah.sgid_index = grh->sgid_index / 2; 1280 qp->qplib_qp.ah.host_sgid_index = grh->sgid_index; 1281 qp->qplib_qp.ah.hop_limit = grh->hop_limit; 1282 qp->qplib_qp.ah.traffic_class = grh->traffic_class; 1283 qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr); 1284 ether_addr_copy(qp->qplib_qp.ah.dmac, 1285 qp_attr->ah_attr.roce.dmac); 1286 1287 status = ib_get_cached_gid(&rdev->ibdev, 1, 1288 grh->sgid_index, 1289 &sgid, &sgid_attr); 1290 if (!status && sgid_attr.ndev) { 1291 memcpy(qp->qplib_qp.smac, sgid_attr.ndev->dev_addr, 1292 ETH_ALEN); 1293 dev_put(sgid_attr.ndev); 1294 nw_type = ib_gid_to_network_type(sgid_attr.gid_type, 1295 &sgid); 1296 switch (nw_type) { 1297 case RDMA_NETWORK_IPV4: 1298 qp->qplib_qp.nw_type = 1299 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4; 1300 break; 1301 case RDMA_NETWORK_IPV6: 1302 qp->qplib_qp.nw_type = 1303 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6; 1304 break; 1305 default: 1306 qp->qplib_qp.nw_type = 1307 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1; 1308 break; 1309 } 1310 } 1311 } 1312 1313 if (qp_attr_mask & IB_QP_PATH_MTU) { 1314 qp->qplib_qp.modify_flags |= 1315 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 1316 qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu); 1317 } else if (qp_attr->qp_state == IB_QPS_RTR) { 1318 qp->qplib_qp.modify_flags |= 1319 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 1320 qp->qplib_qp.path_mtu = 1321 __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu)); 1322 } 1323 1324 if (qp_attr_mask & IB_QP_TIMEOUT) { 1325 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; 1326 qp->qplib_qp.timeout = qp_attr->timeout; 1327 } 1328 if (qp_attr_mask & IB_QP_RETRY_CNT) { 1329 qp->qplib_qp.modify_flags |= 1330 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT; 1331 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; 1332 } 1333 if (qp_attr_mask & IB_QP_RNR_RETRY) { 1334 qp->qplib_qp.modify_flags |= 1335 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY; 1336 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; 1337 } 1338 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) { 1339 qp->qplib_qp.modify_flags |= 1340 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER; 1341 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; 1342 } 1343 if (qp_attr_mask & IB_QP_RQ_PSN) { 1344 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; 1345 qp->qplib_qp.rq.psn = qp_attr->rq_psn; 1346 } 1347 if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1348 qp->qplib_qp.modify_flags |= 1349 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC; 1350 qp->qplib_qp.max_rd_atomic = qp_attr->max_rd_atomic; 1351 } 1352 if (qp_attr_mask & IB_QP_SQ_PSN) { 1353 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 1354 qp->qplib_qp.sq.psn = qp_attr->sq_psn; 1355 } 1356 if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1357 qp->qplib_qp.modify_flags |= 1358 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC; 1359 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; 1360 } 1361 if (qp_attr_mask & IB_QP_CAP) { 1362 qp->qplib_qp.modify_flags |= 1363 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE | 1364 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE | 1365 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE | 1366 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE | 1367 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA; 1368 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || 1369 (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || 1370 (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || 1371 (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || 1372 (qp_attr->cap.max_inline_data >= 1373 dev_attr->max_inline_data)) { 1374 dev_err(rdev_to_dev(rdev), 1375 "Create QP failed - max exceeded"); 1376 return -EINVAL; 1377 } 1378 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr); 1379 qp->qplib_qp.sq.max_wqe = min_t(u32, entries, 1380 dev_attr->max_qp_wqes + 1); 1381 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; 1382 if (qp->qplib_qp.rq.max_wqe) { 1383 entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr); 1384 qp->qplib_qp.rq.max_wqe = 1385 min_t(u32, entries, dev_attr->max_qp_wqes + 1); 1386 qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; 1387 } else { 1388 /* SRQ was used prior, just ignore the RQ caps */ 1389 } 1390 } 1391 if (qp_attr_mask & IB_QP_DEST_QPN) { 1392 qp->qplib_qp.modify_flags |= 1393 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID; 1394 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; 1395 } 1396 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 1397 if (rc) { 1398 dev_err(rdev_to_dev(rdev), "Failed to modify HW QP"); 1399 return rc; 1400 } 1401 if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) 1402 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); 1403 return rc; 1404 } 1405 1406 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 1407 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) 1408 { 1409 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 1410 struct bnxt_re_dev *rdev = qp->rdev; 1411 struct bnxt_qplib_qp qplib_qp; 1412 int rc; 1413 1414 memset(&qplib_qp, 0, sizeof(struct bnxt_qplib_qp)); 1415 qplib_qp.id = qp->qplib_qp.id; 1416 qplib_qp.ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; 1417 1418 rc = bnxt_qplib_query_qp(&rdev->qplib_res, &qplib_qp); 1419 if (rc) { 1420 dev_err(rdev_to_dev(rdev), "Failed to query HW QP"); 1421 return rc; 1422 } 1423 qp_attr->qp_state = __to_ib_qp_state(qplib_qp.state); 1424 qp_attr->en_sqd_async_notify = qplib_qp.en_sqd_async_notify ? 1 : 0; 1425 qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp.access); 1426 qp_attr->pkey_index = qplib_qp.pkey_index; 1427 qp_attr->qkey = qplib_qp.qkey; 1428 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 1429 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp.ah.flow_label, 1430 qplib_qp.ah.host_sgid_index, 1431 qplib_qp.ah.hop_limit, 1432 qplib_qp.ah.traffic_class); 1433 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp.ah.dgid.data); 1434 rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp.ah.sl); 1435 ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp.ah.dmac); 1436 qp_attr->path_mtu = __to_ib_mtu(qplib_qp.path_mtu); 1437 qp_attr->timeout = qplib_qp.timeout; 1438 qp_attr->retry_cnt = qplib_qp.retry_cnt; 1439 qp_attr->rnr_retry = qplib_qp.rnr_retry; 1440 qp_attr->min_rnr_timer = qplib_qp.min_rnr_timer; 1441 qp_attr->rq_psn = qplib_qp.rq.psn; 1442 qp_attr->max_rd_atomic = qplib_qp.max_rd_atomic; 1443 qp_attr->sq_psn = qplib_qp.sq.psn; 1444 qp_attr->max_dest_rd_atomic = qplib_qp.max_dest_rd_atomic; 1445 qp_init_attr->sq_sig_type = qplib_qp.sig_type ? IB_SIGNAL_ALL_WR : 1446 IB_SIGNAL_REQ_WR; 1447 qp_attr->dest_qp_num = qplib_qp.dest_qpn; 1448 1449 qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; 1450 qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; 1451 qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; 1452 qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; 1453 qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; 1454 qp_init_attr->cap = qp_attr->cap; 1455 1456 return 0; 1457 } 1458 1459 /* Routine for sending QP1 packets for RoCE V1 an V2 1460 */ 1461 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp, 1462 struct ib_send_wr *wr, 1463 struct bnxt_qplib_swqe *wqe, 1464 int payload_size) 1465 { 1466 struct ib_device *ibdev = &qp->rdev->ibdev; 1467 struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, 1468 ib_ah); 1469 struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; 1470 struct bnxt_qplib_sge sge; 1471 union ib_gid sgid; 1472 u8 nw_type; 1473 u16 ether_type; 1474 struct ib_gid_attr sgid_attr; 1475 union ib_gid dgid; 1476 bool is_eth = false; 1477 bool is_vlan = false; 1478 bool is_grh = false; 1479 bool is_udp = false; 1480 u8 ip_version = 0; 1481 u16 vlan_id = 0xFFFF; 1482 void *buf; 1483 int i, rc = 0, size; 1484 1485 memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); 1486 1487 rc = ib_get_cached_gid(ibdev, 1, 1488 qplib_ah->host_sgid_index, &sgid, 1489 &sgid_attr); 1490 if (rc) { 1491 dev_err(rdev_to_dev(qp->rdev), 1492 "Failed to query gid at index %d", 1493 qplib_ah->host_sgid_index); 1494 return rc; 1495 } 1496 if (sgid_attr.ndev) { 1497 if (is_vlan_dev(sgid_attr.ndev)) 1498 vlan_id = vlan_dev_vlan_id(sgid_attr.ndev); 1499 dev_put(sgid_attr.ndev); 1500 } 1501 /* Get network header type for this GID */ 1502 nw_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid); 1503 switch (nw_type) { 1504 case RDMA_NETWORK_IPV4: 1505 nw_type = BNXT_RE_ROCEV2_IPV4_PACKET; 1506 break; 1507 case RDMA_NETWORK_IPV6: 1508 nw_type = BNXT_RE_ROCEV2_IPV6_PACKET; 1509 break; 1510 default: 1511 nw_type = BNXT_RE_ROCE_V1_PACKET; 1512 break; 1513 } 1514 memcpy(&dgid.raw, &qplib_ah->dgid, 16); 1515 is_udp = sgid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP; 1516 if (is_udp) { 1517 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid)) { 1518 ip_version = 4; 1519 ether_type = ETH_P_IP; 1520 } else { 1521 ip_version = 6; 1522 ether_type = ETH_P_IPV6; 1523 } 1524 is_grh = false; 1525 } else { 1526 ether_type = ETH_P_IBOE; 1527 is_grh = true; 1528 } 1529 1530 is_eth = true; 1531 is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false; 1532 1533 ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh, 1534 ip_version, is_udp, 0, &qp->qp1_hdr); 1535 1536 /* ETH */ 1537 ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac); 1538 ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac); 1539 1540 /* For vlan, check the sgid for vlan existence */ 1541 1542 if (!is_vlan) { 1543 qp->qp1_hdr.eth.type = cpu_to_be16(ether_type); 1544 } else { 1545 qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type); 1546 qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id); 1547 } 1548 1549 if (is_grh || (ip_version == 6)) { 1550 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid)); 1551 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, 1552 sizeof(sgid)); 1553 qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit; 1554 } 1555 1556 if (ip_version == 4) { 1557 qp->qp1_hdr.ip4.tos = 0; 1558 qp->qp1_hdr.ip4.id = 0; 1559 qp->qp1_hdr.ip4.frag_off = htons(IP_DF); 1560 qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit; 1561 1562 memcpy(&qp->qp1_hdr.ip4.saddr, sgid.raw + 12, 4); 1563 memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4); 1564 qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr); 1565 } 1566 1567 if (is_udp) { 1568 qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT); 1569 qp->qp1_hdr.udp.sport = htons(0x8CD1); 1570 qp->qp1_hdr.udp.csum = 0; 1571 } 1572 1573 /* BTH */ 1574 if (wr->opcode == IB_WR_SEND_WITH_IMM) { 1575 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; 1576 qp->qp1_hdr.immediate_present = 1; 1577 } else { 1578 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; 1579 } 1580 if (wr->send_flags & IB_SEND_SOLICITED) 1581 qp->qp1_hdr.bth.solicited_event = 1; 1582 /* pad_count */ 1583 qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; 1584 1585 /* P_key for QP1 is for all members */ 1586 qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); 1587 qp->qp1_hdr.bth.destination_qpn = IB_QP1; 1588 qp->qp1_hdr.bth.ack_req = 0; 1589 qp->send_psn++; 1590 qp->send_psn &= BTH_PSN_MASK; 1591 qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); 1592 /* DETH */ 1593 /* Use the priviledged Q_Key for QP1 */ 1594 qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); 1595 qp->qp1_hdr.deth.source_qpn = IB_QP1; 1596 1597 /* Pack the QP1 to the transmit buffer */ 1598 buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); 1599 if (buf) { 1600 size = ib_ud_header_pack(&qp->qp1_hdr, buf); 1601 for (i = wqe->num_sge; i; i--) { 1602 wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; 1603 wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; 1604 wqe->sg_list[i].size = wqe->sg_list[i - 1].size; 1605 } 1606 1607 /* 1608 * Max Header buf size for IPV6 RoCE V2 is 86, 1609 * which is same as the QP1 SQ header buffer. 1610 * Header buf size for IPV4 RoCE V2 can be 66. 1611 * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20). 1612 * Subtract 20 bytes from QP1 SQ header buf size 1613 */ 1614 if (is_udp && ip_version == 4) 1615 sge.size -= 20; 1616 /* 1617 * Max Header buf size for RoCE V1 is 78. 1618 * ETH(14) + VLAN(4) + GRH(40) + BTH(20). 1619 * Subtract 8 bytes from QP1 SQ header buf size 1620 */ 1621 if (!is_udp) 1622 sge.size -= 8; 1623 1624 /* Subtract 4 bytes for non vlan packets */ 1625 if (!is_vlan) 1626 sge.size -= 4; 1627 1628 wqe->sg_list[0].addr = sge.addr; 1629 wqe->sg_list[0].lkey = sge.lkey; 1630 wqe->sg_list[0].size = sge.size; 1631 wqe->num_sge++; 1632 1633 } else { 1634 dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!"); 1635 rc = -ENOMEM; 1636 } 1637 return rc; 1638 } 1639 1640 /* For the MAD layer, it only provides the recv SGE the size of 1641 * ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH, 1642 * nor RoCE iCRC. The Cu+ solution must provide buffer for the entire 1643 * receive packet (334 bytes) with no VLAN and then copy the GRH 1644 * and the MAD datagram out to the provided SGE. 1645 */ 1646 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, 1647 struct ib_recv_wr *wr, 1648 struct bnxt_qplib_swqe *wqe, 1649 int payload_size) 1650 { 1651 struct bnxt_qplib_sge ref, sge; 1652 u32 rq_prod_index; 1653 struct bnxt_re_sqp_entries *sqp_entry; 1654 1655 rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); 1656 1657 if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) 1658 return -ENOMEM; 1659 1660 /* Create 1 SGE to receive the entire 1661 * ethernet packet 1662 */ 1663 /* Save the reference from ULP */ 1664 ref.addr = wqe->sg_list[0].addr; 1665 ref.lkey = wqe->sg_list[0].lkey; 1666 ref.size = wqe->sg_list[0].size; 1667 1668 sqp_entry = &qp->rdev->sqp_tbl[rq_prod_index]; 1669 1670 /* SGE 1 */ 1671 wqe->sg_list[0].addr = sge.addr; 1672 wqe->sg_list[0].lkey = sge.lkey; 1673 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; 1674 sge.size -= wqe->sg_list[0].size; 1675 1676 sqp_entry->sge.addr = ref.addr; 1677 sqp_entry->sge.lkey = ref.lkey; 1678 sqp_entry->sge.size = ref.size; 1679 /* Store the wrid for reporting completion */ 1680 sqp_entry->wrid = wqe->wr_id; 1681 /* change the wqe->wrid to table index */ 1682 wqe->wr_id = rq_prod_index; 1683 return 0; 1684 } 1685 1686 static int is_ud_qp(struct bnxt_re_qp *qp) 1687 { 1688 return qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD; 1689 } 1690 1691 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, 1692 struct ib_send_wr *wr, 1693 struct bnxt_qplib_swqe *wqe) 1694 { 1695 struct bnxt_re_ah *ah = NULL; 1696 1697 if (is_ud_qp(qp)) { 1698 ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah); 1699 wqe->send.q_key = ud_wr(wr)->remote_qkey; 1700 wqe->send.dst_qp = ud_wr(wr)->remote_qpn; 1701 wqe->send.avid = ah->qplib_ah.id; 1702 } 1703 switch (wr->opcode) { 1704 case IB_WR_SEND: 1705 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; 1706 break; 1707 case IB_WR_SEND_WITH_IMM: 1708 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; 1709 wqe->send.imm_data = wr->ex.imm_data; 1710 break; 1711 case IB_WR_SEND_WITH_INV: 1712 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; 1713 wqe->send.inv_key = wr->ex.invalidate_rkey; 1714 break; 1715 default: 1716 return -EINVAL; 1717 } 1718 if (wr->send_flags & IB_SEND_SIGNALED) 1719 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 1720 if (wr->send_flags & IB_SEND_FENCE) 1721 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 1722 if (wr->send_flags & IB_SEND_SOLICITED) 1723 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 1724 if (wr->send_flags & IB_SEND_INLINE) 1725 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 1726 1727 return 0; 1728 } 1729 1730 static int bnxt_re_build_rdma_wqe(struct ib_send_wr *wr, 1731 struct bnxt_qplib_swqe *wqe) 1732 { 1733 switch (wr->opcode) { 1734 case IB_WR_RDMA_WRITE: 1735 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; 1736 break; 1737 case IB_WR_RDMA_WRITE_WITH_IMM: 1738 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; 1739 wqe->rdma.imm_data = wr->ex.imm_data; 1740 break; 1741 case IB_WR_RDMA_READ: 1742 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; 1743 wqe->rdma.inv_key = wr->ex.invalidate_rkey; 1744 break; 1745 default: 1746 return -EINVAL; 1747 } 1748 wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; 1749 wqe->rdma.r_key = rdma_wr(wr)->rkey; 1750 if (wr->send_flags & IB_SEND_SIGNALED) 1751 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 1752 if (wr->send_flags & IB_SEND_FENCE) 1753 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 1754 if (wr->send_flags & IB_SEND_SOLICITED) 1755 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 1756 if (wr->send_flags & IB_SEND_INLINE) 1757 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 1758 1759 return 0; 1760 } 1761 1762 static int bnxt_re_build_atomic_wqe(struct ib_send_wr *wr, 1763 struct bnxt_qplib_swqe *wqe) 1764 { 1765 switch (wr->opcode) { 1766 case IB_WR_ATOMIC_CMP_AND_SWP: 1767 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; 1768 wqe->atomic.swap_data = atomic_wr(wr)->swap; 1769 break; 1770 case IB_WR_ATOMIC_FETCH_AND_ADD: 1771 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; 1772 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 1773 break; 1774 default: 1775 return -EINVAL; 1776 } 1777 wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; 1778 wqe->atomic.r_key = atomic_wr(wr)->rkey; 1779 if (wr->send_flags & IB_SEND_SIGNALED) 1780 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 1781 if (wr->send_flags & IB_SEND_FENCE) 1782 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 1783 if (wr->send_flags & IB_SEND_SOLICITED) 1784 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 1785 return 0; 1786 } 1787 1788 static int bnxt_re_build_inv_wqe(struct ib_send_wr *wr, 1789 struct bnxt_qplib_swqe *wqe) 1790 { 1791 wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; 1792 wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; 1793 1794 if (wr->send_flags & IB_SEND_SIGNALED) 1795 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 1796 if (wr->send_flags & IB_SEND_FENCE) 1797 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 1798 if (wr->send_flags & IB_SEND_SOLICITED) 1799 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 1800 1801 return 0; 1802 } 1803 1804 static int bnxt_re_build_reg_wqe(struct ib_reg_wr *wr, 1805 struct bnxt_qplib_swqe *wqe) 1806 { 1807 struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr); 1808 struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; 1809 int access = wr->access; 1810 1811 wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; 1812 wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; 1813 wqe->frmr.page_list = mr->pages; 1814 wqe->frmr.page_list_len = mr->npages; 1815 wqe->frmr.levels = qplib_frpl->hwq.level + 1; 1816 wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; 1817 1818 if (wr->wr.send_flags & IB_SEND_FENCE) 1819 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 1820 if (wr->wr.send_flags & IB_SEND_SIGNALED) 1821 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 1822 1823 if (access & IB_ACCESS_LOCAL_WRITE) 1824 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; 1825 if (access & IB_ACCESS_REMOTE_READ) 1826 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; 1827 if (access & IB_ACCESS_REMOTE_WRITE) 1828 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; 1829 if (access & IB_ACCESS_REMOTE_ATOMIC) 1830 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; 1831 if (access & IB_ACCESS_MW_BIND) 1832 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; 1833 1834 wqe->frmr.l_key = wr->key; 1835 wqe->frmr.length = wr->mr->length; 1836 wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1; 1837 wqe->frmr.va = wr->mr->iova; 1838 return 0; 1839 } 1840 1841 static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev, 1842 struct ib_send_wr *wr, 1843 struct bnxt_qplib_swqe *wqe) 1844 { 1845 /* Copy the inline data to the data field */ 1846 u8 *in_data; 1847 u32 i, sge_len; 1848 void *sge_addr; 1849 1850 in_data = wqe->inline_data; 1851 for (i = 0; i < wr->num_sge; i++) { 1852 sge_addr = (void *)(unsigned long) 1853 wr->sg_list[i].addr; 1854 sge_len = wr->sg_list[i].length; 1855 1856 if ((sge_len + wqe->inline_len) > 1857 BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { 1858 dev_err(rdev_to_dev(rdev), 1859 "Inline data size requested > supported value"); 1860 return -EINVAL; 1861 } 1862 sge_len = wr->sg_list[i].length; 1863 1864 memcpy(in_data, sge_addr, sge_len); 1865 in_data += wr->sg_list[i].length; 1866 wqe->inline_len += wr->sg_list[i].length; 1867 } 1868 return wqe->inline_len; 1869 } 1870 1871 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev, 1872 struct ib_send_wr *wr, 1873 struct bnxt_qplib_swqe *wqe) 1874 { 1875 int payload_sz = 0; 1876 1877 if (wr->send_flags & IB_SEND_INLINE) 1878 payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe); 1879 else 1880 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list, 1881 wqe->num_sge); 1882 1883 return payload_sz; 1884 } 1885 1886 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, 1887 struct bnxt_re_qp *qp, 1888 struct ib_send_wr *wr) 1889 { 1890 struct bnxt_qplib_swqe wqe; 1891 int rc = 0, payload_sz = 0; 1892 unsigned long flags; 1893 1894 spin_lock_irqsave(&qp->sq_lock, flags); 1895 memset(&wqe, 0, sizeof(wqe)); 1896 while (wr) { 1897 /* House keeping */ 1898 memset(&wqe, 0, sizeof(wqe)); 1899 1900 /* Common */ 1901 wqe.num_sge = wr->num_sge; 1902 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 1903 dev_err(rdev_to_dev(rdev), 1904 "Limit exceeded for Send SGEs"); 1905 rc = -EINVAL; 1906 goto bad; 1907 } 1908 1909 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); 1910 if (payload_sz < 0) { 1911 rc = -EINVAL; 1912 goto bad; 1913 } 1914 wqe.wr_id = wr->wr_id; 1915 1916 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; 1917 1918 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 1919 if (!rc) 1920 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 1921 bad: 1922 if (rc) { 1923 dev_err(rdev_to_dev(rdev), 1924 "Post send failed opcode = %#x rc = %d", 1925 wr->opcode, rc); 1926 break; 1927 } 1928 wr = wr->next; 1929 } 1930 bnxt_qplib_post_send_db(&qp->qplib_qp); 1931 spin_unlock_irqrestore(&qp->sq_lock, flags); 1932 return rc; 1933 } 1934 1935 int bnxt_re_post_send(struct ib_qp *ib_qp, struct ib_send_wr *wr, 1936 struct ib_send_wr **bad_wr) 1937 { 1938 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 1939 struct bnxt_qplib_swqe wqe; 1940 int rc = 0, payload_sz = 0; 1941 unsigned long flags; 1942 1943 spin_lock_irqsave(&qp->sq_lock, flags); 1944 while (wr) { 1945 /* House keeping */ 1946 memset(&wqe, 0, sizeof(wqe)); 1947 1948 /* Common */ 1949 wqe.num_sge = wr->num_sge; 1950 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 1951 dev_err(rdev_to_dev(qp->rdev), 1952 "Limit exceeded for Send SGEs"); 1953 rc = -EINVAL; 1954 goto bad; 1955 } 1956 1957 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); 1958 if (payload_sz < 0) { 1959 rc = -EINVAL; 1960 goto bad; 1961 } 1962 wqe.wr_id = wr->wr_id; 1963 1964 switch (wr->opcode) { 1965 case IB_WR_SEND: 1966 case IB_WR_SEND_WITH_IMM: 1967 if (ib_qp->qp_type == IB_QPT_GSI) { 1968 rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe, 1969 payload_sz); 1970 if (rc) 1971 goto bad; 1972 wqe.rawqp1.lflags |= 1973 SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; 1974 } 1975 switch (wr->send_flags) { 1976 case IB_SEND_IP_CSUM: 1977 wqe.rawqp1.lflags |= 1978 SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; 1979 break; 1980 default: 1981 break; 1982 } 1983 /* Fall thru to build the wqe */ 1984 case IB_WR_SEND_WITH_INV: 1985 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 1986 break; 1987 case IB_WR_RDMA_WRITE: 1988 case IB_WR_RDMA_WRITE_WITH_IMM: 1989 case IB_WR_RDMA_READ: 1990 rc = bnxt_re_build_rdma_wqe(wr, &wqe); 1991 break; 1992 case IB_WR_ATOMIC_CMP_AND_SWP: 1993 case IB_WR_ATOMIC_FETCH_AND_ADD: 1994 rc = bnxt_re_build_atomic_wqe(wr, &wqe); 1995 break; 1996 case IB_WR_RDMA_READ_WITH_INV: 1997 dev_err(rdev_to_dev(qp->rdev), 1998 "RDMA Read with Invalidate is not supported"); 1999 rc = -EINVAL; 2000 goto bad; 2001 case IB_WR_LOCAL_INV: 2002 rc = bnxt_re_build_inv_wqe(wr, &wqe); 2003 break; 2004 case IB_WR_REG_MR: 2005 rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); 2006 break; 2007 default: 2008 /* Unsupported WRs */ 2009 dev_err(rdev_to_dev(qp->rdev), 2010 "WR (%#x) is not supported", wr->opcode); 2011 rc = -EINVAL; 2012 goto bad; 2013 } 2014 if (!rc) 2015 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 2016 bad: 2017 if (rc) { 2018 dev_err(rdev_to_dev(qp->rdev), 2019 "post_send failed op:%#x qps = %#x rc = %d\n", 2020 wr->opcode, qp->qplib_qp.state, rc); 2021 *bad_wr = wr; 2022 break; 2023 } 2024 wr = wr->next; 2025 } 2026 bnxt_qplib_post_send_db(&qp->qplib_qp); 2027 spin_unlock_irqrestore(&qp->sq_lock, flags); 2028 2029 return rc; 2030 } 2031 2032 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, 2033 struct bnxt_re_qp *qp, 2034 struct ib_recv_wr *wr) 2035 { 2036 struct bnxt_qplib_swqe wqe; 2037 int rc = 0, payload_sz = 0; 2038 2039 memset(&wqe, 0, sizeof(wqe)); 2040 while (wr) { 2041 /* House keeping */ 2042 memset(&wqe, 0, sizeof(wqe)); 2043 2044 /* Common */ 2045 wqe.num_sge = wr->num_sge; 2046 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 2047 dev_err(rdev_to_dev(rdev), 2048 "Limit exceeded for Receive SGEs"); 2049 rc = -EINVAL; 2050 break; 2051 } 2052 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, 2053 wr->num_sge); 2054 wqe.wr_id = wr->wr_id; 2055 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 2056 2057 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 2058 if (rc) 2059 break; 2060 2061 wr = wr->next; 2062 } 2063 if (!rc) 2064 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2065 return rc; 2066 } 2067 2068 int bnxt_re_post_recv(struct ib_qp *ib_qp, struct ib_recv_wr *wr, 2069 struct ib_recv_wr **bad_wr) 2070 { 2071 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 2072 struct bnxt_qplib_swqe wqe; 2073 int rc = 0, payload_sz = 0; 2074 2075 while (wr) { 2076 /* House keeping */ 2077 memset(&wqe, 0, sizeof(wqe)); 2078 2079 /* Common */ 2080 wqe.num_sge = wr->num_sge; 2081 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 2082 dev_err(rdev_to_dev(qp->rdev), 2083 "Limit exceeded for Receive SGEs"); 2084 rc = -EINVAL; 2085 *bad_wr = wr; 2086 break; 2087 } 2088 2089 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, 2090 wr->num_sge); 2091 wqe.wr_id = wr->wr_id; 2092 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 2093 2094 if (ib_qp->qp_type == IB_QPT_GSI) 2095 rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe, 2096 payload_sz); 2097 if (!rc) 2098 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 2099 if (rc) { 2100 *bad_wr = wr; 2101 break; 2102 } 2103 wr = wr->next; 2104 } 2105 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2106 return rc; 2107 } 2108 2109 /* Completion Queues */ 2110 int bnxt_re_destroy_cq(struct ib_cq *ib_cq) 2111 { 2112 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 2113 struct bnxt_re_dev *rdev = cq->rdev; 2114 int rc; 2115 2116 rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); 2117 if (rc) { 2118 dev_err(rdev_to_dev(rdev), "Failed to destroy HW CQ"); 2119 return rc; 2120 } 2121 if (!IS_ERR_OR_NULL(cq->umem)) 2122 ib_umem_release(cq->umem); 2123 2124 if (cq) { 2125 kfree(cq->cql); 2126 kfree(cq); 2127 } 2128 atomic_dec(&rdev->cq_count); 2129 rdev->nq.budget--; 2130 return 0; 2131 } 2132 2133 struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, 2134 const struct ib_cq_init_attr *attr, 2135 struct ib_ucontext *context, 2136 struct ib_udata *udata) 2137 { 2138 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 2139 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 2140 struct bnxt_re_cq *cq = NULL; 2141 int rc, entries; 2142 int cqe = attr->cqe; 2143 2144 /* Validate CQ fields */ 2145 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { 2146 dev_err(rdev_to_dev(rdev), "Failed to create CQ -max exceeded"); 2147 return ERR_PTR(-EINVAL); 2148 } 2149 cq = kzalloc(sizeof(*cq), GFP_KERNEL); 2150 if (!cq) 2151 return ERR_PTR(-ENOMEM); 2152 2153 cq->rdev = rdev; 2154 cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq); 2155 2156 entries = roundup_pow_of_two(cqe + 1); 2157 if (entries > dev_attr->max_cq_wqes + 1) 2158 entries = dev_attr->max_cq_wqes + 1; 2159 2160 if (context) { 2161 struct bnxt_re_cq_req req; 2162 struct bnxt_re_ucontext *uctx = container_of 2163 (context, 2164 struct bnxt_re_ucontext, 2165 ib_uctx); 2166 if (ib_copy_from_udata(&req, udata, sizeof(req))) { 2167 rc = -EFAULT; 2168 goto fail; 2169 } 2170 2171 cq->umem = ib_umem_get(context, req.cq_va, 2172 entries * sizeof(struct cq_base), 2173 IB_ACCESS_LOCAL_WRITE, 1); 2174 if (IS_ERR(cq->umem)) { 2175 rc = PTR_ERR(cq->umem); 2176 goto fail; 2177 } 2178 cq->qplib_cq.sghead = cq->umem->sg_head.sgl; 2179 cq->qplib_cq.nmap = cq->umem->nmap; 2180 cq->qplib_cq.dpi = uctx->dpi; 2181 } else { 2182 cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL); 2183 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), 2184 GFP_KERNEL); 2185 if (!cq->cql) { 2186 rc = -ENOMEM; 2187 goto fail; 2188 } 2189 2190 cq->qplib_cq.dpi = &rdev->dpi_privileged; 2191 cq->qplib_cq.sghead = NULL; 2192 cq->qplib_cq.nmap = 0; 2193 } 2194 cq->qplib_cq.max_wqe = entries; 2195 cq->qplib_cq.cnq_hw_ring_id = rdev->nq.ring_id; 2196 2197 rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq); 2198 if (rc) { 2199 dev_err(rdev_to_dev(rdev), "Failed to create HW CQ"); 2200 goto fail; 2201 } 2202 2203 cq->ib_cq.cqe = entries; 2204 cq->cq_period = cq->qplib_cq.period; 2205 rdev->nq.budget++; 2206 2207 atomic_inc(&rdev->cq_count); 2208 2209 if (context) { 2210 struct bnxt_re_cq_resp resp; 2211 2212 resp.cqid = cq->qplib_cq.id; 2213 resp.tail = cq->qplib_cq.hwq.cons; 2214 resp.phase = cq->qplib_cq.period; 2215 resp.rsvd = 0; 2216 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 2217 if (rc) { 2218 dev_err(rdev_to_dev(rdev), "Failed to copy CQ udata"); 2219 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); 2220 goto c2fail; 2221 } 2222 } 2223 2224 return &cq->ib_cq; 2225 2226 c2fail: 2227 if (context) 2228 ib_umem_release(cq->umem); 2229 fail: 2230 kfree(cq->cql); 2231 kfree(cq); 2232 return ERR_PTR(rc); 2233 } 2234 2235 static u8 __req_to_ib_wc_status(u8 qstatus) 2236 { 2237 switch (qstatus) { 2238 case CQ_REQ_STATUS_OK: 2239 return IB_WC_SUCCESS; 2240 case CQ_REQ_STATUS_BAD_RESPONSE_ERR: 2241 return IB_WC_BAD_RESP_ERR; 2242 case CQ_REQ_STATUS_LOCAL_LENGTH_ERR: 2243 return IB_WC_LOC_LEN_ERR; 2244 case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR: 2245 return IB_WC_LOC_QP_OP_ERR; 2246 case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR: 2247 return IB_WC_LOC_PROT_ERR; 2248 case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR: 2249 return IB_WC_GENERAL_ERR; 2250 case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR: 2251 return IB_WC_REM_INV_REQ_ERR; 2252 case CQ_REQ_STATUS_REMOTE_ACCESS_ERR: 2253 return IB_WC_REM_ACCESS_ERR; 2254 case CQ_REQ_STATUS_REMOTE_OPERATION_ERR: 2255 return IB_WC_REM_OP_ERR; 2256 case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR: 2257 return IB_WC_RNR_RETRY_EXC_ERR; 2258 case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR: 2259 return IB_WC_RETRY_EXC_ERR; 2260 case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR: 2261 return IB_WC_WR_FLUSH_ERR; 2262 default: 2263 return IB_WC_GENERAL_ERR; 2264 } 2265 return 0; 2266 } 2267 2268 static u8 __rawqp1_to_ib_wc_status(u8 qstatus) 2269 { 2270 switch (qstatus) { 2271 case CQ_RES_RAWETH_QP1_STATUS_OK: 2272 return IB_WC_SUCCESS; 2273 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR: 2274 return IB_WC_LOC_ACCESS_ERR; 2275 case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR: 2276 return IB_WC_LOC_LEN_ERR; 2277 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR: 2278 return IB_WC_LOC_PROT_ERR; 2279 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR: 2280 return IB_WC_LOC_QP_OP_ERR; 2281 case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR: 2282 return IB_WC_GENERAL_ERR; 2283 case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR: 2284 return IB_WC_WR_FLUSH_ERR; 2285 case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR: 2286 return IB_WC_WR_FLUSH_ERR; 2287 default: 2288 return IB_WC_GENERAL_ERR; 2289 } 2290 } 2291 2292 static u8 __rc_to_ib_wc_status(u8 qstatus) 2293 { 2294 switch (qstatus) { 2295 case CQ_RES_RC_STATUS_OK: 2296 return IB_WC_SUCCESS; 2297 case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR: 2298 return IB_WC_LOC_ACCESS_ERR; 2299 case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR: 2300 return IB_WC_LOC_LEN_ERR; 2301 case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR: 2302 return IB_WC_LOC_PROT_ERR; 2303 case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR: 2304 return IB_WC_LOC_QP_OP_ERR; 2305 case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR: 2306 return IB_WC_GENERAL_ERR; 2307 case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR: 2308 return IB_WC_REM_INV_REQ_ERR; 2309 case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR: 2310 return IB_WC_WR_FLUSH_ERR; 2311 case CQ_RES_RC_STATUS_HW_FLUSH_ERR: 2312 return IB_WC_WR_FLUSH_ERR; 2313 default: 2314 return IB_WC_GENERAL_ERR; 2315 } 2316 } 2317 2318 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) 2319 { 2320 switch (cqe->type) { 2321 case BNXT_QPLIB_SWQE_TYPE_SEND: 2322 wc->opcode = IB_WC_SEND; 2323 break; 2324 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: 2325 wc->opcode = IB_WC_SEND; 2326 wc->wc_flags |= IB_WC_WITH_IMM; 2327 break; 2328 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: 2329 wc->opcode = IB_WC_SEND; 2330 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 2331 break; 2332 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: 2333 wc->opcode = IB_WC_RDMA_WRITE; 2334 break; 2335 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: 2336 wc->opcode = IB_WC_RDMA_WRITE; 2337 wc->wc_flags |= IB_WC_WITH_IMM; 2338 break; 2339 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: 2340 wc->opcode = IB_WC_RDMA_READ; 2341 break; 2342 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: 2343 wc->opcode = IB_WC_COMP_SWAP; 2344 break; 2345 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: 2346 wc->opcode = IB_WC_FETCH_ADD; 2347 break; 2348 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: 2349 wc->opcode = IB_WC_LOCAL_INV; 2350 break; 2351 case BNXT_QPLIB_SWQE_TYPE_REG_MR: 2352 wc->opcode = IB_WC_REG_MR; 2353 break; 2354 default: 2355 wc->opcode = IB_WC_SEND; 2356 break; 2357 } 2358 2359 wc->status = __req_to_ib_wc_status(cqe->status); 2360 } 2361 2362 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, 2363 u16 raweth_qp1_flags2) 2364 { 2365 bool is_udp = false, is_ipv6 = false, is_ipv4 = false; 2366 2367 /* raweth_qp1_flags Bit 9-6 indicates itype */ 2368 if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 2369 != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 2370 return -1; 2371 2372 if (raweth_qp1_flags2 & 2373 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC && 2374 raweth_qp1_flags2 & 2375 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) { 2376 is_udp = true; 2377 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ 2378 (raweth_qp1_flags2 & 2379 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ? 2380 (is_ipv6 = true) : (is_ipv4 = true); 2381 return ((is_ipv6) ? 2382 BNXT_RE_ROCEV2_IPV6_PACKET : 2383 BNXT_RE_ROCEV2_IPV4_PACKET); 2384 } else { 2385 return BNXT_RE_ROCE_V1_PACKET; 2386 } 2387 } 2388 2389 static int bnxt_re_to_ib_nw_type(int nw_type) 2390 { 2391 u8 nw_hdr_type = 0xFF; 2392 2393 switch (nw_type) { 2394 case BNXT_RE_ROCE_V1_PACKET: 2395 nw_hdr_type = RDMA_NETWORK_ROCE_V1; 2396 break; 2397 case BNXT_RE_ROCEV2_IPV4_PACKET: 2398 nw_hdr_type = RDMA_NETWORK_IPV4; 2399 break; 2400 case BNXT_RE_ROCEV2_IPV6_PACKET: 2401 nw_hdr_type = RDMA_NETWORK_IPV6; 2402 break; 2403 } 2404 return nw_hdr_type; 2405 } 2406 2407 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev, 2408 void *rq_hdr_buf) 2409 { 2410 u8 *tmp_buf = NULL; 2411 struct ethhdr *eth_hdr; 2412 u16 eth_type; 2413 bool rc = false; 2414 2415 tmp_buf = (u8 *)rq_hdr_buf; 2416 /* 2417 * If dest mac is not same as I/F mac, this could be a 2418 * loopback address or multicast address, check whether 2419 * it is a loopback packet 2420 */ 2421 if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) { 2422 tmp_buf += 4; 2423 /* Check the ether type */ 2424 eth_hdr = (struct ethhdr *)tmp_buf; 2425 eth_type = ntohs(eth_hdr->h_proto); 2426 switch (eth_type) { 2427 case ETH_P_IBOE: 2428 rc = true; 2429 break; 2430 case ETH_P_IP: 2431 case ETH_P_IPV6: { 2432 u32 len; 2433 struct udphdr *udp_hdr; 2434 2435 len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) : 2436 sizeof(struct ipv6hdr)); 2437 tmp_buf += sizeof(struct ethhdr) + len; 2438 udp_hdr = (struct udphdr *)tmp_buf; 2439 if (ntohs(udp_hdr->dest) == 2440 ROCE_V2_UDP_DPORT) 2441 rc = true; 2442 break; 2443 } 2444 default: 2445 break; 2446 } 2447 } 2448 2449 return rc; 2450 } 2451 2452 static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *qp1_qp, 2453 struct bnxt_qplib_cqe *cqe) 2454 { 2455 struct bnxt_re_dev *rdev = qp1_qp->rdev; 2456 struct bnxt_re_sqp_entries *sqp_entry = NULL; 2457 struct bnxt_re_qp *qp = rdev->qp1_sqp; 2458 struct ib_send_wr *swr; 2459 struct ib_ud_wr udwr; 2460 struct ib_recv_wr rwr; 2461 int pkt_type = 0; 2462 u32 tbl_idx; 2463 void *rq_hdr_buf; 2464 dma_addr_t rq_hdr_buf_map; 2465 dma_addr_t shrq_hdr_buf_map; 2466 u32 offset = 0; 2467 u32 skip_bytes = 0; 2468 struct ib_sge s_sge[2]; 2469 struct ib_sge r_sge[2]; 2470 int rc; 2471 2472 memset(&udwr, 0, sizeof(udwr)); 2473 memset(&rwr, 0, sizeof(rwr)); 2474 memset(&s_sge, 0, sizeof(s_sge)); 2475 memset(&r_sge, 0, sizeof(r_sge)); 2476 2477 swr = &udwr.wr; 2478 tbl_idx = cqe->wr_id; 2479 2480 rq_hdr_buf = qp1_qp->qplib_qp.rq_hdr_buf + 2481 (tbl_idx * qp1_qp->qplib_qp.rq_hdr_buf_size); 2482 rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp1_qp->qplib_qp, 2483 tbl_idx); 2484 2485 /* Shadow QP header buffer */ 2486 shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp->qplib_qp, 2487 tbl_idx); 2488 sqp_entry = &rdev->sqp_tbl[tbl_idx]; 2489 2490 /* Store this cqe */ 2491 memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); 2492 sqp_entry->qp1_qp = qp1_qp; 2493 2494 /* Find packet type from the cqe */ 2495 2496 pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, 2497 cqe->raweth_qp1_flags2); 2498 if (pkt_type < 0) { 2499 dev_err(rdev_to_dev(rdev), "Invalid packet\n"); 2500 return -EINVAL; 2501 } 2502 2503 /* Adjust the offset for the user buffer and post in the rq */ 2504 2505 if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET) 2506 offset = 20; 2507 2508 /* 2509 * QP1 loopback packet has 4 bytes of internal header before 2510 * ether header. Skip these four bytes. 2511 */ 2512 if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf)) 2513 skip_bytes = 4; 2514 2515 /* First send SGE . Skip the ether header*/ 2516 s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 2517 + skip_bytes; 2518 s_sge[0].lkey = 0xFFFFFFFF; 2519 s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 : 2520 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; 2521 2522 /* Second Send SGE */ 2523 s_sge[1].addr = s_sge[0].addr + s_sge[0].length + 2524 BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE; 2525 if (pkt_type != BNXT_RE_ROCE_V1_PACKET) 2526 s_sge[1].addr += 8; 2527 s_sge[1].lkey = 0xFFFFFFFF; 2528 s_sge[1].length = 256; 2529 2530 /* First recv SGE */ 2531 2532 r_sge[0].addr = shrq_hdr_buf_map; 2533 r_sge[0].lkey = 0xFFFFFFFF; 2534 r_sge[0].length = 40; 2535 2536 r_sge[1].addr = sqp_entry->sge.addr + offset; 2537 r_sge[1].lkey = sqp_entry->sge.lkey; 2538 r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; 2539 2540 /* Create receive work request */ 2541 rwr.num_sge = 2; 2542 rwr.sg_list = r_sge; 2543 rwr.wr_id = tbl_idx; 2544 rwr.next = NULL; 2545 2546 rc = bnxt_re_post_recv_shadow_qp(rdev, qp, &rwr); 2547 if (rc) { 2548 dev_err(rdev_to_dev(rdev), 2549 "Failed to post Rx buffers to shadow QP"); 2550 return -ENOMEM; 2551 } 2552 2553 swr->num_sge = 2; 2554 swr->sg_list = s_sge; 2555 swr->wr_id = tbl_idx; 2556 swr->opcode = IB_WR_SEND; 2557 swr->next = NULL; 2558 2559 udwr.ah = &rdev->sqp_ah->ib_ah; 2560 udwr.remote_qpn = rdev->qp1_sqp->qplib_qp.id; 2561 udwr.remote_qkey = rdev->qp1_sqp->qplib_qp.qkey; 2562 2563 /* post data received in the send queue */ 2564 rc = bnxt_re_post_send_shadow_qp(rdev, qp, swr); 2565 2566 return 0; 2567 } 2568 2569 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc, 2570 struct bnxt_qplib_cqe *cqe) 2571 { 2572 wc->opcode = IB_WC_RECV; 2573 wc->status = __rawqp1_to_ib_wc_status(cqe->status); 2574 wc->wc_flags |= IB_WC_GRH; 2575 } 2576 2577 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc, 2578 struct bnxt_qplib_cqe *cqe) 2579 { 2580 wc->opcode = IB_WC_RECV; 2581 wc->status = __rc_to_ib_wc_status(cqe->status); 2582 2583 if (cqe->flags & CQ_RES_RC_FLAGS_IMM) 2584 wc->wc_flags |= IB_WC_WITH_IMM; 2585 if (cqe->flags & CQ_RES_RC_FLAGS_INV) 2586 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 2587 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == 2588 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) 2589 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 2590 } 2591 2592 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *qp, 2593 struct ib_wc *wc, 2594 struct bnxt_qplib_cqe *cqe) 2595 { 2596 u32 tbl_idx; 2597 struct bnxt_re_dev *rdev = qp->rdev; 2598 struct bnxt_re_qp *qp1_qp = NULL; 2599 struct bnxt_qplib_cqe *orig_cqe = NULL; 2600 struct bnxt_re_sqp_entries *sqp_entry = NULL; 2601 int nw_type; 2602 2603 tbl_idx = cqe->wr_id; 2604 2605 sqp_entry = &rdev->sqp_tbl[tbl_idx]; 2606 qp1_qp = sqp_entry->qp1_qp; 2607 orig_cqe = &sqp_entry->cqe; 2608 2609 wc->wr_id = sqp_entry->wrid; 2610 wc->byte_len = orig_cqe->length; 2611 wc->qp = &qp1_qp->ib_qp; 2612 2613 wc->ex.imm_data = orig_cqe->immdata; 2614 wc->src_qp = orig_cqe->src_qp; 2615 memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); 2616 wc->port_num = 1; 2617 wc->vendor_err = orig_cqe->status; 2618 2619 wc->opcode = IB_WC_RECV; 2620 wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); 2621 wc->wc_flags |= IB_WC_GRH; 2622 2623 nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, 2624 orig_cqe->raweth_qp1_flags2); 2625 if (nw_type >= 0) { 2626 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type); 2627 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 2628 } 2629 } 2630 2631 static void bnxt_re_process_res_ud_wc(struct ib_wc *wc, 2632 struct bnxt_qplib_cqe *cqe) 2633 { 2634 wc->opcode = IB_WC_RECV; 2635 wc->status = __rc_to_ib_wc_status(cqe->status); 2636 2637 if (cqe->flags & CQ_RES_RC_FLAGS_IMM) 2638 wc->wc_flags |= IB_WC_WITH_IMM; 2639 if (cqe->flags & CQ_RES_RC_FLAGS_INV) 2640 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 2641 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == 2642 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) 2643 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 2644 } 2645 2646 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) 2647 { 2648 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 2649 struct bnxt_re_qp *qp; 2650 struct bnxt_qplib_cqe *cqe; 2651 int i, ncqe, budget; 2652 u32 tbl_idx; 2653 struct bnxt_re_sqp_entries *sqp_entry = NULL; 2654 unsigned long flags; 2655 2656 spin_lock_irqsave(&cq->cq_lock, flags); 2657 budget = min_t(u32, num_entries, cq->max_cql); 2658 if (!cq->cql) { 2659 dev_err(rdev_to_dev(cq->rdev), "POLL CQ : no CQL to use"); 2660 goto exit; 2661 } 2662 cqe = &cq->cql[0]; 2663 while (budget) { 2664 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget); 2665 if (!ncqe) 2666 break; 2667 2668 for (i = 0; i < ncqe; i++, cqe++) { 2669 /* Transcribe each qplib_wqe back to ib_wc */ 2670 memset(wc, 0, sizeof(*wc)); 2671 2672 wc->wr_id = cqe->wr_id; 2673 wc->byte_len = cqe->length; 2674 qp = container_of 2675 ((struct bnxt_qplib_qp *) 2676 (unsigned long)(cqe->qp_handle), 2677 struct bnxt_re_qp, qplib_qp); 2678 if (!qp) { 2679 dev_err(rdev_to_dev(cq->rdev), 2680 "POLL CQ : bad QP handle"); 2681 continue; 2682 } 2683 wc->qp = &qp->ib_qp; 2684 wc->ex.imm_data = cqe->immdata; 2685 wc->src_qp = cqe->src_qp; 2686 memcpy(wc->smac, cqe->smac, ETH_ALEN); 2687 wc->port_num = 1; 2688 wc->vendor_err = cqe->status; 2689 2690 switch (cqe->opcode) { 2691 case CQ_BASE_CQE_TYPE_REQ: 2692 if (qp->qplib_qp.id == 2693 qp->rdev->qp1_sqp->qplib_qp.id) { 2694 /* Handle this completion with 2695 * the stored completion 2696 */ 2697 memset(wc, 0, sizeof(*wc)); 2698 continue; 2699 } 2700 bnxt_re_process_req_wc(wc, cqe); 2701 break; 2702 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 2703 if (!cqe->status) { 2704 int rc = 0; 2705 2706 rc = bnxt_re_process_raw_qp_pkt_rx 2707 (qp, cqe); 2708 if (!rc) { 2709 memset(wc, 0, sizeof(*wc)); 2710 continue; 2711 } 2712 cqe->status = -1; 2713 } 2714 /* Errors need not be looped back. 2715 * But change the wr_id to the one 2716 * stored in the table 2717 */ 2718 tbl_idx = cqe->wr_id; 2719 sqp_entry = &cq->rdev->sqp_tbl[tbl_idx]; 2720 wc->wr_id = sqp_entry->wrid; 2721 bnxt_re_process_res_rawqp1_wc(wc, cqe); 2722 break; 2723 case CQ_BASE_CQE_TYPE_RES_RC: 2724 bnxt_re_process_res_rc_wc(wc, cqe); 2725 break; 2726 case CQ_BASE_CQE_TYPE_RES_UD: 2727 if (qp->qplib_qp.id == 2728 qp->rdev->qp1_sqp->qplib_qp.id) { 2729 /* Handle this completion with 2730 * the stored completion 2731 */ 2732 if (cqe->status) { 2733 continue; 2734 } else { 2735 bnxt_re_process_res_shadow_qp_wc 2736 (qp, wc, cqe); 2737 break; 2738 } 2739 } 2740 bnxt_re_process_res_ud_wc(wc, cqe); 2741 break; 2742 default: 2743 dev_err(rdev_to_dev(cq->rdev), 2744 "POLL CQ : type 0x%x not handled", 2745 cqe->opcode); 2746 continue; 2747 } 2748 wc++; 2749 budget--; 2750 } 2751 } 2752 exit: 2753 spin_unlock_irqrestore(&cq->cq_lock, flags); 2754 return num_entries - budget; 2755 } 2756 2757 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq, 2758 enum ib_cq_notify_flags ib_cqn_flags) 2759 { 2760 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 2761 int type = 0; 2762 2763 /* Trigger on the very next completion */ 2764 if (ib_cqn_flags & IB_CQ_NEXT_COMP) 2765 type = DBR_DBR_TYPE_CQ_ARMALL; 2766 /* Trigger on the next solicited completion */ 2767 else if (ib_cqn_flags & IB_CQ_SOLICITED) 2768 type = DBR_DBR_TYPE_CQ_ARMSE; 2769 2770 bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); 2771 2772 return 0; 2773 } 2774 2775 /* Memory Regions */ 2776 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) 2777 { 2778 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 2779 struct bnxt_re_dev *rdev = pd->rdev; 2780 struct bnxt_re_mr *mr; 2781 u64 pbl = 0; 2782 int rc; 2783 2784 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2785 if (!mr) 2786 return ERR_PTR(-ENOMEM); 2787 2788 mr->rdev = rdev; 2789 mr->qplib_mr.pd = &pd->qplib_pd; 2790 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 2791 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 2792 2793 /* Allocate and register 0 as the address */ 2794 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 2795 if (rc) 2796 goto fail; 2797 2798 mr->qplib_mr.hwq.level = PBL_LVL_MAX; 2799 mr->qplib_mr.total_size = -1; /* Infinte length */ 2800 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl, 0, false); 2801 if (rc) 2802 goto fail_mr; 2803 2804 mr->ib_mr.lkey = mr->qplib_mr.lkey; 2805 if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | 2806 IB_ACCESS_REMOTE_ATOMIC)) 2807 mr->ib_mr.rkey = mr->ib_mr.lkey; 2808 atomic_inc(&rdev->mr_count); 2809 2810 return &mr->ib_mr; 2811 2812 fail_mr: 2813 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 2814 fail: 2815 kfree(mr); 2816 return ERR_PTR(rc); 2817 } 2818 2819 int bnxt_re_dereg_mr(struct ib_mr *ib_mr) 2820 { 2821 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 2822 struct bnxt_re_dev *rdev = mr->rdev; 2823 int rc; 2824 2825 if (mr->npages && mr->pages) { 2826 rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, 2827 &mr->qplib_frpl); 2828 kfree(mr->pages); 2829 mr->npages = 0; 2830 mr->pages = NULL; 2831 } 2832 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 2833 2834 if (!IS_ERR_OR_NULL(mr->ib_umem)) 2835 ib_umem_release(mr->ib_umem); 2836 2837 kfree(mr); 2838 atomic_dec(&rdev->mr_count); 2839 return rc; 2840 } 2841 2842 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) 2843 { 2844 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 2845 2846 if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) 2847 return -ENOMEM; 2848 2849 mr->pages[mr->npages++] = addr; 2850 return 0; 2851 } 2852 2853 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, 2854 unsigned int *sg_offset) 2855 { 2856 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 2857 2858 mr->npages = 0; 2859 return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page); 2860 } 2861 2862 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, 2863 u32 max_num_sg) 2864 { 2865 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 2866 struct bnxt_re_dev *rdev = pd->rdev; 2867 struct bnxt_re_mr *mr = NULL; 2868 int rc; 2869 2870 if (type != IB_MR_TYPE_MEM_REG) { 2871 dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported", type); 2872 return ERR_PTR(-EINVAL); 2873 } 2874 if (max_num_sg > MAX_PBL_LVL_1_PGS) 2875 return ERR_PTR(-EINVAL); 2876 2877 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2878 if (!mr) 2879 return ERR_PTR(-ENOMEM); 2880 2881 mr->rdev = rdev; 2882 mr->qplib_mr.pd = &pd->qplib_pd; 2883 mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; 2884 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 2885 2886 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 2887 if (rc) 2888 goto fail; 2889 2890 mr->ib_mr.lkey = mr->qplib_mr.lkey; 2891 mr->ib_mr.rkey = mr->ib_mr.lkey; 2892 2893 mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); 2894 if (!mr->pages) { 2895 rc = -ENOMEM; 2896 goto fail; 2897 } 2898 rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, 2899 &mr->qplib_frpl, max_num_sg); 2900 if (rc) { 2901 dev_err(rdev_to_dev(rdev), 2902 "Failed to allocate HW FR page list"); 2903 goto fail_mr; 2904 } 2905 2906 atomic_inc(&rdev->mr_count); 2907 return &mr->ib_mr; 2908 2909 fail_mr: 2910 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 2911 fail: 2912 kfree(mr->pages); 2913 kfree(mr); 2914 return ERR_PTR(rc); 2915 } 2916 2917 /* Fast Memory Regions */ 2918 struct ib_fmr *bnxt_re_alloc_fmr(struct ib_pd *ib_pd, int mr_access_flags, 2919 struct ib_fmr_attr *fmr_attr) 2920 { 2921 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 2922 struct bnxt_re_dev *rdev = pd->rdev; 2923 struct bnxt_re_fmr *fmr; 2924 int rc; 2925 2926 if (fmr_attr->max_pages > MAX_PBL_LVL_2_PGS || 2927 fmr_attr->max_maps > rdev->dev_attr.max_map_per_fmr) { 2928 dev_err(rdev_to_dev(rdev), "Allocate FMR exceeded Max limit"); 2929 return ERR_PTR(-ENOMEM); 2930 } 2931 fmr = kzalloc(sizeof(*fmr), GFP_KERNEL); 2932 if (!fmr) 2933 return ERR_PTR(-ENOMEM); 2934 2935 fmr->rdev = rdev; 2936 fmr->qplib_fmr.pd = &pd->qplib_pd; 2937 fmr->qplib_fmr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 2938 2939 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &fmr->qplib_fmr); 2940 if (rc) 2941 goto fail; 2942 2943 fmr->qplib_fmr.flags = __from_ib_access_flags(mr_access_flags); 2944 fmr->ib_fmr.lkey = fmr->qplib_fmr.lkey; 2945 fmr->ib_fmr.rkey = fmr->ib_fmr.lkey; 2946 2947 atomic_inc(&rdev->mr_count); 2948 return &fmr->ib_fmr; 2949 fail: 2950 kfree(fmr); 2951 return ERR_PTR(rc); 2952 } 2953 2954 int bnxt_re_map_phys_fmr(struct ib_fmr *ib_fmr, u64 *page_list, int list_len, 2955 u64 iova) 2956 { 2957 struct bnxt_re_fmr *fmr = container_of(ib_fmr, struct bnxt_re_fmr, 2958 ib_fmr); 2959 struct bnxt_re_dev *rdev = fmr->rdev; 2960 int rc; 2961 2962 fmr->qplib_fmr.va = iova; 2963 fmr->qplib_fmr.total_size = list_len * PAGE_SIZE; 2964 2965 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &fmr->qplib_fmr, page_list, 2966 list_len, true); 2967 if (rc) 2968 dev_err(rdev_to_dev(rdev), "Failed to map FMR for lkey = 0x%x!", 2969 fmr->ib_fmr.lkey); 2970 return rc; 2971 } 2972 2973 int bnxt_re_unmap_fmr(struct list_head *fmr_list) 2974 { 2975 struct bnxt_re_dev *rdev; 2976 struct bnxt_re_fmr *fmr; 2977 struct ib_fmr *ib_fmr; 2978 int rc = 0; 2979 2980 /* Validate each FMRs inside the fmr_list */ 2981 list_for_each_entry(ib_fmr, fmr_list, list) { 2982 fmr = container_of(ib_fmr, struct bnxt_re_fmr, ib_fmr); 2983 rdev = fmr->rdev; 2984 2985 if (rdev) { 2986 rc = bnxt_qplib_dereg_mrw(&rdev->qplib_res, 2987 &fmr->qplib_fmr, true); 2988 if (rc) 2989 break; 2990 } 2991 } 2992 return rc; 2993 } 2994 2995 int bnxt_re_dealloc_fmr(struct ib_fmr *ib_fmr) 2996 { 2997 struct bnxt_re_fmr *fmr = container_of(ib_fmr, struct bnxt_re_fmr, 2998 ib_fmr); 2999 struct bnxt_re_dev *rdev = fmr->rdev; 3000 int rc; 3001 3002 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &fmr->qplib_fmr); 3003 if (rc) 3004 dev_err(rdev_to_dev(rdev), "Failed to free FMR"); 3005 3006 kfree(fmr); 3007 atomic_dec(&rdev->mr_count); 3008 return rc; 3009 } 3010 3011 /* uverbs */ 3012 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, 3013 u64 virt_addr, int mr_access_flags, 3014 struct ib_udata *udata) 3015 { 3016 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3017 struct bnxt_re_dev *rdev = pd->rdev; 3018 struct bnxt_re_mr *mr; 3019 struct ib_umem *umem; 3020 u64 *pbl_tbl, *pbl_tbl_orig; 3021 int i, umem_pgs, pages, rc; 3022 struct scatterlist *sg; 3023 int entry; 3024 3025 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3026 if (!mr) 3027 return ERR_PTR(-ENOMEM); 3028 3029 mr->rdev = rdev; 3030 mr->qplib_mr.pd = &pd->qplib_pd; 3031 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 3032 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; 3033 3034 umem = ib_umem_get(ib_pd->uobject->context, start, length, 3035 mr_access_flags, 0); 3036 if (IS_ERR(umem)) { 3037 dev_err(rdev_to_dev(rdev), "Failed to get umem"); 3038 rc = -EFAULT; 3039 goto free_mr; 3040 } 3041 mr->ib_umem = umem; 3042 3043 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3044 if (rc) { 3045 dev_err(rdev_to_dev(rdev), "Failed to allocate MR"); 3046 goto release_umem; 3047 } 3048 /* The fixed portion of the rkey is the same as the lkey */ 3049 mr->ib_mr.rkey = mr->qplib_mr.rkey; 3050 3051 mr->qplib_mr.va = virt_addr; 3052 umem_pgs = ib_umem_page_count(umem); 3053 if (!umem_pgs) { 3054 dev_err(rdev_to_dev(rdev), "umem is invalid!"); 3055 rc = -EINVAL; 3056 goto free_mrw; 3057 } 3058 mr->qplib_mr.total_size = length; 3059 3060 pbl_tbl = kcalloc(umem_pgs, sizeof(u64 *), GFP_KERNEL); 3061 if (!pbl_tbl) { 3062 rc = -EINVAL; 3063 goto free_mrw; 3064 } 3065 pbl_tbl_orig = pbl_tbl; 3066 3067 if (umem->hugetlb) { 3068 dev_err(rdev_to_dev(rdev), "umem hugetlb not supported!"); 3069 rc = -EFAULT; 3070 goto fail; 3071 } 3072 3073 if (umem->page_shift != PAGE_SHIFT) { 3074 dev_err(rdev_to_dev(rdev), "umem page shift unsupported!"); 3075 rc = -EFAULT; 3076 goto fail; 3077 } 3078 /* Map umem buf ptrs to the PBL */ 3079 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 3080 pages = sg_dma_len(sg) >> umem->page_shift; 3081 for (i = 0; i < pages; i++, pbl_tbl++) 3082 *pbl_tbl = sg_dma_address(sg) + (i << umem->page_shift); 3083 } 3084 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl_orig, 3085 umem_pgs, false); 3086 if (rc) { 3087 dev_err(rdev_to_dev(rdev), "Failed to register user MR"); 3088 goto fail; 3089 } 3090 3091 kfree(pbl_tbl_orig); 3092 3093 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3094 mr->ib_mr.rkey = mr->qplib_mr.lkey; 3095 atomic_inc(&rdev->mr_count); 3096 3097 return &mr->ib_mr; 3098 fail: 3099 kfree(pbl_tbl_orig); 3100 free_mrw: 3101 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3102 release_umem: 3103 ib_umem_release(umem); 3104 free_mr: 3105 kfree(mr); 3106 return ERR_PTR(rc); 3107 } 3108 3109 struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev, 3110 struct ib_udata *udata) 3111 { 3112 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 3113 struct bnxt_re_uctx_resp resp; 3114 struct bnxt_re_ucontext *uctx; 3115 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 3116 int rc; 3117 3118 dev_dbg(rdev_to_dev(rdev), "ABI version requested %d", 3119 ibdev->uverbs_abi_ver); 3120 3121 if (ibdev->uverbs_abi_ver != BNXT_RE_ABI_VERSION) { 3122 dev_dbg(rdev_to_dev(rdev), " is different from the device %d ", 3123 BNXT_RE_ABI_VERSION); 3124 return ERR_PTR(-EPERM); 3125 } 3126 3127 uctx = kzalloc(sizeof(*uctx), GFP_KERNEL); 3128 if (!uctx) 3129 return ERR_PTR(-ENOMEM); 3130 3131 uctx->rdev = rdev; 3132 3133 uctx->shpg = (void *)__get_free_page(GFP_KERNEL); 3134 if (!uctx->shpg) { 3135 rc = -ENOMEM; 3136 goto fail; 3137 } 3138 spin_lock_init(&uctx->sh_lock); 3139 3140 resp.dev_id = rdev->en_dev->pdev->devfn; /*Temp, Use idr_alloc instead*/ 3141 resp.max_qp = rdev->qplib_ctx.qpc_count; 3142 resp.pg_size = PAGE_SIZE; 3143 resp.cqe_sz = sizeof(struct cq_base); 3144 resp.max_cqd = dev_attr->max_cq_wqes; 3145 resp.rsvd = 0; 3146 3147 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 3148 if (rc) { 3149 dev_err(rdev_to_dev(rdev), "Failed to copy user context"); 3150 rc = -EFAULT; 3151 goto cfail; 3152 } 3153 3154 return &uctx->ib_uctx; 3155 cfail: 3156 free_page((unsigned long)uctx->shpg); 3157 uctx->shpg = NULL; 3158 fail: 3159 kfree(uctx); 3160 return ERR_PTR(rc); 3161 } 3162 3163 int bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx) 3164 { 3165 struct bnxt_re_ucontext *uctx = container_of(ib_uctx, 3166 struct bnxt_re_ucontext, 3167 ib_uctx); 3168 if (uctx->shpg) 3169 free_page((unsigned long)uctx->shpg); 3170 kfree(uctx); 3171 return 0; 3172 } 3173 3174 /* Helper function to mmap the virtual memory from user app */ 3175 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) 3176 { 3177 struct bnxt_re_ucontext *uctx = container_of(ib_uctx, 3178 struct bnxt_re_ucontext, 3179 ib_uctx); 3180 struct bnxt_re_dev *rdev = uctx->rdev; 3181 u64 pfn; 3182 3183 if (vma->vm_end - vma->vm_start != PAGE_SIZE) 3184 return -EINVAL; 3185 3186 if (vma->vm_pgoff) { 3187 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 3188 if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 3189 PAGE_SIZE, vma->vm_page_prot)) { 3190 dev_err(rdev_to_dev(rdev), "Failed to map DPI"); 3191 return -EAGAIN; 3192 } 3193 } else { 3194 pfn = virt_to_phys(uctx->shpg) >> PAGE_SHIFT; 3195 if (remap_pfn_range(vma, vma->vm_start, 3196 pfn, PAGE_SIZE, vma->vm_page_prot)) { 3197 dev_err(rdev_to_dev(rdev), 3198 "Failed to map shared page"); 3199 return -EAGAIN; 3200 } 3201 } 3202 3203 return 0; 3204 } 3205