1 /* 2 * Broadcom NetXtreme-E RoCE driver. 3 * 4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term 5 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Description: Main component of the bnxt_re driver 37 */ 38 39 #include <linux/module.h> 40 #include <linux/netdevice.h> 41 #include <linux/ethtool.h> 42 #include <linux/mutex.h> 43 #include <linux/list.h> 44 #include <linux/rculist.h> 45 #include <linux/spinlock.h> 46 #include <linux/pci.h> 47 #include <net/dcbnl.h> 48 #include <net/ipv6.h> 49 #include <net/addrconf.h> 50 #include <linux/if_ether.h> 51 52 #include <rdma/ib_verbs.h> 53 #include <rdma/ib_user_verbs.h> 54 #include <rdma/ib_umem.h> 55 #include <rdma/ib_addr.h> 56 57 #include "bnxt_ulp.h" 58 #include "roce_hsi.h" 59 #include "qplib_res.h" 60 #include "qplib_sp.h" 61 #include "qplib_fp.h" 62 #include "qplib_rcfw.h" 63 #include "bnxt_re.h" 64 #include "ib_verbs.h" 65 #include <rdma/bnxt_re-abi.h> 66 #include "bnxt.h" 67 #include "hw_counters.h" 68 69 static char version[] = 70 BNXT_RE_DESC " v" ROCE_DRV_MODULE_VERSION "\n"; 71 72 MODULE_AUTHOR("Eddie Wai <eddie.wai@broadcom.com>"); 73 MODULE_DESCRIPTION(BNXT_RE_DESC " Driver"); 74 MODULE_LICENSE("Dual BSD/GPL"); 75 76 /* globals */ 77 static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list); 78 /* Mutex to protect the list of bnxt_re devices added */ 79 static DEFINE_MUTEX(bnxt_re_dev_lock); 80 static struct workqueue_struct *bnxt_re_wq; 81 static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev, bool lock_wait); 82 83 /* for handling bnxt_en callbacks later */ 84 static void bnxt_re_stop(void *p) 85 { 86 } 87 88 static void bnxt_re_start(void *p) 89 { 90 } 91 92 static void bnxt_re_sriov_config(void *p, int num_vfs) 93 { 94 } 95 96 static void bnxt_re_shutdown(void *p) 97 { 98 struct bnxt_re_dev *rdev = p; 99 100 if (!rdev) 101 return; 102 103 bnxt_re_ib_unreg(rdev, false); 104 } 105 106 static struct bnxt_ulp_ops bnxt_re_ulp_ops = { 107 .ulp_async_notifier = NULL, 108 .ulp_stop = bnxt_re_stop, 109 .ulp_start = bnxt_re_start, 110 .ulp_sriov_config = bnxt_re_sriov_config, 111 .ulp_shutdown = bnxt_re_shutdown 112 }; 113 114 /* RoCE -> Net driver */ 115 116 /* Driver registration routines used to let the networking driver (bnxt_en) 117 * to know that the RoCE driver is now installed 118 */ 119 static int bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev, bool lock_wait) 120 { 121 struct bnxt_en_dev *en_dev; 122 int rc; 123 124 if (!rdev) 125 return -EINVAL; 126 127 en_dev = rdev->en_dev; 128 /* Acquire rtnl lock if it is not invokded from netdev event */ 129 if (lock_wait) 130 rtnl_lock(); 131 132 rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev, 133 BNXT_ROCE_ULP); 134 if (lock_wait) 135 rtnl_unlock(); 136 return rc; 137 } 138 139 static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev) 140 { 141 struct bnxt_en_dev *en_dev; 142 int rc = 0; 143 144 if (!rdev) 145 return -EINVAL; 146 147 en_dev = rdev->en_dev; 148 149 rtnl_lock(); 150 rc = en_dev->en_ops->bnxt_register_device(en_dev, BNXT_ROCE_ULP, 151 &bnxt_re_ulp_ops, rdev); 152 rtnl_unlock(); 153 return rc; 154 } 155 156 static int bnxt_re_free_msix(struct bnxt_re_dev *rdev, bool lock_wait) 157 { 158 struct bnxt_en_dev *en_dev; 159 int rc; 160 161 if (!rdev) 162 return -EINVAL; 163 164 en_dev = rdev->en_dev; 165 166 if (lock_wait) 167 rtnl_lock(); 168 169 rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP); 170 171 if (lock_wait) 172 rtnl_unlock(); 173 return rc; 174 } 175 176 static int bnxt_re_request_msix(struct bnxt_re_dev *rdev) 177 { 178 int rc = 0, num_msix_want = BNXT_RE_MAX_MSIX, num_msix_got; 179 struct bnxt_en_dev *en_dev; 180 181 if (!rdev) 182 return -EINVAL; 183 184 en_dev = rdev->en_dev; 185 186 num_msix_want = min_t(u32, BNXT_RE_MAX_MSIX, num_online_cpus()); 187 188 rtnl_lock(); 189 num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP, 190 rdev->msix_entries, 191 num_msix_want); 192 if (num_msix_got < BNXT_RE_MIN_MSIX) { 193 rc = -EINVAL; 194 goto done; 195 } 196 if (num_msix_got != num_msix_want) { 197 dev_warn(rdev_to_dev(rdev), 198 "Requested %d MSI-X vectors, got %d\n", 199 num_msix_want, num_msix_got); 200 } 201 rdev->num_msix = num_msix_got; 202 done: 203 rtnl_unlock(); 204 return rc; 205 } 206 207 static void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev, struct input *hdr, 208 u16 opcd, u16 crid, u16 trid) 209 { 210 hdr->req_type = cpu_to_le16(opcd); 211 hdr->cmpl_ring = cpu_to_le16(crid); 212 hdr->target_id = cpu_to_le16(trid); 213 } 214 215 static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg, 216 int msg_len, void *resp, int resp_max_len, 217 int timeout) 218 { 219 fw_msg->msg = msg; 220 fw_msg->msg_len = msg_len; 221 fw_msg->resp = resp; 222 fw_msg->resp_max_len = resp_max_len; 223 fw_msg->timeout = timeout; 224 } 225 226 static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id, 227 bool lock_wait) 228 { 229 struct bnxt_en_dev *en_dev = rdev->en_dev; 230 struct hwrm_ring_free_input req = {0}; 231 struct hwrm_ring_free_output resp; 232 struct bnxt_fw_msg fw_msg; 233 bool do_unlock = false; 234 int rc = -EINVAL; 235 236 if (!en_dev) 237 return rc; 238 239 memset(&fw_msg, 0, sizeof(fw_msg)); 240 if (lock_wait) { 241 rtnl_lock(); 242 do_unlock = true; 243 } 244 245 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1); 246 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 247 req.ring_id = cpu_to_le16(fw_ring_id); 248 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 249 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 250 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 251 if (rc) 252 dev_err(rdev_to_dev(rdev), 253 "Failed to free HW ring:%d :%#x", req.ring_id, rc); 254 if (do_unlock) 255 rtnl_unlock(); 256 return rc; 257 } 258 259 static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, dma_addr_t *dma_arr, 260 int pages, int type, u32 ring_mask, 261 u32 map_index, u16 *fw_ring_id) 262 { 263 struct bnxt_en_dev *en_dev = rdev->en_dev; 264 struct hwrm_ring_alloc_input req = {0}; 265 struct hwrm_ring_alloc_output resp; 266 struct bnxt_fw_msg fw_msg; 267 int rc = -EINVAL; 268 269 if (!en_dev) 270 return rc; 271 272 memset(&fw_msg, 0, sizeof(fw_msg)); 273 rtnl_lock(); 274 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1); 275 req.enables = 0; 276 req.page_tbl_addr = cpu_to_le64(dma_arr[0]); 277 if (pages > 1) { 278 /* Page size is in log2 units */ 279 req.page_size = BNXT_PAGE_SHIFT; 280 req.page_tbl_depth = 1; 281 } 282 req.fbo = 0; 283 /* Association of ring index with doorbell index and MSIX number */ 284 req.logical_id = cpu_to_le16(map_index); 285 req.length = cpu_to_le32(ring_mask + 1); 286 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 287 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 288 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 289 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 290 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 291 if (!rc) 292 *fw_ring_id = le16_to_cpu(resp.ring_id); 293 294 rtnl_unlock(); 295 return rc; 296 } 297 298 static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev, 299 u32 fw_stats_ctx_id, bool lock_wait) 300 { 301 struct bnxt_en_dev *en_dev = rdev->en_dev; 302 struct hwrm_stat_ctx_free_input req = {0}; 303 struct bnxt_fw_msg fw_msg; 304 bool do_unlock = false; 305 int rc = -EINVAL; 306 307 if (!en_dev) 308 return rc; 309 310 memset(&fw_msg, 0, sizeof(fw_msg)); 311 if (lock_wait) { 312 rtnl_lock(); 313 do_unlock = true; 314 } 315 316 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, -1); 317 req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id); 318 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&req, 319 sizeof(req), DFLT_HWRM_CMD_TIMEOUT); 320 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 321 if (rc) 322 dev_err(rdev_to_dev(rdev), 323 "Failed to free HW stats context %#x", rc); 324 325 if (do_unlock) 326 rtnl_unlock(); 327 return rc; 328 } 329 330 static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, 331 dma_addr_t dma_map, 332 u32 *fw_stats_ctx_id) 333 { 334 struct hwrm_stat_ctx_alloc_output resp = {0}; 335 struct hwrm_stat_ctx_alloc_input req = {0}; 336 struct bnxt_en_dev *en_dev = rdev->en_dev; 337 struct bnxt_fw_msg fw_msg; 338 int rc = -EINVAL; 339 340 *fw_stats_ctx_id = INVALID_STATS_CTX_ID; 341 342 if (!en_dev) 343 return rc; 344 345 memset(&fw_msg, 0, sizeof(fw_msg)); 346 rtnl_lock(); 347 348 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_ALLOC, -1, -1); 349 req.update_period_ms = cpu_to_le32(1000); 350 req.stats_dma_addr = cpu_to_le64(dma_map); 351 req.stat_ctx_flags = STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE; 352 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 353 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 354 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 355 if (!rc) 356 *fw_stats_ctx_id = le32_to_cpu(resp.stat_ctx_id); 357 358 rtnl_unlock(); 359 return rc; 360 } 361 362 /* Device */ 363 364 static bool is_bnxt_re_dev(struct net_device *netdev) 365 { 366 struct ethtool_drvinfo drvinfo; 367 368 if (netdev->ethtool_ops && netdev->ethtool_ops->get_drvinfo) { 369 memset(&drvinfo, 0, sizeof(drvinfo)); 370 netdev->ethtool_ops->get_drvinfo(netdev, &drvinfo); 371 372 if (strcmp(drvinfo.driver, "bnxt_en")) 373 return false; 374 return true; 375 } 376 return false; 377 } 378 379 static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev) 380 { 381 struct bnxt_re_dev *rdev; 382 383 rcu_read_lock(); 384 list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { 385 if (rdev->netdev == netdev) { 386 rcu_read_unlock(); 387 return rdev; 388 } 389 } 390 rcu_read_unlock(); 391 return NULL; 392 } 393 394 static void bnxt_re_dev_unprobe(struct net_device *netdev, 395 struct bnxt_en_dev *en_dev) 396 { 397 dev_put(netdev); 398 module_put(en_dev->pdev->driver->driver.owner); 399 } 400 401 static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev) 402 { 403 struct bnxt *bp = netdev_priv(netdev); 404 struct bnxt_en_dev *en_dev; 405 struct pci_dev *pdev; 406 407 /* Call bnxt_en's RoCE probe via indirect API */ 408 if (!bp->ulp_probe) 409 return ERR_PTR(-EINVAL); 410 411 en_dev = bp->ulp_probe(netdev); 412 if (IS_ERR(en_dev)) 413 return en_dev; 414 415 pdev = en_dev->pdev; 416 if (!pdev) 417 return ERR_PTR(-EINVAL); 418 419 if (!(en_dev->flags & BNXT_EN_FLAG_ROCE_CAP)) { 420 dev_dbg(&pdev->dev, 421 "%s: probe error: RoCE is not supported on this device", 422 ROCE_DRV_MODULE_NAME); 423 return ERR_PTR(-ENODEV); 424 } 425 426 /* Bump net device reference count */ 427 if (!try_module_get(pdev->driver->driver.owner)) 428 return ERR_PTR(-ENODEV); 429 430 dev_hold(netdev); 431 432 return en_dev; 433 } 434 435 static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev) 436 { 437 ib_unregister_device(&rdev->ibdev); 438 } 439 440 static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) 441 { 442 struct ib_device *ibdev = &rdev->ibdev; 443 444 /* ib device init */ 445 ibdev->owner = THIS_MODULE; 446 ibdev->node_type = RDMA_NODE_IB_CA; 447 strlcpy(ibdev->name, "bnxt_re%d", IB_DEVICE_NAME_MAX); 448 strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", 449 strlen(BNXT_RE_DESC) + 5); 450 ibdev->phys_port_cnt = 1; 451 452 bnxt_qplib_get_guid(rdev->netdev->dev_addr, (u8 *)&ibdev->node_guid); 453 454 ibdev->num_comp_vectors = 1; 455 ibdev->dev.parent = &rdev->en_dev->pdev->dev; 456 ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; 457 458 /* User space */ 459 ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION; 460 ibdev->uverbs_cmd_mask = 461 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | 462 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | 463 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | 464 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | 465 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | 466 (1ull << IB_USER_VERBS_CMD_REG_MR) | 467 (1ull << IB_USER_VERBS_CMD_REREG_MR) | 468 (1ull << IB_USER_VERBS_CMD_DEREG_MR) | 469 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | 470 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | 471 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | 472 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | 473 (1ull << IB_USER_VERBS_CMD_CREATE_QP) | 474 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | 475 (1ull << IB_USER_VERBS_CMD_QUERY_QP) | 476 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | 477 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | 478 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | 479 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | 480 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | 481 (1ull << IB_USER_VERBS_CMD_CREATE_AH) | 482 (1ull << IB_USER_VERBS_CMD_MODIFY_AH) | 483 (1ull << IB_USER_VERBS_CMD_QUERY_AH) | 484 (1ull << IB_USER_VERBS_CMD_DESTROY_AH); 485 /* POLL_CQ and REQ_NOTIFY_CQ is directly handled in libbnxt_re */ 486 487 /* Kernel verbs */ 488 ibdev->query_device = bnxt_re_query_device; 489 ibdev->modify_device = bnxt_re_modify_device; 490 491 ibdev->query_port = bnxt_re_query_port; 492 ibdev->get_port_immutable = bnxt_re_get_port_immutable; 493 ibdev->query_pkey = bnxt_re_query_pkey; 494 ibdev->query_gid = bnxt_re_query_gid; 495 ibdev->get_netdev = bnxt_re_get_netdev; 496 ibdev->add_gid = bnxt_re_add_gid; 497 ibdev->del_gid = bnxt_re_del_gid; 498 ibdev->get_link_layer = bnxt_re_get_link_layer; 499 500 ibdev->alloc_pd = bnxt_re_alloc_pd; 501 ibdev->dealloc_pd = bnxt_re_dealloc_pd; 502 503 ibdev->create_ah = bnxt_re_create_ah; 504 ibdev->modify_ah = bnxt_re_modify_ah; 505 ibdev->query_ah = bnxt_re_query_ah; 506 ibdev->destroy_ah = bnxt_re_destroy_ah; 507 508 ibdev->create_qp = bnxt_re_create_qp; 509 ibdev->modify_qp = bnxt_re_modify_qp; 510 ibdev->query_qp = bnxt_re_query_qp; 511 ibdev->destroy_qp = bnxt_re_destroy_qp; 512 513 ibdev->post_send = bnxt_re_post_send; 514 ibdev->post_recv = bnxt_re_post_recv; 515 516 ibdev->create_cq = bnxt_re_create_cq; 517 ibdev->destroy_cq = bnxt_re_destroy_cq; 518 ibdev->poll_cq = bnxt_re_poll_cq; 519 ibdev->req_notify_cq = bnxt_re_req_notify_cq; 520 521 ibdev->get_dma_mr = bnxt_re_get_dma_mr; 522 ibdev->dereg_mr = bnxt_re_dereg_mr; 523 ibdev->alloc_mr = bnxt_re_alloc_mr; 524 ibdev->map_mr_sg = bnxt_re_map_mr_sg; 525 526 ibdev->reg_user_mr = bnxt_re_reg_user_mr; 527 ibdev->alloc_ucontext = bnxt_re_alloc_ucontext; 528 ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext; 529 ibdev->mmap = bnxt_re_mmap; 530 ibdev->get_hw_stats = bnxt_re_ib_get_hw_stats; 531 ibdev->alloc_hw_stats = bnxt_re_ib_alloc_hw_stats; 532 533 return ib_register_device(ibdev, NULL); 534 } 535 536 static ssize_t show_rev(struct device *device, struct device_attribute *attr, 537 char *buf) 538 { 539 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); 540 541 return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor); 542 } 543 544 static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr, 545 char *buf) 546 { 547 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); 548 549 return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->dev_attr.fw_ver); 550 } 551 552 static ssize_t show_hca(struct device *device, struct device_attribute *attr, 553 char *buf) 554 { 555 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); 556 557 return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc); 558 } 559 560 static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL); 561 static DEVICE_ATTR(fw_rev, 0444, show_fw_ver, NULL); 562 static DEVICE_ATTR(hca_type, 0444, show_hca, NULL); 563 564 static struct device_attribute *bnxt_re_attributes[] = { 565 &dev_attr_hw_rev, 566 &dev_attr_fw_rev, 567 &dev_attr_hca_type 568 }; 569 570 static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev) 571 { 572 dev_put(rdev->netdev); 573 rdev->netdev = NULL; 574 575 mutex_lock(&bnxt_re_dev_lock); 576 list_del_rcu(&rdev->list); 577 mutex_unlock(&bnxt_re_dev_lock); 578 579 synchronize_rcu(); 580 flush_workqueue(bnxt_re_wq); 581 582 ib_dealloc_device(&rdev->ibdev); 583 /* rdev is gone */ 584 } 585 586 static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev, 587 struct bnxt_en_dev *en_dev) 588 { 589 struct bnxt_re_dev *rdev; 590 591 /* Allocate bnxt_re_dev instance here */ 592 rdev = (struct bnxt_re_dev *)ib_alloc_device(sizeof(*rdev)); 593 if (!rdev) { 594 dev_err(NULL, "%s: bnxt_re_dev allocation failure!", 595 ROCE_DRV_MODULE_NAME); 596 return NULL; 597 } 598 /* Default values */ 599 rdev->netdev = netdev; 600 dev_hold(rdev->netdev); 601 rdev->en_dev = en_dev; 602 rdev->id = rdev->en_dev->pdev->devfn; 603 INIT_LIST_HEAD(&rdev->qp_list); 604 mutex_init(&rdev->qp_lock); 605 atomic_set(&rdev->qp_count, 0); 606 atomic_set(&rdev->cq_count, 0); 607 atomic_set(&rdev->srq_count, 0); 608 atomic_set(&rdev->mr_count, 0); 609 atomic_set(&rdev->mw_count, 0); 610 rdev->cosq[0] = 0xFFFF; 611 rdev->cosq[1] = 0xFFFF; 612 613 mutex_lock(&bnxt_re_dev_lock); 614 list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list); 615 mutex_unlock(&bnxt_re_dev_lock); 616 return rdev; 617 } 618 619 static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw, 620 struct creq_func_event *aeqe) 621 { 622 switch (aeqe->event) { 623 case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: 624 break; 625 case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: 626 break; 627 case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: 628 break; 629 case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: 630 break; 631 case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: 632 break; 633 case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: 634 break; 635 case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: 636 break; 637 case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: 638 break; 639 case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: 640 break; 641 case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: 642 break; 643 case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: 644 break; 645 default: 646 return -EINVAL; 647 } 648 return 0; 649 } 650 651 static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq, 652 struct bnxt_qplib_cq *handle) 653 { 654 struct bnxt_re_cq *cq = container_of(handle, struct bnxt_re_cq, 655 qplib_cq); 656 657 if (!cq) { 658 dev_err(NULL, "%s: CQ is NULL, CQN not handled", 659 ROCE_DRV_MODULE_NAME); 660 return -EINVAL; 661 } 662 if (cq->ib_cq.comp_handler) { 663 /* Lock comp_handler? */ 664 (*cq->ib_cq.comp_handler)(&cq->ib_cq, cq->ib_cq.cq_context); 665 } 666 667 return 0; 668 } 669 670 static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev) 671 { 672 int i; 673 674 if (rdev->nq[0].hwq.max_elements) { 675 for (i = 1; i < rdev->num_msix; i++) 676 bnxt_qplib_disable_nq(&rdev->nq[i - 1]); 677 } 678 679 if (rdev->qplib_res.rcfw) 680 bnxt_qplib_cleanup_res(&rdev->qplib_res); 681 } 682 683 static int bnxt_re_init_res(struct bnxt_re_dev *rdev) 684 { 685 int rc = 0, i; 686 687 bnxt_qplib_init_res(&rdev->qplib_res); 688 689 for (i = 1; i < rdev->num_msix ; i++) { 690 rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq[i - 1], 691 i - 1, rdev->msix_entries[i].vector, 692 rdev->msix_entries[i].db_offset, 693 &bnxt_re_cqn_handler, NULL); 694 695 if (rc) { 696 dev_err(rdev_to_dev(rdev), 697 "Failed to enable NQ with rc = 0x%x", rc); 698 goto fail; 699 } 700 } 701 return 0; 702 fail: 703 return rc; 704 } 705 706 static void bnxt_re_free_nq_res(struct bnxt_re_dev *rdev, bool lock_wait) 707 { 708 int i; 709 710 for (i = 0; i < rdev->num_msix - 1; i++) { 711 bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id, lock_wait); 712 bnxt_qplib_free_nq(&rdev->nq[i]); 713 } 714 } 715 716 static void bnxt_re_free_res(struct bnxt_re_dev *rdev, bool lock_wait) 717 { 718 bnxt_re_free_nq_res(rdev, lock_wait); 719 720 if (rdev->qplib_res.dpi_tbl.max) { 721 bnxt_qplib_dealloc_dpi(&rdev->qplib_res, 722 &rdev->qplib_res.dpi_tbl, 723 &rdev->dpi_privileged); 724 } 725 if (rdev->qplib_res.rcfw) { 726 bnxt_qplib_free_res(&rdev->qplib_res); 727 rdev->qplib_res.rcfw = NULL; 728 } 729 } 730 731 static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev) 732 { 733 int rc = 0, i; 734 735 /* Configure and allocate resources for qplib */ 736 rdev->qplib_res.rcfw = &rdev->rcfw; 737 rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr); 738 if (rc) 739 goto fail; 740 741 rc = bnxt_qplib_alloc_res(&rdev->qplib_res, rdev->en_dev->pdev, 742 rdev->netdev, &rdev->dev_attr); 743 if (rc) 744 goto fail; 745 746 rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl, 747 &rdev->dpi_privileged, 748 rdev); 749 if (rc) 750 goto dealloc_res; 751 752 for (i = 0; i < rdev->num_msix - 1; i++) { 753 rdev->nq[i].hwq.max_elements = BNXT_RE_MAX_CQ_COUNT + 754 BNXT_RE_MAX_SRQC_COUNT + 2; 755 rc = bnxt_qplib_alloc_nq(rdev->en_dev->pdev, &rdev->nq[i]); 756 if (rc) { 757 dev_err(rdev_to_dev(rdev), "Alloc Failed NQ%d rc:%#x", 758 i, rc); 759 goto dealloc_dpi; 760 } 761 rc = bnxt_re_net_ring_alloc 762 (rdev, rdev->nq[i].hwq.pbl[PBL_LVL_0].pg_map_arr, 763 rdev->nq[i].hwq.pbl[rdev->nq[i].hwq.level].pg_count, 764 HWRM_RING_ALLOC_CMPL, 765 BNXT_QPLIB_NQE_MAX_CNT - 1, 766 rdev->msix_entries[i + 1].ring_idx, 767 &rdev->nq[i].ring_id); 768 if (rc) { 769 dev_err(rdev_to_dev(rdev), 770 "Failed to allocate NQ fw id with rc = 0x%x", 771 rc); 772 goto free_nq; 773 } 774 } 775 return 0; 776 free_nq: 777 for (i = 0; i < rdev->num_msix - 1; i++) 778 bnxt_qplib_free_nq(&rdev->nq[i]); 779 dealloc_dpi: 780 bnxt_qplib_dealloc_dpi(&rdev->qplib_res, 781 &rdev->qplib_res.dpi_tbl, 782 &rdev->dpi_privileged); 783 dealloc_res: 784 bnxt_qplib_free_res(&rdev->qplib_res); 785 786 fail: 787 rdev->qplib_res.rcfw = NULL; 788 return rc; 789 } 790 791 static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp, 792 u8 port_num, enum ib_event_type event) 793 { 794 struct ib_event ib_event; 795 796 ib_event.device = ibdev; 797 if (qp) 798 ib_event.element.qp = qp; 799 else 800 ib_event.element.port_num = port_num; 801 ib_event.event = event; 802 ib_dispatch_event(&ib_event); 803 } 804 805 #define HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN 0x02 806 static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev *rdev, u8 dir, 807 u64 *cid_map) 808 { 809 struct hwrm_queue_pri2cos_qcfg_input req = {0}; 810 struct bnxt *bp = netdev_priv(rdev->netdev); 811 struct hwrm_queue_pri2cos_qcfg_output resp; 812 struct bnxt_en_dev *en_dev = rdev->en_dev; 813 struct bnxt_fw_msg fw_msg; 814 u32 flags = 0; 815 u8 *qcfgmap, *tmp_map; 816 int rc = 0, i; 817 818 if (!cid_map) 819 return -EINVAL; 820 821 memset(&fw_msg, 0, sizeof(fw_msg)); 822 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 823 HWRM_QUEUE_PRI2COS_QCFG, -1, -1); 824 flags |= (dir & 0x01); 825 flags |= HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN; 826 req.flags = cpu_to_le32(flags); 827 req.port_id = bp->pf.port_id; 828 829 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 830 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 831 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 832 if (rc) 833 return rc; 834 835 if (resp.queue_cfg_info) { 836 dev_warn(rdev_to_dev(rdev), 837 "Asymmetric cos queue configuration detected"); 838 dev_warn(rdev_to_dev(rdev), 839 " on device, QoS may not be fully functional\n"); 840 } 841 qcfgmap = &resp.pri0_cos_queue_id; 842 tmp_map = (u8 *)cid_map; 843 for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) 844 tmp_map[i] = qcfgmap[i]; 845 846 return rc; 847 } 848 849 static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev, 850 struct bnxt_re_qp *qp) 851 { 852 return (qp->ib_qp.qp_type == IB_QPT_GSI) || (qp == rdev->qp1_sqp); 853 } 854 855 static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev) 856 { 857 int mask = IB_QP_STATE; 858 struct ib_qp_attr qp_attr; 859 struct bnxt_re_qp *qp; 860 861 qp_attr.qp_state = IB_QPS_ERR; 862 mutex_lock(&rdev->qp_lock); 863 list_for_each_entry(qp, &rdev->qp_list, list) { 864 /* Modify the state of all QPs except QP1/Shadow QP */ 865 if (!bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) { 866 if (qp->qplib_qp.state != 867 CMDQ_MODIFY_QP_NEW_STATE_RESET && 868 qp->qplib_qp.state != 869 CMDQ_MODIFY_QP_NEW_STATE_ERR) { 870 bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp, 871 1, IB_EVENT_QP_FATAL); 872 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, mask, 873 NULL); 874 } 875 } 876 } 877 mutex_unlock(&rdev->qp_lock); 878 } 879 880 static int bnxt_re_update_gid(struct bnxt_re_dev *rdev) 881 { 882 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 883 struct bnxt_qplib_gid gid; 884 u16 gid_idx, index; 885 int rc = 0; 886 887 if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) 888 return 0; 889 890 if (!sgid_tbl) { 891 dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated"); 892 return -EINVAL; 893 } 894 895 for (index = 0; index < sgid_tbl->active; index++) { 896 gid_idx = sgid_tbl->hw_id[index]; 897 898 if (!memcmp(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero, 899 sizeof(bnxt_qplib_gid_zero))) 900 continue; 901 /* need to modify the VLAN enable setting of non VLAN GID only 902 * as setting is done for VLAN GID while adding GID 903 */ 904 if (sgid_tbl->vlan[index]) 905 continue; 906 907 memcpy(&gid, &sgid_tbl->tbl[index], sizeof(gid)); 908 909 rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx, 910 rdev->qplib_res.netdev->dev_addr); 911 } 912 913 return rc; 914 } 915 916 static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev) 917 { 918 u32 prio_map = 0, tmp_map = 0; 919 struct net_device *netdev; 920 struct dcb_app app; 921 922 netdev = rdev->netdev; 923 924 memset(&app, 0, sizeof(app)); 925 app.selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE; 926 app.protocol = ETH_P_IBOE; 927 tmp_map = dcb_ieee_getapp_mask(netdev, &app); 928 prio_map = tmp_map; 929 930 app.selector = IEEE_8021QAZ_APP_SEL_DGRAM; 931 app.protocol = ROCE_V2_UDP_DPORT; 932 tmp_map = dcb_ieee_getapp_mask(netdev, &app); 933 prio_map |= tmp_map; 934 935 return prio_map; 936 } 937 938 static void bnxt_re_parse_cid_map(u8 prio_map, u8 *cid_map, u16 *cosq) 939 { 940 u16 prio; 941 u8 id; 942 943 for (prio = 0, id = 0; prio < 8; prio++) { 944 if (prio_map & (1 << prio)) { 945 cosq[id] = cid_map[prio]; 946 id++; 947 if (id == 2) /* Max 2 tcs supported */ 948 break; 949 } 950 } 951 } 952 953 static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev) 954 { 955 u8 prio_map = 0; 956 u64 cid_map; 957 int rc; 958 959 /* Get priority for roce */ 960 prio_map = bnxt_re_get_priority_mask(rdev); 961 962 if (prio_map == rdev->cur_prio_map) 963 return 0; 964 rdev->cur_prio_map = prio_map; 965 /* Get cosq id for this priority */ 966 rc = bnxt_re_query_hwrm_pri2cos(rdev, 0, &cid_map); 967 if (rc) { 968 dev_warn(rdev_to_dev(rdev), "no cos for p_mask %x\n", prio_map); 969 return rc; 970 } 971 /* Parse CoS IDs for app priority */ 972 bnxt_re_parse_cid_map(prio_map, (u8 *)&cid_map, rdev->cosq); 973 974 /* Config BONO. */ 975 rc = bnxt_qplib_map_tc2cos(&rdev->qplib_res, rdev->cosq); 976 if (rc) { 977 dev_warn(rdev_to_dev(rdev), "no tc for cos{%x, %x}\n", 978 rdev->cosq[0], rdev->cosq[1]); 979 return rc; 980 } 981 982 /* Actual priorities are not programmed as they are already 983 * done by L2 driver; just enable or disable priority vlan tagging 984 */ 985 if ((prio_map == 0 && rdev->qplib_res.prio) || 986 (prio_map != 0 && !rdev->qplib_res.prio)) { 987 rdev->qplib_res.prio = prio_map ? true : false; 988 989 bnxt_re_update_gid(rdev); 990 } 991 992 return 0; 993 } 994 995 static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev, bool lock_wait) 996 { 997 int i, rc; 998 999 if (test_and_clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { 1000 for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) 1001 device_remove_file(&rdev->ibdev.dev, 1002 bnxt_re_attributes[i]); 1003 /* Cleanup ib dev */ 1004 bnxt_re_unregister_ib(rdev); 1005 } 1006 if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags)) 1007 cancel_delayed_work(&rdev->worker); 1008 1009 bnxt_re_cleanup_res(rdev); 1010 bnxt_re_free_res(rdev, lock_wait); 1011 1012 if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) { 1013 rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw); 1014 if (rc) 1015 dev_warn(rdev_to_dev(rdev), 1016 "Failed to deinitialize RCFW: %#x", rc); 1017 bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id, 1018 lock_wait); 1019 bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx); 1020 bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); 1021 bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, lock_wait); 1022 bnxt_qplib_free_rcfw_channel(&rdev->rcfw); 1023 } 1024 if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) { 1025 rc = bnxt_re_free_msix(rdev, lock_wait); 1026 if (rc) 1027 dev_warn(rdev_to_dev(rdev), 1028 "Failed to free MSI-X vectors: %#x", rc); 1029 } 1030 if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) { 1031 rc = bnxt_re_unregister_netdev(rdev, lock_wait); 1032 if (rc) 1033 dev_warn(rdev_to_dev(rdev), 1034 "Failed to unregister with netdev: %#x", rc); 1035 } 1036 } 1037 1038 static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev) 1039 { 1040 u32 i; 1041 1042 rdev->qplib_ctx.qpc_count = BNXT_RE_MAX_QPC_COUNT; 1043 rdev->qplib_ctx.mrw_count = BNXT_RE_MAX_MRW_COUNT; 1044 rdev->qplib_ctx.srqc_count = BNXT_RE_MAX_SRQC_COUNT; 1045 rdev->qplib_ctx.cq_count = BNXT_RE_MAX_CQ_COUNT; 1046 for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) 1047 rdev->qplib_ctx.tqm_count[i] = 1048 rdev->dev_attr.tqm_alloc_reqs[i]; 1049 } 1050 1051 /* worker thread for polling periodic events. Now used for QoS programming*/ 1052 static void bnxt_re_worker(struct work_struct *work) 1053 { 1054 struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, 1055 worker.work); 1056 1057 bnxt_re_setup_qos(rdev); 1058 schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000)); 1059 } 1060 1061 static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev) 1062 { 1063 int i, j, rc; 1064 1065 /* Registered a new RoCE device instance to netdev */ 1066 rc = bnxt_re_register_netdev(rdev); 1067 if (rc) { 1068 pr_err("Failed to register with netedev: %#x\n", rc); 1069 return -EINVAL; 1070 } 1071 set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); 1072 1073 rc = bnxt_re_request_msix(rdev); 1074 if (rc) { 1075 pr_err("Failed to get MSI-X vectors: %#x\n", rc); 1076 rc = -EINVAL; 1077 goto fail; 1078 } 1079 set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags); 1080 1081 /* Establish RCFW Communication Channel to initialize the context 1082 * memory for the function and all child VFs 1083 */ 1084 rc = bnxt_qplib_alloc_rcfw_channel(rdev->en_dev->pdev, &rdev->rcfw, 1085 BNXT_RE_MAX_QPC_COUNT); 1086 if (rc) { 1087 pr_err("Failed to allocate RCFW Channel: %#x\n", rc); 1088 goto fail; 1089 } 1090 rc = bnxt_re_net_ring_alloc 1091 (rdev, rdev->rcfw.creq.pbl[PBL_LVL_0].pg_map_arr, 1092 rdev->rcfw.creq.pbl[rdev->rcfw.creq.level].pg_count, 1093 HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_CREQE_MAX_CNT - 1, 1094 rdev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx, 1095 &rdev->rcfw.creq_ring_id); 1096 if (rc) { 1097 pr_err("Failed to allocate CREQ: %#x\n", rc); 1098 goto free_rcfw; 1099 } 1100 rc = bnxt_qplib_enable_rcfw_channel 1101 (rdev->en_dev->pdev, &rdev->rcfw, 1102 rdev->msix_entries[BNXT_RE_AEQ_IDX].vector, 1103 rdev->msix_entries[BNXT_RE_AEQ_IDX].db_offset, 1104 0, &bnxt_re_aeq_handler); 1105 if (rc) { 1106 pr_err("Failed to enable RCFW channel: %#x\n", rc); 1107 goto free_ring; 1108 } 1109 1110 rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr); 1111 if (rc) 1112 goto disable_rcfw; 1113 bnxt_re_set_resource_limits(rdev); 1114 1115 rc = bnxt_qplib_alloc_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx, 0); 1116 if (rc) { 1117 pr_err("Failed to allocate QPLIB context: %#x\n", rc); 1118 goto disable_rcfw; 1119 } 1120 rc = bnxt_re_net_stats_ctx_alloc(rdev, 1121 rdev->qplib_ctx.stats.dma_map, 1122 &rdev->qplib_ctx.stats.fw_id); 1123 if (rc) { 1124 pr_err("Failed to allocate stats context: %#x\n", rc); 1125 goto free_ctx; 1126 } 1127 1128 rc = bnxt_qplib_init_rcfw(&rdev->rcfw, &rdev->qplib_ctx, 0); 1129 if (rc) { 1130 pr_err("Failed to initialize RCFW: %#x\n", rc); 1131 goto free_sctx; 1132 } 1133 set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags); 1134 1135 /* Resources based on the 'new' device caps */ 1136 rc = bnxt_re_alloc_res(rdev); 1137 if (rc) { 1138 pr_err("Failed to allocate resources: %#x\n", rc); 1139 goto fail; 1140 } 1141 rc = bnxt_re_init_res(rdev); 1142 if (rc) { 1143 pr_err("Failed to initialize resources: %#x\n", rc); 1144 goto fail; 1145 } 1146 1147 rc = bnxt_re_setup_qos(rdev); 1148 if (rc) 1149 pr_info("RoCE priority not yet configured\n"); 1150 1151 INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker); 1152 set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags); 1153 schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000)); 1154 1155 /* Register ib dev */ 1156 rc = bnxt_re_register_ib(rdev); 1157 if (rc) { 1158 pr_err("Failed to register with IB: %#x\n", rc); 1159 goto fail; 1160 } 1161 dev_info(rdev_to_dev(rdev), "Device registered successfully"); 1162 for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) { 1163 rc = device_create_file(&rdev->ibdev.dev, 1164 bnxt_re_attributes[i]); 1165 if (rc) { 1166 dev_err(rdev_to_dev(rdev), 1167 "Failed to create IB sysfs: %#x", rc); 1168 /* Must clean up all created device files */ 1169 for (j = 0; j < i; j++) 1170 device_remove_file(&rdev->ibdev.dev, 1171 bnxt_re_attributes[j]); 1172 bnxt_re_unregister_ib(rdev); 1173 goto fail; 1174 } 1175 } 1176 set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); 1177 ib_get_eth_speed(&rdev->ibdev, 1, &rdev->active_speed, 1178 &rdev->active_width); 1179 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); 1180 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE); 1181 1182 return 0; 1183 free_sctx: 1184 bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id, true); 1185 free_ctx: 1186 bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx); 1187 disable_rcfw: 1188 bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); 1189 free_ring: 1190 bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, true); 1191 free_rcfw: 1192 bnxt_qplib_free_rcfw_channel(&rdev->rcfw); 1193 fail: 1194 bnxt_re_ib_unreg(rdev, true); 1195 return rc; 1196 } 1197 1198 static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev) 1199 { 1200 struct bnxt_en_dev *en_dev = rdev->en_dev; 1201 struct net_device *netdev = rdev->netdev; 1202 1203 bnxt_re_dev_remove(rdev); 1204 1205 if (netdev) 1206 bnxt_re_dev_unprobe(netdev, en_dev); 1207 } 1208 1209 static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev) 1210 { 1211 struct bnxt_en_dev *en_dev; 1212 int rc = 0; 1213 1214 if (!is_bnxt_re_dev(netdev)) 1215 return -ENODEV; 1216 1217 en_dev = bnxt_re_dev_probe(netdev); 1218 if (IS_ERR(en_dev)) { 1219 if (en_dev != ERR_PTR(-ENODEV)) 1220 pr_err("%s: Failed to probe\n", ROCE_DRV_MODULE_NAME); 1221 rc = PTR_ERR(en_dev); 1222 goto exit; 1223 } 1224 *rdev = bnxt_re_dev_add(netdev, en_dev); 1225 if (!*rdev) { 1226 rc = -ENOMEM; 1227 bnxt_re_dev_unprobe(netdev, en_dev); 1228 goto exit; 1229 } 1230 exit: 1231 return rc; 1232 } 1233 1234 static void bnxt_re_remove_one(struct bnxt_re_dev *rdev) 1235 { 1236 pci_dev_put(rdev->en_dev->pdev); 1237 } 1238 1239 /* Handle all deferred netevents tasks */ 1240 static void bnxt_re_task(struct work_struct *work) 1241 { 1242 struct bnxt_re_work *re_work; 1243 struct bnxt_re_dev *rdev; 1244 int rc = 0; 1245 1246 re_work = container_of(work, struct bnxt_re_work, work); 1247 rdev = re_work->rdev; 1248 1249 if (re_work->event != NETDEV_REGISTER && 1250 !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) 1251 return; 1252 1253 switch (re_work->event) { 1254 case NETDEV_REGISTER: 1255 rc = bnxt_re_ib_reg(rdev); 1256 if (rc) 1257 dev_err(rdev_to_dev(rdev), 1258 "Failed to register with IB: %#x", rc); 1259 break; 1260 case NETDEV_UP: 1261 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, 1262 IB_EVENT_PORT_ACTIVE); 1263 break; 1264 case NETDEV_DOWN: 1265 bnxt_re_dev_stop(rdev); 1266 break; 1267 case NETDEV_CHANGE: 1268 if (!netif_carrier_ok(rdev->netdev)) 1269 bnxt_re_dev_stop(rdev); 1270 else if (netif_carrier_ok(rdev->netdev)) 1271 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, 1272 IB_EVENT_PORT_ACTIVE); 1273 ib_get_eth_speed(&rdev->ibdev, 1, &rdev->active_speed, 1274 &rdev->active_width); 1275 break; 1276 default: 1277 break; 1278 } 1279 smp_mb__before_atomic(); 1280 clear_bit(BNXT_RE_FLAG_TASK_IN_PROG, &rdev->flags); 1281 kfree(re_work); 1282 } 1283 1284 static void bnxt_re_init_one(struct bnxt_re_dev *rdev) 1285 { 1286 pci_dev_get(rdev->en_dev->pdev); 1287 } 1288 1289 /* 1290 * "Notifier chain callback can be invoked for the same chain from 1291 * different CPUs at the same time". 1292 * 1293 * For cases when the netdev is already present, our call to the 1294 * register_netdevice_notifier() will actually get the rtnl_lock() 1295 * before sending NETDEV_REGISTER and (if up) NETDEV_UP 1296 * events. 1297 * 1298 * But for cases when the netdev is not already present, the notifier 1299 * chain is subjected to be invoked from different CPUs simultaneously. 1300 * 1301 * This is protected by the netdev_mutex. 1302 */ 1303 static int bnxt_re_netdev_event(struct notifier_block *notifier, 1304 unsigned long event, void *ptr) 1305 { 1306 struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(ptr); 1307 struct bnxt_re_work *re_work; 1308 struct bnxt_re_dev *rdev; 1309 int rc = 0; 1310 bool sch_work = false; 1311 1312 real_dev = rdma_vlan_dev_real_dev(netdev); 1313 if (!real_dev) 1314 real_dev = netdev; 1315 1316 rdev = bnxt_re_from_netdev(real_dev); 1317 if (!rdev && event != NETDEV_REGISTER) 1318 goto exit; 1319 if (real_dev != netdev) 1320 goto exit; 1321 1322 switch (event) { 1323 case NETDEV_REGISTER: 1324 if (rdev) 1325 break; 1326 rc = bnxt_re_dev_reg(&rdev, real_dev); 1327 if (rc == -ENODEV) 1328 break; 1329 if (rc) { 1330 pr_err("Failed to register with the device %s: %#x\n", 1331 real_dev->name, rc); 1332 break; 1333 } 1334 bnxt_re_init_one(rdev); 1335 sch_work = true; 1336 break; 1337 1338 case NETDEV_UNREGISTER: 1339 /* netdev notifier will call NETDEV_UNREGISTER again later since 1340 * we are still holding the reference to the netdev 1341 */ 1342 if (test_bit(BNXT_RE_FLAG_TASK_IN_PROG, &rdev->flags)) 1343 goto exit; 1344 bnxt_re_ib_unreg(rdev, false); 1345 bnxt_re_remove_one(rdev); 1346 bnxt_re_dev_unreg(rdev); 1347 break; 1348 1349 default: 1350 sch_work = true; 1351 break; 1352 } 1353 if (sch_work) { 1354 /* Allocate for the deferred task */ 1355 re_work = kzalloc(sizeof(*re_work), GFP_ATOMIC); 1356 if (re_work) { 1357 re_work->rdev = rdev; 1358 re_work->event = event; 1359 re_work->vlan_dev = (real_dev == netdev ? 1360 NULL : netdev); 1361 INIT_WORK(&re_work->work, bnxt_re_task); 1362 set_bit(BNXT_RE_FLAG_TASK_IN_PROG, &rdev->flags); 1363 queue_work(bnxt_re_wq, &re_work->work); 1364 } 1365 } 1366 1367 exit: 1368 return NOTIFY_DONE; 1369 } 1370 1371 static struct notifier_block bnxt_re_netdev_notifier = { 1372 .notifier_call = bnxt_re_netdev_event 1373 }; 1374 1375 static int __init bnxt_re_mod_init(void) 1376 { 1377 int rc = 0; 1378 1379 pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version); 1380 1381 bnxt_re_wq = create_singlethread_workqueue("bnxt_re"); 1382 if (!bnxt_re_wq) 1383 return -ENOMEM; 1384 1385 INIT_LIST_HEAD(&bnxt_re_dev_list); 1386 1387 rc = register_netdevice_notifier(&bnxt_re_netdev_notifier); 1388 if (rc) { 1389 pr_err("%s: Cannot register to netdevice_notifier", 1390 ROCE_DRV_MODULE_NAME); 1391 goto err_netdev; 1392 } 1393 return 0; 1394 1395 err_netdev: 1396 destroy_workqueue(bnxt_re_wq); 1397 1398 return rc; 1399 } 1400 1401 static void __exit bnxt_re_mod_exit(void) 1402 { 1403 struct bnxt_re_dev *rdev; 1404 LIST_HEAD(to_be_deleted); 1405 1406 mutex_lock(&bnxt_re_dev_lock); 1407 /* Free all adapter allocated resources */ 1408 if (!list_empty(&bnxt_re_dev_list)) 1409 list_splice_init(&bnxt_re_dev_list, &to_be_deleted); 1410 mutex_unlock(&bnxt_re_dev_lock); 1411 1412 list_for_each_entry(rdev, &to_be_deleted, list) { 1413 dev_info(rdev_to_dev(rdev), "Unregistering Device"); 1414 bnxt_re_dev_stop(rdev); 1415 bnxt_re_ib_unreg(rdev, true); 1416 bnxt_re_remove_one(rdev); 1417 bnxt_re_dev_unreg(rdev); 1418 } 1419 unregister_netdevice_notifier(&bnxt_re_netdev_notifier); 1420 if (bnxt_re_wq) 1421 destroy_workqueue(bnxt_re_wq); 1422 } 1423 1424 module_init(bnxt_re_mod_init); 1425 module_exit(bnxt_re_mod_exit); 1426