1 /******************************************************************* 2 * This file is part of the Emulex RoCE Device Driver for * 3 * RoCE (RDMA over Converged Ethernet) adapters. * 4 * Copyright (C) 2008-2012 Emulex. All rights reserved. * 5 * EMULEX and SLI are trademarks of Emulex. * 6 * www.emulex.com * 7 * * 8 * This program is free software; you can redistribute it and/or * 9 * modify it under the terms of version 2 of the GNU General * 10 * Public License as published by the Free Software Foundation. * 11 * This program is distributed in the hope that it will be useful. * 12 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 13 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 14 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 15 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 16 * TO BE LEGALLY INVALID. See the GNU General Public License for * 17 * more details, a copy of which can be found in the file COPYING * 18 * included with this package. * 19 * 20 * Contact Information: 21 * linux-drivers@emulex.com 22 * 23 * Emulex 24 * 3333 Susan Street 25 * Costa Mesa, CA 92626 26 *******************************************************************/ 27 28 #include <linux/module.h> 29 #include <linux/idr.h> 30 #include <rdma/ib_verbs.h> 31 #include <rdma/ib_user_verbs.h> 32 #include <rdma/ib_addr.h> 33 34 #include <linux/netdevice.h> 35 #include <net/addrconf.h> 36 37 #include "ocrdma.h" 38 #include "ocrdma_verbs.h" 39 #include "ocrdma_ah.h" 40 #include "be_roce.h" 41 #include "ocrdma_hw.h" 42 43 MODULE_VERSION(OCRDMA_ROCE_DEV_VERSION); 44 MODULE_DESCRIPTION("Emulex RoCE HCA Driver"); 45 MODULE_AUTHOR("Emulex Corporation"); 46 MODULE_LICENSE("GPL"); 47 48 static LIST_HEAD(ocrdma_dev_list); 49 static DEFINE_SPINLOCK(ocrdma_devlist_lock); 50 static DEFINE_IDR(ocrdma_dev_id); 51 52 static union ib_gid ocrdma_zero_sgid; 53 54 void ocrdma_get_guid(struct ocrdma_dev *dev, u8 *guid) 55 { 56 u8 mac_addr[6]; 57 58 memcpy(&mac_addr[0], &dev->nic_info.mac_addr[0], ETH_ALEN); 59 guid[0] = mac_addr[0] ^ 2; 60 guid[1] = mac_addr[1]; 61 guid[2] = mac_addr[2]; 62 guid[3] = 0xff; 63 guid[4] = 0xfe; 64 guid[5] = mac_addr[3]; 65 guid[6] = mac_addr[4]; 66 guid[7] = mac_addr[5]; 67 } 68 69 static void ocrdma_build_sgid_mac(union ib_gid *sgid, unsigned char *mac_addr, 70 bool is_vlan, u16 vlan_id) 71 { 72 sgid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL); 73 sgid->raw[8] = mac_addr[0] ^ 2; 74 sgid->raw[9] = mac_addr[1]; 75 sgid->raw[10] = mac_addr[2]; 76 if (is_vlan) { 77 sgid->raw[11] = vlan_id >> 8; 78 sgid->raw[12] = vlan_id & 0xff; 79 } else { 80 sgid->raw[11] = 0xff; 81 sgid->raw[12] = 0xfe; 82 } 83 sgid->raw[13] = mac_addr[3]; 84 sgid->raw[14] = mac_addr[4]; 85 sgid->raw[15] = mac_addr[5]; 86 } 87 88 static bool ocrdma_add_sgid(struct ocrdma_dev *dev, unsigned char *mac_addr, 89 bool is_vlan, u16 vlan_id) 90 { 91 int i; 92 union ib_gid new_sgid; 93 unsigned long flags; 94 95 memset(&ocrdma_zero_sgid, 0, sizeof(union ib_gid)); 96 97 ocrdma_build_sgid_mac(&new_sgid, mac_addr, is_vlan, vlan_id); 98 99 spin_lock_irqsave(&dev->sgid_lock, flags); 100 for (i = 0; i < OCRDMA_MAX_SGID; i++) { 101 if (!memcmp(&dev->sgid_tbl[i], &ocrdma_zero_sgid, 102 sizeof(union ib_gid))) { 103 /* found free entry */ 104 memcpy(&dev->sgid_tbl[i], &new_sgid, 105 sizeof(union ib_gid)); 106 spin_unlock_irqrestore(&dev->sgid_lock, flags); 107 return true; 108 } else if (!memcmp(&dev->sgid_tbl[i], &new_sgid, 109 sizeof(union ib_gid))) { 110 /* entry already present, no addition is required. */ 111 spin_unlock_irqrestore(&dev->sgid_lock, flags); 112 return false; 113 } 114 } 115 spin_unlock_irqrestore(&dev->sgid_lock, flags); 116 return false; 117 } 118 119 static bool ocrdma_del_sgid(struct ocrdma_dev *dev, unsigned char *mac_addr, 120 bool is_vlan, u16 vlan_id) 121 { 122 int found = false; 123 int i; 124 union ib_gid sgid; 125 unsigned long flags; 126 127 ocrdma_build_sgid_mac(&sgid, mac_addr, is_vlan, vlan_id); 128 129 spin_lock_irqsave(&dev->sgid_lock, flags); 130 /* first is default sgid, which cannot be deleted. */ 131 for (i = 1; i < OCRDMA_MAX_SGID; i++) { 132 if (!memcmp(&dev->sgid_tbl[i], &sgid, sizeof(union ib_gid))) { 133 /* found matching entry */ 134 memset(&dev->sgid_tbl[i], 0, sizeof(union ib_gid)); 135 found = true; 136 break; 137 } 138 } 139 spin_unlock_irqrestore(&dev->sgid_lock, flags); 140 return found; 141 } 142 143 static void ocrdma_add_default_sgid(struct ocrdma_dev *dev) 144 { 145 /* GID Index 0 - Invariant manufacturer-assigned EUI-64 */ 146 union ib_gid *sgid = &dev->sgid_tbl[0]; 147 148 sgid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL); 149 ocrdma_get_guid(dev, &sgid->raw[8]); 150 } 151 152 #if IS_ENABLED(CONFIG_VLAN_8021Q) 153 static void ocrdma_add_vlan_sgids(struct ocrdma_dev *dev) 154 { 155 struct net_device *netdev, *tmp; 156 u16 vlan_id; 157 bool is_vlan; 158 159 netdev = dev->nic_info.netdev; 160 161 rcu_read_lock(); 162 for_each_netdev_rcu(&init_net, tmp) { 163 if (netdev == tmp || vlan_dev_real_dev(tmp) == netdev) { 164 if (!netif_running(tmp) || !netif_oper_up(tmp)) 165 continue; 166 if (netdev != tmp) { 167 vlan_id = vlan_dev_vlan_id(tmp); 168 is_vlan = true; 169 } else { 170 is_vlan = false; 171 vlan_id = 0; 172 tmp = netdev; 173 } 174 ocrdma_add_sgid(dev, tmp->dev_addr, is_vlan, vlan_id); 175 } 176 } 177 rcu_read_unlock(); 178 } 179 #else 180 static void ocrdma_add_vlan_sgids(struct ocrdma_dev *dev) 181 { 182 183 } 184 #endif /* VLAN */ 185 186 static int ocrdma_build_sgid_tbl(struct ocrdma_dev *dev) 187 { 188 ocrdma_add_default_sgid(dev); 189 ocrdma_add_vlan_sgids(dev); 190 return 0; 191 } 192 193 #if IS_ENABLED(CONFIG_IPV6) 194 195 static int ocrdma_inet6addr_event(struct notifier_block *notifier, 196 unsigned long event, void *ptr) 197 { 198 struct inet6_ifaddr *ifa = (struct inet6_ifaddr *)ptr; 199 struct net_device *netdev = ifa->idev->dev; 200 struct ib_event gid_event; 201 struct ocrdma_dev *dev; 202 bool found = false; 203 bool updated = false; 204 bool is_vlan = false; 205 u16 vid = 0; 206 207 is_vlan = netdev->priv_flags & IFF_802_1Q_VLAN; 208 if (is_vlan) { 209 vid = vlan_dev_vlan_id(netdev); 210 netdev = vlan_dev_real_dev(netdev); 211 } 212 213 rcu_read_lock(); 214 list_for_each_entry_rcu(dev, &ocrdma_dev_list, entry) { 215 if (dev->nic_info.netdev == netdev) { 216 found = true; 217 break; 218 } 219 } 220 rcu_read_unlock(); 221 222 if (!found) 223 return NOTIFY_DONE; 224 if (!rdma_link_local_addr((struct in6_addr *)&ifa->addr)) 225 return NOTIFY_DONE; 226 227 mutex_lock(&dev->dev_lock); 228 switch (event) { 229 case NETDEV_UP: 230 updated = ocrdma_add_sgid(dev, netdev->dev_addr, is_vlan, vid); 231 break; 232 case NETDEV_DOWN: 233 updated = ocrdma_del_sgid(dev, netdev->dev_addr, is_vlan, vid); 234 break; 235 default: 236 break; 237 } 238 if (updated) { 239 /* GID table updated, notify the consumers about it */ 240 gid_event.device = &dev->ibdev; 241 gid_event.element.port_num = 1; 242 gid_event.event = IB_EVENT_GID_CHANGE; 243 ib_dispatch_event(&gid_event); 244 } 245 mutex_unlock(&dev->dev_lock); 246 return NOTIFY_OK; 247 } 248 249 static struct notifier_block ocrdma_inet6addr_notifier = { 250 .notifier_call = ocrdma_inet6addr_event 251 }; 252 253 #endif /* IPV6 and VLAN */ 254 255 static enum rdma_link_layer ocrdma_link_layer(struct ib_device *device, 256 u8 port_num) 257 { 258 return IB_LINK_LAYER_ETHERNET; 259 } 260 261 static int ocrdma_register_device(struct ocrdma_dev *dev) 262 { 263 strlcpy(dev->ibdev.name, "ocrdma%d", IB_DEVICE_NAME_MAX); 264 ocrdma_get_guid(dev, (u8 *)&dev->ibdev.node_guid); 265 memcpy(dev->ibdev.node_desc, OCRDMA_NODE_DESC, 266 sizeof(OCRDMA_NODE_DESC)); 267 dev->ibdev.owner = THIS_MODULE; 268 dev->ibdev.uverbs_cmd_mask = 269 OCRDMA_UVERBS(GET_CONTEXT) | 270 OCRDMA_UVERBS(QUERY_DEVICE) | 271 OCRDMA_UVERBS(QUERY_PORT) | 272 OCRDMA_UVERBS(ALLOC_PD) | 273 OCRDMA_UVERBS(DEALLOC_PD) | 274 OCRDMA_UVERBS(REG_MR) | 275 OCRDMA_UVERBS(DEREG_MR) | 276 OCRDMA_UVERBS(CREATE_COMP_CHANNEL) | 277 OCRDMA_UVERBS(CREATE_CQ) | 278 OCRDMA_UVERBS(RESIZE_CQ) | 279 OCRDMA_UVERBS(DESTROY_CQ) | 280 OCRDMA_UVERBS(REQ_NOTIFY_CQ) | 281 OCRDMA_UVERBS(CREATE_QP) | 282 OCRDMA_UVERBS(MODIFY_QP) | 283 OCRDMA_UVERBS(QUERY_QP) | 284 OCRDMA_UVERBS(DESTROY_QP) | 285 OCRDMA_UVERBS(POLL_CQ) | 286 OCRDMA_UVERBS(POST_SEND) | 287 OCRDMA_UVERBS(POST_RECV); 288 289 dev->ibdev.uverbs_cmd_mask |= 290 OCRDMA_UVERBS(CREATE_AH) | 291 OCRDMA_UVERBS(MODIFY_AH) | 292 OCRDMA_UVERBS(QUERY_AH) | 293 OCRDMA_UVERBS(DESTROY_AH); 294 295 dev->ibdev.node_type = RDMA_NODE_IB_CA; 296 dev->ibdev.phys_port_cnt = 1; 297 dev->ibdev.num_comp_vectors = 1; 298 299 /* mandatory verbs. */ 300 dev->ibdev.query_device = ocrdma_query_device; 301 dev->ibdev.query_port = ocrdma_query_port; 302 dev->ibdev.modify_port = ocrdma_modify_port; 303 dev->ibdev.query_gid = ocrdma_query_gid; 304 dev->ibdev.get_link_layer = ocrdma_link_layer; 305 dev->ibdev.alloc_pd = ocrdma_alloc_pd; 306 dev->ibdev.dealloc_pd = ocrdma_dealloc_pd; 307 308 dev->ibdev.create_cq = ocrdma_create_cq; 309 dev->ibdev.destroy_cq = ocrdma_destroy_cq; 310 dev->ibdev.resize_cq = ocrdma_resize_cq; 311 312 dev->ibdev.create_qp = ocrdma_create_qp; 313 dev->ibdev.modify_qp = ocrdma_modify_qp; 314 dev->ibdev.query_qp = ocrdma_query_qp; 315 dev->ibdev.destroy_qp = ocrdma_destroy_qp; 316 317 dev->ibdev.query_pkey = ocrdma_query_pkey; 318 dev->ibdev.create_ah = ocrdma_create_ah; 319 dev->ibdev.destroy_ah = ocrdma_destroy_ah; 320 dev->ibdev.query_ah = ocrdma_query_ah; 321 dev->ibdev.modify_ah = ocrdma_modify_ah; 322 323 dev->ibdev.poll_cq = ocrdma_poll_cq; 324 dev->ibdev.post_send = ocrdma_post_send; 325 dev->ibdev.post_recv = ocrdma_post_recv; 326 dev->ibdev.req_notify_cq = ocrdma_arm_cq; 327 328 dev->ibdev.get_dma_mr = ocrdma_get_dma_mr; 329 dev->ibdev.dereg_mr = ocrdma_dereg_mr; 330 dev->ibdev.reg_user_mr = ocrdma_reg_user_mr; 331 332 /* mandatory to support user space verbs consumer. */ 333 dev->ibdev.alloc_ucontext = ocrdma_alloc_ucontext; 334 dev->ibdev.dealloc_ucontext = ocrdma_dealloc_ucontext; 335 dev->ibdev.mmap = ocrdma_mmap; 336 dev->ibdev.dma_device = &dev->nic_info.pdev->dev; 337 338 dev->ibdev.process_mad = ocrdma_process_mad; 339 340 if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) { 341 dev->ibdev.uverbs_cmd_mask |= 342 OCRDMA_UVERBS(CREATE_SRQ) | 343 OCRDMA_UVERBS(MODIFY_SRQ) | 344 OCRDMA_UVERBS(QUERY_SRQ) | 345 OCRDMA_UVERBS(DESTROY_SRQ) | 346 OCRDMA_UVERBS(POST_SRQ_RECV); 347 348 dev->ibdev.create_srq = ocrdma_create_srq; 349 dev->ibdev.modify_srq = ocrdma_modify_srq; 350 dev->ibdev.query_srq = ocrdma_query_srq; 351 dev->ibdev.destroy_srq = ocrdma_destroy_srq; 352 dev->ibdev.post_srq_recv = ocrdma_post_srq_recv; 353 } 354 return ib_register_device(&dev->ibdev, NULL); 355 } 356 357 static int ocrdma_alloc_resources(struct ocrdma_dev *dev) 358 { 359 mutex_init(&dev->dev_lock); 360 dev->sgid_tbl = kzalloc(sizeof(union ib_gid) * 361 OCRDMA_MAX_SGID, GFP_KERNEL); 362 if (!dev->sgid_tbl) 363 goto alloc_err; 364 spin_lock_init(&dev->sgid_lock); 365 366 dev->cq_tbl = kzalloc(sizeof(struct ocrdma_cq *) * 367 OCRDMA_MAX_CQ, GFP_KERNEL); 368 if (!dev->cq_tbl) 369 goto alloc_err; 370 371 if (dev->attr.max_qp) { 372 dev->qp_tbl = kzalloc(sizeof(struct ocrdma_qp *) * 373 OCRDMA_MAX_QP, GFP_KERNEL); 374 if (!dev->qp_tbl) 375 goto alloc_err; 376 } 377 spin_lock_init(&dev->av_tbl.lock); 378 spin_lock_init(&dev->flush_q_lock); 379 return 0; 380 alloc_err: 381 ocrdma_err("%s(%d) error.\n", __func__, dev->id); 382 return -ENOMEM; 383 } 384 385 static void ocrdma_free_resources(struct ocrdma_dev *dev) 386 { 387 kfree(dev->qp_tbl); 388 kfree(dev->cq_tbl); 389 kfree(dev->sgid_tbl); 390 } 391 392 static struct ocrdma_dev *ocrdma_add(struct be_dev_info *dev_info) 393 { 394 int status = 0; 395 struct ocrdma_dev *dev; 396 397 dev = (struct ocrdma_dev *)ib_alloc_device(sizeof(struct ocrdma_dev)); 398 if (!dev) { 399 ocrdma_err("Unable to allocate ib device\n"); 400 return NULL; 401 } 402 dev->mbx_cmd = kzalloc(sizeof(struct ocrdma_mqe_emb_cmd), GFP_KERNEL); 403 if (!dev->mbx_cmd) 404 goto idr_err; 405 406 memcpy(&dev->nic_info, dev_info, sizeof(*dev_info)); 407 dev->id = idr_alloc(&ocrdma_dev_id, NULL, 0, 0, GFP_KERNEL); 408 if (dev->id < 0) 409 goto idr_err; 410 411 status = ocrdma_init_hw(dev); 412 if (status) 413 goto init_err; 414 415 status = ocrdma_alloc_resources(dev); 416 if (status) 417 goto alloc_err; 418 419 status = ocrdma_build_sgid_tbl(dev); 420 if (status) 421 goto alloc_err; 422 423 status = ocrdma_register_device(dev); 424 if (status) 425 goto alloc_err; 426 427 spin_lock(&ocrdma_devlist_lock); 428 list_add_tail_rcu(&dev->entry, &ocrdma_dev_list); 429 spin_unlock(&ocrdma_devlist_lock); 430 return dev; 431 432 alloc_err: 433 ocrdma_free_resources(dev); 434 ocrdma_cleanup_hw(dev); 435 init_err: 436 idr_remove(&ocrdma_dev_id, dev->id); 437 idr_err: 438 kfree(dev->mbx_cmd); 439 ib_dealloc_device(&dev->ibdev); 440 ocrdma_err("%s() leaving. ret=%d\n", __func__, status); 441 return NULL; 442 } 443 444 static void ocrdma_remove_free(struct rcu_head *rcu) 445 { 446 struct ocrdma_dev *dev = container_of(rcu, struct ocrdma_dev, rcu); 447 448 ocrdma_free_resources(dev); 449 ocrdma_cleanup_hw(dev); 450 451 idr_remove(&ocrdma_dev_id, dev->id); 452 kfree(dev->mbx_cmd); 453 ib_dealloc_device(&dev->ibdev); 454 } 455 456 static void ocrdma_remove(struct ocrdma_dev *dev) 457 { 458 /* first unregister with stack to stop all the active traffic 459 * of the registered clients. 460 */ 461 ib_unregister_device(&dev->ibdev); 462 463 spin_lock(&ocrdma_devlist_lock); 464 list_del_rcu(&dev->entry); 465 spin_unlock(&ocrdma_devlist_lock); 466 call_rcu(&dev->rcu, ocrdma_remove_free); 467 } 468 469 static int ocrdma_open(struct ocrdma_dev *dev) 470 { 471 struct ib_event port_event; 472 473 port_event.event = IB_EVENT_PORT_ACTIVE; 474 port_event.element.port_num = 1; 475 port_event.device = &dev->ibdev; 476 ib_dispatch_event(&port_event); 477 return 0; 478 } 479 480 static int ocrdma_close(struct ocrdma_dev *dev) 481 { 482 int i; 483 struct ocrdma_qp *qp, **cur_qp; 484 struct ib_event err_event; 485 struct ib_qp_attr attrs; 486 int attr_mask = IB_QP_STATE; 487 488 attrs.qp_state = IB_QPS_ERR; 489 mutex_lock(&dev->dev_lock); 490 if (dev->qp_tbl) { 491 cur_qp = dev->qp_tbl; 492 for (i = 0; i < OCRDMA_MAX_QP; i++) { 493 qp = cur_qp[i]; 494 if (qp) { 495 /* change the QP state to ERROR */ 496 _ocrdma_modify_qp(&qp->ibqp, &attrs, attr_mask); 497 498 err_event.event = IB_EVENT_QP_FATAL; 499 err_event.element.qp = &qp->ibqp; 500 err_event.device = &dev->ibdev; 501 ib_dispatch_event(&err_event); 502 } 503 } 504 } 505 mutex_unlock(&dev->dev_lock); 506 507 err_event.event = IB_EVENT_PORT_ERR; 508 err_event.element.port_num = 1; 509 err_event.device = &dev->ibdev; 510 ib_dispatch_event(&err_event); 511 return 0; 512 } 513 514 /* event handling via NIC driver ensures that all the NIC specific 515 * initialization done before RoCE driver notifies 516 * event to stack. 517 */ 518 static void ocrdma_event_handler(struct ocrdma_dev *dev, u32 event) 519 { 520 switch (event) { 521 case BE_DEV_UP: 522 ocrdma_open(dev); 523 break; 524 case BE_DEV_DOWN: 525 ocrdma_close(dev); 526 break; 527 }; 528 } 529 530 static struct ocrdma_driver ocrdma_drv = { 531 .name = "ocrdma_driver", 532 .add = ocrdma_add, 533 .remove = ocrdma_remove, 534 .state_change_handler = ocrdma_event_handler, 535 }; 536 537 static void ocrdma_unregister_inet6addr_notifier(void) 538 { 539 #if IS_ENABLED(CONFIG_IPV6) 540 unregister_inet6addr_notifier(&ocrdma_inet6addr_notifier); 541 #endif 542 } 543 544 static int __init ocrdma_init_module(void) 545 { 546 int status; 547 548 #if IS_ENABLED(CONFIG_IPV6) 549 status = register_inet6addr_notifier(&ocrdma_inet6addr_notifier); 550 if (status) 551 return status; 552 #endif 553 554 status = be_roce_register_driver(&ocrdma_drv); 555 if (status) 556 ocrdma_unregister_inet6addr_notifier(); 557 558 return status; 559 } 560 561 static void __exit ocrdma_exit_module(void) 562 { 563 be_roce_unregister_driver(&ocrdma_drv); 564 ocrdma_unregister_inet6addr_notifier(); 565 } 566 567 module_init(ocrdma_init_module); 568 module_exit(ocrdma_exit_module); 569