1 /* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 4 * Copyright (c) 2004 Voltaire, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include "ipoib.h" 36 37 #include <linux/module.h> 38 39 #include <linux/init.h> 40 #include <linux/slab.h> 41 #include <linux/kernel.h> 42 #include <linux/vmalloc.h> 43 44 #include <linux/if_arp.h> /* For ARPHRD_xxx */ 45 46 #include <linux/ip.h> 47 #include <linux/in.h> 48 49 #include <linux/jhash.h> 50 #include <net/arp.h> 51 52 #define DRV_VERSION "1.0.0" 53 54 const char ipoib_driver_version[] = DRV_VERSION; 55 56 MODULE_AUTHOR("Roland Dreier"); 57 MODULE_DESCRIPTION("IP-over-InfiniBand net driver"); 58 MODULE_LICENSE("Dual BSD/GPL"); 59 MODULE_VERSION(DRV_VERSION); 60 61 int ipoib_sendq_size __read_mostly = IPOIB_TX_RING_SIZE; 62 int ipoib_recvq_size __read_mostly = IPOIB_RX_RING_SIZE; 63 64 module_param_named(send_queue_size, ipoib_sendq_size, int, 0444); 65 MODULE_PARM_DESC(send_queue_size, "Number of descriptors in send queue"); 66 module_param_named(recv_queue_size, ipoib_recvq_size, int, 0444); 67 MODULE_PARM_DESC(recv_queue_size, "Number of descriptors in receive queue"); 68 69 #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG 70 int ipoib_debug_level; 71 72 module_param_named(debug_level, ipoib_debug_level, int, 0644); 73 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0"); 74 #endif 75 76 struct ipoib_path_iter { 77 struct net_device *dev; 78 struct ipoib_path path; 79 }; 80 81 static const u8 ipv4_bcast_addr[] = { 82 0x00, 0xff, 0xff, 0xff, 83 0xff, 0x12, 0x40, 0x1b, 0x00, 0x00, 0x00, 0x00, 84 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff 85 }; 86 87 struct workqueue_struct *ipoib_workqueue; 88 89 struct ib_sa_client ipoib_sa_client; 90 91 static void ipoib_add_one(struct ib_device *device); 92 static void ipoib_remove_one(struct ib_device *device); 93 static void ipoib_neigh_reclaim(struct rcu_head *rp); 94 95 static struct ib_client ipoib_client = { 96 .name = "ipoib", 97 .add = ipoib_add_one, 98 .remove = ipoib_remove_one 99 }; 100 101 int ipoib_open(struct net_device *dev) 102 { 103 struct ipoib_dev_priv *priv = netdev_priv(dev); 104 105 ipoib_dbg(priv, "bringing up interface\n"); 106 107 netif_carrier_off(dev); 108 109 set_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags); 110 111 if (ipoib_ib_dev_open(dev)) { 112 if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) 113 return 0; 114 goto err_disable; 115 } 116 117 if (ipoib_ib_dev_up(dev)) 118 goto err_stop; 119 120 if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) { 121 struct ipoib_dev_priv *cpriv; 122 123 /* Bring up any child interfaces too */ 124 down_read(&priv->vlan_rwsem); 125 list_for_each_entry(cpriv, &priv->child_intfs, list) { 126 int flags; 127 128 flags = cpriv->dev->flags; 129 if (flags & IFF_UP) 130 continue; 131 132 dev_change_flags(cpriv->dev, flags | IFF_UP); 133 } 134 up_read(&priv->vlan_rwsem); 135 } 136 137 netif_start_queue(dev); 138 139 return 0; 140 141 err_stop: 142 ipoib_ib_dev_stop(dev); 143 144 err_disable: 145 clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags); 146 147 return -EINVAL; 148 } 149 150 static int ipoib_stop(struct net_device *dev) 151 { 152 struct ipoib_dev_priv *priv = netdev_priv(dev); 153 154 ipoib_dbg(priv, "stopping interface\n"); 155 156 clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags); 157 158 netif_stop_queue(dev); 159 160 ipoib_ib_dev_down(dev); 161 ipoib_ib_dev_stop(dev); 162 163 if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) { 164 struct ipoib_dev_priv *cpriv; 165 166 /* Bring down any child interfaces too */ 167 down_read(&priv->vlan_rwsem); 168 list_for_each_entry(cpriv, &priv->child_intfs, list) { 169 int flags; 170 171 flags = cpriv->dev->flags; 172 if (!(flags & IFF_UP)) 173 continue; 174 175 dev_change_flags(cpriv->dev, flags & ~IFF_UP); 176 } 177 up_read(&priv->vlan_rwsem); 178 } 179 180 return 0; 181 } 182 183 static void ipoib_uninit(struct net_device *dev) 184 { 185 ipoib_dev_cleanup(dev); 186 } 187 188 static netdev_features_t ipoib_fix_features(struct net_device *dev, netdev_features_t features) 189 { 190 struct ipoib_dev_priv *priv = netdev_priv(dev); 191 192 if (test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags)) 193 features &= ~(NETIF_F_IP_CSUM | NETIF_F_TSO); 194 195 return features; 196 } 197 198 static int ipoib_change_mtu(struct net_device *dev, int new_mtu) 199 { 200 struct ipoib_dev_priv *priv = netdev_priv(dev); 201 202 /* dev->mtu > 2K ==> connected mode */ 203 if (ipoib_cm_admin_enabled(dev)) { 204 if (new_mtu > ipoib_cm_max_mtu(dev)) 205 return -EINVAL; 206 207 if (new_mtu > priv->mcast_mtu) 208 ipoib_warn(priv, "mtu > %d will cause multicast packet drops.\n", 209 priv->mcast_mtu); 210 211 dev->mtu = new_mtu; 212 return 0; 213 } 214 215 if (new_mtu > IPOIB_UD_MTU(priv->max_ib_mtu)) 216 return -EINVAL; 217 218 priv->admin_mtu = new_mtu; 219 220 dev->mtu = min(priv->mcast_mtu, priv->admin_mtu); 221 222 return 0; 223 } 224 225 int ipoib_set_mode(struct net_device *dev, const char *buf) 226 { 227 struct ipoib_dev_priv *priv = netdev_priv(dev); 228 229 /* flush paths if we switch modes so that connections are restarted */ 230 if (IPOIB_CM_SUPPORTED(dev->dev_addr) && !strcmp(buf, "connected\n")) { 231 set_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags); 232 ipoib_warn(priv, "enabling connected mode " 233 "will cause multicast packet drops\n"); 234 netdev_update_features(dev); 235 dev_set_mtu(dev, ipoib_cm_max_mtu(dev)); 236 rtnl_unlock(); 237 priv->tx_wr.send_flags &= ~IB_SEND_IP_CSUM; 238 239 ipoib_flush_paths(dev); 240 rtnl_lock(); 241 return 0; 242 } 243 244 if (!strcmp(buf, "datagram\n")) { 245 clear_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags); 246 netdev_update_features(dev); 247 dev_set_mtu(dev, min(priv->mcast_mtu, dev->mtu)); 248 rtnl_unlock(); 249 ipoib_flush_paths(dev); 250 rtnl_lock(); 251 return 0; 252 } 253 254 return -EINVAL; 255 } 256 257 static struct ipoib_path *__path_find(struct net_device *dev, void *gid) 258 { 259 struct ipoib_dev_priv *priv = netdev_priv(dev); 260 struct rb_node *n = priv->path_tree.rb_node; 261 struct ipoib_path *path; 262 int ret; 263 264 while (n) { 265 path = rb_entry(n, struct ipoib_path, rb_node); 266 267 ret = memcmp(gid, path->pathrec.dgid.raw, 268 sizeof (union ib_gid)); 269 270 if (ret < 0) 271 n = n->rb_left; 272 else if (ret > 0) 273 n = n->rb_right; 274 else 275 return path; 276 } 277 278 return NULL; 279 } 280 281 static int __path_add(struct net_device *dev, struct ipoib_path *path) 282 { 283 struct ipoib_dev_priv *priv = netdev_priv(dev); 284 struct rb_node **n = &priv->path_tree.rb_node; 285 struct rb_node *pn = NULL; 286 struct ipoib_path *tpath; 287 int ret; 288 289 while (*n) { 290 pn = *n; 291 tpath = rb_entry(pn, struct ipoib_path, rb_node); 292 293 ret = memcmp(path->pathrec.dgid.raw, tpath->pathrec.dgid.raw, 294 sizeof (union ib_gid)); 295 if (ret < 0) 296 n = &pn->rb_left; 297 else if (ret > 0) 298 n = &pn->rb_right; 299 else 300 return -EEXIST; 301 } 302 303 rb_link_node(&path->rb_node, pn, n); 304 rb_insert_color(&path->rb_node, &priv->path_tree); 305 306 list_add_tail(&path->list, &priv->path_list); 307 308 return 0; 309 } 310 311 static void path_free(struct net_device *dev, struct ipoib_path *path) 312 { 313 struct sk_buff *skb; 314 315 while ((skb = __skb_dequeue(&path->queue))) 316 dev_kfree_skb_irq(skb); 317 318 ipoib_dbg(netdev_priv(dev), "path_free\n"); 319 320 /* remove all neigh connected to this path */ 321 ipoib_del_neighs_by_gid(dev, path->pathrec.dgid.raw); 322 323 if (path->ah) 324 ipoib_put_ah(path->ah); 325 326 kfree(path); 327 } 328 329 #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG 330 331 struct ipoib_path_iter *ipoib_path_iter_init(struct net_device *dev) 332 { 333 struct ipoib_path_iter *iter; 334 335 iter = kmalloc(sizeof *iter, GFP_KERNEL); 336 if (!iter) 337 return NULL; 338 339 iter->dev = dev; 340 memset(iter->path.pathrec.dgid.raw, 0, 16); 341 342 if (ipoib_path_iter_next(iter)) { 343 kfree(iter); 344 return NULL; 345 } 346 347 return iter; 348 } 349 350 int ipoib_path_iter_next(struct ipoib_path_iter *iter) 351 { 352 struct ipoib_dev_priv *priv = netdev_priv(iter->dev); 353 struct rb_node *n; 354 struct ipoib_path *path; 355 int ret = 1; 356 357 spin_lock_irq(&priv->lock); 358 359 n = rb_first(&priv->path_tree); 360 361 while (n) { 362 path = rb_entry(n, struct ipoib_path, rb_node); 363 364 if (memcmp(iter->path.pathrec.dgid.raw, path->pathrec.dgid.raw, 365 sizeof (union ib_gid)) < 0) { 366 iter->path = *path; 367 ret = 0; 368 break; 369 } 370 371 n = rb_next(n); 372 } 373 374 spin_unlock_irq(&priv->lock); 375 376 return ret; 377 } 378 379 void ipoib_path_iter_read(struct ipoib_path_iter *iter, 380 struct ipoib_path *path) 381 { 382 *path = iter->path; 383 } 384 385 #endif /* CONFIG_INFINIBAND_IPOIB_DEBUG */ 386 387 void ipoib_mark_paths_invalid(struct net_device *dev) 388 { 389 struct ipoib_dev_priv *priv = netdev_priv(dev); 390 struct ipoib_path *path, *tp; 391 392 spin_lock_irq(&priv->lock); 393 394 list_for_each_entry_safe(path, tp, &priv->path_list, list) { 395 ipoib_dbg(priv, "mark path LID 0x%04x GID %pI6 invalid\n", 396 be16_to_cpu(path->pathrec.dlid), 397 path->pathrec.dgid.raw); 398 path->valid = 0; 399 } 400 401 spin_unlock_irq(&priv->lock); 402 } 403 404 void ipoib_flush_paths(struct net_device *dev) 405 { 406 struct ipoib_dev_priv *priv = netdev_priv(dev); 407 struct ipoib_path *path, *tp; 408 LIST_HEAD(remove_list); 409 unsigned long flags; 410 411 netif_tx_lock_bh(dev); 412 spin_lock_irqsave(&priv->lock, flags); 413 414 list_splice_init(&priv->path_list, &remove_list); 415 416 list_for_each_entry(path, &remove_list, list) 417 rb_erase(&path->rb_node, &priv->path_tree); 418 419 list_for_each_entry_safe(path, tp, &remove_list, list) { 420 if (path->query) 421 ib_sa_cancel_query(path->query_id, path->query); 422 spin_unlock_irqrestore(&priv->lock, flags); 423 netif_tx_unlock_bh(dev); 424 wait_for_completion(&path->done); 425 path_free(dev, path); 426 netif_tx_lock_bh(dev); 427 spin_lock_irqsave(&priv->lock, flags); 428 } 429 430 spin_unlock_irqrestore(&priv->lock, flags); 431 netif_tx_unlock_bh(dev); 432 } 433 434 static void path_rec_completion(int status, 435 struct ib_sa_path_rec *pathrec, 436 void *path_ptr) 437 { 438 struct ipoib_path *path = path_ptr; 439 struct net_device *dev = path->dev; 440 struct ipoib_dev_priv *priv = netdev_priv(dev); 441 struct ipoib_ah *ah = NULL; 442 struct ipoib_ah *old_ah = NULL; 443 struct ipoib_neigh *neigh, *tn; 444 struct sk_buff_head skqueue; 445 struct sk_buff *skb; 446 unsigned long flags; 447 448 if (!status) 449 ipoib_dbg(priv, "PathRec LID 0x%04x for GID %pI6\n", 450 be16_to_cpu(pathrec->dlid), pathrec->dgid.raw); 451 else 452 ipoib_dbg(priv, "PathRec status %d for GID %pI6\n", 453 status, path->pathrec.dgid.raw); 454 455 skb_queue_head_init(&skqueue); 456 457 if (!status) { 458 struct ib_ah_attr av; 459 460 if (!ib_init_ah_from_path(priv->ca, priv->port, pathrec, &av)) 461 ah = ipoib_create_ah(dev, priv->pd, &av); 462 } 463 464 spin_lock_irqsave(&priv->lock, flags); 465 466 if (!IS_ERR_OR_NULL(ah)) { 467 path->pathrec = *pathrec; 468 469 old_ah = path->ah; 470 path->ah = ah; 471 472 ipoib_dbg(priv, "created address handle %p for LID 0x%04x, SL %d\n", 473 ah, be16_to_cpu(pathrec->dlid), pathrec->sl); 474 475 while ((skb = __skb_dequeue(&path->queue))) 476 __skb_queue_tail(&skqueue, skb); 477 478 list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) { 479 if (neigh->ah) { 480 WARN_ON(neigh->ah != old_ah); 481 /* 482 * Dropping the ah reference inside 483 * priv->lock is safe here, because we 484 * will hold one more reference from 485 * the original value of path->ah (ie 486 * old_ah). 487 */ 488 ipoib_put_ah(neigh->ah); 489 } 490 kref_get(&path->ah->ref); 491 neigh->ah = path->ah; 492 493 if (ipoib_cm_enabled(dev, neigh->daddr)) { 494 if (!ipoib_cm_get(neigh)) 495 ipoib_cm_set(neigh, ipoib_cm_create_tx(dev, 496 path, 497 neigh)); 498 if (!ipoib_cm_get(neigh)) { 499 ipoib_neigh_free(neigh); 500 continue; 501 } 502 } 503 504 while ((skb = __skb_dequeue(&neigh->queue))) 505 __skb_queue_tail(&skqueue, skb); 506 } 507 path->valid = 1; 508 } 509 510 path->query = NULL; 511 complete(&path->done); 512 513 spin_unlock_irqrestore(&priv->lock, flags); 514 515 if (IS_ERR_OR_NULL(ah)) 516 ipoib_del_neighs_by_gid(dev, path->pathrec.dgid.raw); 517 518 if (old_ah) 519 ipoib_put_ah(old_ah); 520 521 while ((skb = __skb_dequeue(&skqueue))) { 522 skb->dev = dev; 523 if (dev_queue_xmit(skb)) 524 ipoib_warn(priv, "dev_queue_xmit failed " 525 "to requeue packet\n"); 526 } 527 } 528 529 static struct ipoib_path *path_rec_create(struct net_device *dev, void *gid) 530 { 531 struct ipoib_dev_priv *priv = netdev_priv(dev); 532 struct ipoib_path *path; 533 534 if (!priv->broadcast) 535 return NULL; 536 537 path = kzalloc(sizeof *path, GFP_ATOMIC); 538 if (!path) 539 return NULL; 540 541 path->dev = dev; 542 543 skb_queue_head_init(&path->queue); 544 545 INIT_LIST_HEAD(&path->neigh_list); 546 547 memcpy(path->pathrec.dgid.raw, gid, sizeof (union ib_gid)); 548 path->pathrec.sgid = priv->local_gid; 549 path->pathrec.pkey = cpu_to_be16(priv->pkey); 550 path->pathrec.numb_path = 1; 551 path->pathrec.traffic_class = priv->broadcast->mcmember.traffic_class; 552 553 return path; 554 } 555 556 static int path_rec_start(struct net_device *dev, 557 struct ipoib_path *path) 558 { 559 struct ipoib_dev_priv *priv = netdev_priv(dev); 560 561 ipoib_dbg(priv, "Start path record lookup for %pI6\n", 562 path->pathrec.dgid.raw); 563 564 init_completion(&path->done); 565 566 path->query_id = 567 ib_sa_path_rec_get(&ipoib_sa_client, priv->ca, priv->port, 568 &path->pathrec, 569 IB_SA_PATH_REC_DGID | 570 IB_SA_PATH_REC_SGID | 571 IB_SA_PATH_REC_NUMB_PATH | 572 IB_SA_PATH_REC_TRAFFIC_CLASS | 573 IB_SA_PATH_REC_PKEY, 574 1000, GFP_ATOMIC, 575 path_rec_completion, 576 path, &path->query); 577 if (path->query_id < 0) { 578 ipoib_warn(priv, "ib_sa_path_rec_get failed: %d\n", path->query_id); 579 path->query = NULL; 580 complete(&path->done); 581 return path->query_id; 582 } 583 584 return 0; 585 } 586 587 static void neigh_add_path(struct sk_buff *skb, u8 *daddr, 588 struct net_device *dev) 589 { 590 struct ipoib_dev_priv *priv = netdev_priv(dev); 591 struct ipoib_path *path; 592 struct ipoib_neigh *neigh; 593 unsigned long flags; 594 595 spin_lock_irqsave(&priv->lock, flags); 596 neigh = ipoib_neigh_alloc(daddr, dev); 597 if (!neigh) { 598 spin_unlock_irqrestore(&priv->lock, flags); 599 ++dev->stats.tx_dropped; 600 dev_kfree_skb_any(skb); 601 return; 602 } 603 604 path = __path_find(dev, daddr + 4); 605 if (!path) { 606 path = path_rec_create(dev, daddr + 4); 607 if (!path) 608 goto err_path; 609 610 __path_add(dev, path); 611 } 612 613 list_add_tail(&neigh->list, &path->neigh_list); 614 615 if (path->ah) { 616 kref_get(&path->ah->ref); 617 neigh->ah = path->ah; 618 619 if (ipoib_cm_enabled(dev, neigh->daddr)) { 620 if (!ipoib_cm_get(neigh)) 621 ipoib_cm_set(neigh, ipoib_cm_create_tx(dev, path, neigh)); 622 if (!ipoib_cm_get(neigh)) { 623 ipoib_neigh_free(neigh); 624 goto err_drop; 625 } 626 if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) 627 __skb_queue_tail(&neigh->queue, skb); 628 else { 629 ipoib_warn(priv, "queue length limit %d. Packet drop.\n", 630 skb_queue_len(&neigh->queue)); 631 goto err_drop; 632 } 633 } else { 634 spin_unlock_irqrestore(&priv->lock, flags); 635 ipoib_send(dev, skb, path->ah, IPOIB_QPN(daddr)); 636 ipoib_neigh_put(neigh); 637 return; 638 } 639 } else { 640 neigh->ah = NULL; 641 642 if (!path->query && path_rec_start(dev, path)) 643 goto err_path; 644 if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) 645 __skb_queue_tail(&neigh->queue, skb); 646 else 647 goto err_drop; 648 } 649 650 spin_unlock_irqrestore(&priv->lock, flags); 651 ipoib_neigh_put(neigh); 652 return; 653 654 err_path: 655 ipoib_neigh_free(neigh); 656 err_drop: 657 ++dev->stats.tx_dropped; 658 dev_kfree_skb_any(skb); 659 660 spin_unlock_irqrestore(&priv->lock, flags); 661 ipoib_neigh_put(neigh); 662 } 663 664 static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev, 665 struct ipoib_cb *cb) 666 { 667 struct ipoib_dev_priv *priv = netdev_priv(dev); 668 struct ipoib_path *path; 669 unsigned long flags; 670 671 spin_lock_irqsave(&priv->lock, flags); 672 673 path = __path_find(dev, cb->hwaddr + 4); 674 if (!path || !path->valid) { 675 int new_path = 0; 676 677 if (!path) { 678 path = path_rec_create(dev, cb->hwaddr + 4); 679 new_path = 1; 680 } 681 if (path) { 682 if (skb_queue_len(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE) { 683 __skb_queue_tail(&path->queue, skb); 684 } else { 685 ++dev->stats.tx_dropped; 686 dev_kfree_skb_any(skb); 687 } 688 689 if (!path->query && path_rec_start(dev, path)) { 690 spin_unlock_irqrestore(&priv->lock, flags); 691 if (new_path) 692 path_free(dev, path); 693 return; 694 } else 695 __path_add(dev, path); 696 } else { 697 ++dev->stats.tx_dropped; 698 dev_kfree_skb_any(skb); 699 } 700 701 spin_unlock_irqrestore(&priv->lock, flags); 702 return; 703 } 704 705 if (path->ah) { 706 ipoib_dbg(priv, "Send unicast ARP to %04x\n", 707 be16_to_cpu(path->pathrec.dlid)); 708 709 spin_unlock_irqrestore(&priv->lock, flags); 710 ipoib_send(dev, skb, path->ah, IPOIB_QPN(cb->hwaddr)); 711 return; 712 } else if ((path->query || !path_rec_start(dev, path)) && 713 skb_queue_len(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE) { 714 __skb_queue_tail(&path->queue, skb); 715 } else { 716 ++dev->stats.tx_dropped; 717 dev_kfree_skb_any(skb); 718 } 719 720 spin_unlock_irqrestore(&priv->lock, flags); 721 } 722 723 static int ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev) 724 { 725 struct ipoib_dev_priv *priv = netdev_priv(dev); 726 struct ipoib_neigh *neigh; 727 struct ipoib_cb *cb = ipoib_skb_cb(skb); 728 struct ipoib_header *header; 729 unsigned long flags; 730 731 header = (struct ipoib_header *) skb->data; 732 733 if (unlikely(cb->hwaddr[4] == 0xff)) { 734 /* multicast, arrange "if" according to probability */ 735 if ((header->proto != htons(ETH_P_IP)) && 736 (header->proto != htons(ETH_P_IPV6)) && 737 (header->proto != htons(ETH_P_ARP)) && 738 (header->proto != htons(ETH_P_RARP)) && 739 (header->proto != htons(ETH_P_TIPC))) { 740 /* ethertype not supported by IPoIB */ 741 ++dev->stats.tx_dropped; 742 dev_kfree_skb_any(skb); 743 return NETDEV_TX_OK; 744 } 745 /* Add in the P_Key for multicast*/ 746 cb->hwaddr[8] = (priv->pkey >> 8) & 0xff; 747 cb->hwaddr[9] = priv->pkey & 0xff; 748 749 neigh = ipoib_neigh_get(dev, cb->hwaddr); 750 if (likely(neigh)) 751 goto send_using_neigh; 752 ipoib_mcast_send(dev, cb->hwaddr, skb); 753 return NETDEV_TX_OK; 754 } 755 756 /* unicast, arrange "switch" according to probability */ 757 switch (header->proto) { 758 case htons(ETH_P_IP): 759 case htons(ETH_P_IPV6): 760 case htons(ETH_P_TIPC): 761 neigh = ipoib_neigh_get(dev, cb->hwaddr); 762 if (unlikely(!neigh)) { 763 neigh_add_path(skb, cb->hwaddr, dev); 764 return NETDEV_TX_OK; 765 } 766 break; 767 case htons(ETH_P_ARP): 768 case htons(ETH_P_RARP): 769 /* for unicast ARP and RARP should always perform path find */ 770 unicast_arp_send(skb, dev, cb); 771 return NETDEV_TX_OK; 772 default: 773 /* ethertype not supported by IPoIB */ 774 ++dev->stats.tx_dropped; 775 dev_kfree_skb_any(skb); 776 return NETDEV_TX_OK; 777 } 778 779 send_using_neigh: 780 /* note we now hold a ref to neigh */ 781 if (ipoib_cm_get(neigh)) { 782 if (ipoib_cm_up(neigh)) { 783 ipoib_cm_send(dev, skb, ipoib_cm_get(neigh)); 784 goto unref; 785 } 786 } else if (neigh->ah) { 787 ipoib_send(dev, skb, neigh->ah, IPOIB_QPN(cb->hwaddr)); 788 goto unref; 789 } 790 791 if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) { 792 spin_lock_irqsave(&priv->lock, flags); 793 __skb_queue_tail(&neigh->queue, skb); 794 spin_unlock_irqrestore(&priv->lock, flags); 795 } else { 796 ++dev->stats.tx_dropped; 797 dev_kfree_skb_any(skb); 798 } 799 800 unref: 801 ipoib_neigh_put(neigh); 802 803 return NETDEV_TX_OK; 804 } 805 806 static void ipoib_timeout(struct net_device *dev) 807 { 808 struct ipoib_dev_priv *priv = netdev_priv(dev); 809 810 ipoib_warn(priv, "transmit timeout: latency %d msecs\n", 811 jiffies_to_msecs(jiffies - dev->trans_start)); 812 ipoib_warn(priv, "queue stopped %d, tx_head %u, tx_tail %u\n", 813 netif_queue_stopped(dev), 814 priv->tx_head, priv->tx_tail); 815 /* XXX reset QP, etc. */ 816 } 817 818 static int ipoib_hard_header(struct sk_buff *skb, 819 struct net_device *dev, 820 unsigned short type, 821 const void *daddr, const void *saddr, unsigned len) 822 { 823 struct ipoib_header *header; 824 struct ipoib_cb *cb = ipoib_skb_cb(skb); 825 826 header = (struct ipoib_header *) skb_push(skb, sizeof *header); 827 828 header->proto = htons(type); 829 header->reserved = 0; 830 831 /* 832 * we don't rely on dst_entry structure, always stuff the 833 * destination address into skb->cb so we can figure out where 834 * to send the packet later. 835 */ 836 memcpy(cb->hwaddr, daddr, INFINIBAND_ALEN); 837 838 return sizeof *header; 839 } 840 841 static void ipoib_set_mcast_list(struct net_device *dev) 842 { 843 struct ipoib_dev_priv *priv = netdev_priv(dev); 844 845 if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) { 846 ipoib_dbg(priv, "IPOIB_FLAG_OPER_UP not set"); 847 return; 848 } 849 850 queue_work(priv->wq, &priv->restart_task); 851 } 852 853 static int ipoib_get_iflink(const struct net_device *dev) 854 { 855 struct ipoib_dev_priv *priv = netdev_priv(dev); 856 857 /* parent interface */ 858 if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) 859 return dev->ifindex; 860 861 /* child/vlan interface */ 862 return priv->parent->ifindex; 863 } 864 865 static u32 ipoib_addr_hash(struct ipoib_neigh_hash *htbl, u8 *daddr) 866 { 867 /* 868 * Use only the address parts that contributes to spreading 869 * The subnet prefix is not used as one can not connect to 870 * same remote port (GUID) using the same remote QPN via two 871 * different subnets. 872 */ 873 /* qpn octets[1:4) & port GUID octets[12:20) */ 874 u32 *d32 = (u32 *) daddr; 875 u32 hv; 876 877 hv = jhash_3words(d32[3], d32[4], IPOIB_QPN_MASK & d32[0], 0); 878 return hv & htbl->mask; 879 } 880 881 struct ipoib_neigh *ipoib_neigh_get(struct net_device *dev, u8 *daddr) 882 { 883 struct ipoib_dev_priv *priv = netdev_priv(dev); 884 struct ipoib_neigh_table *ntbl = &priv->ntbl; 885 struct ipoib_neigh_hash *htbl; 886 struct ipoib_neigh *neigh = NULL; 887 u32 hash_val; 888 889 rcu_read_lock_bh(); 890 891 htbl = rcu_dereference_bh(ntbl->htbl); 892 893 if (!htbl) 894 goto out_unlock; 895 896 hash_val = ipoib_addr_hash(htbl, daddr); 897 for (neigh = rcu_dereference_bh(htbl->buckets[hash_val]); 898 neigh != NULL; 899 neigh = rcu_dereference_bh(neigh->hnext)) { 900 if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) { 901 /* found, take one ref on behalf of the caller */ 902 if (!atomic_inc_not_zero(&neigh->refcnt)) { 903 /* deleted */ 904 neigh = NULL; 905 goto out_unlock; 906 } 907 neigh->alive = jiffies; 908 goto out_unlock; 909 } 910 } 911 912 out_unlock: 913 rcu_read_unlock_bh(); 914 return neigh; 915 } 916 917 static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv) 918 { 919 struct ipoib_neigh_table *ntbl = &priv->ntbl; 920 struct ipoib_neigh_hash *htbl; 921 unsigned long neigh_obsolete; 922 unsigned long dt; 923 unsigned long flags; 924 int i; 925 926 if (test_bit(IPOIB_STOP_NEIGH_GC, &priv->flags)) 927 return; 928 929 spin_lock_irqsave(&priv->lock, flags); 930 931 htbl = rcu_dereference_protected(ntbl->htbl, 932 lockdep_is_held(&priv->lock)); 933 934 if (!htbl) 935 goto out_unlock; 936 937 /* neigh is obsolete if it was idle for two GC periods */ 938 dt = 2 * arp_tbl.gc_interval; 939 neigh_obsolete = jiffies - dt; 940 /* handle possible race condition */ 941 if (test_bit(IPOIB_STOP_NEIGH_GC, &priv->flags)) 942 goto out_unlock; 943 944 for (i = 0; i < htbl->size; i++) { 945 struct ipoib_neigh *neigh; 946 struct ipoib_neigh __rcu **np = &htbl->buckets[i]; 947 948 while ((neigh = rcu_dereference_protected(*np, 949 lockdep_is_held(&priv->lock))) != NULL) { 950 /* was the neigh idle for two GC periods */ 951 if (time_after(neigh_obsolete, neigh->alive)) { 952 rcu_assign_pointer(*np, 953 rcu_dereference_protected(neigh->hnext, 954 lockdep_is_held(&priv->lock))); 955 /* remove from path/mc list */ 956 list_del(&neigh->list); 957 call_rcu(&neigh->rcu, ipoib_neigh_reclaim); 958 } else { 959 np = &neigh->hnext; 960 } 961 962 } 963 } 964 965 out_unlock: 966 spin_unlock_irqrestore(&priv->lock, flags); 967 } 968 969 static void ipoib_reap_neigh(struct work_struct *work) 970 { 971 struct ipoib_dev_priv *priv = 972 container_of(work, struct ipoib_dev_priv, neigh_reap_task.work); 973 974 __ipoib_reap_neigh(priv); 975 976 if (!test_bit(IPOIB_STOP_NEIGH_GC, &priv->flags)) 977 queue_delayed_work(priv->wq, &priv->neigh_reap_task, 978 arp_tbl.gc_interval); 979 } 980 981 982 static struct ipoib_neigh *ipoib_neigh_ctor(u8 *daddr, 983 struct net_device *dev) 984 { 985 struct ipoib_neigh *neigh; 986 987 neigh = kzalloc(sizeof *neigh, GFP_ATOMIC); 988 if (!neigh) 989 return NULL; 990 991 neigh->dev = dev; 992 memcpy(&neigh->daddr, daddr, sizeof(neigh->daddr)); 993 skb_queue_head_init(&neigh->queue); 994 INIT_LIST_HEAD(&neigh->list); 995 ipoib_cm_set(neigh, NULL); 996 /* one ref on behalf of the caller */ 997 atomic_set(&neigh->refcnt, 1); 998 999 return neigh; 1000 } 1001 1002 struct ipoib_neigh *ipoib_neigh_alloc(u8 *daddr, 1003 struct net_device *dev) 1004 { 1005 struct ipoib_dev_priv *priv = netdev_priv(dev); 1006 struct ipoib_neigh_table *ntbl = &priv->ntbl; 1007 struct ipoib_neigh_hash *htbl; 1008 struct ipoib_neigh *neigh; 1009 u32 hash_val; 1010 1011 htbl = rcu_dereference_protected(ntbl->htbl, 1012 lockdep_is_held(&priv->lock)); 1013 if (!htbl) { 1014 neigh = NULL; 1015 goto out_unlock; 1016 } 1017 1018 /* need to add a new neigh, but maybe some other thread succeeded? 1019 * recalc hash, maybe hash resize took place so we do a search 1020 */ 1021 hash_val = ipoib_addr_hash(htbl, daddr); 1022 for (neigh = rcu_dereference_protected(htbl->buckets[hash_val], 1023 lockdep_is_held(&priv->lock)); 1024 neigh != NULL; 1025 neigh = rcu_dereference_protected(neigh->hnext, 1026 lockdep_is_held(&priv->lock))) { 1027 if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) { 1028 /* found, take one ref on behalf of the caller */ 1029 if (!atomic_inc_not_zero(&neigh->refcnt)) { 1030 /* deleted */ 1031 neigh = NULL; 1032 break; 1033 } 1034 neigh->alive = jiffies; 1035 goto out_unlock; 1036 } 1037 } 1038 1039 neigh = ipoib_neigh_ctor(daddr, dev); 1040 if (!neigh) 1041 goto out_unlock; 1042 1043 /* one ref on behalf of the hash table */ 1044 atomic_inc(&neigh->refcnt); 1045 neigh->alive = jiffies; 1046 /* put in hash */ 1047 rcu_assign_pointer(neigh->hnext, 1048 rcu_dereference_protected(htbl->buckets[hash_val], 1049 lockdep_is_held(&priv->lock))); 1050 rcu_assign_pointer(htbl->buckets[hash_val], neigh); 1051 atomic_inc(&ntbl->entries); 1052 1053 out_unlock: 1054 1055 return neigh; 1056 } 1057 1058 void ipoib_neigh_dtor(struct ipoib_neigh *neigh) 1059 { 1060 /* neigh reference count was dropprd to zero */ 1061 struct net_device *dev = neigh->dev; 1062 struct ipoib_dev_priv *priv = netdev_priv(dev); 1063 struct sk_buff *skb; 1064 if (neigh->ah) 1065 ipoib_put_ah(neigh->ah); 1066 while ((skb = __skb_dequeue(&neigh->queue))) { 1067 ++dev->stats.tx_dropped; 1068 dev_kfree_skb_any(skb); 1069 } 1070 if (ipoib_cm_get(neigh)) 1071 ipoib_cm_destroy_tx(ipoib_cm_get(neigh)); 1072 ipoib_dbg(netdev_priv(dev), 1073 "neigh free for %06x %pI6\n", 1074 IPOIB_QPN(neigh->daddr), 1075 neigh->daddr + 4); 1076 kfree(neigh); 1077 if (atomic_dec_and_test(&priv->ntbl.entries)) { 1078 if (test_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags)) 1079 complete(&priv->ntbl.flushed); 1080 } 1081 } 1082 1083 static void ipoib_neigh_reclaim(struct rcu_head *rp) 1084 { 1085 /* Called as a result of removal from hash table */ 1086 struct ipoib_neigh *neigh = container_of(rp, struct ipoib_neigh, rcu); 1087 /* note TX context may hold another ref */ 1088 ipoib_neigh_put(neigh); 1089 } 1090 1091 void ipoib_neigh_free(struct ipoib_neigh *neigh) 1092 { 1093 struct net_device *dev = neigh->dev; 1094 struct ipoib_dev_priv *priv = netdev_priv(dev); 1095 struct ipoib_neigh_table *ntbl = &priv->ntbl; 1096 struct ipoib_neigh_hash *htbl; 1097 struct ipoib_neigh __rcu **np; 1098 struct ipoib_neigh *n; 1099 u32 hash_val; 1100 1101 htbl = rcu_dereference_protected(ntbl->htbl, 1102 lockdep_is_held(&priv->lock)); 1103 if (!htbl) 1104 return; 1105 1106 hash_val = ipoib_addr_hash(htbl, neigh->daddr); 1107 np = &htbl->buckets[hash_val]; 1108 for (n = rcu_dereference_protected(*np, 1109 lockdep_is_held(&priv->lock)); 1110 n != NULL; 1111 n = rcu_dereference_protected(*np, 1112 lockdep_is_held(&priv->lock))) { 1113 if (n == neigh) { 1114 /* found */ 1115 rcu_assign_pointer(*np, 1116 rcu_dereference_protected(neigh->hnext, 1117 lockdep_is_held(&priv->lock))); 1118 /* remove from parent list */ 1119 list_del(&neigh->list); 1120 call_rcu(&neigh->rcu, ipoib_neigh_reclaim); 1121 return; 1122 } else { 1123 np = &n->hnext; 1124 } 1125 } 1126 } 1127 1128 static int ipoib_neigh_hash_init(struct ipoib_dev_priv *priv) 1129 { 1130 struct ipoib_neigh_table *ntbl = &priv->ntbl; 1131 struct ipoib_neigh_hash *htbl; 1132 struct ipoib_neigh __rcu **buckets; 1133 u32 size; 1134 1135 clear_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags); 1136 ntbl->htbl = NULL; 1137 htbl = kzalloc(sizeof(*htbl), GFP_KERNEL); 1138 if (!htbl) 1139 return -ENOMEM; 1140 set_bit(IPOIB_STOP_NEIGH_GC, &priv->flags); 1141 size = roundup_pow_of_two(arp_tbl.gc_thresh3); 1142 buckets = kzalloc(size * sizeof(*buckets), GFP_KERNEL); 1143 if (!buckets) { 1144 kfree(htbl); 1145 return -ENOMEM; 1146 } 1147 htbl->size = size; 1148 htbl->mask = (size - 1); 1149 htbl->buckets = buckets; 1150 RCU_INIT_POINTER(ntbl->htbl, htbl); 1151 htbl->ntbl = ntbl; 1152 atomic_set(&ntbl->entries, 0); 1153 1154 /* start garbage collection */ 1155 clear_bit(IPOIB_STOP_NEIGH_GC, &priv->flags); 1156 queue_delayed_work(priv->wq, &priv->neigh_reap_task, 1157 arp_tbl.gc_interval); 1158 1159 return 0; 1160 } 1161 1162 static void neigh_hash_free_rcu(struct rcu_head *head) 1163 { 1164 struct ipoib_neigh_hash *htbl = container_of(head, 1165 struct ipoib_neigh_hash, 1166 rcu); 1167 struct ipoib_neigh __rcu **buckets = htbl->buckets; 1168 struct ipoib_neigh_table *ntbl = htbl->ntbl; 1169 1170 kfree(buckets); 1171 kfree(htbl); 1172 complete(&ntbl->deleted); 1173 } 1174 1175 void ipoib_del_neighs_by_gid(struct net_device *dev, u8 *gid) 1176 { 1177 struct ipoib_dev_priv *priv = netdev_priv(dev); 1178 struct ipoib_neigh_table *ntbl = &priv->ntbl; 1179 struct ipoib_neigh_hash *htbl; 1180 unsigned long flags; 1181 int i; 1182 1183 /* remove all neigh connected to a given path or mcast */ 1184 spin_lock_irqsave(&priv->lock, flags); 1185 1186 htbl = rcu_dereference_protected(ntbl->htbl, 1187 lockdep_is_held(&priv->lock)); 1188 1189 if (!htbl) 1190 goto out_unlock; 1191 1192 for (i = 0; i < htbl->size; i++) { 1193 struct ipoib_neigh *neigh; 1194 struct ipoib_neigh __rcu **np = &htbl->buckets[i]; 1195 1196 while ((neigh = rcu_dereference_protected(*np, 1197 lockdep_is_held(&priv->lock))) != NULL) { 1198 /* delete neighs belong to this parent */ 1199 if (!memcmp(gid, neigh->daddr + 4, sizeof (union ib_gid))) { 1200 rcu_assign_pointer(*np, 1201 rcu_dereference_protected(neigh->hnext, 1202 lockdep_is_held(&priv->lock))); 1203 /* remove from parent list */ 1204 list_del(&neigh->list); 1205 call_rcu(&neigh->rcu, ipoib_neigh_reclaim); 1206 } else { 1207 np = &neigh->hnext; 1208 } 1209 1210 } 1211 } 1212 out_unlock: 1213 spin_unlock_irqrestore(&priv->lock, flags); 1214 } 1215 1216 static void ipoib_flush_neighs(struct ipoib_dev_priv *priv) 1217 { 1218 struct ipoib_neigh_table *ntbl = &priv->ntbl; 1219 struct ipoib_neigh_hash *htbl; 1220 unsigned long flags; 1221 int i, wait_flushed = 0; 1222 1223 init_completion(&priv->ntbl.flushed); 1224 1225 spin_lock_irqsave(&priv->lock, flags); 1226 1227 htbl = rcu_dereference_protected(ntbl->htbl, 1228 lockdep_is_held(&priv->lock)); 1229 if (!htbl) 1230 goto out_unlock; 1231 1232 wait_flushed = atomic_read(&priv->ntbl.entries); 1233 if (!wait_flushed) 1234 goto free_htbl; 1235 1236 for (i = 0; i < htbl->size; i++) { 1237 struct ipoib_neigh *neigh; 1238 struct ipoib_neigh __rcu **np = &htbl->buckets[i]; 1239 1240 while ((neigh = rcu_dereference_protected(*np, 1241 lockdep_is_held(&priv->lock))) != NULL) { 1242 rcu_assign_pointer(*np, 1243 rcu_dereference_protected(neigh->hnext, 1244 lockdep_is_held(&priv->lock))); 1245 /* remove from path/mc list */ 1246 list_del(&neigh->list); 1247 call_rcu(&neigh->rcu, ipoib_neigh_reclaim); 1248 } 1249 } 1250 1251 free_htbl: 1252 rcu_assign_pointer(ntbl->htbl, NULL); 1253 call_rcu(&htbl->rcu, neigh_hash_free_rcu); 1254 1255 out_unlock: 1256 spin_unlock_irqrestore(&priv->lock, flags); 1257 if (wait_flushed) 1258 wait_for_completion(&priv->ntbl.flushed); 1259 } 1260 1261 static void ipoib_neigh_hash_uninit(struct net_device *dev) 1262 { 1263 struct ipoib_dev_priv *priv = netdev_priv(dev); 1264 int stopped; 1265 1266 ipoib_dbg(priv, "ipoib_neigh_hash_uninit\n"); 1267 init_completion(&priv->ntbl.deleted); 1268 set_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags); 1269 1270 /* Stop GC if called at init fail need to cancel work */ 1271 stopped = test_and_set_bit(IPOIB_STOP_NEIGH_GC, &priv->flags); 1272 if (!stopped) 1273 cancel_delayed_work(&priv->neigh_reap_task); 1274 1275 ipoib_flush_neighs(priv); 1276 1277 wait_for_completion(&priv->ntbl.deleted); 1278 } 1279 1280 1281 int ipoib_dev_init(struct net_device *dev, struct ib_device *ca, int port) 1282 { 1283 struct ipoib_dev_priv *priv = netdev_priv(dev); 1284 1285 /* Allocate RX/TX "rings" to hold queued skbs */ 1286 priv->rx_ring = kzalloc(ipoib_recvq_size * sizeof *priv->rx_ring, 1287 GFP_KERNEL); 1288 if (!priv->rx_ring) { 1289 printk(KERN_WARNING "%s: failed to allocate RX ring (%d entries)\n", 1290 ca->name, ipoib_recvq_size); 1291 goto out; 1292 } 1293 1294 priv->tx_ring = vzalloc(ipoib_sendq_size * sizeof *priv->tx_ring); 1295 if (!priv->tx_ring) { 1296 printk(KERN_WARNING "%s: failed to allocate TX ring (%d entries)\n", 1297 ca->name, ipoib_sendq_size); 1298 goto out_rx_ring_cleanup; 1299 } 1300 1301 /* priv->tx_head, tx_tail & tx_outstanding are already 0 */ 1302 1303 if (ipoib_ib_dev_init(dev, ca, port)) 1304 goto out_tx_ring_cleanup; 1305 1306 /* 1307 * Must be after ipoib_ib_dev_init so we can allocate a per 1308 * device wq there and use it here 1309 */ 1310 if (ipoib_neigh_hash_init(priv) < 0) 1311 goto out_dev_uninit; 1312 1313 return 0; 1314 1315 out_dev_uninit: 1316 ipoib_ib_dev_cleanup(dev); 1317 1318 out_tx_ring_cleanup: 1319 vfree(priv->tx_ring); 1320 1321 out_rx_ring_cleanup: 1322 kfree(priv->rx_ring); 1323 1324 out: 1325 return -ENOMEM; 1326 } 1327 1328 void ipoib_dev_cleanup(struct net_device *dev) 1329 { 1330 struct ipoib_dev_priv *priv = netdev_priv(dev), *cpriv, *tcpriv; 1331 LIST_HEAD(head); 1332 1333 ASSERT_RTNL(); 1334 1335 ipoib_delete_debug_files(dev); 1336 1337 /* Delete any child interfaces first */ 1338 list_for_each_entry_safe(cpriv, tcpriv, &priv->child_intfs, list) { 1339 /* Stop GC on child */ 1340 set_bit(IPOIB_STOP_NEIGH_GC, &cpriv->flags); 1341 cancel_delayed_work(&cpriv->neigh_reap_task); 1342 unregister_netdevice_queue(cpriv->dev, &head); 1343 } 1344 unregister_netdevice_many(&head); 1345 1346 /* 1347 * Must be before ipoib_ib_dev_cleanup or we delete an in use 1348 * work queue 1349 */ 1350 ipoib_neigh_hash_uninit(dev); 1351 1352 ipoib_ib_dev_cleanup(dev); 1353 1354 kfree(priv->rx_ring); 1355 vfree(priv->tx_ring); 1356 1357 priv->rx_ring = NULL; 1358 priv->tx_ring = NULL; 1359 } 1360 1361 static const struct header_ops ipoib_header_ops = { 1362 .create = ipoib_hard_header, 1363 }; 1364 1365 static const struct net_device_ops ipoib_netdev_ops = { 1366 .ndo_uninit = ipoib_uninit, 1367 .ndo_open = ipoib_open, 1368 .ndo_stop = ipoib_stop, 1369 .ndo_change_mtu = ipoib_change_mtu, 1370 .ndo_fix_features = ipoib_fix_features, 1371 .ndo_start_xmit = ipoib_start_xmit, 1372 .ndo_tx_timeout = ipoib_timeout, 1373 .ndo_set_rx_mode = ipoib_set_mcast_list, 1374 .ndo_get_iflink = ipoib_get_iflink, 1375 }; 1376 1377 void ipoib_setup(struct net_device *dev) 1378 { 1379 struct ipoib_dev_priv *priv = netdev_priv(dev); 1380 1381 dev->netdev_ops = &ipoib_netdev_ops; 1382 dev->header_ops = &ipoib_header_ops; 1383 1384 ipoib_set_ethtool_ops(dev); 1385 1386 netif_napi_add(dev, &priv->napi, ipoib_poll, NAPI_POLL_WEIGHT); 1387 1388 dev->watchdog_timeo = HZ; 1389 1390 dev->flags |= IFF_BROADCAST | IFF_MULTICAST; 1391 1392 dev->hard_header_len = IPOIB_ENCAP_LEN; 1393 dev->addr_len = INFINIBAND_ALEN; 1394 dev->type = ARPHRD_INFINIBAND; 1395 dev->tx_queue_len = ipoib_sendq_size * 2; 1396 dev->features = (NETIF_F_VLAN_CHALLENGED | 1397 NETIF_F_HIGHDMA); 1398 netif_keep_dst(dev); 1399 1400 memcpy(dev->broadcast, ipv4_bcast_addr, INFINIBAND_ALEN); 1401 1402 priv->dev = dev; 1403 1404 spin_lock_init(&priv->lock); 1405 1406 init_rwsem(&priv->vlan_rwsem); 1407 1408 INIT_LIST_HEAD(&priv->path_list); 1409 INIT_LIST_HEAD(&priv->child_intfs); 1410 INIT_LIST_HEAD(&priv->dead_ahs); 1411 INIT_LIST_HEAD(&priv->multicast_list); 1412 1413 INIT_DELAYED_WORK(&priv->mcast_task, ipoib_mcast_join_task); 1414 INIT_WORK(&priv->carrier_on_task, ipoib_mcast_carrier_on_task); 1415 INIT_WORK(&priv->flush_light, ipoib_ib_dev_flush_light); 1416 INIT_WORK(&priv->flush_normal, ipoib_ib_dev_flush_normal); 1417 INIT_WORK(&priv->flush_heavy, ipoib_ib_dev_flush_heavy); 1418 INIT_WORK(&priv->restart_task, ipoib_mcast_restart_task); 1419 INIT_DELAYED_WORK(&priv->ah_reap_task, ipoib_reap_ah); 1420 INIT_DELAYED_WORK(&priv->neigh_reap_task, ipoib_reap_neigh); 1421 } 1422 1423 struct ipoib_dev_priv *ipoib_intf_alloc(const char *name) 1424 { 1425 struct net_device *dev; 1426 1427 dev = alloc_netdev((int)sizeof(struct ipoib_dev_priv), name, 1428 NET_NAME_UNKNOWN, ipoib_setup); 1429 if (!dev) 1430 return NULL; 1431 1432 return netdev_priv(dev); 1433 } 1434 1435 static ssize_t show_pkey(struct device *dev, 1436 struct device_attribute *attr, char *buf) 1437 { 1438 struct ipoib_dev_priv *priv = netdev_priv(to_net_dev(dev)); 1439 1440 return sprintf(buf, "0x%04x\n", priv->pkey); 1441 } 1442 static DEVICE_ATTR(pkey, S_IRUGO, show_pkey, NULL); 1443 1444 static ssize_t show_umcast(struct device *dev, 1445 struct device_attribute *attr, char *buf) 1446 { 1447 struct ipoib_dev_priv *priv = netdev_priv(to_net_dev(dev)); 1448 1449 return sprintf(buf, "%d\n", test_bit(IPOIB_FLAG_UMCAST, &priv->flags)); 1450 } 1451 1452 void ipoib_set_umcast(struct net_device *ndev, int umcast_val) 1453 { 1454 struct ipoib_dev_priv *priv = netdev_priv(ndev); 1455 1456 if (umcast_val > 0) { 1457 set_bit(IPOIB_FLAG_UMCAST, &priv->flags); 1458 ipoib_warn(priv, "ignoring multicast groups joined directly " 1459 "by userspace\n"); 1460 } else 1461 clear_bit(IPOIB_FLAG_UMCAST, &priv->flags); 1462 } 1463 1464 static ssize_t set_umcast(struct device *dev, 1465 struct device_attribute *attr, 1466 const char *buf, size_t count) 1467 { 1468 unsigned long umcast_val = simple_strtoul(buf, NULL, 0); 1469 1470 ipoib_set_umcast(to_net_dev(dev), umcast_val); 1471 1472 return count; 1473 } 1474 static DEVICE_ATTR(umcast, S_IWUSR | S_IRUGO, show_umcast, set_umcast); 1475 1476 int ipoib_add_umcast_attr(struct net_device *dev) 1477 { 1478 return device_create_file(&dev->dev, &dev_attr_umcast); 1479 } 1480 1481 static ssize_t create_child(struct device *dev, 1482 struct device_attribute *attr, 1483 const char *buf, size_t count) 1484 { 1485 int pkey; 1486 int ret; 1487 1488 if (sscanf(buf, "%i", &pkey) != 1) 1489 return -EINVAL; 1490 1491 if (pkey <= 0 || pkey > 0xffff || pkey == 0x8000) 1492 return -EINVAL; 1493 1494 /* 1495 * Set the full membership bit, so that we join the right 1496 * broadcast group, etc. 1497 */ 1498 pkey |= 0x8000; 1499 1500 ret = ipoib_vlan_add(to_net_dev(dev), pkey); 1501 1502 return ret ? ret : count; 1503 } 1504 static DEVICE_ATTR(create_child, S_IWUSR, NULL, create_child); 1505 1506 static ssize_t delete_child(struct device *dev, 1507 struct device_attribute *attr, 1508 const char *buf, size_t count) 1509 { 1510 int pkey; 1511 int ret; 1512 1513 if (sscanf(buf, "%i", &pkey) != 1) 1514 return -EINVAL; 1515 1516 if (pkey < 0 || pkey > 0xffff) 1517 return -EINVAL; 1518 1519 ret = ipoib_vlan_delete(to_net_dev(dev), pkey); 1520 1521 return ret ? ret : count; 1522 1523 } 1524 static DEVICE_ATTR(delete_child, S_IWUSR, NULL, delete_child); 1525 1526 int ipoib_add_pkey_attr(struct net_device *dev) 1527 { 1528 return device_create_file(&dev->dev, &dev_attr_pkey); 1529 } 1530 1531 int ipoib_set_dev_features(struct ipoib_dev_priv *priv, struct ib_device *hca) 1532 { 1533 struct ib_device_attr *device_attr; 1534 int result = -ENOMEM; 1535 1536 device_attr = kmalloc(sizeof *device_attr, GFP_KERNEL); 1537 if (!device_attr) { 1538 printk(KERN_WARNING "%s: allocation of %zu bytes failed\n", 1539 hca->name, sizeof *device_attr); 1540 return result; 1541 } 1542 1543 result = ib_query_device(hca, device_attr); 1544 if (result) { 1545 printk(KERN_WARNING "%s: ib_query_device failed (ret = %d)\n", 1546 hca->name, result); 1547 kfree(device_attr); 1548 return result; 1549 } 1550 priv->hca_caps = device_attr->device_cap_flags; 1551 1552 kfree(device_attr); 1553 1554 if (priv->hca_caps & IB_DEVICE_UD_IP_CSUM) { 1555 priv->dev->hw_features = NETIF_F_SG | 1556 NETIF_F_IP_CSUM | NETIF_F_RXCSUM; 1557 1558 if (priv->hca_caps & IB_DEVICE_UD_TSO) 1559 priv->dev->hw_features |= NETIF_F_TSO; 1560 1561 priv->dev->features |= priv->dev->hw_features; 1562 } 1563 1564 return 0; 1565 } 1566 1567 static struct net_device *ipoib_add_port(const char *format, 1568 struct ib_device *hca, u8 port) 1569 { 1570 struct ipoib_dev_priv *priv; 1571 struct ib_port_attr attr; 1572 int result = -ENOMEM; 1573 1574 priv = ipoib_intf_alloc(format); 1575 if (!priv) 1576 goto alloc_mem_failed; 1577 1578 SET_NETDEV_DEV(priv->dev, hca->dma_device); 1579 priv->dev->dev_id = port - 1; 1580 1581 result = ib_query_port(hca, port, &attr); 1582 if (!result) 1583 priv->max_ib_mtu = ib_mtu_enum_to_int(attr.max_mtu); 1584 else { 1585 printk(KERN_WARNING "%s: ib_query_port %d failed\n", 1586 hca->name, port); 1587 goto device_init_failed; 1588 } 1589 1590 /* MTU will be reset when mcast join happens */ 1591 priv->dev->mtu = IPOIB_UD_MTU(priv->max_ib_mtu); 1592 priv->mcast_mtu = priv->admin_mtu = priv->dev->mtu; 1593 1594 priv->dev->neigh_priv_len = sizeof(struct ipoib_neigh); 1595 1596 result = ib_query_pkey(hca, port, 0, &priv->pkey); 1597 if (result) { 1598 printk(KERN_WARNING "%s: ib_query_pkey port %d failed (ret = %d)\n", 1599 hca->name, port, result); 1600 goto device_init_failed; 1601 } 1602 1603 result = ipoib_set_dev_features(priv, hca); 1604 if (result) 1605 goto device_init_failed; 1606 1607 /* 1608 * Set the full membership bit, so that we join the right 1609 * broadcast group, etc. 1610 */ 1611 priv->pkey |= 0x8000; 1612 1613 priv->dev->broadcast[8] = priv->pkey >> 8; 1614 priv->dev->broadcast[9] = priv->pkey & 0xff; 1615 1616 result = ib_query_gid(hca, port, 0, &priv->local_gid); 1617 if (result) { 1618 printk(KERN_WARNING "%s: ib_query_gid port %d failed (ret = %d)\n", 1619 hca->name, port, result); 1620 goto device_init_failed; 1621 } else 1622 memcpy(priv->dev->dev_addr + 4, priv->local_gid.raw, sizeof (union ib_gid)); 1623 1624 result = ipoib_dev_init(priv->dev, hca, port); 1625 if (result < 0) { 1626 printk(KERN_WARNING "%s: failed to initialize port %d (ret = %d)\n", 1627 hca->name, port, result); 1628 goto device_init_failed; 1629 } 1630 1631 INIT_IB_EVENT_HANDLER(&priv->event_handler, 1632 priv->ca, ipoib_event); 1633 result = ib_register_event_handler(&priv->event_handler); 1634 if (result < 0) { 1635 printk(KERN_WARNING "%s: ib_register_event_handler failed for " 1636 "port %d (ret = %d)\n", 1637 hca->name, port, result); 1638 goto event_failed; 1639 } 1640 1641 result = register_netdev(priv->dev); 1642 if (result) { 1643 printk(KERN_WARNING "%s: couldn't register ipoib port %d; error %d\n", 1644 hca->name, port, result); 1645 goto register_failed; 1646 } 1647 1648 ipoib_create_debug_files(priv->dev); 1649 1650 if (ipoib_cm_add_mode_attr(priv->dev)) 1651 goto sysfs_failed; 1652 if (ipoib_add_pkey_attr(priv->dev)) 1653 goto sysfs_failed; 1654 if (ipoib_add_umcast_attr(priv->dev)) 1655 goto sysfs_failed; 1656 if (device_create_file(&priv->dev->dev, &dev_attr_create_child)) 1657 goto sysfs_failed; 1658 if (device_create_file(&priv->dev->dev, &dev_attr_delete_child)) 1659 goto sysfs_failed; 1660 1661 return priv->dev; 1662 1663 sysfs_failed: 1664 ipoib_delete_debug_files(priv->dev); 1665 unregister_netdev(priv->dev); 1666 1667 register_failed: 1668 ib_unregister_event_handler(&priv->event_handler); 1669 flush_workqueue(ipoib_workqueue); 1670 /* Stop GC if started before flush */ 1671 set_bit(IPOIB_STOP_NEIGH_GC, &priv->flags); 1672 cancel_delayed_work(&priv->neigh_reap_task); 1673 flush_workqueue(priv->wq); 1674 1675 event_failed: 1676 ipoib_dev_cleanup(priv->dev); 1677 1678 device_init_failed: 1679 free_netdev(priv->dev); 1680 1681 alloc_mem_failed: 1682 return ERR_PTR(result); 1683 } 1684 1685 static void ipoib_add_one(struct ib_device *device) 1686 { 1687 struct list_head *dev_list; 1688 struct net_device *dev; 1689 struct ipoib_dev_priv *priv; 1690 int p; 1691 int count = 0; 1692 1693 dev_list = kmalloc(sizeof *dev_list, GFP_KERNEL); 1694 if (!dev_list) 1695 return; 1696 1697 INIT_LIST_HEAD(dev_list); 1698 1699 for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) { 1700 if (!rdma_protocol_ib(device, p)) 1701 continue; 1702 dev = ipoib_add_port("ib%d", device, p); 1703 if (!IS_ERR(dev)) { 1704 priv = netdev_priv(dev); 1705 list_add_tail(&priv->list, dev_list); 1706 count++; 1707 } 1708 } 1709 1710 if (!count) { 1711 kfree(dev_list); 1712 return; 1713 } 1714 1715 ib_set_client_data(device, &ipoib_client, dev_list); 1716 } 1717 1718 static void ipoib_remove_one(struct ib_device *device) 1719 { 1720 struct ipoib_dev_priv *priv, *tmp; 1721 struct list_head *dev_list; 1722 1723 dev_list = ib_get_client_data(device, &ipoib_client); 1724 if (!dev_list) 1725 return; 1726 1727 list_for_each_entry_safe(priv, tmp, dev_list, list) { 1728 ib_unregister_event_handler(&priv->event_handler); 1729 flush_workqueue(ipoib_workqueue); 1730 1731 rtnl_lock(); 1732 dev_change_flags(priv->dev, priv->dev->flags & ~IFF_UP); 1733 rtnl_unlock(); 1734 1735 /* Stop GC */ 1736 set_bit(IPOIB_STOP_NEIGH_GC, &priv->flags); 1737 cancel_delayed_work(&priv->neigh_reap_task); 1738 flush_workqueue(priv->wq); 1739 1740 unregister_netdev(priv->dev); 1741 free_netdev(priv->dev); 1742 } 1743 1744 kfree(dev_list); 1745 } 1746 1747 static int __init ipoib_init_module(void) 1748 { 1749 int ret; 1750 1751 ipoib_recvq_size = roundup_pow_of_two(ipoib_recvq_size); 1752 ipoib_recvq_size = min(ipoib_recvq_size, IPOIB_MAX_QUEUE_SIZE); 1753 ipoib_recvq_size = max(ipoib_recvq_size, IPOIB_MIN_QUEUE_SIZE); 1754 1755 ipoib_sendq_size = roundup_pow_of_two(ipoib_sendq_size); 1756 ipoib_sendq_size = min(ipoib_sendq_size, IPOIB_MAX_QUEUE_SIZE); 1757 ipoib_sendq_size = max3(ipoib_sendq_size, 2 * MAX_SEND_CQE, IPOIB_MIN_QUEUE_SIZE); 1758 #ifdef CONFIG_INFINIBAND_IPOIB_CM 1759 ipoib_max_conn_qp = min(ipoib_max_conn_qp, IPOIB_CM_MAX_CONN_QP); 1760 #endif 1761 1762 /* 1763 * When copying small received packets, we only copy from the 1764 * linear data part of the SKB, so we rely on this condition. 1765 */ 1766 BUILD_BUG_ON(IPOIB_CM_COPYBREAK > IPOIB_CM_HEAD_SIZE); 1767 1768 ret = ipoib_register_debugfs(); 1769 if (ret) 1770 return ret; 1771 1772 /* 1773 * We create a global workqueue here that is used for all flush 1774 * operations. However, if you attempt to flush a workqueue 1775 * from a task on that same workqueue, it deadlocks the system. 1776 * We want to be able to flush the tasks associated with a 1777 * specific net device, so we also create a workqueue for each 1778 * netdevice. We queue up the tasks for that device only on 1779 * its private workqueue, and we only queue up flush events 1780 * on our global flush workqueue. This avoids the deadlocks. 1781 */ 1782 ipoib_workqueue = create_singlethread_workqueue("ipoib_flush"); 1783 if (!ipoib_workqueue) { 1784 ret = -ENOMEM; 1785 goto err_fs; 1786 } 1787 1788 ib_sa_register_client(&ipoib_sa_client); 1789 1790 ret = ib_register_client(&ipoib_client); 1791 if (ret) 1792 goto err_sa; 1793 1794 ret = ipoib_netlink_init(); 1795 if (ret) 1796 goto err_client; 1797 1798 return 0; 1799 1800 err_client: 1801 ib_unregister_client(&ipoib_client); 1802 1803 err_sa: 1804 ib_sa_unregister_client(&ipoib_sa_client); 1805 destroy_workqueue(ipoib_workqueue); 1806 1807 err_fs: 1808 ipoib_unregister_debugfs(); 1809 1810 return ret; 1811 } 1812 1813 static void __exit ipoib_cleanup_module(void) 1814 { 1815 ipoib_netlink_fini(); 1816 ib_unregister_client(&ipoib_client); 1817 ib_sa_unregister_client(&ipoib_sa_client); 1818 ipoib_unregister_debugfs(); 1819 destroy_workqueue(ipoib_workqueue); 1820 } 1821 1822 module_init(ipoib_init_module); 1823 module_exit(ipoib_cleanup_module); 1824