1 /* 2 * NET3 Protocol independent device support routines. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 * 9 * Derived from the non IP parts of dev.c 1.0.19 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Mark Evans, <evansmp@uhura.aston.ac.uk> 13 * 14 * Additional Authors: 15 * Florian la Roche <rzsfl@rz.uni-sb.de> 16 * Alan Cox <gw4pts@gw4pts.ampr.org> 17 * David Hinds <dahinds@users.sourceforge.net> 18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 19 * Adam Sulmicki <adam@cfar.umd.edu> 20 * Pekka Riikonen <priikone@poesidon.pspt.fi> 21 * 22 * Changes: 23 * D.J. Barrow : Fixed bug where dev->refcnt gets set 24 * to 2 if register_netdev gets called 25 * before net_dev_init & also removed a 26 * few lines of code in the process. 27 * Alan Cox : device private ioctl copies fields back. 28 * Alan Cox : Transmit queue code does relevant 29 * stunts to keep the queue safe. 30 * Alan Cox : Fixed double lock. 31 * Alan Cox : Fixed promisc NULL pointer trap 32 * ???????? : Support the full private ioctl range 33 * Alan Cox : Moved ioctl permission check into 34 * drivers 35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI 36 * Alan Cox : 100 backlog just doesn't cut it when 37 * you start doing multicast video 8) 38 * Alan Cox : Rewrote net_bh and list manager. 39 * Alan Cox : Fix ETH_P_ALL echoback lengths. 40 * Alan Cox : Took out transmit every packet pass 41 * Saved a few bytes in the ioctl handler 42 * Alan Cox : Network driver sets packet type before 43 * calling netif_rx. Saves a function 44 * call a packet. 45 * Alan Cox : Hashed net_bh() 46 * Richard Kooijman: Timestamp fixes. 47 * Alan Cox : Wrong field in SIOCGIFDSTADDR 48 * Alan Cox : Device lock protection. 49 * Alan Cox : Fixed nasty side effect of device close 50 * changes. 51 * Rudi Cilibrasi : Pass the right thing to 52 * set_mac_address() 53 * Dave Miller : 32bit quantity for the device lock to 54 * make it work out on a Sparc. 55 * Bjorn Ekwall : Added KERNELD hack. 56 * Alan Cox : Cleaned up the backlog initialise. 57 * Craig Metz : SIOCGIFCONF fix if space for under 58 * 1 device. 59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there 60 * is no device open function. 61 * Andi Kleen : Fix error reporting for SIOCGIFCONF 62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF 63 * Cyrus Durgin : Cleaned for KMOD 64 * Adam Sulmicki : Bug Fix : Network Device Unload 65 * A network device unload needs to purge 66 * the backlog queue. 67 * Paul Rusty Russell : SIOCSIFNAME 68 * Pekka Riikonen : Netdev boot-time settings code 69 * Andrew Morton : Make unregister_netdevice wait 70 * indefinitely on dev->refcnt 71 * J Hadi Salim : - Backlog queue sampling 72 * - netif_rx() feedback 73 */ 74 75 #include <asm/uaccess.h> 76 #include <asm/system.h> 77 #include <linux/bitops.h> 78 #include <linux/capability.h> 79 #include <linux/cpu.h> 80 #include <linux/types.h> 81 #include <linux/kernel.h> 82 #include <linux/hash.h> 83 #include <linux/sched.h> 84 #include <linux/mutex.h> 85 #include <linux/string.h> 86 #include <linux/mm.h> 87 #include <linux/socket.h> 88 #include <linux/sockios.h> 89 #include <linux/errno.h> 90 #include <linux/interrupt.h> 91 #include <linux/if_ether.h> 92 #include <linux/netdevice.h> 93 #include <linux/etherdevice.h> 94 #include <linux/ethtool.h> 95 #include <linux/notifier.h> 96 #include <linux/skbuff.h> 97 #include <net/net_namespace.h> 98 #include <net/sock.h> 99 #include <linux/rtnetlink.h> 100 #include <linux/proc_fs.h> 101 #include <linux/seq_file.h> 102 #include <linux/stat.h> 103 #include <linux/if_bridge.h> 104 #include <linux/if_macvlan.h> 105 #include <net/dst.h> 106 #include <net/pkt_sched.h> 107 #include <net/checksum.h> 108 #include <net/xfrm.h> 109 #include <linux/highmem.h> 110 #include <linux/init.h> 111 #include <linux/kmod.h> 112 #include <linux/module.h> 113 #include <linux/netpoll.h> 114 #include <linux/rcupdate.h> 115 #include <linux/delay.h> 116 #include <net/wext.h> 117 #include <net/iw_handler.h> 118 #include <asm/current.h> 119 #include <linux/audit.h> 120 #include <linux/dmaengine.h> 121 #include <linux/err.h> 122 #include <linux/ctype.h> 123 #include <linux/if_arp.h> 124 #include <linux/if_vlan.h> 125 #include <linux/ip.h> 126 #include <net/ip.h> 127 #include <linux/ipv6.h> 128 #include <linux/in.h> 129 #include <linux/jhash.h> 130 #include <linux/random.h> 131 #include <trace/events/napi.h> 132 133 #include "net-sysfs.h" 134 135 /* Instead of increasing this, you should create a hash table. */ 136 #define MAX_GRO_SKBS 8 137 138 /* This should be increased if a protocol with a bigger head is added. */ 139 #define GRO_MAX_HEAD (MAX_HEADER + 128) 140 141 /* 142 * The list of packet types we will receive (as opposed to discard) 143 * and the routines to invoke. 144 * 145 * Why 16. Because with 16 the only overlap we get on a hash of the 146 * low nibble of the protocol value is RARP/SNAP/X.25. 147 * 148 * NOTE: That is no longer true with the addition of VLAN tags. Not 149 * sure which should go first, but I bet it won't make much 150 * difference if we are running VLANs. The good news is that 151 * this protocol won't be in the list unless compiled in, so 152 * the average user (w/out VLANs) will not be adversely affected. 153 * --BLG 154 * 155 * 0800 IP 156 * 8100 802.1Q VLAN 157 * 0001 802.3 158 * 0002 AX.25 159 * 0004 802.2 160 * 8035 RARP 161 * 0005 SNAP 162 * 0805 X.25 163 * 0806 ARP 164 * 8137 IPX 165 * 0009 Localtalk 166 * 86DD IPv6 167 */ 168 169 #define PTYPE_HASH_SIZE (16) 170 #define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1) 171 172 static DEFINE_SPINLOCK(ptype_lock); 173 static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; 174 static struct list_head ptype_all __read_mostly; /* Taps */ 175 176 /* 177 * The @dev_base_head list is protected by @dev_base_lock and the rtnl 178 * semaphore. 179 * 180 * Pure readers hold dev_base_lock for reading, or rcu_read_lock() 181 * 182 * Writers must hold the rtnl semaphore while they loop through the 183 * dev_base_head list, and hold dev_base_lock for writing when they do the 184 * actual updates. This allows pure readers to access the list even 185 * while a writer is preparing to update it. 186 * 187 * To put it another way, dev_base_lock is held for writing only to 188 * protect against pure readers; the rtnl semaphore provides the 189 * protection against other writers. 190 * 191 * See, for example usages, register_netdevice() and 192 * unregister_netdevice(), which must be called with the rtnl 193 * semaphore held. 194 */ 195 DEFINE_RWLOCK(dev_base_lock); 196 EXPORT_SYMBOL(dev_base_lock); 197 198 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name) 199 { 200 unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ)); 201 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; 202 } 203 204 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) 205 { 206 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; 207 } 208 209 /* Device list insertion */ 210 static int list_netdevice(struct net_device *dev) 211 { 212 struct net *net = dev_net(dev); 213 214 ASSERT_RTNL(); 215 216 write_lock_bh(&dev_base_lock); 217 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); 218 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); 219 hlist_add_head_rcu(&dev->index_hlist, 220 dev_index_hash(net, dev->ifindex)); 221 write_unlock_bh(&dev_base_lock); 222 return 0; 223 } 224 225 /* Device list removal 226 * caller must respect a RCU grace period before freeing/reusing dev 227 */ 228 static void unlist_netdevice(struct net_device *dev) 229 { 230 ASSERT_RTNL(); 231 232 /* Unlink dev from the device chain */ 233 write_lock_bh(&dev_base_lock); 234 list_del_rcu(&dev->dev_list); 235 hlist_del_rcu(&dev->name_hlist); 236 hlist_del_rcu(&dev->index_hlist); 237 write_unlock_bh(&dev_base_lock); 238 } 239 240 /* 241 * Our notifier list 242 */ 243 244 static RAW_NOTIFIER_HEAD(netdev_chain); 245 246 /* 247 * Device drivers call our routines to queue packets here. We empty the 248 * queue in the local softnet handler. 249 */ 250 251 DEFINE_PER_CPU(struct softnet_data, softnet_data); 252 EXPORT_PER_CPU_SYMBOL(softnet_data); 253 254 #ifdef CONFIG_LOCKDEP 255 /* 256 * register_netdevice() inits txq->_xmit_lock and sets lockdep class 257 * according to dev->type 258 */ 259 static const unsigned short netdev_lock_type[] = 260 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25, 261 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET, 262 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM, 263 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, 264 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, 265 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, 266 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, 267 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, 268 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, 269 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, 270 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, 271 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, 272 ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211, 273 ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, 274 ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154, 275 ARPHRD_VOID, ARPHRD_NONE}; 276 277 static const char *const netdev_lock_name[] = 278 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", 279 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET", 280 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM", 281 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", 282 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", 283 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", 284 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", 285 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", 286 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", 287 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", 288 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", 289 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", 290 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211", 291 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", 292 "_xmit_PHONET_PIPE", "_xmit_IEEE802154", 293 "_xmit_VOID", "_xmit_NONE"}; 294 295 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; 296 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; 297 298 static inline unsigned short netdev_lock_pos(unsigned short dev_type) 299 { 300 int i; 301 302 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++) 303 if (netdev_lock_type[i] == dev_type) 304 return i; 305 /* the last key is used by default */ 306 return ARRAY_SIZE(netdev_lock_type) - 1; 307 } 308 309 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 310 unsigned short dev_type) 311 { 312 int i; 313 314 i = netdev_lock_pos(dev_type); 315 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i], 316 netdev_lock_name[i]); 317 } 318 319 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 320 { 321 int i; 322 323 i = netdev_lock_pos(dev->type); 324 lockdep_set_class_and_name(&dev->addr_list_lock, 325 &netdev_addr_lock_key[i], 326 netdev_lock_name[i]); 327 } 328 #else 329 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 330 unsigned short dev_type) 331 { 332 } 333 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 334 { 335 } 336 #endif 337 338 /******************************************************************************* 339 340 Protocol management and registration routines 341 342 *******************************************************************************/ 343 344 /* 345 * Add a protocol ID to the list. Now that the input handler is 346 * smarter we can dispense with all the messy stuff that used to be 347 * here. 348 * 349 * BEWARE!!! Protocol handlers, mangling input packets, 350 * MUST BE last in hash buckets and checking protocol handlers 351 * MUST start from promiscuous ptype_all chain in net_bh. 352 * It is true now, do not change it. 353 * Explanation follows: if protocol handler, mangling packet, will 354 * be the first on list, it is not able to sense, that packet 355 * is cloned and should be copied-on-write, so that it will 356 * change it and subsequent readers will get broken packet. 357 * --ANK (980803) 358 */ 359 360 /** 361 * dev_add_pack - add packet handler 362 * @pt: packet type declaration 363 * 364 * Add a protocol handler to the networking stack. The passed &packet_type 365 * is linked into kernel lists and may not be freed until it has been 366 * removed from the kernel lists. 367 * 368 * This call does not sleep therefore it can not 369 * guarantee all CPU's that are in middle of receiving packets 370 * will see the new packet type (until the next received packet). 371 */ 372 373 void dev_add_pack(struct packet_type *pt) 374 { 375 int hash; 376 377 spin_lock_bh(&ptype_lock); 378 if (pt->type == htons(ETH_P_ALL)) 379 list_add_rcu(&pt->list, &ptype_all); 380 else { 381 hash = ntohs(pt->type) & PTYPE_HASH_MASK; 382 list_add_rcu(&pt->list, &ptype_base[hash]); 383 } 384 spin_unlock_bh(&ptype_lock); 385 } 386 EXPORT_SYMBOL(dev_add_pack); 387 388 /** 389 * __dev_remove_pack - remove packet handler 390 * @pt: packet type declaration 391 * 392 * Remove a protocol handler that was previously added to the kernel 393 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 394 * from the kernel lists and can be freed or reused once this function 395 * returns. 396 * 397 * The packet type might still be in use by receivers 398 * and must not be freed until after all the CPU's have gone 399 * through a quiescent state. 400 */ 401 void __dev_remove_pack(struct packet_type *pt) 402 { 403 struct list_head *head; 404 struct packet_type *pt1; 405 406 spin_lock_bh(&ptype_lock); 407 408 if (pt->type == htons(ETH_P_ALL)) 409 head = &ptype_all; 410 else 411 head = &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; 412 413 list_for_each_entry(pt1, head, list) { 414 if (pt == pt1) { 415 list_del_rcu(&pt->list); 416 goto out; 417 } 418 } 419 420 printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt); 421 out: 422 spin_unlock_bh(&ptype_lock); 423 } 424 EXPORT_SYMBOL(__dev_remove_pack); 425 426 /** 427 * dev_remove_pack - remove packet handler 428 * @pt: packet type declaration 429 * 430 * Remove a protocol handler that was previously added to the kernel 431 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 432 * from the kernel lists and can be freed or reused once this function 433 * returns. 434 * 435 * This call sleeps to guarantee that no CPU is looking at the packet 436 * type after return. 437 */ 438 void dev_remove_pack(struct packet_type *pt) 439 { 440 __dev_remove_pack(pt); 441 442 synchronize_net(); 443 } 444 EXPORT_SYMBOL(dev_remove_pack); 445 446 /****************************************************************************** 447 448 Device Boot-time Settings Routines 449 450 *******************************************************************************/ 451 452 /* Boot time configuration table */ 453 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX]; 454 455 /** 456 * netdev_boot_setup_add - add new setup entry 457 * @name: name of the device 458 * @map: configured settings for the device 459 * 460 * Adds new setup entry to the dev_boot_setup list. The function 461 * returns 0 on error and 1 on success. This is a generic routine to 462 * all netdevices. 463 */ 464 static int netdev_boot_setup_add(char *name, struct ifmap *map) 465 { 466 struct netdev_boot_setup *s; 467 int i; 468 469 s = dev_boot_setup; 470 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 471 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') { 472 memset(s[i].name, 0, sizeof(s[i].name)); 473 strlcpy(s[i].name, name, IFNAMSIZ); 474 memcpy(&s[i].map, map, sizeof(s[i].map)); 475 break; 476 } 477 } 478 479 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1; 480 } 481 482 /** 483 * netdev_boot_setup_check - check boot time settings 484 * @dev: the netdevice 485 * 486 * Check boot time settings for the device. 487 * The found settings are set for the device to be used 488 * later in the device probing. 489 * Returns 0 if no settings found, 1 if they are. 490 */ 491 int netdev_boot_setup_check(struct net_device *dev) 492 { 493 struct netdev_boot_setup *s = dev_boot_setup; 494 int i; 495 496 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 497 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' && 498 !strcmp(dev->name, s[i].name)) { 499 dev->irq = s[i].map.irq; 500 dev->base_addr = s[i].map.base_addr; 501 dev->mem_start = s[i].map.mem_start; 502 dev->mem_end = s[i].map.mem_end; 503 return 1; 504 } 505 } 506 return 0; 507 } 508 EXPORT_SYMBOL(netdev_boot_setup_check); 509 510 511 /** 512 * netdev_boot_base - get address from boot time settings 513 * @prefix: prefix for network device 514 * @unit: id for network device 515 * 516 * Check boot time settings for the base address of device. 517 * The found settings are set for the device to be used 518 * later in the device probing. 519 * Returns 0 if no settings found. 520 */ 521 unsigned long netdev_boot_base(const char *prefix, int unit) 522 { 523 const struct netdev_boot_setup *s = dev_boot_setup; 524 char name[IFNAMSIZ]; 525 int i; 526 527 sprintf(name, "%s%d", prefix, unit); 528 529 /* 530 * If device already registered then return base of 1 531 * to indicate not to probe for this interface 532 */ 533 if (__dev_get_by_name(&init_net, name)) 534 return 1; 535 536 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) 537 if (!strcmp(name, s[i].name)) 538 return s[i].map.base_addr; 539 return 0; 540 } 541 542 /* 543 * Saves at boot time configured settings for any netdevice. 544 */ 545 int __init netdev_boot_setup(char *str) 546 { 547 int ints[5]; 548 struct ifmap map; 549 550 str = get_options(str, ARRAY_SIZE(ints), ints); 551 if (!str || !*str) 552 return 0; 553 554 /* Save settings */ 555 memset(&map, 0, sizeof(map)); 556 if (ints[0] > 0) 557 map.irq = ints[1]; 558 if (ints[0] > 1) 559 map.base_addr = ints[2]; 560 if (ints[0] > 2) 561 map.mem_start = ints[3]; 562 if (ints[0] > 3) 563 map.mem_end = ints[4]; 564 565 /* Add new entry to the list */ 566 return netdev_boot_setup_add(str, &map); 567 } 568 569 __setup("netdev=", netdev_boot_setup); 570 571 /******************************************************************************* 572 573 Device Interface Subroutines 574 575 *******************************************************************************/ 576 577 /** 578 * __dev_get_by_name - find a device by its name 579 * @net: the applicable net namespace 580 * @name: name to find 581 * 582 * Find an interface by name. Must be called under RTNL semaphore 583 * or @dev_base_lock. If the name is found a pointer to the device 584 * is returned. If the name is not found then %NULL is returned. The 585 * reference counters are not incremented so the caller must be 586 * careful with locks. 587 */ 588 589 struct net_device *__dev_get_by_name(struct net *net, const char *name) 590 { 591 struct hlist_node *p; 592 struct net_device *dev; 593 struct hlist_head *head = dev_name_hash(net, name); 594 595 hlist_for_each_entry(dev, p, head, name_hlist) 596 if (!strncmp(dev->name, name, IFNAMSIZ)) 597 return dev; 598 599 return NULL; 600 } 601 EXPORT_SYMBOL(__dev_get_by_name); 602 603 /** 604 * dev_get_by_name_rcu - find a device by its name 605 * @net: the applicable net namespace 606 * @name: name to find 607 * 608 * Find an interface by name. 609 * If the name is found a pointer to the device is returned. 610 * If the name is not found then %NULL is returned. 611 * The reference counters are not incremented so the caller must be 612 * careful with locks. The caller must hold RCU lock. 613 */ 614 615 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name) 616 { 617 struct hlist_node *p; 618 struct net_device *dev; 619 struct hlist_head *head = dev_name_hash(net, name); 620 621 hlist_for_each_entry_rcu(dev, p, head, name_hlist) 622 if (!strncmp(dev->name, name, IFNAMSIZ)) 623 return dev; 624 625 return NULL; 626 } 627 EXPORT_SYMBOL(dev_get_by_name_rcu); 628 629 /** 630 * dev_get_by_name - find a device by its name 631 * @net: the applicable net namespace 632 * @name: name to find 633 * 634 * Find an interface by name. This can be called from any 635 * context and does its own locking. The returned handle has 636 * the usage count incremented and the caller must use dev_put() to 637 * release it when it is no longer needed. %NULL is returned if no 638 * matching device is found. 639 */ 640 641 struct net_device *dev_get_by_name(struct net *net, const char *name) 642 { 643 struct net_device *dev; 644 645 rcu_read_lock(); 646 dev = dev_get_by_name_rcu(net, name); 647 if (dev) 648 dev_hold(dev); 649 rcu_read_unlock(); 650 return dev; 651 } 652 EXPORT_SYMBOL(dev_get_by_name); 653 654 /** 655 * __dev_get_by_index - find a device by its ifindex 656 * @net: the applicable net namespace 657 * @ifindex: index of device 658 * 659 * Search for an interface by index. Returns %NULL if the device 660 * is not found or a pointer to the device. The device has not 661 * had its reference counter increased so the caller must be careful 662 * about locking. The caller must hold either the RTNL semaphore 663 * or @dev_base_lock. 664 */ 665 666 struct net_device *__dev_get_by_index(struct net *net, int ifindex) 667 { 668 struct hlist_node *p; 669 struct net_device *dev; 670 struct hlist_head *head = dev_index_hash(net, ifindex); 671 672 hlist_for_each_entry(dev, p, head, index_hlist) 673 if (dev->ifindex == ifindex) 674 return dev; 675 676 return NULL; 677 } 678 EXPORT_SYMBOL(__dev_get_by_index); 679 680 /** 681 * dev_get_by_index_rcu - find a device by its ifindex 682 * @net: the applicable net namespace 683 * @ifindex: index of device 684 * 685 * Search for an interface by index. Returns %NULL if the device 686 * is not found or a pointer to the device. The device has not 687 * had its reference counter increased so the caller must be careful 688 * about locking. The caller must hold RCU lock. 689 */ 690 691 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex) 692 { 693 struct hlist_node *p; 694 struct net_device *dev; 695 struct hlist_head *head = dev_index_hash(net, ifindex); 696 697 hlist_for_each_entry_rcu(dev, p, head, index_hlist) 698 if (dev->ifindex == ifindex) 699 return dev; 700 701 return NULL; 702 } 703 EXPORT_SYMBOL(dev_get_by_index_rcu); 704 705 706 /** 707 * dev_get_by_index - find a device by its ifindex 708 * @net: the applicable net namespace 709 * @ifindex: index of device 710 * 711 * Search for an interface by index. Returns NULL if the device 712 * is not found or a pointer to the device. The device returned has 713 * had a reference added and the pointer is safe until the user calls 714 * dev_put to indicate they have finished with it. 715 */ 716 717 struct net_device *dev_get_by_index(struct net *net, int ifindex) 718 { 719 struct net_device *dev; 720 721 rcu_read_lock(); 722 dev = dev_get_by_index_rcu(net, ifindex); 723 if (dev) 724 dev_hold(dev); 725 rcu_read_unlock(); 726 return dev; 727 } 728 EXPORT_SYMBOL(dev_get_by_index); 729 730 /** 731 * dev_getbyhwaddr - find a device by its hardware address 732 * @net: the applicable net namespace 733 * @type: media type of device 734 * @ha: hardware address 735 * 736 * Search for an interface by MAC address. Returns NULL if the device 737 * is not found or a pointer to the device. The caller must hold the 738 * rtnl semaphore. The returned device has not had its ref count increased 739 * and the caller must therefore be careful about locking 740 * 741 * BUGS: 742 * If the API was consistent this would be __dev_get_by_hwaddr 743 */ 744 745 struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type, char *ha) 746 { 747 struct net_device *dev; 748 749 ASSERT_RTNL(); 750 751 for_each_netdev(net, dev) 752 if (dev->type == type && 753 !memcmp(dev->dev_addr, ha, dev->addr_len)) 754 return dev; 755 756 return NULL; 757 } 758 EXPORT_SYMBOL(dev_getbyhwaddr); 759 760 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type) 761 { 762 struct net_device *dev; 763 764 ASSERT_RTNL(); 765 for_each_netdev(net, dev) 766 if (dev->type == type) 767 return dev; 768 769 return NULL; 770 } 771 EXPORT_SYMBOL(__dev_getfirstbyhwtype); 772 773 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) 774 { 775 struct net_device *dev; 776 777 rtnl_lock(); 778 dev = __dev_getfirstbyhwtype(net, type); 779 if (dev) 780 dev_hold(dev); 781 rtnl_unlock(); 782 return dev; 783 } 784 EXPORT_SYMBOL(dev_getfirstbyhwtype); 785 786 /** 787 * dev_get_by_flags - find any device with given flags 788 * @net: the applicable net namespace 789 * @if_flags: IFF_* values 790 * @mask: bitmask of bits in if_flags to check 791 * 792 * Search for any interface with the given flags. Returns NULL if a device 793 * is not found or a pointer to the device. The device returned has 794 * had a reference added and the pointer is safe until the user calls 795 * dev_put to indicate they have finished with it. 796 */ 797 798 struct net_device *dev_get_by_flags(struct net *net, unsigned short if_flags, 799 unsigned short mask) 800 { 801 struct net_device *dev, *ret; 802 803 ret = NULL; 804 rcu_read_lock(); 805 for_each_netdev_rcu(net, dev) { 806 if (((dev->flags ^ if_flags) & mask) == 0) { 807 dev_hold(dev); 808 ret = dev; 809 break; 810 } 811 } 812 rcu_read_unlock(); 813 return ret; 814 } 815 EXPORT_SYMBOL(dev_get_by_flags); 816 817 /** 818 * dev_valid_name - check if name is okay for network device 819 * @name: name string 820 * 821 * Network device names need to be valid file names to 822 * to allow sysfs to work. We also disallow any kind of 823 * whitespace. 824 */ 825 int dev_valid_name(const char *name) 826 { 827 if (*name == '\0') 828 return 0; 829 if (strlen(name) >= IFNAMSIZ) 830 return 0; 831 if (!strcmp(name, ".") || !strcmp(name, "..")) 832 return 0; 833 834 while (*name) { 835 if (*name == '/' || isspace(*name)) 836 return 0; 837 name++; 838 } 839 return 1; 840 } 841 EXPORT_SYMBOL(dev_valid_name); 842 843 /** 844 * __dev_alloc_name - allocate a name for a device 845 * @net: network namespace to allocate the device name in 846 * @name: name format string 847 * @buf: scratch buffer and result name string 848 * 849 * Passed a format string - eg "lt%d" it will try and find a suitable 850 * id. It scans list of devices to build up a free map, then chooses 851 * the first empty slot. The caller must hold the dev_base or rtnl lock 852 * while allocating the name and adding the device in order to avoid 853 * duplicates. 854 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 855 * Returns the number of the unit assigned or a negative errno code. 856 */ 857 858 static int __dev_alloc_name(struct net *net, const char *name, char *buf) 859 { 860 int i = 0; 861 const char *p; 862 const int max_netdevices = 8*PAGE_SIZE; 863 unsigned long *inuse; 864 struct net_device *d; 865 866 p = strnchr(name, IFNAMSIZ-1, '%'); 867 if (p) { 868 /* 869 * Verify the string as this thing may have come from 870 * the user. There must be either one "%d" and no other "%" 871 * characters. 872 */ 873 if (p[1] != 'd' || strchr(p + 2, '%')) 874 return -EINVAL; 875 876 /* Use one page as a bit array of possible slots */ 877 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC); 878 if (!inuse) 879 return -ENOMEM; 880 881 for_each_netdev(net, d) { 882 if (!sscanf(d->name, name, &i)) 883 continue; 884 if (i < 0 || i >= max_netdevices) 885 continue; 886 887 /* avoid cases where sscanf is not exact inverse of printf */ 888 snprintf(buf, IFNAMSIZ, name, i); 889 if (!strncmp(buf, d->name, IFNAMSIZ)) 890 set_bit(i, inuse); 891 } 892 893 i = find_first_zero_bit(inuse, max_netdevices); 894 free_page((unsigned long) inuse); 895 } 896 897 if (buf != name) 898 snprintf(buf, IFNAMSIZ, name, i); 899 if (!__dev_get_by_name(net, buf)) 900 return i; 901 902 /* It is possible to run out of possible slots 903 * when the name is long and there isn't enough space left 904 * for the digits, or if all bits are used. 905 */ 906 return -ENFILE; 907 } 908 909 /** 910 * dev_alloc_name - allocate a name for a device 911 * @dev: device 912 * @name: name format string 913 * 914 * Passed a format string - eg "lt%d" it will try and find a suitable 915 * id. It scans list of devices to build up a free map, then chooses 916 * the first empty slot. The caller must hold the dev_base or rtnl lock 917 * while allocating the name and adding the device in order to avoid 918 * duplicates. 919 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 920 * Returns the number of the unit assigned or a negative errno code. 921 */ 922 923 int dev_alloc_name(struct net_device *dev, const char *name) 924 { 925 char buf[IFNAMSIZ]; 926 struct net *net; 927 int ret; 928 929 BUG_ON(!dev_net(dev)); 930 net = dev_net(dev); 931 ret = __dev_alloc_name(net, name, buf); 932 if (ret >= 0) 933 strlcpy(dev->name, buf, IFNAMSIZ); 934 return ret; 935 } 936 EXPORT_SYMBOL(dev_alloc_name); 937 938 static int dev_get_valid_name(struct net *net, const char *name, char *buf, 939 bool fmt) 940 { 941 if (!dev_valid_name(name)) 942 return -EINVAL; 943 944 if (fmt && strchr(name, '%')) 945 return __dev_alloc_name(net, name, buf); 946 else if (__dev_get_by_name(net, name)) 947 return -EEXIST; 948 else if (buf != name) 949 strlcpy(buf, name, IFNAMSIZ); 950 951 return 0; 952 } 953 954 /** 955 * dev_change_name - change name of a device 956 * @dev: device 957 * @newname: name (or format string) must be at least IFNAMSIZ 958 * 959 * Change name of a device, can pass format strings "eth%d". 960 * for wildcarding. 961 */ 962 int dev_change_name(struct net_device *dev, const char *newname) 963 { 964 char oldname[IFNAMSIZ]; 965 int err = 0; 966 int ret; 967 struct net *net; 968 969 ASSERT_RTNL(); 970 BUG_ON(!dev_net(dev)); 971 972 net = dev_net(dev); 973 if (dev->flags & IFF_UP) 974 return -EBUSY; 975 976 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) 977 return 0; 978 979 memcpy(oldname, dev->name, IFNAMSIZ); 980 981 err = dev_get_valid_name(net, newname, dev->name, 1); 982 if (err < 0) 983 return err; 984 985 rollback: 986 /* For now only devices in the initial network namespace 987 * are in sysfs. 988 */ 989 if (net_eq(net, &init_net)) { 990 ret = device_rename(&dev->dev, dev->name); 991 if (ret) { 992 memcpy(dev->name, oldname, IFNAMSIZ); 993 return ret; 994 } 995 } 996 997 write_lock_bh(&dev_base_lock); 998 hlist_del(&dev->name_hlist); 999 write_unlock_bh(&dev_base_lock); 1000 1001 synchronize_rcu(); 1002 1003 write_lock_bh(&dev_base_lock); 1004 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); 1005 write_unlock_bh(&dev_base_lock); 1006 1007 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); 1008 ret = notifier_to_errno(ret); 1009 1010 if (ret) { 1011 /* err >= 0 after dev_alloc_name() or stores the first errno */ 1012 if (err >= 0) { 1013 err = ret; 1014 memcpy(dev->name, oldname, IFNAMSIZ); 1015 goto rollback; 1016 } else { 1017 printk(KERN_ERR 1018 "%s: name change rollback failed: %d.\n", 1019 dev->name, ret); 1020 } 1021 } 1022 1023 return err; 1024 } 1025 1026 /** 1027 * dev_set_alias - change ifalias of a device 1028 * @dev: device 1029 * @alias: name up to IFALIASZ 1030 * @len: limit of bytes to copy from info 1031 * 1032 * Set ifalias for a device, 1033 */ 1034 int dev_set_alias(struct net_device *dev, const char *alias, size_t len) 1035 { 1036 ASSERT_RTNL(); 1037 1038 if (len >= IFALIASZ) 1039 return -EINVAL; 1040 1041 if (!len) { 1042 if (dev->ifalias) { 1043 kfree(dev->ifalias); 1044 dev->ifalias = NULL; 1045 } 1046 return 0; 1047 } 1048 1049 dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL); 1050 if (!dev->ifalias) 1051 return -ENOMEM; 1052 1053 strlcpy(dev->ifalias, alias, len+1); 1054 return len; 1055 } 1056 1057 1058 /** 1059 * netdev_features_change - device changes features 1060 * @dev: device to cause notification 1061 * 1062 * Called to indicate a device has changed features. 1063 */ 1064 void netdev_features_change(struct net_device *dev) 1065 { 1066 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); 1067 } 1068 EXPORT_SYMBOL(netdev_features_change); 1069 1070 /** 1071 * netdev_state_change - device changes state 1072 * @dev: device to cause notification 1073 * 1074 * Called to indicate a device has changed state. This function calls 1075 * the notifier chains for netdev_chain and sends a NEWLINK message 1076 * to the routing socket. 1077 */ 1078 void netdev_state_change(struct net_device *dev) 1079 { 1080 if (dev->flags & IFF_UP) { 1081 call_netdevice_notifiers(NETDEV_CHANGE, dev); 1082 rtmsg_ifinfo(RTM_NEWLINK, dev, 0); 1083 } 1084 } 1085 EXPORT_SYMBOL(netdev_state_change); 1086 1087 void netdev_bonding_change(struct net_device *dev, unsigned long event) 1088 { 1089 call_netdevice_notifiers(event, dev); 1090 } 1091 EXPORT_SYMBOL(netdev_bonding_change); 1092 1093 /** 1094 * dev_load - load a network module 1095 * @net: the applicable net namespace 1096 * @name: name of interface 1097 * 1098 * If a network interface is not present and the process has suitable 1099 * privileges this function loads the module. If module loading is not 1100 * available in this kernel then it becomes a nop. 1101 */ 1102 1103 void dev_load(struct net *net, const char *name) 1104 { 1105 struct net_device *dev; 1106 1107 rcu_read_lock(); 1108 dev = dev_get_by_name_rcu(net, name); 1109 rcu_read_unlock(); 1110 1111 if (!dev && capable(CAP_NET_ADMIN)) 1112 request_module("%s", name); 1113 } 1114 EXPORT_SYMBOL(dev_load); 1115 1116 /** 1117 * dev_open - prepare an interface for use. 1118 * @dev: device to open 1119 * 1120 * Takes a device from down to up state. The device's private open 1121 * function is invoked and then the multicast lists are loaded. Finally 1122 * the device is moved into the up state and a %NETDEV_UP message is 1123 * sent to the netdev notifier chain. 1124 * 1125 * Calling this function on an active interface is a nop. On a failure 1126 * a negative errno code is returned. 1127 */ 1128 int dev_open(struct net_device *dev) 1129 { 1130 const struct net_device_ops *ops = dev->netdev_ops; 1131 int ret; 1132 1133 ASSERT_RTNL(); 1134 1135 /* 1136 * Is it already up? 1137 */ 1138 1139 if (dev->flags & IFF_UP) 1140 return 0; 1141 1142 /* 1143 * Is it even present? 1144 */ 1145 if (!netif_device_present(dev)) 1146 return -ENODEV; 1147 1148 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev); 1149 ret = notifier_to_errno(ret); 1150 if (ret) 1151 return ret; 1152 1153 /* 1154 * Call device private open method 1155 */ 1156 set_bit(__LINK_STATE_START, &dev->state); 1157 1158 if (ops->ndo_validate_addr) 1159 ret = ops->ndo_validate_addr(dev); 1160 1161 if (!ret && ops->ndo_open) 1162 ret = ops->ndo_open(dev); 1163 1164 /* 1165 * If it went open OK then: 1166 */ 1167 1168 if (ret) 1169 clear_bit(__LINK_STATE_START, &dev->state); 1170 else { 1171 /* 1172 * Set the flags. 1173 */ 1174 dev->flags |= IFF_UP; 1175 1176 /* 1177 * Enable NET_DMA 1178 */ 1179 net_dmaengine_get(); 1180 1181 /* 1182 * Initialize multicasting status 1183 */ 1184 dev_set_rx_mode(dev); 1185 1186 /* 1187 * Wakeup transmit queue engine 1188 */ 1189 dev_activate(dev); 1190 1191 /* 1192 * ... and announce new interface. 1193 */ 1194 call_netdevice_notifiers(NETDEV_UP, dev); 1195 } 1196 1197 return ret; 1198 } 1199 EXPORT_SYMBOL(dev_open); 1200 1201 /** 1202 * dev_close - shutdown an interface. 1203 * @dev: device to shutdown 1204 * 1205 * This function moves an active device into down state. A 1206 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device 1207 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier 1208 * chain. 1209 */ 1210 int dev_close(struct net_device *dev) 1211 { 1212 const struct net_device_ops *ops = dev->netdev_ops; 1213 ASSERT_RTNL(); 1214 1215 might_sleep(); 1216 1217 if (!(dev->flags & IFF_UP)) 1218 return 0; 1219 1220 /* 1221 * Tell people we are going down, so that they can 1222 * prepare to death, when device is still operating. 1223 */ 1224 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); 1225 1226 clear_bit(__LINK_STATE_START, &dev->state); 1227 1228 /* Synchronize to scheduled poll. We cannot touch poll list, 1229 * it can be even on different cpu. So just clear netif_running(). 1230 * 1231 * dev->stop() will invoke napi_disable() on all of it's 1232 * napi_struct instances on this device. 1233 */ 1234 smp_mb__after_clear_bit(); /* Commit netif_running(). */ 1235 1236 dev_deactivate(dev); 1237 1238 /* 1239 * Call the device specific close. This cannot fail. 1240 * Only if device is UP 1241 * 1242 * We allow it to be called even after a DETACH hot-plug 1243 * event. 1244 */ 1245 if (ops->ndo_stop) 1246 ops->ndo_stop(dev); 1247 1248 /* 1249 * Device is now down. 1250 */ 1251 1252 dev->flags &= ~IFF_UP; 1253 1254 /* 1255 * Tell people we are down 1256 */ 1257 call_netdevice_notifiers(NETDEV_DOWN, dev); 1258 1259 /* 1260 * Shutdown NET_DMA 1261 */ 1262 net_dmaengine_put(); 1263 1264 return 0; 1265 } 1266 EXPORT_SYMBOL(dev_close); 1267 1268 1269 /** 1270 * dev_disable_lro - disable Large Receive Offload on a device 1271 * @dev: device 1272 * 1273 * Disable Large Receive Offload (LRO) on a net device. Must be 1274 * called under RTNL. This is needed if received packets may be 1275 * forwarded to another interface. 1276 */ 1277 void dev_disable_lro(struct net_device *dev) 1278 { 1279 if (dev->ethtool_ops && dev->ethtool_ops->get_flags && 1280 dev->ethtool_ops->set_flags) { 1281 u32 flags = dev->ethtool_ops->get_flags(dev); 1282 if (flags & ETH_FLAG_LRO) { 1283 flags &= ~ETH_FLAG_LRO; 1284 dev->ethtool_ops->set_flags(dev, flags); 1285 } 1286 } 1287 WARN_ON(dev->features & NETIF_F_LRO); 1288 } 1289 EXPORT_SYMBOL(dev_disable_lro); 1290 1291 1292 static int dev_boot_phase = 1; 1293 1294 /* 1295 * Device change register/unregister. These are not inline or static 1296 * as we export them to the world. 1297 */ 1298 1299 /** 1300 * register_netdevice_notifier - register a network notifier block 1301 * @nb: notifier 1302 * 1303 * Register a notifier to be called when network device events occur. 1304 * The notifier passed is linked into the kernel structures and must 1305 * not be reused until it has been unregistered. A negative errno code 1306 * is returned on a failure. 1307 * 1308 * When registered all registration and up events are replayed 1309 * to the new notifier to allow device to have a race free 1310 * view of the network device list. 1311 */ 1312 1313 int register_netdevice_notifier(struct notifier_block *nb) 1314 { 1315 struct net_device *dev; 1316 struct net_device *last; 1317 struct net *net; 1318 int err; 1319 1320 rtnl_lock(); 1321 err = raw_notifier_chain_register(&netdev_chain, nb); 1322 if (err) 1323 goto unlock; 1324 if (dev_boot_phase) 1325 goto unlock; 1326 for_each_net(net) { 1327 for_each_netdev(net, dev) { 1328 err = nb->notifier_call(nb, NETDEV_REGISTER, dev); 1329 err = notifier_to_errno(err); 1330 if (err) 1331 goto rollback; 1332 1333 if (!(dev->flags & IFF_UP)) 1334 continue; 1335 1336 nb->notifier_call(nb, NETDEV_UP, dev); 1337 } 1338 } 1339 1340 unlock: 1341 rtnl_unlock(); 1342 return err; 1343 1344 rollback: 1345 last = dev; 1346 for_each_net(net) { 1347 for_each_netdev(net, dev) { 1348 if (dev == last) 1349 break; 1350 1351 if (dev->flags & IFF_UP) { 1352 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev); 1353 nb->notifier_call(nb, NETDEV_DOWN, dev); 1354 } 1355 nb->notifier_call(nb, NETDEV_UNREGISTER, dev); 1356 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev); 1357 } 1358 } 1359 1360 raw_notifier_chain_unregister(&netdev_chain, nb); 1361 goto unlock; 1362 } 1363 EXPORT_SYMBOL(register_netdevice_notifier); 1364 1365 /** 1366 * unregister_netdevice_notifier - unregister a network notifier block 1367 * @nb: notifier 1368 * 1369 * Unregister a notifier previously registered by 1370 * register_netdevice_notifier(). The notifier is unlinked into the 1371 * kernel structures and may then be reused. A negative errno code 1372 * is returned on a failure. 1373 */ 1374 1375 int unregister_netdevice_notifier(struct notifier_block *nb) 1376 { 1377 int err; 1378 1379 rtnl_lock(); 1380 err = raw_notifier_chain_unregister(&netdev_chain, nb); 1381 rtnl_unlock(); 1382 return err; 1383 } 1384 EXPORT_SYMBOL(unregister_netdevice_notifier); 1385 1386 /** 1387 * call_netdevice_notifiers - call all network notifier blocks 1388 * @val: value passed unmodified to notifier function 1389 * @dev: net_device pointer passed unmodified to notifier function 1390 * 1391 * Call all network notifier blocks. Parameters and return value 1392 * are as for raw_notifier_call_chain(). 1393 */ 1394 1395 int call_netdevice_notifiers(unsigned long val, struct net_device *dev) 1396 { 1397 return raw_notifier_call_chain(&netdev_chain, val, dev); 1398 } 1399 1400 /* When > 0 there are consumers of rx skb time stamps */ 1401 static atomic_t netstamp_needed = ATOMIC_INIT(0); 1402 1403 void net_enable_timestamp(void) 1404 { 1405 atomic_inc(&netstamp_needed); 1406 } 1407 EXPORT_SYMBOL(net_enable_timestamp); 1408 1409 void net_disable_timestamp(void) 1410 { 1411 atomic_dec(&netstamp_needed); 1412 } 1413 EXPORT_SYMBOL(net_disable_timestamp); 1414 1415 static inline void net_timestamp(struct sk_buff *skb) 1416 { 1417 if (atomic_read(&netstamp_needed)) 1418 __net_timestamp(skb); 1419 else 1420 skb->tstamp.tv64 = 0; 1421 } 1422 1423 /** 1424 * dev_forward_skb - loopback an skb to another netif 1425 * 1426 * @dev: destination network device 1427 * @skb: buffer to forward 1428 * 1429 * return values: 1430 * NET_RX_SUCCESS (no congestion) 1431 * NET_RX_DROP (packet was dropped) 1432 * 1433 * dev_forward_skb can be used for injecting an skb from the 1434 * start_xmit function of one device into the receive queue 1435 * of another device. 1436 * 1437 * The receiving device may be in another namespace, so 1438 * we have to clear all information in the skb that could 1439 * impact namespace isolation. 1440 */ 1441 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 1442 { 1443 skb_orphan(skb); 1444 1445 if (!(dev->flags & IFF_UP)) 1446 return NET_RX_DROP; 1447 1448 if (skb->len > (dev->mtu + dev->hard_header_len)) 1449 return NET_RX_DROP; 1450 1451 skb_dst_drop(skb); 1452 skb->tstamp.tv64 = 0; 1453 skb->pkt_type = PACKET_HOST; 1454 skb->protocol = eth_type_trans(skb, dev); 1455 skb->mark = 0; 1456 secpath_reset(skb); 1457 nf_reset(skb); 1458 return netif_rx(skb); 1459 } 1460 EXPORT_SYMBOL_GPL(dev_forward_skb); 1461 1462 /* 1463 * Support routine. Sends outgoing frames to any network 1464 * taps currently in use. 1465 */ 1466 1467 static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) 1468 { 1469 struct packet_type *ptype; 1470 1471 #ifdef CONFIG_NET_CLS_ACT 1472 if (!(skb->tstamp.tv64 && (G_TC_FROM(skb->tc_verd) & AT_INGRESS))) 1473 net_timestamp(skb); 1474 #else 1475 net_timestamp(skb); 1476 #endif 1477 1478 rcu_read_lock(); 1479 list_for_each_entry_rcu(ptype, &ptype_all, list) { 1480 /* Never send packets back to the socket 1481 * they originated from - MvS (miquels@drinkel.ow.org) 1482 */ 1483 if ((ptype->dev == dev || !ptype->dev) && 1484 (ptype->af_packet_priv == NULL || 1485 (struct sock *)ptype->af_packet_priv != skb->sk)) { 1486 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC); 1487 if (!skb2) 1488 break; 1489 1490 /* skb->nh should be correctly 1491 set by sender, so that the second statement is 1492 just protection against buggy protocols. 1493 */ 1494 skb_reset_mac_header(skb2); 1495 1496 if (skb_network_header(skb2) < skb2->data || 1497 skb2->network_header > skb2->tail) { 1498 if (net_ratelimit()) 1499 printk(KERN_CRIT "protocol %04x is " 1500 "buggy, dev %s\n", 1501 skb2->protocol, dev->name); 1502 skb_reset_network_header(skb2); 1503 } 1504 1505 skb2->transport_header = skb2->network_header; 1506 skb2->pkt_type = PACKET_OUTGOING; 1507 ptype->func(skb2, skb->dev, ptype, skb->dev); 1508 } 1509 } 1510 rcu_read_unlock(); 1511 } 1512 1513 1514 static inline void __netif_reschedule(struct Qdisc *q) 1515 { 1516 struct softnet_data *sd; 1517 unsigned long flags; 1518 1519 local_irq_save(flags); 1520 sd = &__get_cpu_var(softnet_data); 1521 q->next_sched = sd->output_queue; 1522 sd->output_queue = q; 1523 raise_softirq_irqoff(NET_TX_SOFTIRQ); 1524 local_irq_restore(flags); 1525 } 1526 1527 void __netif_schedule(struct Qdisc *q) 1528 { 1529 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) 1530 __netif_reschedule(q); 1531 } 1532 EXPORT_SYMBOL(__netif_schedule); 1533 1534 void dev_kfree_skb_irq(struct sk_buff *skb) 1535 { 1536 if (atomic_dec_and_test(&skb->users)) { 1537 struct softnet_data *sd; 1538 unsigned long flags; 1539 1540 local_irq_save(flags); 1541 sd = &__get_cpu_var(softnet_data); 1542 skb->next = sd->completion_queue; 1543 sd->completion_queue = skb; 1544 raise_softirq_irqoff(NET_TX_SOFTIRQ); 1545 local_irq_restore(flags); 1546 } 1547 } 1548 EXPORT_SYMBOL(dev_kfree_skb_irq); 1549 1550 void dev_kfree_skb_any(struct sk_buff *skb) 1551 { 1552 if (in_irq() || irqs_disabled()) 1553 dev_kfree_skb_irq(skb); 1554 else 1555 dev_kfree_skb(skb); 1556 } 1557 EXPORT_SYMBOL(dev_kfree_skb_any); 1558 1559 1560 /** 1561 * netif_device_detach - mark device as removed 1562 * @dev: network device 1563 * 1564 * Mark device as removed from system and therefore no longer available. 1565 */ 1566 void netif_device_detach(struct net_device *dev) 1567 { 1568 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && 1569 netif_running(dev)) { 1570 netif_tx_stop_all_queues(dev); 1571 } 1572 } 1573 EXPORT_SYMBOL(netif_device_detach); 1574 1575 /** 1576 * netif_device_attach - mark device as attached 1577 * @dev: network device 1578 * 1579 * Mark device as attached from system and restart if needed. 1580 */ 1581 void netif_device_attach(struct net_device *dev) 1582 { 1583 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && 1584 netif_running(dev)) { 1585 netif_tx_wake_all_queues(dev); 1586 __netdev_watchdog_up(dev); 1587 } 1588 } 1589 EXPORT_SYMBOL(netif_device_attach); 1590 1591 static bool can_checksum_protocol(unsigned long features, __be16 protocol) 1592 { 1593 return ((features & NETIF_F_GEN_CSUM) || 1594 ((features & NETIF_F_IP_CSUM) && 1595 protocol == htons(ETH_P_IP)) || 1596 ((features & NETIF_F_IPV6_CSUM) && 1597 protocol == htons(ETH_P_IPV6)) || 1598 ((features & NETIF_F_FCOE_CRC) && 1599 protocol == htons(ETH_P_FCOE))); 1600 } 1601 1602 static bool dev_can_checksum(struct net_device *dev, struct sk_buff *skb) 1603 { 1604 if (can_checksum_protocol(dev->features, skb->protocol)) 1605 return true; 1606 1607 if (skb->protocol == htons(ETH_P_8021Q)) { 1608 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data; 1609 if (can_checksum_protocol(dev->features & dev->vlan_features, 1610 veh->h_vlan_encapsulated_proto)) 1611 return true; 1612 } 1613 1614 return false; 1615 } 1616 1617 /* 1618 * Invalidate hardware checksum when packet is to be mangled, and 1619 * complete checksum manually on outgoing path. 1620 */ 1621 int skb_checksum_help(struct sk_buff *skb) 1622 { 1623 __wsum csum; 1624 int ret = 0, offset; 1625 1626 if (skb->ip_summed == CHECKSUM_COMPLETE) 1627 goto out_set_summed; 1628 1629 if (unlikely(skb_shinfo(skb)->gso_size)) { 1630 /* Let GSO fix up the checksum. */ 1631 goto out_set_summed; 1632 } 1633 1634 offset = skb->csum_start - skb_headroom(skb); 1635 BUG_ON(offset >= skb_headlen(skb)); 1636 csum = skb_checksum(skb, offset, skb->len - offset, 0); 1637 1638 offset += skb->csum_offset; 1639 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb)); 1640 1641 if (skb_cloned(skb) && 1642 !skb_clone_writable(skb, offset + sizeof(__sum16))) { 1643 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 1644 if (ret) 1645 goto out; 1646 } 1647 1648 *(__sum16 *)(skb->data + offset) = csum_fold(csum); 1649 out_set_summed: 1650 skb->ip_summed = CHECKSUM_NONE; 1651 out: 1652 return ret; 1653 } 1654 EXPORT_SYMBOL(skb_checksum_help); 1655 1656 /** 1657 * skb_gso_segment - Perform segmentation on skb. 1658 * @skb: buffer to segment 1659 * @features: features for the output path (see dev->features) 1660 * 1661 * This function segments the given skb and returns a list of segments. 1662 * 1663 * It may return NULL if the skb requires no segmentation. This is 1664 * only possible when GSO is used for verifying header integrity. 1665 */ 1666 struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features) 1667 { 1668 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); 1669 struct packet_type *ptype; 1670 __be16 type = skb->protocol; 1671 int err; 1672 1673 skb_reset_mac_header(skb); 1674 skb->mac_len = skb->network_header - skb->mac_header; 1675 __skb_pull(skb, skb->mac_len); 1676 1677 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) { 1678 struct net_device *dev = skb->dev; 1679 struct ethtool_drvinfo info = {}; 1680 1681 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo) 1682 dev->ethtool_ops->get_drvinfo(dev, &info); 1683 1684 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d " 1685 "ip_summed=%d", 1686 info.driver, dev ? dev->features : 0L, 1687 skb->sk ? skb->sk->sk_route_caps : 0L, 1688 skb->len, skb->data_len, skb->ip_summed); 1689 1690 if (skb_header_cloned(skb) && 1691 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))) 1692 return ERR_PTR(err); 1693 } 1694 1695 rcu_read_lock(); 1696 list_for_each_entry_rcu(ptype, 1697 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) { 1698 if (ptype->type == type && !ptype->dev && ptype->gso_segment) { 1699 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) { 1700 err = ptype->gso_send_check(skb); 1701 segs = ERR_PTR(err); 1702 if (err || skb_gso_ok(skb, features)) 1703 break; 1704 __skb_push(skb, (skb->data - 1705 skb_network_header(skb))); 1706 } 1707 segs = ptype->gso_segment(skb, features); 1708 break; 1709 } 1710 } 1711 rcu_read_unlock(); 1712 1713 __skb_push(skb, skb->data - skb_mac_header(skb)); 1714 1715 return segs; 1716 } 1717 EXPORT_SYMBOL(skb_gso_segment); 1718 1719 /* Take action when hardware reception checksum errors are detected. */ 1720 #ifdef CONFIG_BUG 1721 void netdev_rx_csum_fault(struct net_device *dev) 1722 { 1723 if (net_ratelimit()) { 1724 printk(KERN_ERR "%s: hw csum failure.\n", 1725 dev ? dev->name : "<unknown>"); 1726 dump_stack(); 1727 } 1728 } 1729 EXPORT_SYMBOL(netdev_rx_csum_fault); 1730 #endif 1731 1732 /* Actually, we should eliminate this check as soon as we know, that: 1733 * 1. IOMMU is present and allows to map all the memory. 1734 * 2. No high memory really exists on this machine. 1735 */ 1736 1737 static inline int illegal_highdma(struct net_device *dev, struct sk_buff *skb) 1738 { 1739 #ifdef CONFIG_HIGHMEM 1740 int i; 1741 1742 if (dev->features & NETIF_F_HIGHDMA) 1743 return 0; 1744 1745 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1746 if (PageHighMem(skb_shinfo(skb)->frags[i].page)) 1747 return 1; 1748 1749 #endif 1750 return 0; 1751 } 1752 1753 struct dev_gso_cb { 1754 void (*destructor)(struct sk_buff *skb); 1755 }; 1756 1757 #define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb) 1758 1759 static void dev_gso_skb_destructor(struct sk_buff *skb) 1760 { 1761 struct dev_gso_cb *cb; 1762 1763 do { 1764 struct sk_buff *nskb = skb->next; 1765 1766 skb->next = nskb->next; 1767 nskb->next = NULL; 1768 kfree_skb(nskb); 1769 } while (skb->next); 1770 1771 cb = DEV_GSO_CB(skb); 1772 if (cb->destructor) 1773 cb->destructor(skb); 1774 } 1775 1776 /** 1777 * dev_gso_segment - Perform emulated hardware segmentation on skb. 1778 * @skb: buffer to segment 1779 * 1780 * This function segments the given skb and stores the list of segments 1781 * in skb->next. 1782 */ 1783 static int dev_gso_segment(struct sk_buff *skb) 1784 { 1785 struct net_device *dev = skb->dev; 1786 struct sk_buff *segs; 1787 int features = dev->features & ~(illegal_highdma(dev, skb) ? 1788 NETIF_F_SG : 0); 1789 1790 segs = skb_gso_segment(skb, features); 1791 1792 /* Verifying header integrity only. */ 1793 if (!segs) 1794 return 0; 1795 1796 if (IS_ERR(segs)) 1797 return PTR_ERR(segs); 1798 1799 skb->next = segs; 1800 DEV_GSO_CB(skb)->destructor = skb->destructor; 1801 skb->destructor = dev_gso_skb_destructor; 1802 1803 return 0; 1804 } 1805 1806 int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev, 1807 struct netdev_queue *txq) 1808 { 1809 const struct net_device_ops *ops = dev->netdev_ops; 1810 int rc = NETDEV_TX_OK; 1811 1812 if (likely(!skb->next)) { 1813 if (!list_empty(&ptype_all)) 1814 dev_queue_xmit_nit(skb, dev); 1815 1816 if (netif_needs_gso(dev, skb)) { 1817 if (unlikely(dev_gso_segment(skb))) 1818 goto out_kfree_skb; 1819 if (skb->next) 1820 goto gso; 1821 } 1822 1823 /* 1824 * If device doesnt need skb->dst, release it right now while 1825 * its hot in this cpu cache 1826 */ 1827 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) 1828 skb_dst_drop(skb); 1829 1830 rc = ops->ndo_start_xmit(skb, dev); 1831 if (rc == NETDEV_TX_OK) 1832 txq_trans_update(txq); 1833 /* 1834 * TODO: if skb_orphan() was called by 1835 * dev->hard_start_xmit() (for example, the unmodified 1836 * igb driver does that; bnx2 doesn't), then 1837 * skb_tx_software_timestamp() will be unable to send 1838 * back the time stamp. 1839 * 1840 * How can this be prevented? Always create another 1841 * reference to the socket before calling 1842 * dev->hard_start_xmit()? Prevent that skb_orphan() 1843 * does anything in dev->hard_start_xmit() by clearing 1844 * the skb destructor before the call and restoring it 1845 * afterwards, then doing the skb_orphan() ourselves? 1846 */ 1847 return rc; 1848 } 1849 1850 gso: 1851 do { 1852 struct sk_buff *nskb = skb->next; 1853 1854 skb->next = nskb->next; 1855 nskb->next = NULL; 1856 rc = ops->ndo_start_xmit(nskb, dev); 1857 if (unlikely(rc != NETDEV_TX_OK)) { 1858 if (rc & ~NETDEV_TX_MASK) 1859 goto out_kfree_gso_skb; 1860 nskb->next = skb->next; 1861 skb->next = nskb; 1862 return rc; 1863 } 1864 txq_trans_update(txq); 1865 if (unlikely(netif_tx_queue_stopped(txq) && skb->next)) 1866 return NETDEV_TX_BUSY; 1867 } while (skb->next); 1868 1869 out_kfree_gso_skb: 1870 if (likely(skb->next == NULL)) 1871 skb->destructor = DEV_GSO_CB(skb)->destructor; 1872 out_kfree_skb: 1873 kfree_skb(skb); 1874 return rc; 1875 } 1876 1877 static u32 skb_tx_hashrnd; 1878 1879 u16 skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb) 1880 { 1881 u32 hash; 1882 1883 if (skb_rx_queue_recorded(skb)) { 1884 hash = skb_get_rx_queue(skb); 1885 while (unlikely(hash >= dev->real_num_tx_queues)) 1886 hash -= dev->real_num_tx_queues; 1887 return hash; 1888 } 1889 1890 if (skb->sk && skb->sk->sk_hash) 1891 hash = skb->sk->sk_hash; 1892 else 1893 hash = skb->protocol; 1894 1895 hash = jhash_1word(hash, skb_tx_hashrnd); 1896 1897 return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32); 1898 } 1899 EXPORT_SYMBOL(skb_tx_hash); 1900 1901 static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index) 1902 { 1903 if (unlikely(queue_index >= dev->real_num_tx_queues)) { 1904 if (net_ratelimit()) { 1905 WARN(1, "%s selects TX queue %d, but " 1906 "real number of TX queues is %d\n", 1907 dev->name, queue_index, 1908 dev->real_num_tx_queues); 1909 } 1910 return 0; 1911 } 1912 return queue_index; 1913 } 1914 1915 static struct netdev_queue *dev_pick_tx(struct net_device *dev, 1916 struct sk_buff *skb) 1917 { 1918 u16 queue_index; 1919 struct sock *sk = skb->sk; 1920 1921 if (sk_tx_queue_recorded(sk)) { 1922 queue_index = sk_tx_queue_get(sk); 1923 } else { 1924 const struct net_device_ops *ops = dev->netdev_ops; 1925 1926 if (ops->ndo_select_queue) { 1927 queue_index = ops->ndo_select_queue(dev, skb); 1928 queue_index = dev_cap_txqueue(dev, queue_index); 1929 } else { 1930 queue_index = 0; 1931 if (dev->real_num_tx_queues > 1) 1932 queue_index = skb_tx_hash(dev, skb); 1933 1934 if (sk && sk->sk_dst_cache) 1935 sk_tx_queue_set(sk, queue_index); 1936 } 1937 } 1938 1939 skb_set_queue_mapping(skb, queue_index); 1940 return netdev_get_tx_queue(dev, queue_index); 1941 } 1942 1943 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 1944 struct net_device *dev, 1945 struct netdev_queue *txq) 1946 { 1947 spinlock_t *root_lock = qdisc_lock(q); 1948 int rc; 1949 1950 spin_lock(root_lock); 1951 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { 1952 kfree_skb(skb); 1953 rc = NET_XMIT_DROP; 1954 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && 1955 !test_and_set_bit(__QDISC_STATE_RUNNING, &q->state)) { 1956 /* 1957 * This is a work-conserving queue; there are no old skbs 1958 * waiting to be sent out; and the qdisc is not running - 1959 * xmit the skb directly. 1960 */ 1961 __qdisc_update_bstats(q, skb->len); 1962 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) 1963 __qdisc_run(q); 1964 else 1965 clear_bit(__QDISC_STATE_RUNNING, &q->state); 1966 1967 rc = NET_XMIT_SUCCESS; 1968 } else { 1969 rc = qdisc_enqueue_root(skb, q); 1970 qdisc_run(q); 1971 } 1972 spin_unlock(root_lock); 1973 1974 return rc; 1975 } 1976 1977 /** 1978 * dev_queue_xmit - transmit a buffer 1979 * @skb: buffer to transmit 1980 * 1981 * Queue a buffer for transmission to a network device. The caller must 1982 * have set the device and priority and built the buffer before calling 1983 * this function. The function can be called from an interrupt. 1984 * 1985 * A negative errno code is returned on a failure. A success does not 1986 * guarantee the frame will be transmitted as it may be dropped due 1987 * to congestion or traffic shaping. 1988 * 1989 * ----------------------------------------------------------------------------------- 1990 * I notice this method can also return errors from the queue disciplines, 1991 * including NET_XMIT_DROP, which is a positive value. So, errors can also 1992 * be positive. 1993 * 1994 * Regardless of the return value, the skb is consumed, so it is currently 1995 * difficult to retry a send to this method. (You can bump the ref count 1996 * before sending to hold a reference for retry if you are careful.) 1997 * 1998 * When calling this method, interrupts MUST be enabled. This is because 1999 * the BH enable code must have IRQs enabled so that it will not deadlock. 2000 * --BLG 2001 */ 2002 int dev_queue_xmit(struct sk_buff *skb) 2003 { 2004 struct net_device *dev = skb->dev; 2005 struct netdev_queue *txq; 2006 struct Qdisc *q; 2007 int rc = -ENOMEM; 2008 2009 /* GSO will handle the following emulations directly. */ 2010 if (netif_needs_gso(dev, skb)) 2011 goto gso; 2012 2013 if (skb_has_frags(skb) && 2014 !(dev->features & NETIF_F_FRAGLIST) && 2015 __skb_linearize(skb)) 2016 goto out_kfree_skb; 2017 2018 /* Fragmented skb is linearized if device does not support SG, 2019 * or if at least one of fragments is in highmem and device 2020 * does not support DMA from it. 2021 */ 2022 if (skb_shinfo(skb)->nr_frags && 2023 (!(dev->features & NETIF_F_SG) || illegal_highdma(dev, skb)) && 2024 __skb_linearize(skb)) 2025 goto out_kfree_skb; 2026 2027 /* If packet is not checksummed and device does not support 2028 * checksumming for this protocol, complete checksumming here. 2029 */ 2030 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2031 skb_set_transport_header(skb, skb->csum_start - 2032 skb_headroom(skb)); 2033 if (!dev_can_checksum(dev, skb) && skb_checksum_help(skb)) 2034 goto out_kfree_skb; 2035 } 2036 2037 gso: 2038 /* Disable soft irqs for various locks below. Also 2039 * stops preemption for RCU. 2040 */ 2041 rcu_read_lock_bh(); 2042 2043 txq = dev_pick_tx(dev, skb); 2044 q = rcu_dereference(txq->qdisc); 2045 2046 #ifdef CONFIG_NET_CLS_ACT 2047 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS); 2048 #endif 2049 if (q->enqueue) { 2050 rc = __dev_xmit_skb(skb, q, dev, txq); 2051 goto out; 2052 } 2053 2054 /* The device has no queue. Common case for software devices: 2055 loopback, all the sorts of tunnels... 2056 2057 Really, it is unlikely that netif_tx_lock protection is necessary 2058 here. (f.e. loopback and IP tunnels are clean ignoring statistics 2059 counters.) 2060 However, it is possible, that they rely on protection 2061 made by us here. 2062 2063 Check this and shot the lock. It is not prone from deadlocks. 2064 Either shot noqueue qdisc, it is even simpler 8) 2065 */ 2066 if (dev->flags & IFF_UP) { 2067 int cpu = smp_processor_id(); /* ok because BHs are off */ 2068 2069 if (txq->xmit_lock_owner != cpu) { 2070 2071 HARD_TX_LOCK(dev, txq, cpu); 2072 2073 if (!netif_tx_queue_stopped(txq)) { 2074 rc = dev_hard_start_xmit(skb, dev, txq); 2075 if (dev_xmit_complete(rc)) { 2076 HARD_TX_UNLOCK(dev, txq); 2077 goto out; 2078 } 2079 } 2080 HARD_TX_UNLOCK(dev, txq); 2081 if (net_ratelimit()) 2082 printk(KERN_CRIT "Virtual device %s asks to " 2083 "queue packet!\n", dev->name); 2084 } else { 2085 /* Recursion is detected! It is possible, 2086 * unfortunately */ 2087 if (net_ratelimit()) 2088 printk(KERN_CRIT "Dead loop on virtual device " 2089 "%s, fix it urgently!\n", dev->name); 2090 } 2091 } 2092 2093 rc = -ENETDOWN; 2094 rcu_read_unlock_bh(); 2095 2096 out_kfree_skb: 2097 kfree_skb(skb); 2098 return rc; 2099 out: 2100 rcu_read_unlock_bh(); 2101 return rc; 2102 } 2103 EXPORT_SYMBOL(dev_queue_xmit); 2104 2105 2106 /*======================================================================= 2107 Receiver routines 2108 =======================================================================*/ 2109 2110 int netdev_max_backlog __read_mostly = 1000; 2111 int netdev_budget __read_mostly = 300; 2112 int weight_p __read_mostly = 64; /* old backlog weight */ 2113 2114 DEFINE_PER_CPU(struct netif_rx_stats, netdev_rx_stat) = { 0, }; 2115 2116 2117 /** 2118 * netif_rx - post buffer to the network code 2119 * @skb: buffer to post 2120 * 2121 * This function receives a packet from a device driver and queues it for 2122 * the upper (protocol) levels to process. It always succeeds. The buffer 2123 * may be dropped during processing for congestion control or by the 2124 * protocol layers. 2125 * 2126 * return values: 2127 * NET_RX_SUCCESS (no congestion) 2128 * NET_RX_DROP (packet was dropped) 2129 * 2130 */ 2131 2132 int netif_rx(struct sk_buff *skb) 2133 { 2134 struct softnet_data *queue; 2135 unsigned long flags; 2136 2137 /* if netpoll wants it, pretend we never saw it */ 2138 if (netpoll_rx(skb)) 2139 return NET_RX_DROP; 2140 2141 if (!skb->tstamp.tv64) 2142 net_timestamp(skb); 2143 2144 /* 2145 * The code is rearranged so that the path is the most 2146 * short when CPU is congested, but is still operating. 2147 */ 2148 local_irq_save(flags); 2149 queue = &__get_cpu_var(softnet_data); 2150 2151 __get_cpu_var(netdev_rx_stat).total++; 2152 if (queue->input_pkt_queue.qlen <= netdev_max_backlog) { 2153 if (queue->input_pkt_queue.qlen) { 2154 enqueue: 2155 __skb_queue_tail(&queue->input_pkt_queue, skb); 2156 local_irq_restore(flags); 2157 return NET_RX_SUCCESS; 2158 } 2159 2160 napi_schedule(&queue->backlog); 2161 goto enqueue; 2162 } 2163 2164 __get_cpu_var(netdev_rx_stat).dropped++; 2165 local_irq_restore(flags); 2166 2167 kfree_skb(skb); 2168 return NET_RX_DROP; 2169 } 2170 EXPORT_SYMBOL(netif_rx); 2171 2172 int netif_rx_ni(struct sk_buff *skb) 2173 { 2174 int err; 2175 2176 preempt_disable(); 2177 err = netif_rx(skb); 2178 if (local_softirq_pending()) 2179 do_softirq(); 2180 preempt_enable(); 2181 2182 return err; 2183 } 2184 EXPORT_SYMBOL(netif_rx_ni); 2185 2186 static void net_tx_action(struct softirq_action *h) 2187 { 2188 struct softnet_data *sd = &__get_cpu_var(softnet_data); 2189 2190 if (sd->completion_queue) { 2191 struct sk_buff *clist; 2192 2193 local_irq_disable(); 2194 clist = sd->completion_queue; 2195 sd->completion_queue = NULL; 2196 local_irq_enable(); 2197 2198 while (clist) { 2199 struct sk_buff *skb = clist; 2200 clist = clist->next; 2201 2202 WARN_ON(atomic_read(&skb->users)); 2203 __kfree_skb(skb); 2204 } 2205 } 2206 2207 if (sd->output_queue) { 2208 struct Qdisc *head; 2209 2210 local_irq_disable(); 2211 head = sd->output_queue; 2212 sd->output_queue = NULL; 2213 local_irq_enable(); 2214 2215 while (head) { 2216 struct Qdisc *q = head; 2217 spinlock_t *root_lock; 2218 2219 head = head->next_sched; 2220 2221 root_lock = qdisc_lock(q); 2222 if (spin_trylock(root_lock)) { 2223 smp_mb__before_clear_bit(); 2224 clear_bit(__QDISC_STATE_SCHED, 2225 &q->state); 2226 qdisc_run(q); 2227 spin_unlock(root_lock); 2228 } else { 2229 if (!test_bit(__QDISC_STATE_DEACTIVATED, 2230 &q->state)) { 2231 __netif_reschedule(q); 2232 } else { 2233 smp_mb__before_clear_bit(); 2234 clear_bit(__QDISC_STATE_SCHED, 2235 &q->state); 2236 } 2237 } 2238 } 2239 } 2240 } 2241 2242 static inline int deliver_skb(struct sk_buff *skb, 2243 struct packet_type *pt_prev, 2244 struct net_device *orig_dev) 2245 { 2246 atomic_inc(&skb->users); 2247 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 2248 } 2249 2250 #if defined(CONFIG_BRIDGE) || defined (CONFIG_BRIDGE_MODULE) 2251 2252 #if defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE) 2253 /* This hook is defined here for ATM LANE */ 2254 int (*br_fdb_test_addr_hook)(struct net_device *dev, 2255 unsigned char *addr) __read_mostly; 2256 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); 2257 #endif 2258 2259 /* 2260 * If bridge module is loaded call bridging hook. 2261 * returns NULL if packet was consumed. 2262 */ 2263 struct sk_buff *(*br_handle_frame_hook)(struct net_bridge_port *p, 2264 struct sk_buff *skb) __read_mostly; 2265 EXPORT_SYMBOL_GPL(br_handle_frame_hook); 2266 2267 static inline struct sk_buff *handle_bridge(struct sk_buff *skb, 2268 struct packet_type **pt_prev, int *ret, 2269 struct net_device *orig_dev) 2270 { 2271 struct net_bridge_port *port; 2272 2273 if (skb->pkt_type == PACKET_LOOPBACK || 2274 (port = rcu_dereference(skb->dev->br_port)) == NULL) 2275 return skb; 2276 2277 if (*pt_prev) { 2278 *ret = deliver_skb(skb, *pt_prev, orig_dev); 2279 *pt_prev = NULL; 2280 } 2281 2282 return br_handle_frame_hook(port, skb); 2283 } 2284 #else 2285 #define handle_bridge(skb, pt_prev, ret, orig_dev) (skb) 2286 #endif 2287 2288 #if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE) 2289 struct sk_buff *(*macvlan_handle_frame_hook)(struct sk_buff *skb) __read_mostly; 2290 EXPORT_SYMBOL_GPL(macvlan_handle_frame_hook); 2291 2292 static inline struct sk_buff *handle_macvlan(struct sk_buff *skb, 2293 struct packet_type **pt_prev, 2294 int *ret, 2295 struct net_device *orig_dev) 2296 { 2297 if (skb->dev->macvlan_port == NULL) 2298 return skb; 2299 2300 if (*pt_prev) { 2301 *ret = deliver_skb(skb, *pt_prev, orig_dev); 2302 *pt_prev = NULL; 2303 } 2304 return macvlan_handle_frame_hook(skb); 2305 } 2306 #else 2307 #define handle_macvlan(skb, pt_prev, ret, orig_dev) (skb) 2308 #endif 2309 2310 #ifdef CONFIG_NET_CLS_ACT 2311 /* TODO: Maybe we should just force sch_ingress to be compiled in 2312 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions 2313 * a compare and 2 stores extra right now if we dont have it on 2314 * but have CONFIG_NET_CLS_ACT 2315 * NOTE: This doesnt stop any functionality; if you dont have 2316 * the ingress scheduler, you just cant add policies on ingress. 2317 * 2318 */ 2319 static int ing_filter(struct sk_buff *skb) 2320 { 2321 struct net_device *dev = skb->dev; 2322 u32 ttl = G_TC_RTTL(skb->tc_verd); 2323 struct netdev_queue *rxq; 2324 int result = TC_ACT_OK; 2325 struct Qdisc *q; 2326 2327 if (MAX_RED_LOOP < ttl++) { 2328 printk(KERN_WARNING 2329 "Redir loop detected Dropping packet (%d->%d)\n", 2330 skb->skb_iif, dev->ifindex); 2331 return TC_ACT_SHOT; 2332 } 2333 2334 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl); 2335 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS); 2336 2337 rxq = &dev->rx_queue; 2338 2339 q = rxq->qdisc; 2340 if (q != &noop_qdisc) { 2341 spin_lock(qdisc_lock(q)); 2342 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) 2343 result = qdisc_enqueue_root(skb, q); 2344 spin_unlock(qdisc_lock(q)); 2345 } 2346 2347 return result; 2348 } 2349 2350 static inline struct sk_buff *handle_ing(struct sk_buff *skb, 2351 struct packet_type **pt_prev, 2352 int *ret, struct net_device *orig_dev) 2353 { 2354 if (skb->dev->rx_queue.qdisc == &noop_qdisc) 2355 goto out; 2356 2357 if (*pt_prev) { 2358 *ret = deliver_skb(skb, *pt_prev, orig_dev); 2359 *pt_prev = NULL; 2360 } else { 2361 /* Huh? Why does turning on AF_PACKET affect this? */ 2362 skb->tc_verd = SET_TC_OK2MUNGE(skb->tc_verd); 2363 } 2364 2365 switch (ing_filter(skb)) { 2366 case TC_ACT_SHOT: 2367 case TC_ACT_STOLEN: 2368 kfree_skb(skb); 2369 return NULL; 2370 } 2371 2372 out: 2373 skb->tc_verd = 0; 2374 return skb; 2375 } 2376 #endif 2377 2378 /* 2379 * netif_nit_deliver - deliver received packets to network taps 2380 * @skb: buffer 2381 * 2382 * This function is used to deliver incoming packets to network 2383 * taps. It should be used when the normal netif_receive_skb path 2384 * is bypassed, for example because of VLAN acceleration. 2385 */ 2386 void netif_nit_deliver(struct sk_buff *skb) 2387 { 2388 struct packet_type *ptype; 2389 2390 if (list_empty(&ptype_all)) 2391 return; 2392 2393 skb_reset_network_header(skb); 2394 skb_reset_transport_header(skb); 2395 skb->mac_len = skb->network_header - skb->mac_header; 2396 2397 rcu_read_lock(); 2398 list_for_each_entry_rcu(ptype, &ptype_all, list) { 2399 if (!ptype->dev || ptype->dev == skb->dev) 2400 deliver_skb(skb, ptype, skb->dev); 2401 } 2402 rcu_read_unlock(); 2403 } 2404 2405 /** 2406 * netif_receive_skb - process receive buffer from network 2407 * @skb: buffer to process 2408 * 2409 * netif_receive_skb() is the main receive data processing function. 2410 * It always succeeds. The buffer may be dropped during processing 2411 * for congestion control or by the protocol layers. 2412 * 2413 * This function may only be called from softirq context and interrupts 2414 * should be enabled. 2415 * 2416 * Return values (usually ignored): 2417 * NET_RX_SUCCESS: no congestion 2418 * NET_RX_DROP: packet was dropped 2419 */ 2420 int netif_receive_skb(struct sk_buff *skb) 2421 { 2422 struct packet_type *ptype, *pt_prev; 2423 struct net_device *orig_dev; 2424 struct net_device *null_or_orig; 2425 int ret = NET_RX_DROP; 2426 __be16 type; 2427 2428 if (!skb->tstamp.tv64) 2429 net_timestamp(skb); 2430 2431 if (vlan_tx_tag_present(skb) && vlan_hwaccel_do_receive(skb)) 2432 return NET_RX_SUCCESS; 2433 2434 /* if we've gotten here through NAPI, check netpoll */ 2435 if (netpoll_receive_skb(skb)) 2436 return NET_RX_DROP; 2437 2438 if (!skb->skb_iif) 2439 skb->skb_iif = skb->dev->ifindex; 2440 2441 null_or_orig = NULL; 2442 orig_dev = skb->dev; 2443 if (orig_dev->master) { 2444 if (skb_bond_should_drop(skb)) 2445 null_or_orig = orig_dev; /* deliver only exact match */ 2446 else 2447 skb->dev = orig_dev->master; 2448 } 2449 2450 __get_cpu_var(netdev_rx_stat).total++; 2451 2452 skb_reset_network_header(skb); 2453 skb_reset_transport_header(skb); 2454 skb->mac_len = skb->network_header - skb->mac_header; 2455 2456 pt_prev = NULL; 2457 2458 rcu_read_lock(); 2459 2460 #ifdef CONFIG_NET_CLS_ACT 2461 if (skb->tc_verd & TC_NCLS) { 2462 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd); 2463 goto ncls; 2464 } 2465 #endif 2466 2467 list_for_each_entry_rcu(ptype, &ptype_all, list) { 2468 if (ptype->dev == null_or_orig || ptype->dev == skb->dev || 2469 ptype->dev == orig_dev) { 2470 if (pt_prev) 2471 ret = deliver_skb(skb, pt_prev, orig_dev); 2472 pt_prev = ptype; 2473 } 2474 } 2475 2476 #ifdef CONFIG_NET_CLS_ACT 2477 skb = handle_ing(skb, &pt_prev, &ret, orig_dev); 2478 if (!skb) 2479 goto out; 2480 ncls: 2481 #endif 2482 2483 skb = handle_bridge(skb, &pt_prev, &ret, orig_dev); 2484 if (!skb) 2485 goto out; 2486 skb = handle_macvlan(skb, &pt_prev, &ret, orig_dev); 2487 if (!skb) 2488 goto out; 2489 2490 type = skb->protocol; 2491 list_for_each_entry_rcu(ptype, 2492 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) { 2493 if (ptype->type == type && 2494 (ptype->dev == null_or_orig || ptype->dev == skb->dev || 2495 ptype->dev == orig_dev)) { 2496 if (pt_prev) 2497 ret = deliver_skb(skb, pt_prev, orig_dev); 2498 pt_prev = ptype; 2499 } 2500 } 2501 2502 if (pt_prev) { 2503 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 2504 } else { 2505 kfree_skb(skb); 2506 /* Jamal, now you will not able to escape explaining 2507 * me how you were going to use this. :-) 2508 */ 2509 ret = NET_RX_DROP; 2510 } 2511 2512 out: 2513 rcu_read_unlock(); 2514 return ret; 2515 } 2516 EXPORT_SYMBOL(netif_receive_skb); 2517 2518 /* Network device is going away, flush any packets still pending */ 2519 static void flush_backlog(void *arg) 2520 { 2521 struct net_device *dev = arg; 2522 struct softnet_data *queue = &__get_cpu_var(softnet_data); 2523 struct sk_buff *skb, *tmp; 2524 2525 skb_queue_walk_safe(&queue->input_pkt_queue, skb, tmp) 2526 if (skb->dev == dev) { 2527 __skb_unlink(skb, &queue->input_pkt_queue); 2528 kfree_skb(skb); 2529 } 2530 } 2531 2532 static int napi_gro_complete(struct sk_buff *skb) 2533 { 2534 struct packet_type *ptype; 2535 __be16 type = skb->protocol; 2536 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK]; 2537 int err = -ENOENT; 2538 2539 if (NAPI_GRO_CB(skb)->count == 1) { 2540 skb_shinfo(skb)->gso_size = 0; 2541 goto out; 2542 } 2543 2544 rcu_read_lock(); 2545 list_for_each_entry_rcu(ptype, head, list) { 2546 if (ptype->type != type || ptype->dev || !ptype->gro_complete) 2547 continue; 2548 2549 err = ptype->gro_complete(skb); 2550 break; 2551 } 2552 rcu_read_unlock(); 2553 2554 if (err) { 2555 WARN_ON(&ptype->list == head); 2556 kfree_skb(skb); 2557 return NET_RX_SUCCESS; 2558 } 2559 2560 out: 2561 return netif_receive_skb(skb); 2562 } 2563 2564 void napi_gro_flush(struct napi_struct *napi) 2565 { 2566 struct sk_buff *skb, *next; 2567 2568 for (skb = napi->gro_list; skb; skb = next) { 2569 next = skb->next; 2570 skb->next = NULL; 2571 napi_gro_complete(skb); 2572 } 2573 2574 napi->gro_count = 0; 2575 napi->gro_list = NULL; 2576 } 2577 EXPORT_SYMBOL(napi_gro_flush); 2578 2579 enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 2580 { 2581 struct sk_buff **pp = NULL; 2582 struct packet_type *ptype; 2583 __be16 type = skb->protocol; 2584 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK]; 2585 int same_flow; 2586 int mac_len; 2587 enum gro_result ret; 2588 2589 if (!(skb->dev->features & NETIF_F_GRO)) 2590 goto normal; 2591 2592 if (skb_is_gso(skb) || skb_has_frags(skb)) 2593 goto normal; 2594 2595 rcu_read_lock(); 2596 list_for_each_entry_rcu(ptype, head, list) { 2597 if (ptype->type != type || ptype->dev || !ptype->gro_receive) 2598 continue; 2599 2600 skb_set_network_header(skb, skb_gro_offset(skb)); 2601 mac_len = skb->network_header - skb->mac_header; 2602 skb->mac_len = mac_len; 2603 NAPI_GRO_CB(skb)->same_flow = 0; 2604 NAPI_GRO_CB(skb)->flush = 0; 2605 NAPI_GRO_CB(skb)->free = 0; 2606 2607 pp = ptype->gro_receive(&napi->gro_list, skb); 2608 break; 2609 } 2610 rcu_read_unlock(); 2611 2612 if (&ptype->list == head) 2613 goto normal; 2614 2615 same_flow = NAPI_GRO_CB(skb)->same_flow; 2616 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; 2617 2618 if (pp) { 2619 struct sk_buff *nskb = *pp; 2620 2621 *pp = nskb->next; 2622 nskb->next = NULL; 2623 napi_gro_complete(nskb); 2624 napi->gro_count--; 2625 } 2626 2627 if (same_flow) 2628 goto ok; 2629 2630 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS) 2631 goto normal; 2632 2633 napi->gro_count++; 2634 NAPI_GRO_CB(skb)->count = 1; 2635 skb_shinfo(skb)->gso_size = skb_gro_len(skb); 2636 skb->next = napi->gro_list; 2637 napi->gro_list = skb; 2638 ret = GRO_HELD; 2639 2640 pull: 2641 if (skb_headlen(skb) < skb_gro_offset(skb)) { 2642 int grow = skb_gro_offset(skb) - skb_headlen(skb); 2643 2644 BUG_ON(skb->end - skb->tail < grow); 2645 2646 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); 2647 2648 skb->tail += grow; 2649 skb->data_len -= grow; 2650 2651 skb_shinfo(skb)->frags[0].page_offset += grow; 2652 skb_shinfo(skb)->frags[0].size -= grow; 2653 2654 if (unlikely(!skb_shinfo(skb)->frags[0].size)) { 2655 put_page(skb_shinfo(skb)->frags[0].page); 2656 memmove(skb_shinfo(skb)->frags, 2657 skb_shinfo(skb)->frags + 1, 2658 --skb_shinfo(skb)->nr_frags); 2659 } 2660 } 2661 2662 ok: 2663 return ret; 2664 2665 normal: 2666 ret = GRO_NORMAL; 2667 goto pull; 2668 } 2669 EXPORT_SYMBOL(dev_gro_receive); 2670 2671 static gro_result_t 2672 __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 2673 { 2674 struct sk_buff *p; 2675 2676 if (netpoll_rx_on(skb)) 2677 return GRO_NORMAL; 2678 2679 for (p = napi->gro_list; p; p = p->next) { 2680 NAPI_GRO_CB(p)->same_flow = 2681 (p->dev == skb->dev) && 2682 !compare_ether_header(skb_mac_header(p), 2683 skb_gro_mac_header(skb)); 2684 NAPI_GRO_CB(p)->flush = 0; 2685 } 2686 2687 return dev_gro_receive(napi, skb); 2688 } 2689 2690 gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) 2691 { 2692 switch (ret) { 2693 case GRO_NORMAL: 2694 if (netif_receive_skb(skb)) 2695 ret = GRO_DROP; 2696 break; 2697 2698 case GRO_DROP: 2699 case GRO_MERGED_FREE: 2700 kfree_skb(skb); 2701 break; 2702 2703 case GRO_HELD: 2704 case GRO_MERGED: 2705 break; 2706 } 2707 2708 return ret; 2709 } 2710 EXPORT_SYMBOL(napi_skb_finish); 2711 2712 void skb_gro_reset_offset(struct sk_buff *skb) 2713 { 2714 NAPI_GRO_CB(skb)->data_offset = 0; 2715 NAPI_GRO_CB(skb)->frag0 = NULL; 2716 NAPI_GRO_CB(skb)->frag0_len = 0; 2717 2718 if (skb->mac_header == skb->tail && 2719 !PageHighMem(skb_shinfo(skb)->frags[0].page)) { 2720 NAPI_GRO_CB(skb)->frag0 = 2721 page_address(skb_shinfo(skb)->frags[0].page) + 2722 skb_shinfo(skb)->frags[0].page_offset; 2723 NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size; 2724 } 2725 } 2726 EXPORT_SYMBOL(skb_gro_reset_offset); 2727 2728 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 2729 { 2730 skb_gro_reset_offset(skb); 2731 2732 return napi_skb_finish(__napi_gro_receive(napi, skb), skb); 2733 } 2734 EXPORT_SYMBOL(napi_gro_receive); 2735 2736 void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) 2737 { 2738 __skb_pull(skb, skb_headlen(skb)); 2739 skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb)); 2740 2741 napi->skb = skb; 2742 } 2743 EXPORT_SYMBOL(napi_reuse_skb); 2744 2745 struct sk_buff *napi_get_frags(struct napi_struct *napi) 2746 { 2747 struct sk_buff *skb = napi->skb; 2748 2749 if (!skb) { 2750 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD); 2751 if (skb) 2752 napi->skb = skb; 2753 } 2754 return skb; 2755 } 2756 EXPORT_SYMBOL(napi_get_frags); 2757 2758 gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb, 2759 gro_result_t ret) 2760 { 2761 switch (ret) { 2762 case GRO_NORMAL: 2763 case GRO_HELD: 2764 skb->protocol = eth_type_trans(skb, napi->dev); 2765 2766 if (ret == GRO_HELD) 2767 skb_gro_pull(skb, -ETH_HLEN); 2768 else if (netif_receive_skb(skb)) 2769 ret = GRO_DROP; 2770 break; 2771 2772 case GRO_DROP: 2773 case GRO_MERGED_FREE: 2774 napi_reuse_skb(napi, skb); 2775 break; 2776 2777 case GRO_MERGED: 2778 break; 2779 } 2780 2781 return ret; 2782 } 2783 EXPORT_SYMBOL(napi_frags_finish); 2784 2785 struct sk_buff *napi_frags_skb(struct napi_struct *napi) 2786 { 2787 struct sk_buff *skb = napi->skb; 2788 struct ethhdr *eth; 2789 unsigned int hlen; 2790 unsigned int off; 2791 2792 napi->skb = NULL; 2793 2794 skb_reset_mac_header(skb); 2795 skb_gro_reset_offset(skb); 2796 2797 off = skb_gro_offset(skb); 2798 hlen = off + sizeof(*eth); 2799 eth = skb_gro_header_fast(skb, off); 2800 if (skb_gro_header_hard(skb, hlen)) { 2801 eth = skb_gro_header_slow(skb, hlen, off); 2802 if (unlikely(!eth)) { 2803 napi_reuse_skb(napi, skb); 2804 skb = NULL; 2805 goto out; 2806 } 2807 } 2808 2809 skb_gro_pull(skb, sizeof(*eth)); 2810 2811 /* 2812 * This works because the only protocols we care about don't require 2813 * special handling. We'll fix it up properly at the end. 2814 */ 2815 skb->protocol = eth->h_proto; 2816 2817 out: 2818 return skb; 2819 } 2820 EXPORT_SYMBOL(napi_frags_skb); 2821 2822 gro_result_t napi_gro_frags(struct napi_struct *napi) 2823 { 2824 struct sk_buff *skb = napi_frags_skb(napi); 2825 2826 if (!skb) 2827 return GRO_DROP; 2828 2829 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb)); 2830 } 2831 EXPORT_SYMBOL(napi_gro_frags); 2832 2833 static int process_backlog(struct napi_struct *napi, int quota) 2834 { 2835 int work = 0; 2836 struct softnet_data *queue = &__get_cpu_var(softnet_data); 2837 unsigned long start_time = jiffies; 2838 2839 napi->weight = weight_p; 2840 do { 2841 struct sk_buff *skb; 2842 2843 local_irq_disable(); 2844 skb = __skb_dequeue(&queue->input_pkt_queue); 2845 if (!skb) { 2846 __napi_complete(napi); 2847 local_irq_enable(); 2848 break; 2849 } 2850 local_irq_enable(); 2851 2852 netif_receive_skb(skb); 2853 } while (++work < quota && jiffies == start_time); 2854 2855 return work; 2856 } 2857 2858 /** 2859 * __napi_schedule - schedule for receive 2860 * @n: entry to schedule 2861 * 2862 * The entry's receive function will be scheduled to run 2863 */ 2864 void __napi_schedule(struct napi_struct *n) 2865 { 2866 unsigned long flags; 2867 2868 local_irq_save(flags); 2869 list_add_tail(&n->poll_list, &__get_cpu_var(softnet_data).poll_list); 2870 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 2871 local_irq_restore(flags); 2872 } 2873 EXPORT_SYMBOL(__napi_schedule); 2874 2875 void __napi_complete(struct napi_struct *n) 2876 { 2877 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state)); 2878 BUG_ON(n->gro_list); 2879 2880 list_del(&n->poll_list); 2881 smp_mb__before_clear_bit(); 2882 clear_bit(NAPI_STATE_SCHED, &n->state); 2883 } 2884 EXPORT_SYMBOL(__napi_complete); 2885 2886 void napi_complete(struct napi_struct *n) 2887 { 2888 unsigned long flags; 2889 2890 /* 2891 * don't let napi dequeue from the cpu poll list 2892 * just in case its running on a different cpu 2893 */ 2894 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state))) 2895 return; 2896 2897 napi_gro_flush(n); 2898 local_irq_save(flags); 2899 __napi_complete(n); 2900 local_irq_restore(flags); 2901 } 2902 EXPORT_SYMBOL(napi_complete); 2903 2904 void netif_napi_add(struct net_device *dev, struct napi_struct *napi, 2905 int (*poll)(struct napi_struct *, int), int weight) 2906 { 2907 INIT_LIST_HEAD(&napi->poll_list); 2908 napi->gro_count = 0; 2909 napi->gro_list = NULL; 2910 napi->skb = NULL; 2911 napi->poll = poll; 2912 napi->weight = weight; 2913 list_add(&napi->dev_list, &dev->napi_list); 2914 napi->dev = dev; 2915 #ifdef CONFIG_NETPOLL 2916 spin_lock_init(&napi->poll_lock); 2917 napi->poll_owner = -1; 2918 #endif 2919 set_bit(NAPI_STATE_SCHED, &napi->state); 2920 } 2921 EXPORT_SYMBOL(netif_napi_add); 2922 2923 void netif_napi_del(struct napi_struct *napi) 2924 { 2925 struct sk_buff *skb, *next; 2926 2927 list_del_init(&napi->dev_list); 2928 napi_free_frags(napi); 2929 2930 for (skb = napi->gro_list; skb; skb = next) { 2931 next = skb->next; 2932 skb->next = NULL; 2933 kfree_skb(skb); 2934 } 2935 2936 napi->gro_list = NULL; 2937 napi->gro_count = 0; 2938 } 2939 EXPORT_SYMBOL(netif_napi_del); 2940 2941 2942 static void net_rx_action(struct softirq_action *h) 2943 { 2944 struct list_head *list = &__get_cpu_var(softnet_data).poll_list; 2945 unsigned long time_limit = jiffies + 2; 2946 int budget = netdev_budget; 2947 void *have; 2948 2949 local_irq_disable(); 2950 2951 while (!list_empty(list)) { 2952 struct napi_struct *n; 2953 int work, weight; 2954 2955 /* If softirq window is exhuasted then punt. 2956 * Allow this to run for 2 jiffies since which will allow 2957 * an average latency of 1.5/HZ. 2958 */ 2959 if (unlikely(budget <= 0 || time_after(jiffies, time_limit))) 2960 goto softnet_break; 2961 2962 local_irq_enable(); 2963 2964 /* Even though interrupts have been re-enabled, this 2965 * access is safe because interrupts can only add new 2966 * entries to the tail of this list, and only ->poll() 2967 * calls can remove this head entry from the list. 2968 */ 2969 n = list_entry(list->next, struct napi_struct, poll_list); 2970 2971 have = netpoll_poll_lock(n); 2972 2973 weight = n->weight; 2974 2975 /* This NAPI_STATE_SCHED test is for avoiding a race 2976 * with netpoll's poll_napi(). Only the entity which 2977 * obtains the lock and sees NAPI_STATE_SCHED set will 2978 * actually make the ->poll() call. Therefore we avoid 2979 * accidently calling ->poll() when NAPI is not scheduled. 2980 */ 2981 work = 0; 2982 if (test_bit(NAPI_STATE_SCHED, &n->state)) { 2983 work = n->poll(n, weight); 2984 trace_napi_poll(n); 2985 } 2986 2987 WARN_ON_ONCE(work > weight); 2988 2989 budget -= work; 2990 2991 local_irq_disable(); 2992 2993 /* Drivers must not modify the NAPI state if they 2994 * consume the entire weight. In such cases this code 2995 * still "owns" the NAPI instance and therefore can 2996 * move the instance around on the list at-will. 2997 */ 2998 if (unlikely(work == weight)) { 2999 if (unlikely(napi_disable_pending(n))) { 3000 local_irq_enable(); 3001 napi_complete(n); 3002 local_irq_disable(); 3003 } else 3004 list_move_tail(&n->poll_list, list); 3005 } 3006 3007 netpoll_poll_unlock(have); 3008 } 3009 out: 3010 local_irq_enable(); 3011 3012 #ifdef CONFIG_NET_DMA 3013 /* 3014 * There may not be any more sk_buffs coming right now, so push 3015 * any pending DMA copies to hardware 3016 */ 3017 dma_issue_pending_all(); 3018 #endif 3019 3020 return; 3021 3022 softnet_break: 3023 __get_cpu_var(netdev_rx_stat).time_squeeze++; 3024 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 3025 goto out; 3026 } 3027 3028 static gifconf_func_t *gifconf_list[NPROTO]; 3029 3030 /** 3031 * register_gifconf - register a SIOCGIF handler 3032 * @family: Address family 3033 * @gifconf: Function handler 3034 * 3035 * Register protocol dependent address dumping routines. The handler 3036 * that is passed must not be freed or reused until it has been replaced 3037 * by another handler. 3038 */ 3039 int register_gifconf(unsigned int family, gifconf_func_t *gifconf) 3040 { 3041 if (family >= NPROTO) 3042 return -EINVAL; 3043 gifconf_list[family] = gifconf; 3044 return 0; 3045 } 3046 EXPORT_SYMBOL(register_gifconf); 3047 3048 3049 /* 3050 * Map an interface index to its name (SIOCGIFNAME) 3051 */ 3052 3053 /* 3054 * We need this ioctl for efficient implementation of the 3055 * if_indextoname() function required by the IPv6 API. Without 3056 * it, we would have to search all the interfaces to find a 3057 * match. --pb 3058 */ 3059 3060 static int dev_ifname(struct net *net, struct ifreq __user *arg) 3061 { 3062 struct net_device *dev; 3063 struct ifreq ifr; 3064 3065 /* 3066 * Fetch the caller's info block. 3067 */ 3068 3069 if (copy_from_user(&ifr, arg, sizeof(struct ifreq))) 3070 return -EFAULT; 3071 3072 rcu_read_lock(); 3073 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex); 3074 if (!dev) { 3075 rcu_read_unlock(); 3076 return -ENODEV; 3077 } 3078 3079 strcpy(ifr.ifr_name, dev->name); 3080 rcu_read_unlock(); 3081 3082 if (copy_to_user(arg, &ifr, sizeof(struct ifreq))) 3083 return -EFAULT; 3084 return 0; 3085 } 3086 3087 /* 3088 * Perform a SIOCGIFCONF call. This structure will change 3089 * size eventually, and there is nothing I can do about it. 3090 * Thus we will need a 'compatibility mode'. 3091 */ 3092 3093 static int dev_ifconf(struct net *net, char __user *arg) 3094 { 3095 struct ifconf ifc; 3096 struct net_device *dev; 3097 char __user *pos; 3098 int len; 3099 int total; 3100 int i; 3101 3102 /* 3103 * Fetch the caller's info block. 3104 */ 3105 3106 if (copy_from_user(&ifc, arg, sizeof(struct ifconf))) 3107 return -EFAULT; 3108 3109 pos = ifc.ifc_buf; 3110 len = ifc.ifc_len; 3111 3112 /* 3113 * Loop over the interfaces, and write an info block for each. 3114 */ 3115 3116 total = 0; 3117 for_each_netdev(net, dev) { 3118 for (i = 0; i < NPROTO; i++) { 3119 if (gifconf_list[i]) { 3120 int done; 3121 if (!pos) 3122 done = gifconf_list[i](dev, NULL, 0); 3123 else 3124 done = gifconf_list[i](dev, pos + total, 3125 len - total); 3126 if (done < 0) 3127 return -EFAULT; 3128 total += done; 3129 } 3130 } 3131 } 3132 3133 /* 3134 * All done. Write the updated control block back to the caller. 3135 */ 3136 ifc.ifc_len = total; 3137 3138 /* 3139 * Both BSD and Solaris return 0 here, so we do too. 3140 */ 3141 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0; 3142 } 3143 3144 #ifdef CONFIG_PROC_FS 3145 /* 3146 * This is invoked by the /proc filesystem handler to display a device 3147 * in detail. 3148 */ 3149 void *dev_seq_start(struct seq_file *seq, loff_t *pos) 3150 __acquires(RCU) 3151 { 3152 struct net *net = seq_file_net(seq); 3153 loff_t off; 3154 struct net_device *dev; 3155 3156 rcu_read_lock(); 3157 if (!*pos) 3158 return SEQ_START_TOKEN; 3159 3160 off = 1; 3161 for_each_netdev_rcu(net, dev) 3162 if (off++ == *pos) 3163 return dev; 3164 3165 return NULL; 3166 } 3167 3168 void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos) 3169 { 3170 struct net_device *dev = (v == SEQ_START_TOKEN) ? 3171 first_net_device(seq_file_net(seq)) : 3172 next_net_device((struct net_device *)v); 3173 3174 ++*pos; 3175 return rcu_dereference(dev); 3176 } 3177 3178 void dev_seq_stop(struct seq_file *seq, void *v) 3179 __releases(RCU) 3180 { 3181 rcu_read_unlock(); 3182 } 3183 3184 static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev) 3185 { 3186 const struct net_device_stats *stats = dev_get_stats(dev); 3187 3188 seq_printf(seq, "%6s:%8lu %7lu %4lu %4lu %4lu %5lu %10lu %9lu " 3189 "%8lu %7lu %4lu %4lu %4lu %5lu %7lu %10lu\n", 3190 dev->name, stats->rx_bytes, stats->rx_packets, 3191 stats->rx_errors, 3192 stats->rx_dropped + stats->rx_missed_errors, 3193 stats->rx_fifo_errors, 3194 stats->rx_length_errors + stats->rx_over_errors + 3195 stats->rx_crc_errors + stats->rx_frame_errors, 3196 stats->rx_compressed, stats->multicast, 3197 stats->tx_bytes, stats->tx_packets, 3198 stats->tx_errors, stats->tx_dropped, 3199 stats->tx_fifo_errors, stats->collisions, 3200 stats->tx_carrier_errors + 3201 stats->tx_aborted_errors + 3202 stats->tx_window_errors + 3203 stats->tx_heartbeat_errors, 3204 stats->tx_compressed); 3205 } 3206 3207 /* 3208 * Called from the PROCfs module. This now uses the new arbitrary sized 3209 * /proc/net interface to create /proc/net/dev 3210 */ 3211 static int dev_seq_show(struct seq_file *seq, void *v) 3212 { 3213 if (v == SEQ_START_TOKEN) 3214 seq_puts(seq, "Inter-| Receive " 3215 " | Transmit\n" 3216 " face |bytes packets errs drop fifo frame " 3217 "compressed multicast|bytes packets errs " 3218 "drop fifo colls carrier compressed\n"); 3219 else 3220 dev_seq_printf_stats(seq, v); 3221 return 0; 3222 } 3223 3224 static struct netif_rx_stats *softnet_get_online(loff_t *pos) 3225 { 3226 struct netif_rx_stats *rc = NULL; 3227 3228 while (*pos < nr_cpu_ids) 3229 if (cpu_online(*pos)) { 3230 rc = &per_cpu(netdev_rx_stat, *pos); 3231 break; 3232 } else 3233 ++*pos; 3234 return rc; 3235 } 3236 3237 static void *softnet_seq_start(struct seq_file *seq, loff_t *pos) 3238 { 3239 return softnet_get_online(pos); 3240 } 3241 3242 static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos) 3243 { 3244 ++*pos; 3245 return softnet_get_online(pos); 3246 } 3247 3248 static void softnet_seq_stop(struct seq_file *seq, void *v) 3249 { 3250 } 3251 3252 static int softnet_seq_show(struct seq_file *seq, void *v) 3253 { 3254 struct netif_rx_stats *s = v; 3255 3256 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x\n", 3257 s->total, s->dropped, s->time_squeeze, 0, 3258 0, 0, 0, 0, /* was fastroute */ 3259 s->cpu_collision); 3260 return 0; 3261 } 3262 3263 static const struct seq_operations dev_seq_ops = { 3264 .start = dev_seq_start, 3265 .next = dev_seq_next, 3266 .stop = dev_seq_stop, 3267 .show = dev_seq_show, 3268 }; 3269 3270 static int dev_seq_open(struct inode *inode, struct file *file) 3271 { 3272 return seq_open_net(inode, file, &dev_seq_ops, 3273 sizeof(struct seq_net_private)); 3274 } 3275 3276 static const struct file_operations dev_seq_fops = { 3277 .owner = THIS_MODULE, 3278 .open = dev_seq_open, 3279 .read = seq_read, 3280 .llseek = seq_lseek, 3281 .release = seq_release_net, 3282 }; 3283 3284 static const struct seq_operations softnet_seq_ops = { 3285 .start = softnet_seq_start, 3286 .next = softnet_seq_next, 3287 .stop = softnet_seq_stop, 3288 .show = softnet_seq_show, 3289 }; 3290 3291 static int softnet_seq_open(struct inode *inode, struct file *file) 3292 { 3293 return seq_open(file, &softnet_seq_ops); 3294 } 3295 3296 static const struct file_operations softnet_seq_fops = { 3297 .owner = THIS_MODULE, 3298 .open = softnet_seq_open, 3299 .read = seq_read, 3300 .llseek = seq_lseek, 3301 .release = seq_release, 3302 }; 3303 3304 static void *ptype_get_idx(loff_t pos) 3305 { 3306 struct packet_type *pt = NULL; 3307 loff_t i = 0; 3308 int t; 3309 3310 list_for_each_entry_rcu(pt, &ptype_all, list) { 3311 if (i == pos) 3312 return pt; 3313 ++i; 3314 } 3315 3316 for (t = 0; t < PTYPE_HASH_SIZE; t++) { 3317 list_for_each_entry_rcu(pt, &ptype_base[t], list) { 3318 if (i == pos) 3319 return pt; 3320 ++i; 3321 } 3322 } 3323 return NULL; 3324 } 3325 3326 static void *ptype_seq_start(struct seq_file *seq, loff_t *pos) 3327 __acquires(RCU) 3328 { 3329 rcu_read_lock(); 3330 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN; 3331 } 3332 3333 static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos) 3334 { 3335 struct packet_type *pt; 3336 struct list_head *nxt; 3337 int hash; 3338 3339 ++*pos; 3340 if (v == SEQ_START_TOKEN) 3341 return ptype_get_idx(0); 3342 3343 pt = v; 3344 nxt = pt->list.next; 3345 if (pt->type == htons(ETH_P_ALL)) { 3346 if (nxt != &ptype_all) 3347 goto found; 3348 hash = 0; 3349 nxt = ptype_base[0].next; 3350 } else 3351 hash = ntohs(pt->type) & PTYPE_HASH_MASK; 3352 3353 while (nxt == &ptype_base[hash]) { 3354 if (++hash >= PTYPE_HASH_SIZE) 3355 return NULL; 3356 nxt = ptype_base[hash].next; 3357 } 3358 found: 3359 return list_entry(nxt, struct packet_type, list); 3360 } 3361 3362 static void ptype_seq_stop(struct seq_file *seq, void *v) 3363 __releases(RCU) 3364 { 3365 rcu_read_unlock(); 3366 } 3367 3368 static int ptype_seq_show(struct seq_file *seq, void *v) 3369 { 3370 struct packet_type *pt = v; 3371 3372 if (v == SEQ_START_TOKEN) 3373 seq_puts(seq, "Type Device Function\n"); 3374 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) { 3375 if (pt->type == htons(ETH_P_ALL)) 3376 seq_puts(seq, "ALL "); 3377 else 3378 seq_printf(seq, "%04x", ntohs(pt->type)); 3379 3380 seq_printf(seq, " %-8s %pF\n", 3381 pt->dev ? pt->dev->name : "", pt->func); 3382 } 3383 3384 return 0; 3385 } 3386 3387 static const struct seq_operations ptype_seq_ops = { 3388 .start = ptype_seq_start, 3389 .next = ptype_seq_next, 3390 .stop = ptype_seq_stop, 3391 .show = ptype_seq_show, 3392 }; 3393 3394 static int ptype_seq_open(struct inode *inode, struct file *file) 3395 { 3396 return seq_open_net(inode, file, &ptype_seq_ops, 3397 sizeof(struct seq_net_private)); 3398 } 3399 3400 static const struct file_operations ptype_seq_fops = { 3401 .owner = THIS_MODULE, 3402 .open = ptype_seq_open, 3403 .read = seq_read, 3404 .llseek = seq_lseek, 3405 .release = seq_release_net, 3406 }; 3407 3408 3409 static int __net_init dev_proc_net_init(struct net *net) 3410 { 3411 int rc = -ENOMEM; 3412 3413 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops)) 3414 goto out; 3415 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops)) 3416 goto out_dev; 3417 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops)) 3418 goto out_softnet; 3419 3420 if (wext_proc_init(net)) 3421 goto out_ptype; 3422 rc = 0; 3423 out: 3424 return rc; 3425 out_ptype: 3426 proc_net_remove(net, "ptype"); 3427 out_softnet: 3428 proc_net_remove(net, "softnet_stat"); 3429 out_dev: 3430 proc_net_remove(net, "dev"); 3431 goto out; 3432 } 3433 3434 static void __net_exit dev_proc_net_exit(struct net *net) 3435 { 3436 wext_proc_exit(net); 3437 3438 proc_net_remove(net, "ptype"); 3439 proc_net_remove(net, "softnet_stat"); 3440 proc_net_remove(net, "dev"); 3441 } 3442 3443 static struct pernet_operations __net_initdata dev_proc_ops = { 3444 .init = dev_proc_net_init, 3445 .exit = dev_proc_net_exit, 3446 }; 3447 3448 static int __init dev_proc_init(void) 3449 { 3450 return register_pernet_subsys(&dev_proc_ops); 3451 } 3452 #else 3453 #define dev_proc_init() 0 3454 #endif /* CONFIG_PROC_FS */ 3455 3456 3457 /** 3458 * netdev_set_master - set up master/slave pair 3459 * @slave: slave device 3460 * @master: new master device 3461 * 3462 * Changes the master device of the slave. Pass %NULL to break the 3463 * bonding. The caller must hold the RTNL semaphore. On a failure 3464 * a negative errno code is returned. On success the reference counts 3465 * are adjusted, %RTM_NEWLINK is sent to the routing socket and the 3466 * function returns zero. 3467 */ 3468 int netdev_set_master(struct net_device *slave, struct net_device *master) 3469 { 3470 struct net_device *old = slave->master; 3471 3472 ASSERT_RTNL(); 3473 3474 if (master) { 3475 if (old) 3476 return -EBUSY; 3477 dev_hold(master); 3478 } 3479 3480 slave->master = master; 3481 3482 synchronize_net(); 3483 3484 if (old) 3485 dev_put(old); 3486 3487 if (master) 3488 slave->flags |= IFF_SLAVE; 3489 else 3490 slave->flags &= ~IFF_SLAVE; 3491 3492 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE); 3493 return 0; 3494 } 3495 EXPORT_SYMBOL(netdev_set_master); 3496 3497 static void dev_change_rx_flags(struct net_device *dev, int flags) 3498 { 3499 const struct net_device_ops *ops = dev->netdev_ops; 3500 3501 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags) 3502 ops->ndo_change_rx_flags(dev, flags); 3503 } 3504 3505 static int __dev_set_promiscuity(struct net_device *dev, int inc) 3506 { 3507 unsigned short old_flags = dev->flags; 3508 uid_t uid; 3509 gid_t gid; 3510 3511 ASSERT_RTNL(); 3512 3513 dev->flags |= IFF_PROMISC; 3514 dev->promiscuity += inc; 3515 if (dev->promiscuity == 0) { 3516 /* 3517 * Avoid overflow. 3518 * If inc causes overflow, untouch promisc and return error. 3519 */ 3520 if (inc < 0) 3521 dev->flags &= ~IFF_PROMISC; 3522 else { 3523 dev->promiscuity -= inc; 3524 printk(KERN_WARNING "%s: promiscuity touches roof, " 3525 "set promiscuity failed, promiscuity feature " 3526 "of device might be broken.\n", dev->name); 3527 return -EOVERFLOW; 3528 } 3529 } 3530 if (dev->flags != old_flags) { 3531 printk(KERN_INFO "device %s %s promiscuous mode\n", 3532 dev->name, (dev->flags & IFF_PROMISC) ? "entered" : 3533 "left"); 3534 if (audit_enabled) { 3535 current_uid_gid(&uid, &gid); 3536 audit_log(current->audit_context, GFP_ATOMIC, 3537 AUDIT_ANOM_PROMISCUOUS, 3538 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u", 3539 dev->name, (dev->flags & IFF_PROMISC), 3540 (old_flags & IFF_PROMISC), 3541 audit_get_loginuid(current), 3542 uid, gid, 3543 audit_get_sessionid(current)); 3544 } 3545 3546 dev_change_rx_flags(dev, IFF_PROMISC); 3547 } 3548 return 0; 3549 } 3550 3551 /** 3552 * dev_set_promiscuity - update promiscuity count on a device 3553 * @dev: device 3554 * @inc: modifier 3555 * 3556 * Add or remove promiscuity from a device. While the count in the device 3557 * remains above zero the interface remains promiscuous. Once it hits zero 3558 * the device reverts back to normal filtering operation. A negative inc 3559 * value is used to drop promiscuity on the device. 3560 * Return 0 if successful or a negative errno code on error. 3561 */ 3562 int dev_set_promiscuity(struct net_device *dev, int inc) 3563 { 3564 unsigned short old_flags = dev->flags; 3565 int err; 3566 3567 err = __dev_set_promiscuity(dev, inc); 3568 if (err < 0) 3569 return err; 3570 if (dev->flags != old_flags) 3571 dev_set_rx_mode(dev); 3572 return err; 3573 } 3574 EXPORT_SYMBOL(dev_set_promiscuity); 3575 3576 /** 3577 * dev_set_allmulti - update allmulti count on a device 3578 * @dev: device 3579 * @inc: modifier 3580 * 3581 * Add or remove reception of all multicast frames to a device. While the 3582 * count in the device remains above zero the interface remains listening 3583 * to all interfaces. Once it hits zero the device reverts back to normal 3584 * filtering operation. A negative @inc value is used to drop the counter 3585 * when releasing a resource needing all multicasts. 3586 * Return 0 if successful or a negative errno code on error. 3587 */ 3588 3589 int dev_set_allmulti(struct net_device *dev, int inc) 3590 { 3591 unsigned short old_flags = dev->flags; 3592 3593 ASSERT_RTNL(); 3594 3595 dev->flags |= IFF_ALLMULTI; 3596 dev->allmulti += inc; 3597 if (dev->allmulti == 0) { 3598 /* 3599 * Avoid overflow. 3600 * If inc causes overflow, untouch allmulti and return error. 3601 */ 3602 if (inc < 0) 3603 dev->flags &= ~IFF_ALLMULTI; 3604 else { 3605 dev->allmulti -= inc; 3606 printk(KERN_WARNING "%s: allmulti touches roof, " 3607 "set allmulti failed, allmulti feature of " 3608 "device might be broken.\n", dev->name); 3609 return -EOVERFLOW; 3610 } 3611 } 3612 if (dev->flags ^ old_flags) { 3613 dev_change_rx_flags(dev, IFF_ALLMULTI); 3614 dev_set_rx_mode(dev); 3615 } 3616 return 0; 3617 } 3618 EXPORT_SYMBOL(dev_set_allmulti); 3619 3620 /* 3621 * Upload unicast and multicast address lists to device and 3622 * configure RX filtering. When the device doesn't support unicast 3623 * filtering it is put in promiscuous mode while unicast addresses 3624 * are present. 3625 */ 3626 void __dev_set_rx_mode(struct net_device *dev) 3627 { 3628 const struct net_device_ops *ops = dev->netdev_ops; 3629 3630 /* dev_open will call this function so the list will stay sane. */ 3631 if (!(dev->flags&IFF_UP)) 3632 return; 3633 3634 if (!netif_device_present(dev)) 3635 return; 3636 3637 if (ops->ndo_set_rx_mode) 3638 ops->ndo_set_rx_mode(dev); 3639 else { 3640 /* Unicast addresses changes may only happen under the rtnl, 3641 * therefore calling __dev_set_promiscuity here is safe. 3642 */ 3643 if (dev->uc.count > 0 && !dev->uc_promisc) { 3644 __dev_set_promiscuity(dev, 1); 3645 dev->uc_promisc = 1; 3646 } else if (dev->uc.count == 0 && dev->uc_promisc) { 3647 __dev_set_promiscuity(dev, -1); 3648 dev->uc_promisc = 0; 3649 } 3650 3651 if (ops->ndo_set_multicast_list) 3652 ops->ndo_set_multicast_list(dev); 3653 } 3654 } 3655 3656 void dev_set_rx_mode(struct net_device *dev) 3657 { 3658 netif_addr_lock_bh(dev); 3659 __dev_set_rx_mode(dev); 3660 netif_addr_unlock_bh(dev); 3661 } 3662 3663 /* hw addresses list handling functions */ 3664 3665 static int __hw_addr_add(struct netdev_hw_addr_list *list, unsigned char *addr, 3666 int addr_len, unsigned char addr_type) 3667 { 3668 struct netdev_hw_addr *ha; 3669 int alloc_size; 3670 3671 if (addr_len > MAX_ADDR_LEN) 3672 return -EINVAL; 3673 3674 list_for_each_entry(ha, &list->list, list) { 3675 if (!memcmp(ha->addr, addr, addr_len) && 3676 ha->type == addr_type) { 3677 ha->refcount++; 3678 return 0; 3679 } 3680 } 3681 3682 3683 alloc_size = sizeof(*ha); 3684 if (alloc_size < L1_CACHE_BYTES) 3685 alloc_size = L1_CACHE_BYTES; 3686 ha = kmalloc(alloc_size, GFP_ATOMIC); 3687 if (!ha) 3688 return -ENOMEM; 3689 memcpy(ha->addr, addr, addr_len); 3690 ha->type = addr_type; 3691 ha->refcount = 1; 3692 ha->synced = false; 3693 list_add_tail_rcu(&ha->list, &list->list); 3694 list->count++; 3695 return 0; 3696 } 3697 3698 static void ha_rcu_free(struct rcu_head *head) 3699 { 3700 struct netdev_hw_addr *ha; 3701 3702 ha = container_of(head, struct netdev_hw_addr, rcu_head); 3703 kfree(ha); 3704 } 3705 3706 static int __hw_addr_del(struct netdev_hw_addr_list *list, unsigned char *addr, 3707 int addr_len, unsigned char addr_type) 3708 { 3709 struct netdev_hw_addr *ha; 3710 3711 list_for_each_entry(ha, &list->list, list) { 3712 if (!memcmp(ha->addr, addr, addr_len) && 3713 (ha->type == addr_type || !addr_type)) { 3714 if (--ha->refcount) 3715 return 0; 3716 list_del_rcu(&ha->list); 3717 call_rcu(&ha->rcu_head, ha_rcu_free); 3718 list->count--; 3719 return 0; 3720 } 3721 } 3722 return -ENOENT; 3723 } 3724 3725 static int __hw_addr_add_multiple(struct netdev_hw_addr_list *to_list, 3726 struct netdev_hw_addr_list *from_list, 3727 int addr_len, 3728 unsigned char addr_type) 3729 { 3730 int err; 3731 struct netdev_hw_addr *ha, *ha2; 3732 unsigned char type; 3733 3734 list_for_each_entry(ha, &from_list->list, list) { 3735 type = addr_type ? addr_type : ha->type; 3736 err = __hw_addr_add(to_list, ha->addr, addr_len, type); 3737 if (err) 3738 goto unroll; 3739 } 3740 return 0; 3741 3742 unroll: 3743 list_for_each_entry(ha2, &from_list->list, list) { 3744 if (ha2 == ha) 3745 break; 3746 type = addr_type ? addr_type : ha2->type; 3747 __hw_addr_del(to_list, ha2->addr, addr_len, type); 3748 } 3749 return err; 3750 } 3751 3752 static void __hw_addr_del_multiple(struct netdev_hw_addr_list *to_list, 3753 struct netdev_hw_addr_list *from_list, 3754 int addr_len, 3755 unsigned char addr_type) 3756 { 3757 struct netdev_hw_addr *ha; 3758 unsigned char type; 3759 3760 list_for_each_entry(ha, &from_list->list, list) { 3761 type = addr_type ? addr_type : ha->type; 3762 __hw_addr_del(to_list, ha->addr, addr_len, addr_type); 3763 } 3764 } 3765 3766 static int __hw_addr_sync(struct netdev_hw_addr_list *to_list, 3767 struct netdev_hw_addr_list *from_list, 3768 int addr_len) 3769 { 3770 int err = 0; 3771 struct netdev_hw_addr *ha, *tmp; 3772 3773 list_for_each_entry_safe(ha, tmp, &from_list->list, list) { 3774 if (!ha->synced) { 3775 err = __hw_addr_add(to_list, ha->addr, 3776 addr_len, ha->type); 3777 if (err) 3778 break; 3779 ha->synced = true; 3780 ha->refcount++; 3781 } else if (ha->refcount == 1) { 3782 __hw_addr_del(to_list, ha->addr, addr_len, ha->type); 3783 __hw_addr_del(from_list, ha->addr, addr_len, ha->type); 3784 } 3785 } 3786 return err; 3787 } 3788 3789 static void __hw_addr_unsync(struct netdev_hw_addr_list *to_list, 3790 struct netdev_hw_addr_list *from_list, 3791 int addr_len) 3792 { 3793 struct netdev_hw_addr *ha, *tmp; 3794 3795 list_for_each_entry_safe(ha, tmp, &from_list->list, list) { 3796 if (ha->synced) { 3797 __hw_addr_del(to_list, ha->addr, 3798 addr_len, ha->type); 3799 ha->synced = false; 3800 __hw_addr_del(from_list, ha->addr, 3801 addr_len, ha->type); 3802 } 3803 } 3804 } 3805 3806 static void __hw_addr_flush(struct netdev_hw_addr_list *list) 3807 { 3808 struct netdev_hw_addr *ha, *tmp; 3809 3810 list_for_each_entry_safe(ha, tmp, &list->list, list) { 3811 list_del_rcu(&ha->list); 3812 call_rcu(&ha->rcu_head, ha_rcu_free); 3813 } 3814 list->count = 0; 3815 } 3816 3817 static void __hw_addr_init(struct netdev_hw_addr_list *list) 3818 { 3819 INIT_LIST_HEAD(&list->list); 3820 list->count = 0; 3821 } 3822 3823 /* Device addresses handling functions */ 3824 3825 static void dev_addr_flush(struct net_device *dev) 3826 { 3827 /* rtnl_mutex must be held here */ 3828 3829 __hw_addr_flush(&dev->dev_addrs); 3830 dev->dev_addr = NULL; 3831 } 3832 3833 static int dev_addr_init(struct net_device *dev) 3834 { 3835 unsigned char addr[MAX_ADDR_LEN]; 3836 struct netdev_hw_addr *ha; 3837 int err; 3838 3839 /* rtnl_mutex must be held here */ 3840 3841 __hw_addr_init(&dev->dev_addrs); 3842 memset(addr, 0, sizeof(addr)); 3843 err = __hw_addr_add(&dev->dev_addrs, addr, sizeof(addr), 3844 NETDEV_HW_ADDR_T_LAN); 3845 if (!err) { 3846 /* 3847 * Get the first (previously created) address from the list 3848 * and set dev_addr pointer to this location. 3849 */ 3850 ha = list_first_entry(&dev->dev_addrs.list, 3851 struct netdev_hw_addr, list); 3852 dev->dev_addr = ha->addr; 3853 } 3854 return err; 3855 } 3856 3857 /** 3858 * dev_addr_add - Add a device address 3859 * @dev: device 3860 * @addr: address to add 3861 * @addr_type: address type 3862 * 3863 * Add a device address to the device or increase the reference count if 3864 * it already exists. 3865 * 3866 * The caller must hold the rtnl_mutex. 3867 */ 3868 int dev_addr_add(struct net_device *dev, unsigned char *addr, 3869 unsigned char addr_type) 3870 { 3871 int err; 3872 3873 ASSERT_RTNL(); 3874 3875 err = __hw_addr_add(&dev->dev_addrs, addr, dev->addr_len, addr_type); 3876 if (!err) 3877 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 3878 return err; 3879 } 3880 EXPORT_SYMBOL(dev_addr_add); 3881 3882 /** 3883 * dev_addr_del - Release a device address. 3884 * @dev: device 3885 * @addr: address to delete 3886 * @addr_type: address type 3887 * 3888 * Release reference to a device address and remove it from the device 3889 * if the reference count drops to zero. 3890 * 3891 * The caller must hold the rtnl_mutex. 3892 */ 3893 int dev_addr_del(struct net_device *dev, unsigned char *addr, 3894 unsigned char addr_type) 3895 { 3896 int err; 3897 struct netdev_hw_addr *ha; 3898 3899 ASSERT_RTNL(); 3900 3901 /* 3902 * We can not remove the first address from the list because 3903 * dev->dev_addr points to that. 3904 */ 3905 ha = list_first_entry(&dev->dev_addrs.list, 3906 struct netdev_hw_addr, list); 3907 if (ha->addr == dev->dev_addr && ha->refcount == 1) 3908 return -ENOENT; 3909 3910 err = __hw_addr_del(&dev->dev_addrs, addr, dev->addr_len, 3911 addr_type); 3912 if (!err) 3913 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 3914 return err; 3915 } 3916 EXPORT_SYMBOL(dev_addr_del); 3917 3918 /** 3919 * dev_addr_add_multiple - Add device addresses from another device 3920 * @to_dev: device to which addresses will be added 3921 * @from_dev: device from which addresses will be added 3922 * @addr_type: address type - 0 means type will be used from from_dev 3923 * 3924 * Add device addresses of the one device to another. 3925 ** 3926 * The caller must hold the rtnl_mutex. 3927 */ 3928 int dev_addr_add_multiple(struct net_device *to_dev, 3929 struct net_device *from_dev, 3930 unsigned char addr_type) 3931 { 3932 int err; 3933 3934 ASSERT_RTNL(); 3935 3936 if (from_dev->addr_len != to_dev->addr_len) 3937 return -EINVAL; 3938 err = __hw_addr_add_multiple(&to_dev->dev_addrs, &from_dev->dev_addrs, 3939 to_dev->addr_len, addr_type); 3940 if (!err) 3941 call_netdevice_notifiers(NETDEV_CHANGEADDR, to_dev); 3942 return err; 3943 } 3944 EXPORT_SYMBOL(dev_addr_add_multiple); 3945 3946 /** 3947 * dev_addr_del_multiple - Delete device addresses by another device 3948 * @to_dev: device where the addresses will be deleted 3949 * @from_dev: device by which addresses the addresses will be deleted 3950 * @addr_type: address type - 0 means type will used from from_dev 3951 * 3952 * Deletes addresses in to device by the list of addresses in from device. 3953 * 3954 * The caller must hold the rtnl_mutex. 3955 */ 3956 int dev_addr_del_multiple(struct net_device *to_dev, 3957 struct net_device *from_dev, 3958 unsigned char addr_type) 3959 { 3960 ASSERT_RTNL(); 3961 3962 if (from_dev->addr_len != to_dev->addr_len) 3963 return -EINVAL; 3964 __hw_addr_del_multiple(&to_dev->dev_addrs, &from_dev->dev_addrs, 3965 to_dev->addr_len, addr_type); 3966 call_netdevice_notifiers(NETDEV_CHANGEADDR, to_dev); 3967 return 0; 3968 } 3969 EXPORT_SYMBOL(dev_addr_del_multiple); 3970 3971 /* multicast addresses handling functions */ 3972 3973 int __dev_addr_delete(struct dev_addr_list **list, int *count, 3974 void *addr, int alen, int glbl) 3975 { 3976 struct dev_addr_list *da; 3977 3978 for (; (da = *list) != NULL; list = &da->next) { 3979 if (memcmp(da->da_addr, addr, da->da_addrlen) == 0 && 3980 alen == da->da_addrlen) { 3981 if (glbl) { 3982 int old_glbl = da->da_gusers; 3983 da->da_gusers = 0; 3984 if (old_glbl == 0) 3985 break; 3986 } 3987 if (--da->da_users) 3988 return 0; 3989 3990 *list = da->next; 3991 kfree(da); 3992 (*count)--; 3993 return 0; 3994 } 3995 } 3996 return -ENOENT; 3997 } 3998 3999 int __dev_addr_add(struct dev_addr_list **list, int *count, 4000 void *addr, int alen, int glbl) 4001 { 4002 struct dev_addr_list *da; 4003 4004 for (da = *list; da != NULL; da = da->next) { 4005 if (memcmp(da->da_addr, addr, da->da_addrlen) == 0 && 4006 da->da_addrlen == alen) { 4007 if (glbl) { 4008 int old_glbl = da->da_gusers; 4009 da->da_gusers = 1; 4010 if (old_glbl) 4011 return 0; 4012 } 4013 da->da_users++; 4014 return 0; 4015 } 4016 } 4017 4018 da = kzalloc(sizeof(*da), GFP_ATOMIC); 4019 if (da == NULL) 4020 return -ENOMEM; 4021 memcpy(da->da_addr, addr, alen); 4022 da->da_addrlen = alen; 4023 da->da_users = 1; 4024 da->da_gusers = glbl ? 1 : 0; 4025 da->next = *list; 4026 *list = da; 4027 (*count)++; 4028 return 0; 4029 } 4030 4031 /** 4032 * dev_unicast_delete - Release secondary unicast address. 4033 * @dev: device 4034 * @addr: address to delete 4035 * 4036 * Release reference to a secondary unicast address and remove it 4037 * from the device if the reference count drops to zero. 4038 * 4039 * The caller must hold the rtnl_mutex. 4040 */ 4041 int dev_unicast_delete(struct net_device *dev, void *addr) 4042 { 4043 int err; 4044 4045 ASSERT_RTNL(); 4046 4047 netif_addr_lock_bh(dev); 4048 err = __hw_addr_del(&dev->uc, addr, dev->addr_len, 4049 NETDEV_HW_ADDR_T_UNICAST); 4050 if (!err) 4051 __dev_set_rx_mode(dev); 4052 netif_addr_unlock_bh(dev); 4053 return err; 4054 } 4055 EXPORT_SYMBOL(dev_unicast_delete); 4056 4057 /** 4058 * dev_unicast_add - add a secondary unicast address 4059 * @dev: device 4060 * @addr: address to add 4061 * 4062 * Add a secondary unicast address to the device or increase 4063 * the reference count if it already exists. 4064 * 4065 * The caller must hold the rtnl_mutex. 4066 */ 4067 int dev_unicast_add(struct net_device *dev, void *addr) 4068 { 4069 int err; 4070 4071 ASSERT_RTNL(); 4072 4073 netif_addr_lock_bh(dev); 4074 err = __hw_addr_add(&dev->uc, addr, dev->addr_len, 4075 NETDEV_HW_ADDR_T_UNICAST); 4076 if (!err) 4077 __dev_set_rx_mode(dev); 4078 netif_addr_unlock_bh(dev); 4079 return err; 4080 } 4081 EXPORT_SYMBOL(dev_unicast_add); 4082 4083 int __dev_addr_sync(struct dev_addr_list **to, int *to_count, 4084 struct dev_addr_list **from, int *from_count) 4085 { 4086 struct dev_addr_list *da, *next; 4087 int err = 0; 4088 4089 da = *from; 4090 while (da != NULL) { 4091 next = da->next; 4092 if (!da->da_synced) { 4093 err = __dev_addr_add(to, to_count, 4094 da->da_addr, da->da_addrlen, 0); 4095 if (err < 0) 4096 break; 4097 da->da_synced = 1; 4098 da->da_users++; 4099 } else if (da->da_users == 1) { 4100 __dev_addr_delete(to, to_count, 4101 da->da_addr, da->da_addrlen, 0); 4102 __dev_addr_delete(from, from_count, 4103 da->da_addr, da->da_addrlen, 0); 4104 } 4105 da = next; 4106 } 4107 return err; 4108 } 4109 EXPORT_SYMBOL_GPL(__dev_addr_sync); 4110 4111 void __dev_addr_unsync(struct dev_addr_list **to, int *to_count, 4112 struct dev_addr_list **from, int *from_count) 4113 { 4114 struct dev_addr_list *da, *next; 4115 4116 da = *from; 4117 while (da != NULL) { 4118 next = da->next; 4119 if (da->da_synced) { 4120 __dev_addr_delete(to, to_count, 4121 da->da_addr, da->da_addrlen, 0); 4122 da->da_synced = 0; 4123 __dev_addr_delete(from, from_count, 4124 da->da_addr, da->da_addrlen, 0); 4125 } 4126 da = next; 4127 } 4128 } 4129 EXPORT_SYMBOL_GPL(__dev_addr_unsync); 4130 4131 /** 4132 * dev_unicast_sync - Synchronize device's unicast list to another device 4133 * @to: destination device 4134 * @from: source device 4135 * 4136 * Add newly added addresses to the destination device and release 4137 * addresses that have no users left. The source device must be 4138 * locked by netif_tx_lock_bh. 4139 * 4140 * This function is intended to be called from the dev->set_rx_mode 4141 * function of layered software devices. 4142 */ 4143 int dev_unicast_sync(struct net_device *to, struct net_device *from) 4144 { 4145 int err = 0; 4146 4147 if (to->addr_len != from->addr_len) 4148 return -EINVAL; 4149 4150 netif_addr_lock_bh(to); 4151 err = __hw_addr_sync(&to->uc, &from->uc, to->addr_len); 4152 if (!err) 4153 __dev_set_rx_mode(to); 4154 netif_addr_unlock_bh(to); 4155 return err; 4156 } 4157 EXPORT_SYMBOL(dev_unicast_sync); 4158 4159 /** 4160 * dev_unicast_unsync - Remove synchronized addresses from the destination device 4161 * @to: destination device 4162 * @from: source device 4163 * 4164 * Remove all addresses that were added to the destination device by 4165 * dev_unicast_sync(). This function is intended to be called from the 4166 * dev->stop function of layered software devices. 4167 */ 4168 void dev_unicast_unsync(struct net_device *to, struct net_device *from) 4169 { 4170 if (to->addr_len != from->addr_len) 4171 return; 4172 4173 netif_addr_lock_bh(from); 4174 netif_addr_lock(to); 4175 __hw_addr_unsync(&to->uc, &from->uc, to->addr_len); 4176 __dev_set_rx_mode(to); 4177 netif_addr_unlock(to); 4178 netif_addr_unlock_bh(from); 4179 } 4180 EXPORT_SYMBOL(dev_unicast_unsync); 4181 4182 static void dev_unicast_flush(struct net_device *dev) 4183 { 4184 netif_addr_lock_bh(dev); 4185 __hw_addr_flush(&dev->uc); 4186 netif_addr_unlock_bh(dev); 4187 } 4188 4189 static void dev_unicast_init(struct net_device *dev) 4190 { 4191 __hw_addr_init(&dev->uc); 4192 } 4193 4194 4195 static void __dev_addr_discard(struct dev_addr_list **list) 4196 { 4197 struct dev_addr_list *tmp; 4198 4199 while (*list != NULL) { 4200 tmp = *list; 4201 *list = tmp->next; 4202 if (tmp->da_users > tmp->da_gusers) 4203 printk("__dev_addr_discard: address leakage! " 4204 "da_users=%d\n", tmp->da_users); 4205 kfree(tmp); 4206 } 4207 } 4208 4209 static void dev_addr_discard(struct net_device *dev) 4210 { 4211 netif_addr_lock_bh(dev); 4212 4213 __dev_addr_discard(&dev->mc_list); 4214 dev->mc_count = 0; 4215 4216 netif_addr_unlock_bh(dev); 4217 } 4218 4219 /** 4220 * dev_get_flags - get flags reported to userspace 4221 * @dev: device 4222 * 4223 * Get the combination of flag bits exported through APIs to userspace. 4224 */ 4225 unsigned dev_get_flags(const struct net_device *dev) 4226 { 4227 unsigned flags; 4228 4229 flags = (dev->flags & ~(IFF_PROMISC | 4230 IFF_ALLMULTI | 4231 IFF_RUNNING | 4232 IFF_LOWER_UP | 4233 IFF_DORMANT)) | 4234 (dev->gflags & (IFF_PROMISC | 4235 IFF_ALLMULTI)); 4236 4237 if (netif_running(dev)) { 4238 if (netif_oper_up(dev)) 4239 flags |= IFF_RUNNING; 4240 if (netif_carrier_ok(dev)) 4241 flags |= IFF_LOWER_UP; 4242 if (netif_dormant(dev)) 4243 flags |= IFF_DORMANT; 4244 } 4245 4246 return flags; 4247 } 4248 EXPORT_SYMBOL(dev_get_flags); 4249 4250 /** 4251 * dev_change_flags - change device settings 4252 * @dev: device 4253 * @flags: device state flags 4254 * 4255 * Change settings on device based state flags. The flags are 4256 * in the userspace exported format. 4257 */ 4258 int dev_change_flags(struct net_device *dev, unsigned flags) 4259 { 4260 int ret, changes; 4261 int old_flags = dev->flags; 4262 4263 ASSERT_RTNL(); 4264 4265 /* 4266 * Set the flags on our device. 4267 */ 4268 4269 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | 4270 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL | 4271 IFF_AUTOMEDIA)) | 4272 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | 4273 IFF_ALLMULTI)); 4274 4275 /* 4276 * Load in the correct multicast list now the flags have changed. 4277 */ 4278 4279 if ((old_flags ^ flags) & IFF_MULTICAST) 4280 dev_change_rx_flags(dev, IFF_MULTICAST); 4281 4282 dev_set_rx_mode(dev); 4283 4284 /* 4285 * Have we downed the interface. We handle IFF_UP ourselves 4286 * according to user attempts to set it, rather than blindly 4287 * setting it. 4288 */ 4289 4290 ret = 0; 4291 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */ 4292 ret = ((old_flags & IFF_UP) ? dev_close : dev_open)(dev); 4293 4294 if (!ret) 4295 dev_set_rx_mode(dev); 4296 } 4297 4298 if (dev->flags & IFF_UP && 4299 ((old_flags ^ dev->flags) & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | 4300 IFF_VOLATILE))) 4301 call_netdevice_notifiers(NETDEV_CHANGE, dev); 4302 4303 if ((flags ^ dev->gflags) & IFF_PROMISC) { 4304 int inc = (flags & IFF_PROMISC) ? 1 : -1; 4305 4306 dev->gflags ^= IFF_PROMISC; 4307 dev_set_promiscuity(dev, inc); 4308 } 4309 4310 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI 4311 is important. Some (broken) drivers set IFF_PROMISC, when 4312 IFF_ALLMULTI is requested not asking us and not reporting. 4313 */ 4314 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { 4315 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; 4316 4317 dev->gflags ^= IFF_ALLMULTI; 4318 dev_set_allmulti(dev, inc); 4319 } 4320 4321 /* Exclude state transition flags, already notified */ 4322 changes = (old_flags ^ dev->flags) & ~(IFF_UP | IFF_RUNNING); 4323 if (changes) 4324 rtmsg_ifinfo(RTM_NEWLINK, dev, changes); 4325 4326 return ret; 4327 } 4328 EXPORT_SYMBOL(dev_change_flags); 4329 4330 /** 4331 * dev_set_mtu - Change maximum transfer unit 4332 * @dev: device 4333 * @new_mtu: new transfer unit 4334 * 4335 * Change the maximum transfer size of the network device. 4336 */ 4337 int dev_set_mtu(struct net_device *dev, int new_mtu) 4338 { 4339 const struct net_device_ops *ops = dev->netdev_ops; 4340 int err; 4341 4342 if (new_mtu == dev->mtu) 4343 return 0; 4344 4345 /* MTU must be positive. */ 4346 if (new_mtu < 0) 4347 return -EINVAL; 4348 4349 if (!netif_device_present(dev)) 4350 return -ENODEV; 4351 4352 err = 0; 4353 if (ops->ndo_change_mtu) 4354 err = ops->ndo_change_mtu(dev, new_mtu); 4355 else 4356 dev->mtu = new_mtu; 4357 4358 if (!err && dev->flags & IFF_UP) 4359 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); 4360 return err; 4361 } 4362 EXPORT_SYMBOL(dev_set_mtu); 4363 4364 /** 4365 * dev_set_mac_address - Change Media Access Control Address 4366 * @dev: device 4367 * @sa: new address 4368 * 4369 * Change the hardware (MAC) address of the device 4370 */ 4371 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa) 4372 { 4373 const struct net_device_ops *ops = dev->netdev_ops; 4374 int err; 4375 4376 if (!ops->ndo_set_mac_address) 4377 return -EOPNOTSUPP; 4378 if (sa->sa_family != dev->type) 4379 return -EINVAL; 4380 if (!netif_device_present(dev)) 4381 return -ENODEV; 4382 err = ops->ndo_set_mac_address(dev, sa); 4383 if (!err) 4384 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 4385 return err; 4386 } 4387 EXPORT_SYMBOL(dev_set_mac_address); 4388 4389 /* 4390 * Perform the SIOCxIFxxx calls, inside rcu_read_lock() 4391 */ 4392 static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd) 4393 { 4394 int err; 4395 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name); 4396 4397 if (!dev) 4398 return -ENODEV; 4399 4400 switch (cmd) { 4401 case SIOCGIFFLAGS: /* Get interface flags */ 4402 ifr->ifr_flags = (short) dev_get_flags(dev); 4403 return 0; 4404 4405 case SIOCGIFMETRIC: /* Get the metric on the interface 4406 (currently unused) */ 4407 ifr->ifr_metric = 0; 4408 return 0; 4409 4410 case SIOCGIFMTU: /* Get the MTU of a device */ 4411 ifr->ifr_mtu = dev->mtu; 4412 return 0; 4413 4414 case SIOCGIFHWADDR: 4415 if (!dev->addr_len) 4416 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data); 4417 else 4418 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr, 4419 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len)); 4420 ifr->ifr_hwaddr.sa_family = dev->type; 4421 return 0; 4422 4423 case SIOCGIFSLAVE: 4424 err = -EINVAL; 4425 break; 4426 4427 case SIOCGIFMAP: 4428 ifr->ifr_map.mem_start = dev->mem_start; 4429 ifr->ifr_map.mem_end = dev->mem_end; 4430 ifr->ifr_map.base_addr = dev->base_addr; 4431 ifr->ifr_map.irq = dev->irq; 4432 ifr->ifr_map.dma = dev->dma; 4433 ifr->ifr_map.port = dev->if_port; 4434 return 0; 4435 4436 case SIOCGIFINDEX: 4437 ifr->ifr_ifindex = dev->ifindex; 4438 return 0; 4439 4440 case SIOCGIFTXQLEN: 4441 ifr->ifr_qlen = dev->tx_queue_len; 4442 return 0; 4443 4444 default: 4445 /* dev_ioctl() should ensure this case 4446 * is never reached 4447 */ 4448 WARN_ON(1); 4449 err = -EINVAL; 4450 break; 4451 4452 } 4453 return err; 4454 } 4455 4456 /* 4457 * Perform the SIOCxIFxxx calls, inside rtnl_lock() 4458 */ 4459 static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd) 4460 { 4461 int err; 4462 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name); 4463 const struct net_device_ops *ops; 4464 4465 if (!dev) 4466 return -ENODEV; 4467 4468 ops = dev->netdev_ops; 4469 4470 switch (cmd) { 4471 case SIOCSIFFLAGS: /* Set interface flags */ 4472 return dev_change_flags(dev, ifr->ifr_flags); 4473 4474 case SIOCSIFMETRIC: /* Set the metric on the interface 4475 (currently unused) */ 4476 return -EOPNOTSUPP; 4477 4478 case SIOCSIFMTU: /* Set the MTU of a device */ 4479 return dev_set_mtu(dev, ifr->ifr_mtu); 4480 4481 case SIOCSIFHWADDR: 4482 return dev_set_mac_address(dev, &ifr->ifr_hwaddr); 4483 4484 case SIOCSIFHWBROADCAST: 4485 if (ifr->ifr_hwaddr.sa_family != dev->type) 4486 return -EINVAL; 4487 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data, 4488 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len)); 4489 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 4490 return 0; 4491 4492 case SIOCSIFMAP: 4493 if (ops->ndo_set_config) { 4494 if (!netif_device_present(dev)) 4495 return -ENODEV; 4496 return ops->ndo_set_config(dev, &ifr->ifr_map); 4497 } 4498 return -EOPNOTSUPP; 4499 4500 case SIOCADDMULTI: 4501 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) || 4502 ifr->ifr_hwaddr.sa_family != AF_UNSPEC) 4503 return -EINVAL; 4504 if (!netif_device_present(dev)) 4505 return -ENODEV; 4506 return dev_mc_add(dev, ifr->ifr_hwaddr.sa_data, 4507 dev->addr_len, 1); 4508 4509 case SIOCDELMULTI: 4510 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) || 4511 ifr->ifr_hwaddr.sa_family != AF_UNSPEC) 4512 return -EINVAL; 4513 if (!netif_device_present(dev)) 4514 return -ENODEV; 4515 return dev_mc_delete(dev, ifr->ifr_hwaddr.sa_data, 4516 dev->addr_len, 1); 4517 4518 case SIOCSIFTXQLEN: 4519 if (ifr->ifr_qlen < 0) 4520 return -EINVAL; 4521 dev->tx_queue_len = ifr->ifr_qlen; 4522 return 0; 4523 4524 case SIOCSIFNAME: 4525 ifr->ifr_newname[IFNAMSIZ-1] = '\0'; 4526 return dev_change_name(dev, ifr->ifr_newname); 4527 4528 /* 4529 * Unknown or private ioctl 4530 */ 4531 default: 4532 if ((cmd >= SIOCDEVPRIVATE && 4533 cmd <= SIOCDEVPRIVATE + 15) || 4534 cmd == SIOCBONDENSLAVE || 4535 cmd == SIOCBONDRELEASE || 4536 cmd == SIOCBONDSETHWADDR || 4537 cmd == SIOCBONDSLAVEINFOQUERY || 4538 cmd == SIOCBONDINFOQUERY || 4539 cmd == SIOCBONDCHANGEACTIVE || 4540 cmd == SIOCGMIIPHY || 4541 cmd == SIOCGMIIREG || 4542 cmd == SIOCSMIIREG || 4543 cmd == SIOCBRADDIF || 4544 cmd == SIOCBRDELIF || 4545 cmd == SIOCSHWTSTAMP || 4546 cmd == SIOCWANDEV) { 4547 err = -EOPNOTSUPP; 4548 if (ops->ndo_do_ioctl) { 4549 if (netif_device_present(dev)) 4550 err = ops->ndo_do_ioctl(dev, ifr, cmd); 4551 else 4552 err = -ENODEV; 4553 } 4554 } else 4555 err = -EINVAL; 4556 4557 } 4558 return err; 4559 } 4560 4561 /* 4562 * This function handles all "interface"-type I/O control requests. The actual 4563 * 'doing' part of this is dev_ifsioc above. 4564 */ 4565 4566 /** 4567 * dev_ioctl - network device ioctl 4568 * @net: the applicable net namespace 4569 * @cmd: command to issue 4570 * @arg: pointer to a struct ifreq in user space 4571 * 4572 * Issue ioctl functions to devices. This is normally called by the 4573 * user space syscall interfaces but can sometimes be useful for 4574 * other purposes. The return value is the return from the syscall if 4575 * positive or a negative errno code on error. 4576 */ 4577 4578 int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg) 4579 { 4580 struct ifreq ifr; 4581 int ret; 4582 char *colon; 4583 4584 /* One special case: SIOCGIFCONF takes ifconf argument 4585 and requires shared lock, because it sleeps writing 4586 to user space. 4587 */ 4588 4589 if (cmd == SIOCGIFCONF) { 4590 rtnl_lock(); 4591 ret = dev_ifconf(net, (char __user *) arg); 4592 rtnl_unlock(); 4593 return ret; 4594 } 4595 if (cmd == SIOCGIFNAME) 4596 return dev_ifname(net, (struct ifreq __user *)arg); 4597 4598 if (copy_from_user(&ifr, arg, sizeof(struct ifreq))) 4599 return -EFAULT; 4600 4601 ifr.ifr_name[IFNAMSIZ-1] = 0; 4602 4603 colon = strchr(ifr.ifr_name, ':'); 4604 if (colon) 4605 *colon = 0; 4606 4607 /* 4608 * See which interface the caller is talking about. 4609 */ 4610 4611 switch (cmd) { 4612 /* 4613 * These ioctl calls: 4614 * - can be done by all. 4615 * - atomic and do not require locking. 4616 * - return a value 4617 */ 4618 case SIOCGIFFLAGS: 4619 case SIOCGIFMETRIC: 4620 case SIOCGIFMTU: 4621 case SIOCGIFHWADDR: 4622 case SIOCGIFSLAVE: 4623 case SIOCGIFMAP: 4624 case SIOCGIFINDEX: 4625 case SIOCGIFTXQLEN: 4626 dev_load(net, ifr.ifr_name); 4627 rcu_read_lock(); 4628 ret = dev_ifsioc_locked(net, &ifr, cmd); 4629 rcu_read_unlock(); 4630 if (!ret) { 4631 if (colon) 4632 *colon = ':'; 4633 if (copy_to_user(arg, &ifr, 4634 sizeof(struct ifreq))) 4635 ret = -EFAULT; 4636 } 4637 return ret; 4638 4639 case SIOCETHTOOL: 4640 dev_load(net, ifr.ifr_name); 4641 rtnl_lock(); 4642 ret = dev_ethtool(net, &ifr); 4643 rtnl_unlock(); 4644 if (!ret) { 4645 if (colon) 4646 *colon = ':'; 4647 if (copy_to_user(arg, &ifr, 4648 sizeof(struct ifreq))) 4649 ret = -EFAULT; 4650 } 4651 return ret; 4652 4653 /* 4654 * These ioctl calls: 4655 * - require superuser power. 4656 * - require strict serialization. 4657 * - return a value 4658 */ 4659 case SIOCGMIIPHY: 4660 case SIOCGMIIREG: 4661 case SIOCSIFNAME: 4662 if (!capable(CAP_NET_ADMIN)) 4663 return -EPERM; 4664 dev_load(net, ifr.ifr_name); 4665 rtnl_lock(); 4666 ret = dev_ifsioc(net, &ifr, cmd); 4667 rtnl_unlock(); 4668 if (!ret) { 4669 if (colon) 4670 *colon = ':'; 4671 if (copy_to_user(arg, &ifr, 4672 sizeof(struct ifreq))) 4673 ret = -EFAULT; 4674 } 4675 return ret; 4676 4677 /* 4678 * These ioctl calls: 4679 * - require superuser power. 4680 * - require strict serialization. 4681 * - do not return a value 4682 */ 4683 case SIOCSIFFLAGS: 4684 case SIOCSIFMETRIC: 4685 case SIOCSIFMTU: 4686 case SIOCSIFMAP: 4687 case SIOCSIFHWADDR: 4688 case SIOCSIFSLAVE: 4689 case SIOCADDMULTI: 4690 case SIOCDELMULTI: 4691 case SIOCSIFHWBROADCAST: 4692 case SIOCSIFTXQLEN: 4693 case SIOCSMIIREG: 4694 case SIOCBONDENSLAVE: 4695 case SIOCBONDRELEASE: 4696 case SIOCBONDSETHWADDR: 4697 case SIOCBONDCHANGEACTIVE: 4698 case SIOCBRADDIF: 4699 case SIOCBRDELIF: 4700 case SIOCSHWTSTAMP: 4701 if (!capable(CAP_NET_ADMIN)) 4702 return -EPERM; 4703 /* fall through */ 4704 case SIOCBONDSLAVEINFOQUERY: 4705 case SIOCBONDINFOQUERY: 4706 dev_load(net, ifr.ifr_name); 4707 rtnl_lock(); 4708 ret = dev_ifsioc(net, &ifr, cmd); 4709 rtnl_unlock(); 4710 return ret; 4711 4712 case SIOCGIFMEM: 4713 /* Get the per device memory space. We can add this but 4714 * currently do not support it */ 4715 case SIOCSIFMEM: 4716 /* Set the per device memory buffer space. 4717 * Not applicable in our case */ 4718 case SIOCSIFLINK: 4719 return -EINVAL; 4720 4721 /* 4722 * Unknown or private ioctl. 4723 */ 4724 default: 4725 if (cmd == SIOCWANDEV || 4726 (cmd >= SIOCDEVPRIVATE && 4727 cmd <= SIOCDEVPRIVATE + 15)) { 4728 dev_load(net, ifr.ifr_name); 4729 rtnl_lock(); 4730 ret = dev_ifsioc(net, &ifr, cmd); 4731 rtnl_unlock(); 4732 if (!ret && copy_to_user(arg, &ifr, 4733 sizeof(struct ifreq))) 4734 ret = -EFAULT; 4735 return ret; 4736 } 4737 /* Take care of Wireless Extensions */ 4738 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST) 4739 return wext_handle_ioctl(net, &ifr, cmd, arg); 4740 return -EINVAL; 4741 } 4742 } 4743 4744 4745 /** 4746 * dev_new_index - allocate an ifindex 4747 * @net: the applicable net namespace 4748 * 4749 * Returns a suitable unique value for a new device interface 4750 * number. The caller must hold the rtnl semaphore or the 4751 * dev_base_lock to be sure it remains unique. 4752 */ 4753 static int dev_new_index(struct net *net) 4754 { 4755 static int ifindex; 4756 for (;;) { 4757 if (++ifindex <= 0) 4758 ifindex = 1; 4759 if (!__dev_get_by_index(net, ifindex)) 4760 return ifindex; 4761 } 4762 } 4763 4764 /* Delayed registration/unregisteration */ 4765 static LIST_HEAD(net_todo_list); 4766 4767 static void net_set_todo(struct net_device *dev) 4768 { 4769 list_add_tail(&dev->todo_list, &net_todo_list); 4770 } 4771 4772 static void rollback_registered_many(struct list_head *head) 4773 { 4774 struct net_device *dev, *tmp; 4775 4776 BUG_ON(dev_boot_phase); 4777 ASSERT_RTNL(); 4778 4779 list_for_each_entry_safe(dev, tmp, head, unreg_list) { 4780 /* Some devices call without registering 4781 * for initialization unwind. Remove those 4782 * devices and proceed with the remaining. 4783 */ 4784 if (dev->reg_state == NETREG_UNINITIALIZED) { 4785 pr_debug("unregister_netdevice: device %s/%p never " 4786 "was registered\n", dev->name, dev); 4787 4788 WARN_ON(1); 4789 list_del(&dev->unreg_list); 4790 continue; 4791 } 4792 4793 BUG_ON(dev->reg_state != NETREG_REGISTERED); 4794 4795 /* If device is running, close it first. */ 4796 dev_close(dev); 4797 4798 /* And unlink it from device chain. */ 4799 unlist_netdevice(dev); 4800 4801 dev->reg_state = NETREG_UNREGISTERING; 4802 } 4803 4804 synchronize_net(); 4805 4806 list_for_each_entry(dev, head, unreg_list) { 4807 /* Shutdown queueing discipline. */ 4808 dev_shutdown(dev); 4809 4810 4811 /* Notify protocols, that we are about to destroy 4812 this device. They should clean all the things. 4813 */ 4814 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 4815 4816 /* 4817 * Flush the unicast and multicast chains 4818 */ 4819 dev_unicast_flush(dev); 4820 dev_addr_discard(dev); 4821 4822 if (dev->netdev_ops->ndo_uninit) 4823 dev->netdev_ops->ndo_uninit(dev); 4824 4825 /* Notifier chain MUST detach us from master device. */ 4826 WARN_ON(dev->master); 4827 4828 /* Remove entries from kobject tree */ 4829 netdev_unregister_kobject(dev); 4830 } 4831 4832 /* Process any work delayed until the end of the batch */ 4833 dev = list_entry(head->next, struct net_device, unreg_list); 4834 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev); 4835 4836 synchronize_net(); 4837 4838 list_for_each_entry(dev, head, unreg_list) 4839 dev_put(dev); 4840 } 4841 4842 static void rollback_registered(struct net_device *dev) 4843 { 4844 LIST_HEAD(single); 4845 4846 list_add(&dev->unreg_list, &single); 4847 rollback_registered_many(&single); 4848 } 4849 4850 static void __netdev_init_queue_locks_one(struct net_device *dev, 4851 struct netdev_queue *dev_queue, 4852 void *_unused) 4853 { 4854 spin_lock_init(&dev_queue->_xmit_lock); 4855 netdev_set_xmit_lockdep_class(&dev_queue->_xmit_lock, dev->type); 4856 dev_queue->xmit_lock_owner = -1; 4857 } 4858 4859 static void netdev_init_queue_locks(struct net_device *dev) 4860 { 4861 netdev_for_each_tx_queue(dev, __netdev_init_queue_locks_one, NULL); 4862 __netdev_init_queue_locks_one(dev, &dev->rx_queue, NULL); 4863 } 4864 4865 unsigned long netdev_fix_features(unsigned long features, const char *name) 4866 { 4867 /* Fix illegal SG+CSUM combinations. */ 4868 if ((features & NETIF_F_SG) && 4869 !(features & NETIF_F_ALL_CSUM)) { 4870 if (name) 4871 printk(KERN_NOTICE "%s: Dropping NETIF_F_SG since no " 4872 "checksum feature.\n", name); 4873 features &= ~NETIF_F_SG; 4874 } 4875 4876 /* TSO requires that SG is present as well. */ 4877 if ((features & NETIF_F_TSO) && !(features & NETIF_F_SG)) { 4878 if (name) 4879 printk(KERN_NOTICE "%s: Dropping NETIF_F_TSO since no " 4880 "SG feature.\n", name); 4881 features &= ~NETIF_F_TSO; 4882 } 4883 4884 if (features & NETIF_F_UFO) { 4885 if (!(features & NETIF_F_GEN_CSUM)) { 4886 if (name) 4887 printk(KERN_ERR "%s: Dropping NETIF_F_UFO " 4888 "since no NETIF_F_HW_CSUM feature.\n", 4889 name); 4890 features &= ~NETIF_F_UFO; 4891 } 4892 4893 if (!(features & NETIF_F_SG)) { 4894 if (name) 4895 printk(KERN_ERR "%s: Dropping NETIF_F_UFO " 4896 "since no NETIF_F_SG feature.\n", name); 4897 features &= ~NETIF_F_UFO; 4898 } 4899 } 4900 4901 return features; 4902 } 4903 EXPORT_SYMBOL(netdev_fix_features); 4904 4905 /** 4906 * netif_stacked_transfer_operstate - transfer operstate 4907 * @rootdev: the root or lower level device to transfer state from 4908 * @dev: the device to transfer operstate to 4909 * 4910 * Transfer operational state from root to device. This is normally 4911 * called when a stacking relationship exists between the root 4912 * device and the device(a leaf device). 4913 */ 4914 void netif_stacked_transfer_operstate(const struct net_device *rootdev, 4915 struct net_device *dev) 4916 { 4917 if (rootdev->operstate == IF_OPER_DORMANT) 4918 netif_dormant_on(dev); 4919 else 4920 netif_dormant_off(dev); 4921 4922 if (netif_carrier_ok(rootdev)) { 4923 if (!netif_carrier_ok(dev)) 4924 netif_carrier_on(dev); 4925 } else { 4926 if (netif_carrier_ok(dev)) 4927 netif_carrier_off(dev); 4928 } 4929 } 4930 EXPORT_SYMBOL(netif_stacked_transfer_operstate); 4931 4932 /** 4933 * register_netdevice - register a network device 4934 * @dev: device to register 4935 * 4936 * Take a completed network device structure and add it to the kernel 4937 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 4938 * chain. 0 is returned on success. A negative errno code is returned 4939 * on a failure to set up the device, or if the name is a duplicate. 4940 * 4941 * Callers must hold the rtnl semaphore. You may want 4942 * register_netdev() instead of this. 4943 * 4944 * BUGS: 4945 * The locking appears insufficient to guarantee two parallel registers 4946 * will not get the same name. 4947 */ 4948 4949 int register_netdevice(struct net_device *dev) 4950 { 4951 int ret; 4952 struct net *net = dev_net(dev); 4953 4954 BUG_ON(dev_boot_phase); 4955 ASSERT_RTNL(); 4956 4957 might_sleep(); 4958 4959 /* When net_device's are persistent, this will be fatal. */ 4960 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); 4961 BUG_ON(!net); 4962 4963 spin_lock_init(&dev->addr_list_lock); 4964 netdev_set_addr_lockdep_class(dev); 4965 netdev_init_queue_locks(dev); 4966 4967 dev->iflink = -1; 4968 4969 /* Init, if this function is available */ 4970 if (dev->netdev_ops->ndo_init) { 4971 ret = dev->netdev_ops->ndo_init(dev); 4972 if (ret) { 4973 if (ret > 0) 4974 ret = -EIO; 4975 goto out; 4976 } 4977 } 4978 4979 ret = dev_get_valid_name(net, dev->name, dev->name, 0); 4980 if (ret) 4981 goto err_uninit; 4982 4983 dev->ifindex = dev_new_index(net); 4984 if (dev->iflink == -1) 4985 dev->iflink = dev->ifindex; 4986 4987 /* Fix illegal checksum combinations */ 4988 if ((dev->features & NETIF_F_HW_CSUM) && 4989 (dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { 4990 printk(KERN_NOTICE "%s: mixed HW and IP checksum settings.\n", 4991 dev->name); 4992 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); 4993 } 4994 4995 if ((dev->features & NETIF_F_NO_CSUM) && 4996 (dev->features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { 4997 printk(KERN_NOTICE "%s: mixed no checksumming and other settings.\n", 4998 dev->name); 4999 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM); 5000 } 5001 5002 dev->features = netdev_fix_features(dev->features, dev->name); 5003 5004 /* Enable software GSO if SG is supported. */ 5005 if (dev->features & NETIF_F_SG) 5006 dev->features |= NETIF_F_GSO; 5007 5008 netdev_initialize_kobject(dev); 5009 5010 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); 5011 ret = notifier_to_errno(ret); 5012 if (ret) 5013 goto err_uninit; 5014 5015 ret = netdev_register_kobject(dev); 5016 if (ret) 5017 goto err_uninit; 5018 dev->reg_state = NETREG_REGISTERED; 5019 5020 /* 5021 * Default initial state at registry is that the 5022 * device is present. 5023 */ 5024 5025 set_bit(__LINK_STATE_PRESENT, &dev->state); 5026 5027 dev_init_scheduler(dev); 5028 dev_hold(dev); 5029 list_netdevice(dev); 5030 5031 /* Notify protocols, that a new device appeared. */ 5032 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); 5033 ret = notifier_to_errno(ret); 5034 if (ret) { 5035 rollback_registered(dev); 5036 dev->reg_state = NETREG_UNREGISTERED; 5037 } 5038 /* 5039 * Prevent userspace races by waiting until the network 5040 * device is fully setup before sending notifications. 5041 */ 5042 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U); 5043 5044 out: 5045 return ret; 5046 5047 err_uninit: 5048 if (dev->netdev_ops->ndo_uninit) 5049 dev->netdev_ops->ndo_uninit(dev); 5050 goto out; 5051 } 5052 EXPORT_SYMBOL(register_netdevice); 5053 5054 /** 5055 * init_dummy_netdev - init a dummy network device for NAPI 5056 * @dev: device to init 5057 * 5058 * This takes a network device structure and initialize the minimum 5059 * amount of fields so it can be used to schedule NAPI polls without 5060 * registering a full blown interface. This is to be used by drivers 5061 * that need to tie several hardware interfaces to a single NAPI 5062 * poll scheduler due to HW limitations. 5063 */ 5064 int init_dummy_netdev(struct net_device *dev) 5065 { 5066 /* Clear everything. Note we don't initialize spinlocks 5067 * are they aren't supposed to be taken by any of the 5068 * NAPI code and this dummy netdev is supposed to be 5069 * only ever used for NAPI polls 5070 */ 5071 memset(dev, 0, sizeof(struct net_device)); 5072 5073 /* make sure we BUG if trying to hit standard 5074 * register/unregister code path 5075 */ 5076 dev->reg_state = NETREG_DUMMY; 5077 5078 /* initialize the ref count */ 5079 atomic_set(&dev->refcnt, 1); 5080 5081 /* NAPI wants this */ 5082 INIT_LIST_HEAD(&dev->napi_list); 5083 5084 /* a dummy interface is started by default */ 5085 set_bit(__LINK_STATE_PRESENT, &dev->state); 5086 set_bit(__LINK_STATE_START, &dev->state); 5087 5088 return 0; 5089 } 5090 EXPORT_SYMBOL_GPL(init_dummy_netdev); 5091 5092 5093 /** 5094 * register_netdev - register a network device 5095 * @dev: device to register 5096 * 5097 * Take a completed network device structure and add it to the kernel 5098 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 5099 * chain. 0 is returned on success. A negative errno code is returned 5100 * on a failure to set up the device, or if the name is a duplicate. 5101 * 5102 * This is a wrapper around register_netdevice that takes the rtnl semaphore 5103 * and expands the device name if you passed a format string to 5104 * alloc_netdev. 5105 */ 5106 int register_netdev(struct net_device *dev) 5107 { 5108 int err; 5109 5110 rtnl_lock(); 5111 5112 /* 5113 * If the name is a format string the caller wants us to do a 5114 * name allocation. 5115 */ 5116 if (strchr(dev->name, '%')) { 5117 err = dev_alloc_name(dev, dev->name); 5118 if (err < 0) 5119 goto out; 5120 } 5121 5122 err = register_netdevice(dev); 5123 out: 5124 rtnl_unlock(); 5125 return err; 5126 } 5127 EXPORT_SYMBOL(register_netdev); 5128 5129 /* 5130 * netdev_wait_allrefs - wait until all references are gone. 5131 * 5132 * This is called when unregistering network devices. 5133 * 5134 * Any protocol or device that holds a reference should register 5135 * for netdevice notification, and cleanup and put back the 5136 * reference if they receive an UNREGISTER event. 5137 * We can get stuck here if buggy protocols don't correctly 5138 * call dev_put. 5139 */ 5140 static void netdev_wait_allrefs(struct net_device *dev) 5141 { 5142 unsigned long rebroadcast_time, warning_time; 5143 5144 linkwatch_forget_dev(dev); 5145 5146 rebroadcast_time = warning_time = jiffies; 5147 while (atomic_read(&dev->refcnt) != 0) { 5148 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { 5149 rtnl_lock(); 5150 5151 /* Rebroadcast unregister notification */ 5152 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 5153 /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users 5154 * should have already handle it the first time */ 5155 5156 if (test_bit(__LINK_STATE_LINKWATCH_PENDING, 5157 &dev->state)) { 5158 /* We must not have linkwatch events 5159 * pending on unregister. If this 5160 * happens, we simply run the queue 5161 * unscheduled, resulting in a noop 5162 * for this device. 5163 */ 5164 linkwatch_run_queue(); 5165 } 5166 5167 __rtnl_unlock(); 5168 5169 rebroadcast_time = jiffies; 5170 } 5171 5172 msleep(250); 5173 5174 if (time_after(jiffies, warning_time + 10 * HZ)) { 5175 printk(KERN_EMERG "unregister_netdevice: " 5176 "waiting for %s to become free. Usage " 5177 "count = %d\n", 5178 dev->name, atomic_read(&dev->refcnt)); 5179 warning_time = jiffies; 5180 } 5181 } 5182 } 5183 5184 /* The sequence is: 5185 * 5186 * rtnl_lock(); 5187 * ... 5188 * register_netdevice(x1); 5189 * register_netdevice(x2); 5190 * ... 5191 * unregister_netdevice(y1); 5192 * unregister_netdevice(y2); 5193 * ... 5194 * rtnl_unlock(); 5195 * free_netdev(y1); 5196 * free_netdev(y2); 5197 * 5198 * We are invoked by rtnl_unlock(). 5199 * This allows us to deal with problems: 5200 * 1) We can delete sysfs objects which invoke hotplug 5201 * without deadlocking with linkwatch via keventd. 5202 * 2) Since we run with the RTNL semaphore not held, we can sleep 5203 * safely in order to wait for the netdev refcnt to drop to zero. 5204 * 5205 * We must not return until all unregister events added during 5206 * the interval the lock was held have been completed. 5207 */ 5208 void netdev_run_todo(void) 5209 { 5210 struct list_head list; 5211 5212 /* Snapshot list, allow later requests */ 5213 list_replace_init(&net_todo_list, &list); 5214 5215 __rtnl_unlock(); 5216 5217 while (!list_empty(&list)) { 5218 struct net_device *dev 5219 = list_entry(list.next, struct net_device, todo_list); 5220 list_del(&dev->todo_list); 5221 5222 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { 5223 printk(KERN_ERR "network todo '%s' but state %d\n", 5224 dev->name, dev->reg_state); 5225 dump_stack(); 5226 continue; 5227 } 5228 5229 dev->reg_state = NETREG_UNREGISTERED; 5230 5231 on_each_cpu(flush_backlog, dev, 1); 5232 5233 netdev_wait_allrefs(dev); 5234 5235 /* paranoia */ 5236 BUG_ON(atomic_read(&dev->refcnt)); 5237 WARN_ON(dev->ip_ptr); 5238 WARN_ON(dev->ip6_ptr); 5239 WARN_ON(dev->dn_ptr); 5240 5241 if (dev->destructor) 5242 dev->destructor(dev); 5243 5244 /* Free network device */ 5245 kobject_put(&dev->dev.kobj); 5246 } 5247 } 5248 5249 /** 5250 * dev_txq_stats_fold - fold tx_queues stats 5251 * @dev: device to get statistics from 5252 * @stats: struct net_device_stats to hold results 5253 */ 5254 void dev_txq_stats_fold(const struct net_device *dev, 5255 struct net_device_stats *stats) 5256 { 5257 unsigned long tx_bytes = 0, tx_packets = 0, tx_dropped = 0; 5258 unsigned int i; 5259 struct netdev_queue *txq; 5260 5261 for (i = 0; i < dev->num_tx_queues; i++) { 5262 txq = netdev_get_tx_queue(dev, i); 5263 tx_bytes += txq->tx_bytes; 5264 tx_packets += txq->tx_packets; 5265 tx_dropped += txq->tx_dropped; 5266 } 5267 if (tx_bytes || tx_packets || tx_dropped) { 5268 stats->tx_bytes = tx_bytes; 5269 stats->tx_packets = tx_packets; 5270 stats->tx_dropped = tx_dropped; 5271 } 5272 } 5273 EXPORT_SYMBOL(dev_txq_stats_fold); 5274 5275 /** 5276 * dev_get_stats - get network device statistics 5277 * @dev: device to get statistics from 5278 * 5279 * Get network statistics from device. The device driver may provide 5280 * its own method by setting dev->netdev_ops->get_stats; otherwise 5281 * the internal statistics structure is used. 5282 */ 5283 const struct net_device_stats *dev_get_stats(struct net_device *dev) 5284 { 5285 const struct net_device_ops *ops = dev->netdev_ops; 5286 5287 if (ops->ndo_get_stats) 5288 return ops->ndo_get_stats(dev); 5289 5290 dev_txq_stats_fold(dev, &dev->stats); 5291 return &dev->stats; 5292 } 5293 EXPORT_SYMBOL(dev_get_stats); 5294 5295 static void netdev_init_one_queue(struct net_device *dev, 5296 struct netdev_queue *queue, 5297 void *_unused) 5298 { 5299 queue->dev = dev; 5300 } 5301 5302 static void netdev_init_queues(struct net_device *dev) 5303 { 5304 netdev_init_one_queue(dev, &dev->rx_queue, NULL); 5305 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); 5306 spin_lock_init(&dev->tx_global_lock); 5307 } 5308 5309 /** 5310 * alloc_netdev_mq - allocate network device 5311 * @sizeof_priv: size of private data to allocate space for 5312 * @name: device name format string 5313 * @setup: callback to initialize device 5314 * @queue_count: the number of subqueues to allocate 5315 * 5316 * Allocates a struct net_device with private data area for driver use 5317 * and performs basic initialization. Also allocates subquue structs 5318 * for each queue on the device at the end of the netdevice. 5319 */ 5320 struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name, 5321 void (*setup)(struct net_device *), unsigned int queue_count) 5322 { 5323 struct netdev_queue *tx; 5324 struct net_device *dev; 5325 size_t alloc_size; 5326 struct net_device *p; 5327 5328 BUG_ON(strlen(name) >= sizeof(dev->name)); 5329 5330 alloc_size = sizeof(struct net_device); 5331 if (sizeof_priv) { 5332 /* ensure 32-byte alignment of private area */ 5333 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN); 5334 alloc_size += sizeof_priv; 5335 } 5336 /* ensure 32-byte alignment of whole construct */ 5337 alloc_size += NETDEV_ALIGN - 1; 5338 5339 p = kzalloc(alloc_size, GFP_KERNEL); 5340 if (!p) { 5341 printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n"); 5342 return NULL; 5343 } 5344 5345 tx = kcalloc(queue_count, sizeof(struct netdev_queue), GFP_KERNEL); 5346 if (!tx) { 5347 printk(KERN_ERR "alloc_netdev: Unable to allocate " 5348 "tx qdiscs.\n"); 5349 goto free_p; 5350 } 5351 5352 dev = PTR_ALIGN(p, NETDEV_ALIGN); 5353 dev->padded = (char *)dev - (char *)p; 5354 5355 if (dev_addr_init(dev)) 5356 goto free_tx; 5357 5358 dev_unicast_init(dev); 5359 5360 dev_net_set(dev, &init_net); 5361 5362 dev->_tx = tx; 5363 dev->num_tx_queues = queue_count; 5364 dev->real_num_tx_queues = queue_count; 5365 5366 dev->gso_max_size = GSO_MAX_SIZE; 5367 5368 netdev_init_queues(dev); 5369 5370 INIT_LIST_HEAD(&dev->napi_list); 5371 INIT_LIST_HEAD(&dev->unreg_list); 5372 INIT_LIST_HEAD(&dev->link_watch_list); 5373 dev->priv_flags = IFF_XMIT_DST_RELEASE; 5374 setup(dev); 5375 strcpy(dev->name, name); 5376 return dev; 5377 5378 free_tx: 5379 kfree(tx); 5380 5381 free_p: 5382 kfree(p); 5383 return NULL; 5384 } 5385 EXPORT_SYMBOL(alloc_netdev_mq); 5386 5387 /** 5388 * free_netdev - free network device 5389 * @dev: device 5390 * 5391 * This function does the last stage of destroying an allocated device 5392 * interface. The reference to the device object is released. 5393 * If this is the last reference then it will be freed. 5394 */ 5395 void free_netdev(struct net_device *dev) 5396 { 5397 struct napi_struct *p, *n; 5398 5399 release_net(dev_net(dev)); 5400 5401 kfree(dev->_tx); 5402 5403 /* Flush device addresses */ 5404 dev_addr_flush(dev); 5405 5406 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) 5407 netif_napi_del(p); 5408 5409 /* Compatibility with error handling in drivers */ 5410 if (dev->reg_state == NETREG_UNINITIALIZED) { 5411 kfree((char *)dev - dev->padded); 5412 return; 5413 } 5414 5415 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); 5416 dev->reg_state = NETREG_RELEASED; 5417 5418 /* will free via device release */ 5419 put_device(&dev->dev); 5420 } 5421 EXPORT_SYMBOL(free_netdev); 5422 5423 /** 5424 * synchronize_net - Synchronize with packet receive processing 5425 * 5426 * Wait for packets currently being received to be done. 5427 * Does not block later packets from starting. 5428 */ 5429 void synchronize_net(void) 5430 { 5431 might_sleep(); 5432 synchronize_rcu(); 5433 } 5434 EXPORT_SYMBOL(synchronize_net); 5435 5436 /** 5437 * unregister_netdevice_queue - remove device from the kernel 5438 * @dev: device 5439 * @head: list 5440 * 5441 * This function shuts down a device interface and removes it 5442 * from the kernel tables. 5443 * If head not NULL, device is queued to be unregistered later. 5444 * 5445 * Callers must hold the rtnl semaphore. You may want 5446 * unregister_netdev() instead of this. 5447 */ 5448 5449 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) 5450 { 5451 ASSERT_RTNL(); 5452 5453 if (head) { 5454 list_move_tail(&dev->unreg_list, head); 5455 } else { 5456 rollback_registered(dev); 5457 /* Finish processing unregister after unlock */ 5458 net_set_todo(dev); 5459 } 5460 } 5461 EXPORT_SYMBOL(unregister_netdevice_queue); 5462 5463 /** 5464 * unregister_netdevice_many - unregister many devices 5465 * @head: list of devices 5466 */ 5467 void unregister_netdevice_many(struct list_head *head) 5468 { 5469 struct net_device *dev; 5470 5471 if (!list_empty(head)) { 5472 rollback_registered_many(head); 5473 list_for_each_entry(dev, head, unreg_list) 5474 net_set_todo(dev); 5475 } 5476 } 5477 EXPORT_SYMBOL(unregister_netdevice_many); 5478 5479 /** 5480 * unregister_netdev - remove device from the kernel 5481 * @dev: device 5482 * 5483 * This function shuts down a device interface and removes it 5484 * from the kernel tables. 5485 * 5486 * This is just a wrapper for unregister_netdevice that takes 5487 * the rtnl semaphore. In general you want to use this and not 5488 * unregister_netdevice. 5489 */ 5490 void unregister_netdev(struct net_device *dev) 5491 { 5492 rtnl_lock(); 5493 unregister_netdevice(dev); 5494 rtnl_unlock(); 5495 } 5496 EXPORT_SYMBOL(unregister_netdev); 5497 5498 /** 5499 * dev_change_net_namespace - move device to different nethost namespace 5500 * @dev: device 5501 * @net: network namespace 5502 * @pat: If not NULL name pattern to try if the current device name 5503 * is already taken in the destination network namespace. 5504 * 5505 * This function shuts down a device interface and moves it 5506 * to a new network namespace. On success 0 is returned, on 5507 * a failure a netagive errno code is returned. 5508 * 5509 * Callers must hold the rtnl semaphore. 5510 */ 5511 5512 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat) 5513 { 5514 int err; 5515 5516 ASSERT_RTNL(); 5517 5518 /* Don't allow namespace local devices to be moved. */ 5519 err = -EINVAL; 5520 if (dev->features & NETIF_F_NETNS_LOCAL) 5521 goto out; 5522 5523 #ifdef CONFIG_SYSFS 5524 /* Don't allow real devices to be moved when sysfs 5525 * is enabled. 5526 */ 5527 err = -EINVAL; 5528 if (dev->dev.parent) 5529 goto out; 5530 #endif 5531 5532 /* Ensure the device has been registrered */ 5533 err = -EINVAL; 5534 if (dev->reg_state != NETREG_REGISTERED) 5535 goto out; 5536 5537 /* Get out if there is nothing todo */ 5538 err = 0; 5539 if (net_eq(dev_net(dev), net)) 5540 goto out; 5541 5542 /* Pick the destination device name, and ensure 5543 * we can use it in the destination network namespace. 5544 */ 5545 err = -EEXIST; 5546 if (__dev_get_by_name(net, dev->name)) { 5547 /* We get here if we can't use the current device name */ 5548 if (!pat) 5549 goto out; 5550 if (dev_get_valid_name(net, pat, dev->name, 1)) 5551 goto out; 5552 } 5553 5554 /* 5555 * And now a mini version of register_netdevice unregister_netdevice. 5556 */ 5557 5558 /* If device is running close it first. */ 5559 dev_close(dev); 5560 5561 /* And unlink it from device chain */ 5562 err = -ENODEV; 5563 unlist_netdevice(dev); 5564 5565 synchronize_net(); 5566 5567 /* Shutdown queueing discipline. */ 5568 dev_shutdown(dev); 5569 5570 /* Notify protocols, that we are about to destroy 5571 this device. They should clean all the things. 5572 */ 5573 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 5574 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev); 5575 5576 /* 5577 * Flush the unicast and multicast chains 5578 */ 5579 dev_unicast_flush(dev); 5580 dev_addr_discard(dev); 5581 5582 netdev_unregister_kobject(dev); 5583 5584 /* Actually switch the network namespace */ 5585 dev_net_set(dev, net); 5586 5587 /* If there is an ifindex conflict assign a new one */ 5588 if (__dev_get_by_index(net, dev->ifindex)) { 5589 int iflink = (dev->iflink == dev->ifindex); 5590 dev->ifindex = dev_new_index(net); 5591 if (iflink) 5592 dev->iflink = dev->ifindex; 5593 } 5594 5595 /* Fixup kobjects */ 5596 err = netdev_register_kobject(dev); 5597 WARN_ON(err); 5598 5599 /* Add the device back in the hashes */ 5600 list_netdevice(dev); 5601 5602 /* Notify protocols, that a new device appeared. */ 5603 call_netdevice_notifiers(NETDEV_REGISTER, dev); 5604 5605 /* 5606 * Prevent userspace races by waiting until the network 5607 * device is fully setup before sending notifications. 5608 */ 5609 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U); 5610 5611 synchronize_net(); 5612 err = 0; 5613 out: 5614 return err; 5615 } 5616 EXPORT_SYMBOL_GPL(dev_change_net_namespace); 5617 5618 static int dev_cpu_callback(struct notifier_block *nfb, 5619 unsigned long action, 5620 void *ocpu) 5621 { 5622 struct sk_buff **list_skb; 5623 struct Qdisc **list_net; 5624 struct sk_buff *skb; 5625 unsigned int cpu, oldcpu = (unsigned long)ocpu; 5626 struct softnet_data *sd, *oldsd; 5627 5628 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN) 5629 return NOTIFY_OK; 5630 5631 local_irq_disable(); 5632 cpu = smp_processor_id(); 5633 sd = &per_cpu(softnet_data, cpu); 5634 oldsd = &per_cpu(softnet_data, oldcpu); 5635 5636 /* Find end of our completion_queue. */ 5637 list_skb = &sd->completion_queue; 5638 while (*list_skb) 5639 list_skb = &(*list_skb)->next; 5640 /* Append completion queue from offline CPU. */ 5641 *list_skb = oldsd->completion_queue; 5642 oldsd->completion_queue = NULL; 5643 5644 /* Find end of our output_queue. */ 5645 list_net = &sd->output_queue; 5646 while (*list_net) 5647 list_net = &(*list_net)->next_sched; 5648 /* Append output queue from offline CPU. */ 5649 *list_net = oldsd->output_queue; 5650 oldsd->output_queue = NULL; 5651 5652 raise_softirq_irqoff(NET_TX_SOFTIRQ); 5653 local_irq_enable(); 5654 5655 /* Process offline CPU's input_pkt_queue */ 5656 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) 5657 netif_rx(skb); 5658 5659 return NOTIFY_OK; 5660 } 5661 5662 5663 /** 5664 * netdev_increment_features - increment feature set by one 5665 * @all: current feature set 5666 * @one: new feature set 5667 * @mask: mask feature set 5668 * 5669 * Computes a new feature set after adding a device with feature set 5670 * @one to the master device with current feature set @all. Will not 5671 * enable anything that is off in @mask. Returns the new feature set. 5672 */ 5673 unsigned long netdev_increment_features(unsigned long all, unsigned long one, 5674 unsigned long mask) 5675 { 5676 /* If device needs checksumming, downgrade to it. */ 5677 if (all & NETIF_F_NO_CSUM && !(one & NETIF_F_NO_CSUM)) 5678 all ^= NETIF_F_NO_CSUM | (one & NETIF_F_ALL_CSUM); 5679 else if (mask & NETIF_F_ALL_CSUM) { 5680 /* If one device supports v4/v6 checksumming, set for all. */ 5681 if (one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM) && 5682 !(all & NETIF_F_GEN_CSUM)) { 5683 all &= ~NETIF_F_ALL_CSUM; 5684 all |= one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); 5685 } 5686 5687 /* If one device supports hw checksumming, set for all. */ 5688 if (one & NETIF_F_GEN_CSUM && !(all & NETIF_F_GEN_CSUM)) { 5689 all &= ~NETIF_F_ALL_CSUM; 5690 all |= NETIF_F_HW_CSUM; 5691 } 5692 } 5693 5694 one |= NETIF_F_ALL_CSUM; 5695 5696 one |= all & NETIF_F_ONE_FOR_ALL; 5697 all &= one | NETIF_F_LLTX | NETIF_F_GSO | NETIF_F_UFO; 5698 all |= one & mask & NETIF_F_ONE_FOR_ALL; 5699 5700 return all; 5701 } 5702 EXPORT_SYMBOL(netdev_increment_features); 5703 5704 static struct hlist_head *netdev_create_hash(void) 5705 { 5706 int i; 5707 struct hlist_head *hash; 5708 5709 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL); 5710 if (hash != NULL) 5711 for (i = 0; i < NETDEV_HASHENTRIES; i++) 5712 INIT_HLIST_HEAD(&hash[i]); 5713 5714 return hash; 5715 } 5716 5717 /* Initialize per network namespace state */ 5718 static int __net_init netdev_init(struct net *net) 5719 { 5720 INIT_LIST_HEAD(&net->dev_base_head); 5721 5722 net->dev_name_head = netdev_create_hash(); 5723 if (net->dev_name_head == NULL) 5724 goto err_name; 5725 5726 net->dev_index_head = netdev_create_hash(); 5727 if (net->dev_index_head == NULL) 5728 goto err_idx; 5729 5730 return 0; 5731 5732 err_idx: 5733 kfree(net->dev_name_head); 5734 err_name: 5735 return -ENOMEM; 5736 } 5737 5738 /** 5739 * netdev_drivername - network driver for the device 5740 * @dev: network device 5741 * @buffer: buffer for resulting name 5742 * @len: size of buffer 5743 * 5744 * Determine network driver for device. 5745 */ 5746 char *netdev_drivername(const struct net_device *dev, char *buffer, int len) 5747 { 5748 const struct device_driver *driver; 5749 const struct device *parent; 5750 5751 if (len <= 0 || !buffer) 5752 return buffer; 5753 buffer[0] = 0; 5754 5755 parent = dev->dev.parent; 5756 5757 if (!parent) 5758 return buffer; 5759 5760 driver = parent->driver; 5761 if (driver && driver->name) 5762 strlcpy(buffer, driver->name, len); 5763 return buffer; 5764 } 5765 5766 static void __net_exit netdev_exit(struct net *net) 5767 { 5768 kfree(net->dev_name_head); 5769 kfree(net->dev_index_head); 5770 } 5771 5772 static struct pernet_operations __net_initdata netdev_net_ops = { 5773 .init = netdev_init, 5774 .exit = netdev_exit, 5775 }; 5776 5777 static void __net_exit default_device_exit(struct net *net) 5778 { 5779 struct net_device *dev, *aux; 5780 /* 5781 * Push all migratable network devices back to the 5782 * initial network namespace 5783 */ 5784 rtnl_lock(); 5785 for_each_netdev_safe(net, dev, aux) { 5786 int err; 5787 char fb_name[IFNAMSIZ]; 5788 5789 /* Ignore unmoveable devices (i.e. loopback) */ 5790 if (dev->features & NETIF_F_NETNS_LOCAL) 5791 continue; 5792 5793 /* Leave virtual devices for the generic cleanup */ 5794 if (dev->rtnl_link_ops) 5795 continue; 5796 5797 /* Push remaing network devices to init_net */ 5798 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); 5799 err = dev_change_net_namespace(dev, &init_net, fb_name); 5800 if (err) { 5801 printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n", 5802 __func__, dev->name, err); 5803 BUG(); 5804 } 5805 } 5806 rtnl_unlock(); 5807 } 5808 5809 static void __net_exit default_device_exit_batch(struct list_head *net_list) 5810 { 5811 /* At exit all network devices most be removed from a network 5812 * namespace. Do this in the reverse order of registeration. 5813 * Do this across as many network namespaces as possible to 5814 * improve batching efficiency. 5815 */ 5816 struct net_device *dev; 5817 struct net *net; 5818 LIST_HEAD(dev_kill_list); 5819 5820 rtnl_lock(); 5821 list_for_each_entry(net, net_list, exit_list) { 5822 for_each_netdev_reverse(net, dev) { 5823 if (dev->rtnl_link_ops) 5824 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); 5825 else 5826 unregister_netdevice_queue(dev, &dev_kill_list); 5827 } 5828 } 5829 unregister_netdevice_many(&dev_kill_list); 5830 rtnl_unlock(); 5831 } 5832 5833 static struct pernet_operations __net_initdata default_device_ops = { 5834 .exit = default_device_exit, 5835 .exit_batch = default_device_exit_batch, 5836 }; 5837 5838 /* 5839 * Initialize the DEV module. At boot time this walks the device list and 5840 * unhooks any devices that fail to initialise (normally hardware not 5841 * present) and leaves us with a valid list of present and active devices. 5842 * 5843 */ 5844 5845 /* 5846 * This is called single threaded during boot, so no need 5847 * to take the rtnl semaphore. 5848 */ 5849 static int __init net_dev_init(void) 5850 { 5851 int i, rc = -ENOMEM; 5852 5853 BUG_ON(!dev_boot_phase); 5854 5855 if (dev_proc_init()) 5856 goto out; 5857 5858 if (netdev_kobject_init()) 5859 goto out; 5860 5861 INIT_LIST_HEAD(&ptype_all); 5862 for (i = 0; i < PTYPE_HASH_SIZE; i++) 5863 INIT_LIST_HEAD(&ptype_base[i]); 5864 5865 if (register_pernet_subsys(&netdev_net_ops)) 5866 goto out; 5867 5868 /* 5869 * Initialise the packet receive queues. 5870 */ 5871 5872 for_each_possible_cpu(i) { 5873 struct softnet_data *queue; 5874 5875 queue = &per_cpu(softnet_data, i); 5876 skb_queue_head_init(&queue->input_pkt_queue); 5877 queue->completion_queue = NULL; 5878 INIT_LIST_HEAD(&queue->poll_list); 5879 5880 queue->backlog.poll = process_backlog; 5881 queue->backlog.weight = weight_p; 5882 queue->backlog.gro_list = NULL; 5883 queue->backlog.gro_count = 0; 5884 } 5885 5886 dev_boot_phase = 0; 5887 5888 /* The loopback device is special if any other network devices 5889 * is present in a network namespace the loopback device must 5890 * be present. Since we now dynamically allocate and free the 5891 * loopback device ensure this invariant is maintained by 5892 * keeping the loopback device as the first device on the 5893 * list of network devices. Ensuring the loopback devices 5894 * is the first device that appears and the last network device 5895 * that disappears. 5896 */ 5897 if (register_pernet_device(&loopback_net_ops)) 5898 goto out; 5899 5900 if (register_pernet_device(&default_device_ops)) 5901 goto out; 5902 5903 open_softirq(NET_TX_SOFTIRQ, net_tx_action); 5904 open_softirq(NET_RX_SOFTIRQ, net_rx_action); 5905 5906 hotcpu_notifier(dev_cpu_callback, 0); 5907 dst_init(); 5908 dev_mcast_init(); 5909 rc = 0; 5910 out: 5911 return rc; 5912 } 5913 5914 subsys_initcall(net_dev_init); 5915 5916 static int __init initialize_hashrnd(void) 5917 { 5918 get_random_bytes(&skb_tx_hashrnd, sizeof(skb_tx_hashrnd)); 5919 return 0; 5920 } 5921 5922 late_initcall_sync(initialize_hashrnd); 5923 5924