1 /* 2 * NET3 Protocol independent device support routines. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 * 9 * Derived from the non IP parts of dev.c 1.0.19 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Mark Evans, <evansmp@uhura.aston.ac.uk> 13 * 14 * Additional Authors: 15 * Florian la Roche <rzsfl@rz.uni-sb.de> 16 * Alan Cox <gw4pts@gw4pts.ampr.org> 17 * David Hinds <dahinds@users.sourceforge.net> 18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 19 * Adam Sulmicki <adam@cfar.umd.edu> 20 * Pekka Riikonen <priikone@poesidon.pspt.fi> 21 * 22 * Changes: 23 * D.J. Barrow : Fixed bug where dev->refcnt gets set 24 * to 2 if register_netdev gets called 25 * before net_dev_init & also removed a 26 * few lines of code in the process. 27 * Alan Cox : device private ioctl copies fields back. 28 * Alan Cox : Transmit queue code does relevant 29 * stunts to keep the queue safe. 30 * Alan Cox : Fixed double lock. 31 * Alan Cox : Fixed promisc NULL pointer trap 32 * ???????? : Support the full private ioctl range 33 * Alan Cox : Moved ioctl permission check into 34 * drivers 35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI 36 * Alan Cox : 100 backlog just doesn't cut it when 37 * you start doing multicast video 8) 38 * Alan Cox : Rewrote net_bh and list manager. 39 * Alan Cox : Fix ETH_P_ALL echoback lengths. 40 * Alan Cox : Took out transmit every packet pass 41 * Saved a few bytes in the ioctl handler 42 * Alan Cox : Network driver sets packet type before 43 * calling netif_rx. Saves a function 44 * call a packet. 45 * Alan Cox : Hashed net_bh() 46 * Richard Kooijman: Timestamp fixes. 47 * Alan Cox : Wrong field in SIOCGIFDSTADDR 48 * Alan Cox : Device lock protection. 49 * Alan Cox : Fixed nasty side effect of device close 50 * changes. 51 * Rudi Cilibrasi : Pass the right thing to 52 * set_mac_address() 53 * Dave Miller : 32bit quantity for the device lock to 54 * make it work out on a Sparc. 55 * Bjorn Ekwall : Added KERNELD hack. 56 * Alan Cox : Cleaned up the backlog initialise. 57 * Craig Metz : SIOCGIFCONF fix if space for under 58 * 1 device. 59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there 60 * is no device open function. 61 * Andi Kleen : Fix error reporting for SIOCGIFCONF 62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF 63 * Cyrus Durgin : Cleaned for KMOD 64 * Adam Sulmicki : Bug Fix : Network Device Unload 65 * A network device unload needs to purge 66 * the backlog queue. 67 * Paul Rusty Russell : SIOCSIFNAME 68 * Pekka Riikonen : Netdev boot-time settings code 69 * Andrew Morton : Make unregister_netdevice wait 70 * indefinitely on dev->refcnt 71 * J Hadi Salim : - Backlog queue sampling 72 * - netif_rx() feedback 73 */ 74 75 #include <asm/uaccess.h> 76 #include <linux/bitops.h> 77 #include <linux/capability.h> 78 #include <linux/cpu.h> 79 #include <linux/types.h> 80 #include <linux/kernel.h> 81 #include <linux/hash.h> 82 #include <linux/slab.h> 83 #include <linux/sched.h> 84 #include <linux/mutex.h> 85 #include <linux/string.h> 86 #include <linux/mm.h> 87 #include <linux/socket.h> 88 #include <linux/sockios.h> 89 #include <linux/errno.h> 90 #include <linux/interrupt.h> 91 #include <linux/if_ether.h> 92 #include <linux/netdevice.h> 93 #include <linux/etherdevice.h> 94 #include <linux/ethtool.h> 95 #include <linux/notifier.h> 96 #include <linux/skbuff.h> 97 #include <linux/bpf.h> 98 #include <net/net_namespace.h> 99 #include <net/sock.h> 100 #include <net/busy_poll.h> 101 #include <linux/rtnetlink.h> 102 #include <linux/stat.h> 103 #include <net/dst.h> 104 #include <net/dst_metadata.h> 105 #include <net/pkt_sched.h> 106 #include <net/checksum.h> 107 #include <net/xfrm.h> 108 #include <linux/highmem.h> 109 #include <linux/init.h> 110 #include <linux/module.h> 111 #include <linux/netpoll.h> 112 #include <linux/rcupdate.h> 113 #include <linux/delay.h> 114 #include <net/iw_handler.h> 115 #include <asm/current.h> 116 #include <linux/audit.h> 117 #include <linux/dmaengine.h> 118 #include <linux/err.h> 119 #include <linux/ctype.h> 120 #include <linux/if_arp.h> 121 #include <linux/if_vlan.h> 122 #include <linux/ip.h> 123 #include <net/ip.h> 124 #include <net/mpls.h> 125 #include <linux/ipv6.h> 126 #include <linux/in.h> 127 #include <linux/jhash.h> 128 #include <linux/random.h> 129 #include <trace/events/napi.h> 130 #include <trace/events/net.h> 131 #include <trace/events/skb.h> 132 #include <linux/pci.h> 133 #include <linux/inetdevice.h> 134 #include <linux/cpu_rmap.h> 135 #include <linux/static_key.h> 136 #include <linux/hashtable.h> 137 #include <linux/vmalloc.h> 138 #include <linux/if_macvlan.h> 139 #include <linux/errqueue.h> 140 #include <linux/hrtimer.h> 141 #include <linux/netfilter_ingress.h> 142 #include <linux/sctp.h> 143 #include <linux/crash_dump.h> 144 145 #include "net-sysfs.h" 146 147 /* Instead of increasing this, you should create a hash table. */ 148 #define MAX_GRO_SKBS 8 149 150 /* This should be increased if a protocol with a bigger head is added. */ 151 #define GRO_MAX_HEAD (MAX_HEADER + 128) 152 153 static DEFINE_SPINLOCK(ptype_lock); 154 static DEFINE_SPINLOCK(offload_lock); 155 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; 156 struct list_head ptype_all __read_mostly; /* Taps */ 157 static struct list_head offload_base __read_mostly; 158 159 static int netif_rx_internal(struct sk_buff *skb); 160 static int call_netdevice_notifiers_info(unsigned long val, 161 struct net_device *dev, 162 struct netdev_notifier_info *info); 163 164 /* 165 * The @dev_base_head list is protected by @dev_base_lock and the rtnl 166 * semaphore. 167 * 168 * Pure readers hold dev_base_lock for reading, or rcu_read_lock() 169 * 170 * Writers must hold the rtnl semaphore while they loop through the 171 * dev_base_head list, and hold dev_base_lock for writing when they do the 172 * actual updates. This allows pure readers to access the list even 173 * while a writer is preparing to update it. 174 * 175 * To put it another way, dev_base_lock is held for writing only to 176 * protect against pure readers; the rtnl semaphore provides the 177 * protection against other writers. 178 * 179 * See, for example usages, register_netdevice() and 180 * unregister_netdevice(), which must be called with the rtnl 181 * semaphore held. 182 */ 183 DEFINE_RWLOCK(dev_base_lock); 184 EXPORT_SYMBOL(dev_base_lock); 185 186 /* protects napi_hash addition/deletion and napi_gen_id */ 187 static DEFINE_SPINLOCK(napi_hash_lock); 188 189 static unsigned int napi_gen_id = NR_CPUS; 190 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8); 191 192 static seqcount_t devnet_rename_seq; 193 194 static inline void dev_base_seq_inc(struct net *net) 195 { 196 while (++net->dev_base_seq == 0); 197 } 198 199 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name) 200 { 201 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ)); 202 203 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; 204 } 205 206 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) 207 { 208 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; 209 } 210 211 static inline void rps_lock(struct softnet_data *sd) 212 { 213 #ifdef CONFIG_RPS 214 spin_lock(&sd->input_pkt_queue.lock); 215 #endif 216 } 217 218 static inline void rps_unlock(struct softnet_data *sd) 219 { 220 #ifdef CONFIG_RPS 221 spin_unlock(&sd->input_pkt_queue.lock); 222 #endif 223 } 224 225 /* Device list insertion */ 226 static void list_netdevice(struct net_device *dev) 227 { 228 struct net *net = dev_net(dev); 229 230 ASSERT_RTNL(); 231 232 write_lock_bh(&dev_base_lock); 233 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); 234 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); 235 hlist_add_head_rcu(&dev->index_hlist, 236 dev_index_hash(net, dev->ifindex)); 237 write_unlock_bh(&dev_base_lock); 238 239 dev_base_seq_inc(net); 240 } 241 242 /* Device list removal 243 * caller must respect a RCU grace period before freeing/reusing dev 244 */ 245 static void unlist_netdevice(struct net_device *dev) 246 { 247 ASSERT_RTNL(); 248 249 /* Unlink dev from the device chain */ 250 write_lock_bh(&dev_base_lock); 251 list_del_rcu(&dev->dev_list); 252 hlist_del_rcu(&dev->name_hlist); 253 hlist_del_rcu(&dev->index_hlist); 254 write_unlock_bh(&dev_base_lock); 255 256 dev_base_seq_inc(dev_net(dev)); 257 } 258 259 /* 260 * Our notifier list 261 */ 262 263 static RAW_NOTIFIER_HEAD(netdev_chain); 264 265 /* 266 * Device drivers call our routines to queue packets here. We empty the 267 * queue in the local softnet handler. 268 */ 269 270 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data); 271 EXPORT_PER_CPU_SYMBOL(softnet_data); 272 273 #ifdef CONFIG_LOCKDEP 274 /* 275 * register_netdevice() inits txq->_xmit_lock and sets lockdep class 276 * according to dev->type 277 */ 278 static const unsigned short netdev_lock_type[] = 279 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25, 280 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET, 281 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM, 282 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, 283 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, 284 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, 285 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, 286 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, 287 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, 288 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, 289 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, 290 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, 291 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM, 292 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE, 293 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE}; 294 295 static const char *const netdev_lock_name[] = 296 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", 297 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET", 298 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM", 299 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", 300 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", 301 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", 302 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", 303 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", 304 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", 305 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", 306 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", 307 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", 308 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM", 309 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE", 310 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"}; 311 312 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; 313 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; 314 315 static inline unsigned short netdev_lock_pos(unsigned short dev_type) 316 { 317 int i; 318 319 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++) 320 if (netdev_lock_type[i] == dev_type) 321 return i; 322 /* the last key is used by default */ 323 return ARRAY_SIZE(netdev_lock_type) - 1; 324 } 325 326 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 327 unsigned short dev_type) 328 { 329 int i; 330 331 i = netdev_lock_pos(dev_type); 332 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i], 333 netdev_lock_name[i]); 334 } 335 336 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 337 { 338 int i; 339 340 i = netdev_lock_pos(dev->type); 341 lockdep_set_class_and_name(&dev->addr_list_lock, 342 &netdev_addr_lock_key[i], 343 netdev_lock_name[i]); 344 } 345 #else 346 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 347 unsigned short dev_type) 348 { 349 } 350 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 351 { 352 } 353 #endif 354 355 /******************************************************************************* 356 357 Protocol management and registration routines 358 359 *******************************************************************************/ 360 361 /* 362 * Add a protocol ID to the list. Now that the input handler is 363 * smarter we can dispense with all the messy stuff that used to be 364 * here. 365 * 366 * BEWARE!!! Protocol handlers, mangling input packets, 367 * MUST BE last in hash buckets and checking protocol handlers 368 * MUST start from promiscuous ptype_all chain in net_bh. 369 * It is true now, do not change it. 370 * Explanation follows: if protocol handler, mangling packet, will 371 * be the first on list, it is not able to sense, that packet 372 * is cloned and should be copied-on-write, so that it will 373 * change it and subsequent readers will get broken packet. 374 * --ANK (980803) 375 */ 376 377 static inline struct list_head *ptype_head(const struct packet_type *pt) 378 { 379 if (pt->type == htons(ETH_P_ALL)) 380 return pt->dev ? &pt->dev->ptype_all : &ptype_all; 381 else 382 return pt->dev ? &pt->dev->ptype_specific : 383 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; 384 } 385 386 /** 387 * dev_add_pack - add packet handler 388 * @pt: packet type declaration 389 * 390 * Add a protocol handler to the networking stack. The passed &packet_type 391 * is linked into kernel lists and may not be freed until it has been 392 * removed from the kernel lists. 393 * 394 * This call does not sleep therefore it can not 395 * guarantee all CPU's that are in middle of receiving packets 396 * will see the new packet type (until the next received packet). 397 */ 398 399 void dev_add_pack(struct packet_type *pt) 400 { 401 struct list_head *head = ptype_head(pt); 402 403 spin_lock(&ptype_lock); 404 list_add_rcu(&pt->list, head); 405 spin_unlock(&ptype_lock); 406 } 407 EXPORT_SYMBOL(dev_add_pack); 408 409 /** 410 * __dev_remove_pack - remove packet handler 411 * @pt: packet type declaration 412 * 413 * Remove a protocol handler that was previously added to the kernel 414 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 415 * from the kernel lists and can be freed or reused once this function 416 * returns. 417 * 418 * The packet type might still be in use by receivers 419 * and must not be freed until after all the CPU's have gone 420 * through a quiescent state. 421 */ 422 void __dev_remove_pack(struct packet_type *pt) 423 { 424 struct list_head *head = ptype_head(pt); 425 struct packet_type *pt1; 426 427 spin_lock(&ptype_lock); 428 429 list_for_each_entry(pt1, head, list) { 430 if (pt == pt1) { 431 list_del_rcu(&pt->list); 432 goto out; 433 } 434 } 435 436 pr_warn("dev_remove_pack: %p not found\n", pt); 437 out: 438 spin_unlock(&ptype_lock); 439 } 440 EXPORT_SYMBOL(__dev_remove_pack); 441 442 /** 443 * dev_remove_pack - remove packet handler 444 * @pt: packet type declaration 445 * 446 * Remove a protocol handler that was previously added to the kernel 447 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 448 * from the kernel lists and can be freed or reused once this function 449 * returns. 450 * 451 * This call sleeps to guarantee that no CPU is looking at the packet 452 * type after return. 453 */ 454 void dev_remove_pack(struct packet_type *pt) 455 { 456 __dev_remove_pack(pt); 457 458 synchronize_net(); 459 } 460 EXPORT_SYMBOL(dev_remove_pack); 461 462 463 /** 464 * dev_add_offload - register offload handlers 465 * @po: protocol offload declaration 466 * 467 * Add protocol offload handlers to the networking stack. The passed 468 * &proto_offload is linked into kernel lists and may not be freed until 469 * it has been removed from the kernel lists. 470 * 471 * This call does not sleep therefore it can not 472 * guarantee all CPU's that are in middle of receiving packets 473 * will see the new offload handlers (until the next received packet). 474 */ 475 void dev_add_offload(struct packet_offload *po) 476 { 477 struct packet_offload *elem; 478 479 spin_lock(&offload_lock); 480 list_for_each_entry(elem, &offload_base, list) { 481 if (po->priority < elem->priority) 482 break; 483 } 484 list_add_rcu(&po->list, elem->list.prev); 485 spin_unlock(&offload_lock); 486 } 487 EXPORT_SYMBOL(dev_add_offload); 488 489 /** 490 * __dev_remove_offload - remove offload handler 491 * @po: packet offload declaration 492 * 493 * Remove a protocol offload handler that was previously added to the 494 * kernel offload handlers by dev_add_offload(). The passed &offload_type 495 * is removed from the kernel lists and can be freed or reused once this 496 * function returns. 497 * 498 * The packet type might still be in use by receivers 499 * and must not be freed until after all the CPU's have gone 500 * through a quiescent state. 501 */ 502 static void __dev_remove_offload(struct packet_offload *po) 503 { 504 struct list_head *head = &offload_base; 505 struct packet_offload *po1; 506 507 spin_lock(&offload_lock); 508 509 list_for_each_entry(po1, head, list) { 510 if (po == po1) { 511 list_del_rcu(&po->list); 512 goto out; 513 } 514 } 515 516 pr_warn("dev_remove_offload: %p not found\n", po); 517 out: 518 spin_unlock(&offload_lock); 519 } 520 521 /** 522 * dev_remove_offload - remove packet offload handler 523 * @po: packet offload declaration 524 * 525 * Remove a packet offload handler that was previously added to the kernel 526 * offload handlers by dev_add_offload(). The passed &offload_type is 527 * removed from the kernel lists and can be freed or reused once this 528 * function returns. 529 * 530 * This call sleeps to guarantee that no CPU is looking at the packet 531 * type after return. 532 */ 533 void dev_remove_offload(struct packet_offload *po) 534 { 535 __dev_remove_offload(po); 536 537 synchronize_net(); 538 } 539 EXPORT_SYMBOL(dev_remove_offload); 540 541 /****************************************************************************** 542 543 Device Boot-time Settings Routines 544 545 *******************************************************************************/ 546 547 /* Boot time configuration table */ 548 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX]; 549 550 /** 551 * netdev_boot_setup_add - add new setup entry 552 * @name: name of the device 553 * @map: configured settings for the device 554 * 555 * Adds new setup entry to the dev_boot_setup list. The function 556 * returns 0 on error and 1 on success. This is a generic routine to 557 * all netdevices. 558 */ 559 static int netdev_boot_setup_add(char *name, struct ifmap *map) 560 { 561 struct netdev_boot_setup *s; 562 int i; 563 564 s = dev_boot_setup; 565 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 566 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') { 567 memset(s[i].name, 0, sizeof(s[i].name)); 568 strlcpy(s[i].name, name, IFNAMSIZ); 569 memcpy(&s[i].map, map, sizeof(s[i].map)); 570 break; 571 } 572 } 573 574 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1; 575 } 576 577 /** 578 * netdev_boot_setup_check - check boot time settings 579 * @dev: the netdevice 580 * 581 * Check boot time settings for the device. 582 * The found settings are set for the device to be used 583 * later in the device probing. 584 * Returns 0 if no settings found, 1 if they are. 585 */ 586 int netdev_boot_setup_check(struct net_device *dev) 587 { 588 struct netdev_boot_setup *s = dev_boot_setup; 589 int i; 590 591 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 592 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' && 593 !strcmp(dev->name, s[i].name)) { 594 dev->irq = s[i].map.irq; 595 dev->base_addr = s[i].map.base_addr; 596 dev->mem_start = s[i].map.mem_start; 597 dev->mem_end = s[i].map.mem_end; 598 return 1; 599 } 600 } 601 return 0; 602 } 603 EXPORT_SYMBOL(netdev_boot_setup_check); 604 605 606 /** 607 * netdev_boot_base - get address from boot time settings 608 * @prefix: prefix for network device 609 * @unit: id for network device 610 * 611 * Check boot time settings for the base address of device. 612 * The found settings are set for the device to be used 613 * later in the device probing. 614 * Returns 0 if no settings found. 615 */ 616 unsigned long netdev_boot_base(const char *prefix, int unit) 617 { 618 const struct netdev_boot_setup *s = dev_boot_setup; 619 char name[IFNAMSIZ]; 620 int i; 621 622 sprintf(name, "%s%d", prefix, unit); 623 624 /* 625 * If device already registered then return base of 1 626 * to indicate not to probe for this interface 627 */ 628 if (__dev_get_by_name(&init_net, name)) 629 return 1; 630 631 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) 632 if (!strcmp(name, s[i].name)) 633 return s[i].map.base_addr; 634 return 0; 635 } 636 637 /* 638 * Saves at boot time configured settings for any netdevice. 639 */ 640 int __init netdev_boot_setup(char *str) 641 { 642 int ints[5]; 643 struct ifmap map; 644 645 str = get_options(str, ARRAY_SIZE(ints), ints); 646 if (!str || !*str) 647 return 0; 648 649 /* Save settings */ 650 memset(&map, 0, sizeof(map)); 651 if (ints[0] > 0) 652 map.irq = ints[1]; 653 if (ints[0] > 1) 654 map.base_addr = ints[2]; 655 if (ints[0] > 2) 656 map.mem_start = ints[3]; 657 if (ints[0] > 3) 658 map.mem_end = ints[4]; 659 660 /* Add new entry to the list */ 661 return netdev_boot_setup_add(str, &map); 662 } 663 664 __setup("netdev=", netdev_boot_setup); 665 666 /******************************************************************************* 667 668 Device Interface Subroutines 669 670 *******************************************************************************/ 671 672 /** 673 * dev_get_iflink - get 'iflink' value of a interface 674 * @dev: targeted interface 675 * 676 * Indicates the ifindex the interface is linked to. 677 * Physical interfaces have the same 'ifindex' and 'iflink' values. 678 */ 679 680 int dev_get_iflink(const struct net_device *dev) 681 { 682 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) 683 return dev->netdev_ops->ndo_get_iflink(dev); 684 685 return dev->ifindex; 686 } 687 EXPORT_SYMBOL(dev_get_iflink); 688 689 /** 690 * dev_fill_metadata_dst - Retrieve tunnel egress information. 691 * @dev: targeted interface 692 * @skb: The packet. 693 * 694 * For better visibility of tunnel traffic OVS needs to retrieve 695 * egress tunnel information for a packet. Following API allows 696 * user to get this info. 697 */ 698 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb) 699 { 700 struct ip_tunnel_info *info; 701 702 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst) 703 return -EINVAL; 704 705 info = skb_tunnel_info_unclone(skb); 706 if (!info) 707 return -ENOMEM; 708 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX))) 709 return -EINVAL; 710 711 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb); 712 } 713 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst); 714 715 /** 716 * __dev_get_by_name - find a device by its name 717 * @net: the applicable net namespace 718 * @name: name to find 719 * 720 * Find an interface by name. Must be called under RTNL semaphore 721 * or @dev_base_lock. If the name is found a pointer to the device 722 * is returned. If the name is not found then %NULL is returned. The 723 * reference counters are not incremented so the caller must be 724 * careful with locks. 725 */ 726 727 struct net_device *__dev_get_by_name(struct net *net, const char *name) 728 { 729 struct net_device *dev; 730 struct hlist_head *head = dev_name_hash(net, name); 731 732 hlist_for_each_entry(dev, head, name_hlist) 733 if (!strncmp(dev->name, name, IFNAMSIZ)) 734 return dev; 735 736 return NULL; 737 } 738 EXPORT_SYMBOL(__dev_get_by_name); 739 740 /** 741 * dev_get_by_name_rcu - find a device by its name 742 * @net: the applicable net namespace 743 * @name: name to find 744 * 745 * Find an interface by name. 746 * If the name is found a pointer to the device is returned. 747 * If the name is not found then %NULL is returned. 748 * The reference counters are not incremented so the caller must be 749 * careful with locks. The caller must hold RCU lock. 750 */ 751 752 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name) 753 { 754 struct net_device *dev; 755 struct hlist_head *head = dev_name_hash(net, name); 756 757 hlist_for_each_entry_rcu(dev, head, name_hlist) 758 if (!strncmp(dev->name, name, IFNAMSIZ)) 759 return dev; 760 761 return NULL; 762 } 763 EXPORT_SYMBOL(dev_get_by_name_rcu); 764 765 /** 766 * dev_get_by_name - find a device by its name 767 * @net: the applicable net namespace 768 * @name: name to find 769 * 770 * Find an interface by name. This can be called from any 771 * context and does its own locking. The returned handle has 772 * the usage count incremented and the caller must use dev_put() to 773 * release it when it is no longer needed. %NULL is returned if no 774 * matching device is found. 775 */ 776 777 struct net_device *dev_get_by_name(struct net *net, const char *name) 778 { 779 struct net_device *dev; 780 781 rcu_read_lock(); 782 dev = dev_get_by_name_rcu(net, name); 783 if (dev) 784 dev_hold(dev); 785 rcu_read_unlock(); 786 return dev; 787 } 788 EXPORT_SYMBOL(dev_get_by_name); 789 790 /** 791 * __dev_get_by_index - find a device by its ifindex 792 * @net: the applicable net namespace 793 * @ifindex: index of device 794 * 795 * Search for an interface by index. Returns %NULL if the device 796 * is not found or a pointer to the device. The device has not 797 * had its reference counter increased so the caller must be careful 798 * about locking. The caller must hold either the RTNL semaphore 799 * or @dev_base_lock. 800 */ 801 802 struct net_device *__dev_get_by_index(struct net *net, int ifindex) 803 { 804 struct net_device *dev; 805 struct hlist_head *head = dev_index_hash(net, ifindex); 806 807 hlist_for_each_entry(dev, head, index_hlist) 808 if (dev->ifindex == ifindex) 809 return dev; 810 811 return NULL; 812 } 813 EXPORT_SYMBOL(__dev_get_by_index); 814 815 /** 816 * dev_get_by_index_rcu - find a device by its ifindex 817 * @net: the applicable net namespace 818 * @ifindex: index of device 819 * 820 * Search for an interface by index. Returns %NULL if the device 821 * is not found or a pointer to the device. The device has not 822 * had its reference counter increased so the caller must be careful 823 * about locking. The caller must hold RCU lock. 824 */ 825 826 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex) 827 { 828 struct net_device *dev; 829 struct hlist_head *head = dev_index_hash(net, ifindex); 830 831 hlist_for_each_entry_rcu(dev, head, index_hlist) 832 if (dev->ifindex == ifindex) 833 return dev; 834 835 return NULL; 836 } 837 EXPORT_SYMBOL(dev_get_by_index_rcu); 838 839 840 /** 841 * dev_get_by_index - find a device by its ifindex 842 * @net: the applicable net namespace 843 * @ifindex: index of device 844 * 845 * Search for an interface by index. Returns NULL if the device 846 * is not found or a pointer to the device. The device returned has 847 * had a reference added and the pointer is safe until the user calls 848 * dev_put to indicate they have finished with it. 849 */ 850 851 struct net_device *dev_get_by_index(struct net *net, int ifindex) 852 { 853 struct net_device *dev; 854 855 rcu_read_lock(); 856 dev = dev_get_by_index_rcu(net, ifindex); 857 if (dev) 858 dev_hold(dev); 859 rcu_read_unlock(); 860 return dev; 861 } 862 EXPORT_SYMBOL(dev_get_by_index); 863 864 /** 865 * netdev_get_name - get a netdevice name, knowing its ifindex. 866 * @net: network namespace 867 * @name: a pointer to the buffer where the name will be stored. 868 * @ifindex: the ifindex of the interface to get the name from. 869 * 870 * The use of raw_seqcount_begin() and cond_resched() before 871 * retrying is required as we want to give the writers a chance 872 * to complete when CONFIG_PREEMPT is not set. 873 */ 874 int netdev_get_name(struct net *net, char *name, int ifindex) 875 { 876 struct net_device *dev; 877 unsigned int seq; 878 879 retry: 880 seq = raw_seqcount_begin(&devnet_rename_seq); 881 rcu_read_lock(); 882 dev = dev_get_by_index_rcu(net, ifindex); 883 if (!dev) { 884 rcu_read_unlock(); 885 return -ENODEV; 886 } 887 888 strcpy(name, dev->name); 889 rcu_read_unlock(); 890 if (read_seqcount_retry(&devnet_rename_seq, seq)) { 891 cond_resched(); 892 goto retry; 893 } 894 895 return 0; 896 } 897 898 /** 899 * dev_getbyhwaddr_rcu - find a device by its hardware address 900 * @net: the applicable net namespace 901 * @type: media type of device 902 * @ha: hardware address 903 * 904 * Search for an interface by MAC address. Returns NULL if the device 905 * is not found or a pointer to the device. 906 * The caller must hold RCU or RTNL. 907 * The returned device has not had its ref count increased 908 * and the caller must therefore be careful about locking 909 * 910 */ 911 912 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type, 913 const char *ha) 914 { 915 struct net_device *dev; 916 917 for_each_netdev_rcu(net, dev) 918 if (dev->type == type && 919 !memcmp(dev->dev_addr, ha, dev->addr_len)) 920 return dev; 921 922 return NULL; 923 } 924 EXPORT_SYMBOL(dev_getbyhwaddr_rcu); 925 926 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type) 927 { 928 struct net_device *dev; 929 930 ASSERT_RTNL(); 931 for_each_netdev(net, dev) 932 if (dev->type == type) 933 return dev; 934 935 return NULL; 936 } 937 EXPORT_SYMBOL(__dev_getfirstbyhwtype); 938 939 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) 940 { 941 struct net_device *dev, *ret = NULL; 942 943 rcu_read_lock(); 944 for_each_netdev_rcu(net, dev) 945 if (dev->type == type) { 946 dev_hold(dev); 947 ret = dev; 948 break; 949 } 950 rcu_read_unlock(); 951 return ret; 952 } 953 EXPORT_SYMBOL(dev_getfirstbyhwtype); 954 955 /** 956 * __dev_get_by_flags - find any device with given flags 957 * @net: the applicable net namespace 958 * @if_flags: IFF_* values 959 * @mask: bitmask of bits in if_flags to check 960 * 961 * Search for any interface with the given flags. Returns NULL if a device 962 * is not found or a pointer to the device. Must be called inside 963 * rtnl_lock(), and result refcount is unchanged. 964 */ 965 966 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags, 967 unsigned short mask) 968 { 969 struct net_device *dev, *ret; 970 971 ASSERT_RTNL(); 972 973 ret = NULL; 974 for_each_netdev(net, dev) { 975 if (((dev->flags ^ if_flags) & mask) == 0) { 976 ret = dev; 977 break; 978 } 979 } 980 return ret; 981 } 982 EXPORT_SYMBOL(__dev_get_by_flags); 983 984 /** 985 * dev_valid_name - check if name is okay for network device 986 * @name: name string 987 * 988 * Network device names need to be valid file names to 989 * to allow sysfs to work. We also disallow any kind of 990 * whitespace. 991 */ 992 bool dev_valid_name(const char *name) 993 { 994 if (*name == '\0') 995 return false; 996 if (strlen(name) >= IFNAMSIZ) 997 return false; 998 if (!strcmp(name, ".") || !strcmp(name, "..")) 999 return false; 1000 1001 while (*name) { 1002 if (*name == '/' || *name == ':' || isspace(*name)) 1003 return false; 1004 name++; 1005 } 1006 return true; 1007 } 1008 EXPORT_SYMBOL(dev_valid_name); 1009 1010 /** 1011 * __dev_alloc_name - allocate a name for a device 1012 * @net: network namespace to allocate the device name in 1013 * @name: name format string 1014 * @buf: scratch buffer and result name string 1015 * 1016 * Passed a format string - eg "lt%d" it will try and find a suitable 1017 * id. It scans list of devices to build up a free map, then chooses 1018 * the first empty slot. The caller must hold the dev_base or rtnl lock 1019 * while allocating the name and adding the device in order to avoid 1020 * duplicates. 1021 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1022 * Returns the number of the unit assigned or a negative errno code. 1023 */ 1024 1025 static int __dev_alloc_name(struct net *net, const char *name, char *buf) 1026 { 1027 int i = 0; 1028 const char *p; 1029 const int max_netdevices = 8*PAGE_SIZE; 1030 unsigned long *inuse; 1031 struct net_device *d; 1032 1033 p = strnchr(name, IFNAMSIZ-1, '%'); 1034 if (p) { 1035 /* 1036 * Verify the string as this thing may have come from 1037 * the user. There must be either one "%d" and no other "%" 1038 * characters. 1039 */ 1040 if (p[1] != 'd' || strchr(p + 2, '%')) 1041 return -EINVAL; 1042 1043 /* Use one page as a bit array of possible slots */ 1044 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC); 1045 if (!inuse) 1046 return -ENOMEM; 1047 1048 for_each_netdev(net, d) { 1049 if (!sscanf(d->name, name, &i)) 1050 continue; 1051 if (i < 0 || i >= max_netdevices) 1052 continue; 1053 1054 /* avoid cases where sscanf is not exact inverse of printf */ 1055 snprintf(buf, IFNAMSIZ, name, i); 1056 if (!strncmp(buf, d->name, IFNAMSIZ)) 1057 set_bit(i, inuse); 1058 } 1059 1060 i = find_first_zero_bit(inuse, max_netdevices); 1061 free_page((unsigned long) inuse); 1062 } 1063 1064 if (buf != name) 1065 snprintf(buf, IFNAMSIZ, name, i); 1066 if (!__dev_get_by_name(net, buf)) 1067 return i; 1068 1069 /* It is possible to run out of possible slots 1070 * when the name is long and there isn't enough space left 1071 * for the digits, or if all bits are used. 1072 */ 1073 return -ENFILE; 1074 } 1075 1076 /** 1077 * dev_alloc_name - allocate a name for a device 1078 * @dev: device 1079 * @name: name format string 1080 * 1081 * Passed a format string - eg "lt%d" it will try and find a suitable 1082 * id. It scans list of devices to build up a free map, then chooses 1083 * the first empty slot. The caller must hold the dev_base or rtnl lock 1084 * while allocating the name and adding the device in order to avoid 1085 * duplicates. 1086 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1087 * Returns the number of the unit assigned or a negative errno code. 1088 */ 1089 1090 int dev_alloc_name(struct net_device *dev, const char *name) 1091 { 1092 char buf[IFNAMSIZ]; 1093 struct net *net; 1094 int ret; 1095 1096 BUG_ON(!dev_net(dev)); 1097 net = dev_net(dev); 1098 ret = __dev_alloc_name(net, name, buf); 1099 if (ret >= 0) 1100 strlcpy(dev->name, buf, IFNAMSIZ); 1101 return ret; 1102 } 1103 EXPORT_SYMBOL(dev_alloc_name); 1104 1105 static int dev_alloc_name_ns(struct net *net, 1106 struct net_device *dev, 1107 const char *name) 1108 { 1109 char buf[IFNAMSIZ]; 1110 int ret; 1111 1112 ret = __dev_alloc_name(net, name, buf); 1113 if (ret >= 0) 1114 strlcpy(dev->name, buf, IFNAMSIZ); 1115 return ret; 1116 } 1117 1118 static int dev_get_valid_name(struct net *net, 1119 struct net_device *dev, 1120 const char *name) 1121 { 1122 BUG_ON(!net); 1123 1124 if (!dev_valid_name(name)) 1125 return -EINVAL; 1126 1127 if (strchr(name, '%')) 1128 return dev_alloc_name_ns(net, dev, name); 1129 else if (__dev_get_by_name(net, name)) 1130 return -EEXIST; 1131 else if (dev->name != name) 1132 strlcpy(dev->name, name, IFNAMSIZ); 1133 1134 return 0; 1135 } 1136 1137 /** 1138 * dev_change_name - change name of a device 1139 * @dev: device 1140 * @newname: name (or format string) must be at least IFNAMSIZ 1141 * 1142 * Change name of a device, can pass format strings "eth%d". 1143 * for wildcarding. 1144 */ 1145 int dev_change_name(struct net_device *dev, const char *newname) 1146 { 1147 unsigned char old_assign_type; 1148 char oldname[IFNAMSIZ]; 1149 int err = 0; 1150 int ret; 1151 struct net *net; 1152 1153 ASSERT_RTNL(); 1154 BUG_ON(!dev_net(dev)); 1155 1156 net = dev_net(dev); 1157 if (dev->flags & IFF_UP) 1158 return -EBUSY; 1159 1160 write_seqcount_begin(&devnet_rename_seq); 1161 1162 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) { 1163 write_seqcount_end(&devnet_rename_seq); 1164 return 0; 1165 } 1166 1167 memcpy(oldname, dev->name, IFNAMSIZ); 1168 1169 err = dev_get_valid_name(net, dev, newname); 1170 if (err < 0) { 1171 write_seqcount_end(&devnet_rename_seq); 1172 return err; 1173 } 1174 1175 if (oldname[0] && !strchr(oldname, '%')) 1176 netdev_info(dev, "renamed from %s\n", oldname); 1177 1178 old_assign_type = dev->name_assign_type; 1179 dev->name_assign_type = NET_NAME_RENAMED; 1180 1181 rollback: 1182 ret = device_rename(&dev->dev, dev->name); 1183 if (ret) { 1184 memcpy(dev->name, oldname, IFNAMSIZ); 1185 dev->name_assign_type = old_assign_type; 1186 write_seqcount_end(&devnet_rename_seq); 1187 return ret; 1188 } 1189 1190 write_seqcount_end(&devnet_rename_seq); 1191 1192 netdev_adjacent_rename_links(dev, oldname); 1193 1194 write_lock_bh(&dev_base_lock); 1195 hlist_del_rcu(&dev->name_hlist); 1196 write_unlock_bh(&dev_base_lock); 1197 1198 synchronize_rcu(); 1199 1200 write_lock_bh(&dev_base_lock); 1201 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); 1202 write_unlock_bh(&dev_base_lock); 1203 1204 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); 1205 ret = notifier_to_errno(ret); 1206 1207 if (ret) { 1208 /* err >= 0 after dev_alloc_name() or stores the first errno */ 1209 if (err >= 0) { 1210 err = ret; 1211 write_seqcount_begin(&devnet_rename_seq); 1212 memcpy(dev->name, oldname, IFNAMSIZ); 1213 memcpy(oldname, newname, IFNAMSIZ); 1214 dev->name_assign_type = old_assign_type; 1215 old_assign_type = NET_NAME_RENAMED; 1216 goto rollback; 1217 } else { 1218 pr_err("%s: name change rollback failed: %d\n", 1219 dev->name, ret); 1220 } 1221 } 1222 1223 return err; 1224 } 1225 1226 /** 1227 * dev_set_alias - change ifalias of a device 1228 * @dev: device 1229 * @alias: name up to IFALIASZ 1230 * @len: limit of bytes to copy from info 1231 * 1232 * Set ifalias for a device, 1233 */ 1234 int dev_set_alias(struct net_device *dev, const char *alias, size_t len) 1235 { 1236 char *new_ifalias; 1237 1238 ASSERT_RTNL(); 1239 1240 if (len >= IFALIASZ) 1241 return -EINVAL; 1242 1243 if (!len) { 1244 kfree(dev->ifalias); 1245 dev->ifalias = NULL; 1246 return 0; 1247 } 1248 1249 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL); 1250 if (!new_ifalias) 1251 return -ENOMEM; 1252 dev->ifalias = new_ifalias; 1253 1254 strlcpy(dev->ifalias, alias, len+1); 1255 return len; 1256 } 1257 1258 1259 /** 1260 * netdev_features_change - device changes features 1261 * @dev: device to cause notification 1262 * 1263 * Called to indicate a device has changed features. 1264 */ 1265 void netdev_features_change(struct net_device *dev) 1266 { 1267 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); 1268 } 1269 EXPORT_SYMBOL(netdev_features_change); 1270 1271 /** 1272 * netdev_state_change - device changes state 1273 * @dev: device to cause notification 1274 * 1275 * Called to indicate a device has changed state. This function calls 1276 * the notifier chains for netdev_chain and sends a NEWLINK message 1277 * to the routing socket. 1278 */ 1279 void netdev_state_change(struct net_device *dev) 1280 { 1281 if (dev->flags & IFF_UP) { 1282 struct netdev_notifier_change_info change_info; 1283 1284 change_info.flags_changed = 0; 1285 call_netdevice_notifiers_info(NETDEV_CHANGE, dev, 1286 &change_info.info); 1287 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL); 1288 } 1289 } 1290 EXPORT_SYMBOL(netdev_state_change); 1291 1292 /** 1293 * netdev_notify_peers - notify network peers about existence of @dev 1294 * @dev: network device 1295 * 1296 * Generate traffic such that interested network peers are aware of 1297 * @dev, such as by generating a gratuitous ARP. This may be used when 1298 * a device wants to inform the rest of the network about some sort of 1299 * reconfiguration such as a failover event or virtual machine 1300 * migration. 1301 */ 1302 void netdev_notify_peers(struct net_device *dev) 1303 { 1304 rtnl_lock(); 1305 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev); 1306 rtnl_unlock(); 1307 } 1308 EXPORT_SYMBOL(netdev_notify_peers); 1309 1310 static int __dev_open(struct net_device *dev) 1311 { 1312 const struct net_device_ops *ops = dev->netdev_ops; 1313 int ret; 1314 1315 ASSERT_RTNL(); 1316 1317 if (!netif_device_present(dev)) 1318 return -ENODEV; 1319 1320 /* Block netpoll from trying to do any rx path servicing. 1321 * If we don't do this there is a chance ndo_poll_controller 1322 * or ndo_poll may be running while we open the device 1323 */ 1324 netpoll_poll_disable(dev); 1325 1326 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev); 1327 ret = notifier_to_errno(ret); 1328 if (ret) 1329 return ret; 1330 1331 set_bit(__LINK_STATE_START, &dev->state); 1332 1333 if (ops->ndo_validate_addr) 1334 ret = ops->ndo_validate_addr(dev); 1335 1336 if (!ret && ops->ndo_open) 1337 ret = ops->ndo_open(dev); 1338 1339 netpoll_poll_enable(dev); 1340 1341 if (ret) 1342 clear_bit(__LINK_STATE_START, &dev->state); 1343 else { 1344 dev->flags |= IFF_UP; 1345 dev_set_rx_mode(dev); 1346 dev_activate(dev); 1347 add_device_randomness(dev->dev_addr, dev->addr_len); 1348 } 1349 1350 return ret; 1351 } 1352 1353 /** 1354 * dev_open - prepare an interface for use. 1355 * @dev: device to open 1356 * 1357 * Takes a device from down to up state. The device's private open 1358 * function is invoked and then the multicast lists are loaded. Finally 1359 * the device is moved into the up state and a %NETDEV_UP message is 1360 * sent to the netdev notifier chain. 1361 * 1362 * Calling this function on an active interface is a nop. On a failure 1363 * a negative errno code is returned. 1364 */ 1365 int dev_open(struct net_device *dev) 1366 { 1367 int ret; 1368 1369 if (dev->flags & IFF_UP) 1370 return 0; 1371 1372 ret = __dev_open(dev); 1373 if (ret < 0) 1374 return ret; 1375 1376 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); 1377 call_netdevice_notifiers(NETDEV_UP, dev); 1378 1379 return ret; 1380 } 1381 EXPORT_SYMBOL(dev_open); 1382 1383 static int __dev_close_many(struct list_head *head) 1384 { 1385 struct net_device *dev; 1386 1387 ASSERT_RTNL(); 1388 might_sleep(); 1389 1390 list_for_each_entry(dev, head, close_list) { 1391 /* Temporarily disable netpoll until the interface is down */ 1392 netpoll_poll_disable(dev); 1393 1394 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); 1395 1396 clear_bit(__LINK_STATE_START, &dev->state); 1397 1398 /* Synchronize to scheduled poll. We cannot touch poll list, it 1399 * can be even on different cpu. So just clear netif_running(). 1400 * 1401 * dev->stop() will invoke napi_disable() on all of it's 1402 * napi_struct instances on this device. 1403 */ 1404 smp_mb__after_atomic(); /* Commit netif_running(). */ 1405 } 1406 1407 dev_deactivate_many(head); 1408 1409 list_for_each_entry(dev, head, close_list) { 1410 const struct net_device_ops *ops = dev->netdev_ops; 1411 1412 /* 1413 * Call the device specific close. This cannot fail. 1414 * Only if device is UP 1415 * 1416 * We allow it to be called even after a DETACH hot-plug 1417 * event. 1418 */ 1419 if (ops->ndo_stop) 1420 ops->ndo_stop(dev); 1421 1422 dev->flags &= ~IFF_UP; 1423 netpoll_poll_enable(dev); 1424 } 1425 1426 return 0; 1427 } 1428 1429 static int __dev_close(struct net_device *dev) 1430 { 1431 int retval; 1432 LIST_HEAD(single); 1433 1434 list_add(&dev->close_list, &single); 1435 retval = __dev_close_many(&single); 1436 list_del(&single); 1437 1438 return retval; 1439 } 1440 1441 int dev_close_many(struct list_head *head, bool unlink) 1442 { 1443 struct net_device *dev, *tmp; 1444 1445 /* Remove the devices that don't need to be closed */ 1446 list_for_each_entry_safe(dev, tmp, head, close_list) 1447 if (!(dev->flags & IFF_UP)) 1448 list_del_init(&dev->close_list); 1449 1450 __dev_close_many(head); 1451 1452 list_for_each_entry_safe(dev, tmp, head, close_list) { 1453 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); 1454 call_netdevice_notifiers(NETDEV_DOWN, dev); 1455 if (unlink) 1456 list_del_init(&dev->close_list); 1457 } 1458 1459 return 0; 1460 } 1461 EXPORT_SYMBOL(dev_close_many); 1462 1463 /** 1464 * dev_close - shutdown an interface. 1465 * @dev: device to shutdown 1466 * 1467 * This function moves an active device into down state. A 1468 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device 1469 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier 1470 * chain. 1471 */ 1472 int dev_close(struct net_device *dev) 1473 { 1474 if (dev->flags & IFF_UP) { 1475 LIST_HEAD(single); 1476 1477 list_add(&dev->close_list, &single); 1478 dev_close_many(&single, true); 1479 list_del(&single); 1480 } 1481 return 0; 1482 } 1483 EXPORT_SYMBOL(dev_close); 1484 1485 1486 /** 1487 * dev_disable_lro - disable Large Receive Offload on a device 1488 * @dev: device 1489 * 1490 * Disable Large Receive Offload (LRO) on a net device. Must be 1491 * called under RTNL. This is needed if received packets may be 1492 * forwarded to another interface. 1493 */ 1494 void dev_disable_lro(struct net_device *dev) 1495 { 1496 struct net_device *lower_dev; 1497 struct list_head *iter; 1498 1499 dev->wanted_features &= ~NETIF_F_LRO; 1500 netdev_update_features(dev); 1501 1502 if (unlikely(dev->features & NETIF_F_LRO)) 1503 netdev_WARN(dev, "failed to disable LRO!\n"); 1504 1505 netdev_for_each_lower_dev(dev, lower_dev, iter) 1506 dev_disable_lro(lower_dev); 1507 } 1508 EXPORT_SYMBOL(dev_disable_lro); 1509 1510 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val, 1511 struct net_device *dev) 1512 { 1513 struct netdev_notifier_info info; 1514 1515 netdev_notifier_info_init(&info, dev); 1516 return nb->notifier_call(nb, val, &info); 1517 } 1518 1519 static int dev_boot_phase = 1; 1520 1521 /** 1522 * register_netdevice_notifier - register a network notifier block 1523 * @nb: notifier 1524 * 1525 * Register a notifier to be called when network device events occur. 1526 * The notifier passed is linked into the kernel structures and must 1527 * not be reused until it has been unregistered. A negative errno code 1528 * is returned on a failure. 1529 * 1530 * When registered all registration and up events are replayed 1531 * to the new notifier to allow device to have a race free 1532 * view of the network device list. 1533 */ 1534 1535 int register_netdevice_notifier(struct notifier_block *nb) 1536 { 1537 struct net_device *dev; 1538 struct net_device *last; 1539 struct net *net; 1540 int err; 1541 1542 rtnl_lock(); 1543 err = raw_notifier_chain_register(&netdev_chain, nb); 1544 if (err) 1545 goto unlock; 1546 if (dev_boot_phase) 1547 goto unlock; 1548 for_each_net(net) { 1549 for_each_netdev(net, dev) { 1550 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev); 1551 err = notifier_to_errno(err); 1552 if (err) 1553 goto rollback; 1554 1555 if (!(dev->flags & IFF_UP)) 1556 continue; 1557 1558 call_netdevice_notifier(nb, NETDEV_UP, dev); 1559 } 1560 } 1561 1562 unlock: 1563 rtnl_unlock(); 1564 return err; 1565 1566 rollback: 1567 last = dev; 1568 for_each_net(net) { 1569 for_each_netdev(net, dev) { 1570 if (dev == last) 1571 goto outroll; 1572 1573 if (dev->flags & IFF_UP) { 1574 call_netdevice_notifier(nb, NETDEV_GOING_DOWN, 1575 dev); 1576 call_netdevice_notifier(nb, NETDEV_DOWN, dev); 1577 } 1578 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); 1579 } 1580 } 1581 1582 outroll: 1583 raw_notifier_chain_unregister(&netdev_chain, nb); 1584 goto unlock; 1585 } 1586 EXPORT_SYMBOL(register_netdevice_notifier); 1587 1588 /** 1589 * unregister_netdevice_notifier - unregister a network notifier block 1590 * @nb: notifier 1591 * 1592 * Unregister a notifier previously registered by 1593 * register_netdevice_notifier(). The notifier is unlinked into the 1594 * kernel structures and may then be reused. A negative errno code 1595 * is returned on a failure. 1596 * 1597 * After unregistering unregister and down device events are synthesized 1598 * for all devices on the device list to the removed notifier to remove 1599 * the need for special case cleanup code. 1600 */ 1601 1602 int unregister_netdevice_notifier(struct notifier_block *nb) 1603 { 1604 struct net_device *dev; 1605 struct net *net; 1606 int err; 1607 1608 rtnl_lock(); 1609 err = raw_notifier_chain_unregister(&netdev_chain, nb); 1610 if (err) 1611 goto unlock; 1612 1613 for_each_net(net) { 1614 for_each_netdev(net, dev) { 1615 if (dev->flags & IFF_UP) { 1616 call_netdevice_notifier(nb, NETDEV_GOING_DOWN, 1617 dev); 1618 call_netdevice_notifier(nb, NETDEV_DOWN, dev); 1619 } 1620 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); 1621 } 1622 } 1623 unlock: 1624 rtnl_unlock(); 1625 return err; 1626 } 1627 EXPORT_SYMBOL(unregister_netdevice_notifier); 1628 1629 /** 1630 * call_netdevice_notifiers_info - call all network notifier blocks 1631 * @val: value passed unmodified to notifier function 1632 * @dev: net_device pointer passed unmodified to notifier function 1633 * @info: notifier information data 1634 * 1635 * Call all network notifier blocks. Parameters and return value 1636 * are as for raw_notifier_call_chain(). 1637 */ 1638 1639 static int call_netdevice_notifiers_info(unsigned long val, 1640 struct net_device *dev, 1641 struct netdev_notifier_info *info) 1642 { 1643 ASSERT_RTNL(); 1644 netdev_notifier_info_init(info, dev); 1645 return raw_notifier_call_chain(&netdev_chain, val, info); 1646 } 1647 1648 /** 1649 * call_netdevice_notifiers - call all network notifier blocks 1650 * @val: value passed unmodified to notifier function 1651 * @dev: net_device pointer passed unmodified to notifier function 1652 * 1653 * Call all network notifier blocks. Parameters and return value 1654 * are as for raw_notifier_call_chain(). 1655 */ 1656 1657 int call_netdevice_notifiers(unsigned long val, struct net_device *dev) 1658 { 1659 struct netdev_notifier_info info; 1660 1661 return call_netdevice_notifiers_info(val, dev, &info); 1662 } 1663 EXPORT_SYMBOL(call_netdevice_notifiers); 1664 1665 #ifdef CONFIG_NET_INGRESS 1666 static struct static_key ingress_needed __read_mostly; 1667 1668 void net_inc_ingress_queue(void) 1669 { 1670 static_key_slow_inc(&ingress_needed); 1671 } 1672 EXPORT_SYMBOL_GPL(net_inc_ingress_queue); 1673 1674 void net_dec_ingress_queue(void) 1675 { 1676 static_key_slow_dec(&ingress_needed); 1677 } 1678 EXPORT_SYMBOL_GPL(net_dec_ingress_queue); 1679 #endif 1680 1681 #ifdef CONFIG_NET_EGRESS 1682 static struct static_key egress_needed __read_mostly; 1683 1684 void net_inc_egress_queue(void) 1685 { 1686 static_key_slow_inc(&egress_needed); 1687 } 1688 EXPORT_SYMBOL_GPL(net_inc_egress_queue); 1689 1690 void net_dec_egress_queue(void) 1691 { 1692 static_key_slow_dec(&egress_needed); 1693 } 1694 EXPORT_SYMBOL_GPL(net_dec_egress_queue); 1695 #endif 1696 1697 static struct static_key netstamp_needed __read_mostly; 1698 #ifdef HAVE_JUMP_LABEL 1699 /* We are not allowed to call static_key_slow_dec() from irq context 1700 * If net_disable_timestamp() is called from irq context, defer the 1701 * static_key_slow_dec() calls. 1702 */ 1703 static atomic_t netstamp_needed_deferred; 1704 #endif 1705 1706 void net_enable_timestamp(void) 1707 { 1708 #ifdef HAVE_JUMP_LABEL 1709 int deferred = atomic_xchg(&netstamp_needed_deferred, 0); 1710 1711 if (deferred) { 1712 while (--deferred) 1713 static_key_slow_dec(&netstamp_needed); 1714 return; 1715 } 1716 #endif 1717 static_key_slow_inc(&netstamp_needed); 1718 } 1719 EXPORT_SYMBOL(net_enable_timestamp); 1720 1721 void net_disable_timestamp(void) 1722 { 1723 #ifdef HAVE_JUMP_LABEL 1724 if (in_interrupt()) { 1725 atomic_inc(&netstamp_needed_deferred); 1726 return; 1727 } 1728 #endif 1729 static_key_slow_dec(&netstamp_needed); 1730 } 1731 EXPORT_SYMBOL(net_disable_timestamp); 1732 1733 static inline void net_timestamp_set(struct sk_buff *skb) 1734 { 1735 skb->tstamp.tv64 = 0; 1736 if (static_key_false(&netstamp_needed)) 1737 __net_timestamp(skb); 1738 } 1739 1740 #define net_timestamp_check(COND, SKB) \ 1741 if (static_key_false(&netstamp_needed)) { \ 1742 if ((COND) && !(SKB)->tstamp.tv64) \ 1743 __net_timestamp(SKB); \ 1744 } \ 1745 1746 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb) 1747 { 1748 unsigned int len; 1749 1750 if (!(dev->flags & IFF_UP)) 1751 return false; 1752 1753 len = dev->mtu + dev->hard_header_len + VLAN_HLEN; 1754 if (skb->len <= len) 1755 return true; 1756 1757 /* if TSO is enabled, we don't care about the length as the packet 1758 * could be forwarded without being segmented before 1759 */ 1760 if (skb_is_gso(skb)) 1761 return true; 1762 1763 return false; 1764 } 1765 EXPORT_SYMBOL_GPL(is_skb_forwardable); 1766 1767 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 1768 { 1769 if (skb_orphan_frags(skb, GFP_ATOMIC) || 1770 unlikely(!is_skb_forwardable(dev, skb))) { 1771 atomic_long_inc(&dev->rx_dropped); 1772 kfree_skb(skb); 1773 return NET_RX_DROP; 1774 } 1775 1776 skb_scrub_packet(skb, true); 1777 skb->priority = 0; 1778 skb->protocol = eth_type_trans(skb, dev); 1779 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); 1780 1781 return 0; 1782 } 1783 EXPORT_SYMBOL_GPL(__dev_forward_skb); 1784 1785 /** 1786 * dev_forward_skb - loopback an skb to another netif 1787 * 1788 * @dev: destination network device 1789 * @skb: buffer to forward 1790 * 1791 * return values: 1792 * NET_RX_SUCCESS (no congestion) 1793 * NET_RX_DROP (packet was dropped, but freed) 1794 * 1795 * dev_forward_skb can be used for injecting an skb from the 1796 * start_xmit function of one device into the receive queue 1797 * of another device. 1798 * 1799 * The receiving device may be in another namespace, so 1800 * we have to clear all information in the skb that could 1801 * impact namespace isolation. 1802 */ 1803 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 1804 { 1805 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb); 1806 } 1807 EXPORT_SYMBOL_GPL(dev_forward_skb); 1808 1809 static inline int deliver_skb(struct sk_buff *skb, 1810 struct packet_type *pt_prev, 1811 struct net_device *orig_dev) 1812 { 1813 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) 1814 return -ENOMEM; 1815 atomic_inc(&skb->users); 1816 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 1817 } 1818 1819 static inline void deliver_ptype_list_skb(struct sk_buff *skb, 1820 struct packet_type **pt, 1821 struct net_device *orig_dev, 1822 __be16 type, 1823 struct list_head *ptype_list) 1824 { 1825 struct packet_type *ptype, *pt_prev = *pt; 1826 1827 list_for_each_entry_rcu(ptype, ptype_list, list) { 1828 if (ptype->type != type) 1829 continue; 1830 if (pt_prev) 1831 deliver_skb(skb, pt_prev, orig_dev); 1832 pt_prev = ptype; 1833 } 1834 *pt = pt_prev; 1835 } 1836 1837 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) 1838 { 1839 if (!ptype->af_packet_priv || !skb->sk) 1840 return false; 1841 1842 if (ptype->id_match) 1843 return ptype->id_match(ptype, skb->sk); 1844 else if ((struct sock *)ptype->af_packet_priv == skb->sk) 1845 return true; 1846 1847 return false; 1848 } 1849 1850 /* 1851 * Support routine. Sends outgoing frames to any network 1852 * taps currently in use. 1853 */ 1854 1855 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) 1856 { 1857 struct packet_type *ptype; 1858 struct sk_buff *skb2 = NULL; 1859 struct packet_type *pt_prev = NULL; 1860 struct list_head *ptype_list = &ptype_all; 1861 1862 rcu_read_lock(); 1863 again: 1864 list_for_each_entry_rcu(ptype, ptype_list, list) { 1865 /* Never send packets back to the socket 1866 * they originated from - MvS (miquels@drinkel.ow.org) 1867 */ 1868 if (skb_loop_sk(ptype, skb)) 1869 continue; 1870 1871 if (pt_prev) { 1872 deliver_skb(skb2, pt_prev, skb->dev); 1873 pt_prev = ptype; 1874 continue; 1875 } 1876 1877 /* need to clone skb, done only once */ 1878 skb2 = skb_clone(skb, GFP_ATOMIC); 1879 if (!skb2) 1880 goto out_unlock; 1881 1882 net_timestamp_set(skb2); 1883 1884 /* skb->nh should be correctly 1885 * set by sender, so that the second statement is 1886 * just protection against buggy protocols. 1887 */ 1888 skb_reset_mac_header(skb2); 1889 1890 if (skb_network_header(skb2) < skb2->data || 1891 skb_network_header(skb2) > skb_tail_pointer(skb2)) { 1892 net_crit_ratelimited("protocol %04x is buggy, dev %s\n", 1893 ntohs(skb2->protocol), 1894 dev->name); 1895 skb_reset_network_header(skb2); 1896 } 1897 1898 skb2->transport_header = skb2->network_header; 1899 skb2->pkt_type = PACKET_OUTGOING; 1900 pt_prev = ptype; 1901 } 1902 1903 if (ptype_list == &ptype_all) { 1904 ptype_list = &dev->ptype_all; 1905 goto again; 1906 } 1907 out_unlock: 1908 if (pt_prev) 1909 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); 1910 rcu_read_unlock(); 1911 } 1912 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit); 1913 1914 /** 1915 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change 1916 * @dev: Network device 1917 * @txq: number of queues available 1918 * 1919 * If real_num_tx_queues is changed the tc mappings may no longer be 1920 * valid. To resolve this verify the tc mapping remains valid and if 1921 * not NULL the mapping. With no priorities mapping to this 1922 * offset/count pair it will no longer be used. In the worst case TC0 1923 * is invalid nothing can be done so disable priority mappings. If is 1924 * expected that drivers will fix this mapping if they can before 1925 * calling netif_set_real_num_tx_queues. 1926 */ 1927 static void netif_setup_tc(struct net_device *dev, unsigned int txq) 1928 { 1929 int i; 1930 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 1931 1932 /* If TC0 is invalidated disable TC mapping */ 1933 if (tc->offset + tc->count > txq) { 1934 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n"); 1935 dev->num_tc = 0; 1936 return; 1937 } 1938 1939 /* Invalidated prio to tc mappings set to TC0 */ 1940 for (i = 1; i < TC_BITMASK + 1; i++) { 1941 int q = netdev_get_prio_tc_map(dev, i); 1942 1943 tc = &dev->tc_to_txq[q]; 1944 if (tc->offset + tc->count > txq) { 1945 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n", 1946 i, q); 1947 netdev_set_prio_tc_map(dev, i, 0); 1948 } 1949 } 1950 } 1951 1952 #ifdef CONFIG_XPS 1953 static DEFINE_MUTEX(xps_map_mutex); 1954 #define xmap_dereference(P) \ 1955 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex)) 1956 1957 static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps, 1958 int cpu, u16 index) 1959 { 1960 struct xps_map *map = NULL; 1961 int pos; 1962 1963 if (dev_maps) 1964 map = xmap_dereference(dev_maps->cpu_map[cpu]); 1965 1966 for (pos = 0; map && pos < map->len; pos++) { 1967 if (map->queues[pos] == index) { 1968 if (map->len > 1) { 1969 map->queues[pos] = map->queues[--map->len]; 1970 } else { 1971 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL); 1972 kfree_rcu(map, rcu); 1973 map = NULL; 1974 } 1975 break; 1976 } 1977 } 1978 1979 return map; 1980 } 1981 1982 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index) 1983 { 1984 struct xps_dev_maps *dev_maps; 1985 int cpu, i; 1986 bool active = false; 1987 1988 mutex_lock(&xps_map_mutex); 1989 dev_maps = xmap_dereference(dev->xps_maps); 1990 1991 if (!dev_maps) 1992 goto out_no_maps; 1993 1994 for_each_possible_cpu(cpu) { 1995 for (i = index; i < dev->num_tx_queues; i++) { 1996 if (!remove_xps_queue(dev_maps, cpu, i)) 1997 break; 1998 } 1999 if (i == dev->num_tx_queues) 2000 active = true; 2001 } 2002 2003 if (!active) { 2004 RCU_INIT_POINTER(dev->xps_maps, NULL); 2005 kfree_rcu(dev_maps, rcu); 2006 } 2007 2008 for (i = index; i < dev->num_tx_queues; i++) 2009 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i), 2010 NUMA_NO_NODE); 2011 2012 out_no_maps: 2013 mutex_unlock(&xps_map_mutex); 2014 } 2015 2016 static struct xps_map *expand_xps_map(struct xps_map *map, 2017 int cpu, u16 index) 2018 { 2019 struct xps_map *new_map; 2020 int alloc_len = XPS_MIN_MAP_ALLOC; 2021 int i, pos; 2022 2023 for (pos = 0; map && pos < map->len; pos++) { 2024 if (map->queues[pos] != index) 2025 continue; 2026 return map; 2027 } 2028 2029 /* Need to add queue to this CPU's existing map */ 2030 if (map) { 2031 if (pos < map->alloc_len) 2032 return map; 2033 2034 alloc_len = map->alloc_len * 2; 2035 } 2036 2037 /* Need to allocate new map to store queue on this CPU's map */ 2038 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL, 2039 cpu_to_node(cpu)); 2040 if (!new_map) 2041 return NULL; 2042 2043 for (i = 0; i < pos; i++) 2044 new_map->queues[i] = map->queues[i]; 2045 new_map->alloc_len = alloc_len; 2046 new_map->len = pos; 2047 2048 return new_map; 2049 } 2050 2051 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask, 2052 u16 index) 2053 { 2054 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL; 2055 struct xps_map *map, *new_map; 2056 int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES); 2057 int cpu, numa_node_id = -2; 2058 bool active = false; 2059 2060 mutex_lock(&xps_map_mutex); 2061 2062 dev_maps = xmap_dereference(dev->xps_maps); 2063 2064 /* allocate memory for queue storage */ 2065 for_each_online_cpu(cpu) { 2066 if (!cpumask_test_cpu(cpu, mask)) 2067 continue; 2068 2069 if (!new_dev_maps) 2070 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL); 2071 if (!new_dev_maps) { 2072 mutex_unlock(&xps_map_mutex); 2073 return -ENOMEM; 2074 } 2075 2076 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) : 2077 NULL; 2078 2079 map = expand_xps_map(map, cpu, index); 2080 if (!map) 2081 goto error; 2082 2083 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map); 2084 } 2085 2086 if (!new_dev_maps) 2087 goto out_no_new_maps; 2088 2089 for_each_possible_cpu(cpu) { 2090 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) { 2091 /* add queue to CPU maps */ 2092 int pos = 0; 2093 2094 map = xmap_dereference(new_dev_maps->cpu_map[cpu]); 2095 while ((pos < map->len) && (map->queues[pos] != index)) 2096 pos++; 2097 2098 if (pos == map->len) 2099 map->queues[map->len++] = index; 2100 #ifdef CONFIG_NUMA 2101 if (numa_node_id == -2) 2102 numa_node_id = cpu_to_node(cpu); 2103 else if (numa_node_id != cpu_to_node(cpu)) 2104 numa_node_id = -1; 2105 #endif 2106 } else if (dev_maps) { 2107 /* fill in the new device map from the old device map */ 2108 map = xmap_dereference(dev_maps->cpu_map[cpu]); 2109 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map); 2110 } 2111 2112 } 2113 2114 rcu_assign_pointer(dev->xps_maps, new_dev_maps); 2115 2116 /* Cleanup old maps */ 2117 if (dev_maps) { 2118 for_each_possible_cpu(cpu) { 2119 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]); 2120 map = xmap_dereference(dev_maps->cpu_map[cpu]); 2121 if (map && map != new_map) 2122 kfree_rcu(map, rcu); 2123 } 2124 2125 kfree_rcu(dev_maps, rcu); 2126 } 2127 2128 dev_maps = new_dev_maps; 2129 active = true; 2130 2131 out_no_new_maps: 2132 /* update Tx queue numa node */ 2133 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index), 2134 (numa_node_id >= 0) ? numa_node_id : 2135 NUMA_NO_NODE); 2136 2137 if (!dev_maps) 2138 goto out_no_maps; 2139 2140 /* removes queue from unused CPUs */ 2141 for_each_possible_cpu(cpu) { 2142 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) 2143 continue; 2144 2145 if (remove_xps_queue(dev_maps, cpu, index)) 2146 active = true; 2147 } 2148 2149 /* free map if not active */ 2150 if (!active) { 2151 RCU_INIT_POINTER(dev->xps_maps, NULL); 2152 kfree_rcu(dev_maps, rcu); 2153 } 2154 2155 out_no_maps: 2156 mutex_unlock(&xps_map_mutex); 2157 2158 return 0; 2159 error: 2160 /* remove any maps that we added */ 2161 for_each_possible_cpu(cpu) { 2162 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]); 2163 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) : 2164 NULL; 2165 if (new_map && new_map != map) 2166 kfree(new_map); 2167 } 2168 2169 mutex_unlock(&xps_map_mutex); 2170 2171 kfree(new_dev_maps); 2172 return -ENOMEM; 2173 } 2174 EXPORT_SYMBOL(netif_set_xps_queue); 2175 2176 #endif 2177 /* 2178 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues 2179 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed. 2180 */ 2181 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) 2182 { 2183 int rc; 2184 2185 if (txq < 1 || txq > dev->num_tx_queues) 2186 return -EINVAL; 2187 2188 if (dev->reg_state == NETREG_REGISTERED || 2189 dev->reg_state == NETREG_UNREGISTERING) { 2190 ASSERT_RTNL(); 2191 2192 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, 2193 txq); 2194 if (rc) 2195 return rc; 2196 2197 if (dev->num_tc) 2198 netif_setup_tc(dev, txq); 2199 2200 if (txq < dev->real_num_tx_queues) { 2201 qdisc_reset_all_tx_gt(dev, txq); 2202 #ifdef CONFIG_XPS 2203 netif_reset_xps_queues_gt(dev, txq); 2204 #endif 2205 } 2206 } 2207 2208 dev->real_num_tx_queues = txq; 2209 return 0; 2210 } 2211 EXPORT_SYMBOL(netif_set_real_num_tx_queues); 2212 2213 #ifdef CONFIG_SYSFS 2214 /** 2215 * netif_set_real_num_rx_queues - set actual number of RX queues used 2216 * @dev: Network device 2217 * @rxq: Actual number of RX queues 2218 * 2219 * This must be called either with the rtnl_lock held or before 2220 * registration of the net device. Returns 0 on success, or a 2221 * negative error code. If called before registration, it always 2222 * succeeds. 2223 */ 2224 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) 2225 { 2226 int rc; 2227 2228 if (rxq < 1 || rxq > dev->num_rx_queues) 2229 return -EINVAL; 2230 2231 if (dev->reg_state == NETREG_REGISTERED) { 2232 ASSERT_RTNL(); 2233 2234 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, 2235 rxq); 2236 if (rc) 2237 return rc; 2238 } 2239 2240 dev->real_num_rx_queues = rxq; 2241 return 0; 2242 } 2243 EXPORT_SYMBOL(netif_set_real_num_rx_queues); 2244 #endif 2245 2246 /** 2247 * netif_get_num_default_rss_queues - default number of RSS queues 2248 * 2249 * This routine should set an upper limit on the number of RSS queues 2250 * used by default by multiqueue devices. 2251 */ 2252 int netif_get_num_default_rss_queues(void) 2253 { 2254 return is_kdump_kernel() ? 2255 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus()); 2256 } 2257 EXPORT_SYMBOL(netif_get_num_default_rss_queues); 2258 2259 static void __netif_reschedule(struct Qdisc *q) 2260 { 2261 struct softnet_data *sd; 2262 unsigned long flags; 2263 2264 local_irq_save(flags); 2265 sd = this_cpu_ptr(&softnet_data); 2266 q->next_sched = NULL; 2267 *sd->output_queue_tailp = q; 2268 sd->output_queue_tailp = &q->next_sched; 2269 raise_softirq_irqoff(NET_TX_SOFTIRQ); 2270 local_irq_restore(flags); 2271 } 2272 2273 void __netif_schedule(struct Qdisc *q) 2274 { 2275 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) 2276 __netif_reschedule(q); 2277 } 2278 EXPORT_SYMBOL(__netif_schedule); 2279 2280 struct dev_kfree_skb_cb { 2281 enum skb_free_reason reason; 2282 }; 2283 2284 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb) 2285 { 2286 return (struct dev_kfree_skb_cb *)skb->cb; 2287 } 2288 2289 void netif_schedule_queue(struct netdev_queue *txq) 2290 { 2291 rcu_read_lock(); 2292 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) { 2293 struct Qdisc *q = rcu_dereference(txq->qdisc); 2294 2295 __netif_schedule(q); 2296 } 2297 rcu_read_unlock(); 2298 } 2299 EXPORT_SYMBOL(netif_schedule_queue); 2300 2301 /** 2302 * netif_wake_subqueue - allow sending packets on subqueue 2303 * @dev: network device 2304 * @queue_index: sub queue index 2305 * 2306 * Resume individual transmit queue of a device with multiple transmit queues. 2307 */ 2308 void netif_wake_subqueue(struct net_device *dev, u16 queue_index) 2309 { 2310 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index); 2311 2312 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) { 2313 struct Qdisc *q; 2314 2315 rcu_read_lock(); 2316 q = rcu_dereference(txq->qdisc); 2317 __netif_schedule(q); 2318 rcu_read_unlock(); 2319 } 2320 } 2321 EXPORT_SYMBOL(netif_wake_subqueue); 2322 2323 void netif_tx_wake_queue(struct netdev_queue *dev_queue) 2324 { 2325 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { 2326 struct Qdisc *q; 2327 2328 rcu_read_lock(); 2329 q = rcu_dereference(dev_queue->qdisc); 2330 __netif_schedule(q); 2331 rcu_read_unlock(); 2332 } 2333 } 2334 EXPORT_SYMBOL(netif_tx_wake_queue); 2335 2336 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason) 2337 { 2338 unsigned long flags; 2339 2340 if (likely(atomic_read(&skb->users) == 1)) { 2341 smp_rmb(); 2342 atomic_set(&skb->users, 0); 2343 } else if (likely(!atomic_dec_and_test(&skb->users))) { 2344 return; 2345 } 2346 get_kfree_skb_cb(skb)->reason = reason; 2347 local_irq_save(flags); 2348 skb->next = __this_cpu_read(softnet_data.completion_queue); 2349 __this_cpu_write(softnet_data.completion_queue, skb); 2350 raise_softirq_irqoff(NET_TX_SOFTIRQ); 2351 local_irq_restore(flags); 2352 } 2353 EXPORT_SYMBOL(__dev_kfree_skb_irq); 2354 2355 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason) 2356 { 2357 if (in_irq() || irqs_disabled()) 2358 __dev_kfree_skb_irq(skb, reason); 2359 else 2360 dev_kfree_skb(skb); 2361 } 2362 EXPORT_SYMBOL(__dev_kfree_skb_any); 2363 2364 2365 /** 2366 * netif_device_detach - mark device as removed 2367 * @dev: network device 2368 * 2369 * Mark device as removed from system and therefore no longer available. 2370 */ 2371 void netif_device_detach(struct net_device *dev) 2372 { 2373 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && 2374 netif_running(dev)) { 2375 netif_tx_stop_all_queues(dev); 2376 } 2377 } 2378 EXPORT_SYMBOL(netif_device_detach); 2379 2380 /** 2381 * netif_device_attach - mark device as attached 2382 * @dev: network device 2383 * 2384 * Mark device as attached from system and restart if needed. 2385 */ 2386 void netif_device_attach(struct net_device *dev) 2387 { 2388 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && 2389 netif_running(dev)) { 2390 netif_tx_wake_all_queues(dev); 2391 __netdev_watchdog_up(dev); 2392 } 2393 } 2394 EXPORT_SYMBOL(netif_device_attach); 2395 2396 /* 2397 * Returns a Tx hash based on the given packet descriptor a Tx queues' number 2398 * to be used as a distribution range. 2399 */ 2400 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb, 2401 unsigned int num_tx_queues) 2402 { 2403 u32 hash; 2404 u16 qoffset = 0; 2405 u16 qcount = num_tx_queues; 2406 2407 if (skb_rx_queue_recorded(skb)) { 2408 hash = skb_get_rx_queue(skb); 2409 while (unlikely(hash >= num_tx_queues)) 2410 hash -= num_tx_queues; 2411 return hash; 2412 } 2413 2414 if (dev->num_tc) { 2415 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); 2416 qoffset = dev->tc_to_txq[tc].offset; 2417 qcount = dev->tc_to_txq[tc].count; 2418 } 2419 2420 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset; 2421 } 2422 EXPORT_SYMBOL(__skb_tx_hash); 2423 2424 static void skb_warn_bad_offload(const struct sk_buff *skb) 2425 { 2426 static const netdev_features_t null_features; 2427 struct net_device *dev = skb->dev; 2428 const char *name = ""; 2429 2430 if (!net_ratelimit()) 2431 return; 2432 2433 if (dev) { 2434 if (dev->dev.parent) 2435 name = dev_driver_string(dev->dev.parent); 2436 else 2437 name = netdev_name(dev); 2438 } 2439 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d " 2440 "gso_type=%d ip_summed=%d\n", 2441 name, dev ? &dev->features : &null_features, 2442 skb->sk ? &skb->sk->sk_route_caps : &null_features, 2443 skb->len, skb->data_len, skb_shinfo(skb)->gso_size, 2444 skb_shinfo(skb)->gso_type, skb->ip_summed); 2445 } 2446 2447 /* 2448 * Invalidate hardware checksum when packet is to be mangled, and 2449 * complete checksum manually on outgoing path. 2450 */ 2451 int skb_checksum_help(struct sk_buff *skb) 2452 { 2453 __wsum csum; 2454 int ret = 0, offset; 2455 2456 if (skb->ip_summed == CHECKSUM_COMPLETE) 2457 goto out_set_summed; 2458 2459 if (unlikely(skb_shinfo(skb)->gso_size)) { 2460 skb_warn_bad_offload(skb); 2461 return -EINVAL; 2462 } 2463 2464 /* Before computing a checksum, we should make sure no frag could 2465 * be modified by an external entity : checksum could be wrong. 2466 */ 2467 if (skb_has_shared_frag(skb)) { 2468 ret = __skb_linearize(skb); 2469 if (ret) 2470 goto out; 2471 } 2472 2473 offset = skb_checksum_start_offset(skb); 2474 BUG_ON(offset >= skb_headlen(skb)); 2475 csum = skb_checksum(skb, offset, skb->len - offset, 0); 2476 2477 offset += skb->csum_offset; 2478 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb)); 2479 2480 if (skb_cloned(skb) && 2481 !skb_clone_writable(skb, offset + sizeof(__sum16))) { 2482 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2483 if (ret) 2484 goto out; 2485 } 2486 2487 *(__sum16 *)(skb->data + offset) = csum_fold(csum); 2488 out_set_summed: 2489 skb->ip_summed = CHECKSUM_NONE; 2490 out: 2491 return ret; 2492 } 2493 EXPORT_SYMBOL(skb_checksum_help); 2494 2495 /* skb_csum_offload_check - Driver helper function to determine if a device 2496 * with limited checksum offload capabilities is able to offload the checksum 2497 * for a given packet. 2498 * 2499 * Arguments: 2500 * skb - sk_buff for the packet in question 2501 * spec - contains the description of what device can offload 2502 * csum_encapped - returns true if the checksum being offloaded is 2503 * encpasulated. That is it is checksum for the transport header 2504 * in the inner headers. 2505 * checksum_help - when set indicates that helper function should 2506 * call skb_checksum_help if offload checks fail 2507 * 2508 * Returns: 2509 * true: Packet has passed the checksum checks and should be offloadable to 2510 * the device (a driver may still need to check for additional 2511 * restrictions of its device) 2512 * false: Checksum is not offloadable. If checksum_help was set then 2513 * skb_checksum_help was called to resolve checksum for non-GSO 2514 * packets and when IP protocol is not SCTP 2515 */ 2516 bool __skb_csum_offload_chk(struct sk_buff *skb, 2517 const struct skb_csum_offl_spec *spec, 2518 bool *csum_encapped, 2519 bool csum_help) 2520 { 2521 struct iphdr *iph; 2522 struct ipv6hdr *ipv6; 2523 void *nhdr; 2524 int protocol; 2525 u8 ip_proto; 2526 2527 if (skb->protocol == htons(ETH_P_8021Q) || 2528 skb->protocol == htons(ETH_P_8021AD)) { 2529 if (!spec->vlan_okay) 2530 goto need_help; 2531 } 2532 2533 /* We check whether the checksum refers to a transport layer checksum in 2534 * the outermost header or an encapsulated transport layer checksum that 2535 * corresponds to the inner headers of the skb. If the checksum is for 2536 * something else in the packet we need help. 2537 */ 2538 if (skb_checksum_start_offset(skb) == skb_transport_offset(skb)) { 2539 /* Non-encapsulated checksum */ 2540 protocol = eproto_to_ipproto(vlan_get_protocol(skb)); 2541 nhdr = skb_network_header(skb); 2542 *csum_encapped = false; 2543 if (spec->no_not_encapped) 2544 goto need_help; 2545 } else if (skb->encapsulation && spec->encap_okay && 2546 skb_checksum_start_offset(skb) == 2547 skb_inner_transport_offset(skb)) { 2548 /* Encapsulated checksum */ 2549 *csum_encapped = true; 2550 switch (skb->inner_protocol_type) { 2551 case ENCAP_TYPE_ETHER: 2552 protocol = eproto_to_ipproto(skb->inner_protocol); 2553 break; 2554 case ENCAP_TYPE_IPPROTO: 2555 protocol = skb->inner_protocol; 2556 break; 2557 } 2558 nhdr = skb_inner_network_header(skb); 2559 } else { 2560 goto need_help; 2561 } 2562 2563 switch (protocol) { 2564 case IPPROTO_IP: 2565 if (!spec->ipv4_okay) 2566 goto need_help; 2567 iph = nhdr; 2568 ip_proto = iph->protocol; 2569 if (iph->ihl != 5 && !spec->ip_options_okay) 2570 goto need_help; 2571 break; 2572 case IPPROTO_IPV6: 2573 if (!spec->ipv6_okay) 2574 goto need_help; 2575 if (spec->no_encapped_ipv6 && *csum_encapped) 2576 goto need_help; 2577 ipv6 = nhdr; 2578 nhdr += sizeof(*ipv6); 2579 ip_proto = ipv6->nexthdr; 2580 break; 2581 default: 2582 goto need_help; 2583 } 2584 2585 ip_proto_again: 2586 switch (ip_proto) { 2587 case IPPROTO_TCP: 2588 if (!spec->tcp_okay || 2589 skb->csum_offset != offsetof(struct tcphdr, check)) 2590 goto need_help; 2591 break; 2592 case IPPROTO_UDP: 2593 if (!spec->udp_okay || 2594 skb->csum_offset != offsetof(struct udphdr, check)) 2595 goto need_help; 2596 break; 2597 case IPPROTO_SCTP: 2598 if (!spec->sctp_okay || 2599 skb->csum_offset != offsetof(struct sctphdr, checksum)) 2600 goto cant_help; 2601 break; 2602 case NEXTHDR_HOP: 2603 case NEXTHDR_ROUTING: 2604 case NEXTHDR_DEST: { 2605 u8 *opthdr = nhdr; 2606 2607 if (protocol != IPPROTO_IPV6 || !spec->ext_hdrs_okay) 2608 goto need_help; 2609 2610 ip_proto = opthdr[0]; 2611 nhdr += (opthdr[1] + 1) << 3; 2612 2613 goto ip_proto_again; 2614 } 2615 default: 2616 goto need_help; 2617 } 2618 2619 /* Passed the tests for offloading checksum */ 2620 return true; 2621 2622 need_help: 2623 if (csum_help && !skb_shinfo(skb)->gso_size) 2624 skb_checksum_help(skb); 2625 cant_help: 2626 return false; 2627 } 2628 EXPORT_SYMBOL(__skb_csum_offload_chk); 2629 2630 __be16 skb_network_protocol(struct sk_buff *skb, int *depth) 2631 { 2632 __be16 type = skb->protocol; 2633 2634 /* Tunnel gso handlers can set protocol to ethernet. */ 2635 if (type == htons(ETH_P_TEB)) { 2636 struct ethhdr *eth; 2637 2638 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr)))) 2639 return 0; 2640 2641 eth = (struct ethhdr *)skb_mac_header(skb); 2642 type = eth->h_proto; 2643 } 2644 2645 return __vlan_get_protocol(skb, type, depth); 2646 } 2647 2648 /** 2649 * skb_mac_gso_segment - mac layer segmentation handler. 2650 * @skb: buffer to segment 2651 * @features: features for the output path (see dev->features) 2652 */ 2653 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, 2654 netdev_features_t features) 2655 { 2656 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); 2657 struct packet_offload *ptype; 2658 int vlan_depth = skb->mac_len; 2659 __be16 type = skb_network_protocol(skb, &vlan_depth); 2660 2661 if (unlikely(!type)) 2662 return ERR_PTR(-EINVAL); 2663 2664 __skb_pull(skb, vlan_depth); 2665 2666 rcu_read_lock(); 2667 list_for_each_entry_rcu(ptype, &offload_base, list) { 2668 if (ptype->type == type && ptype->callbacks.gso_segment) { 2669 segs = ptype->callbacks.gso_segment(skb, features); 2670 break; 2671 } 2672 } 2673 rcu_read_unlock(); 2674 2675 __skb_push(skb, skb->data - skb_mac_header(skb)); 2676 2677 return segs; 2678 } 2679 EXPORT_SYMBOL(skb_mac_gso_segment); 2680 2681 2682 /* openvswitch calls this on rx path, so we need a different check. 2683 */ 2684 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) 2685 { 2686 if (tx_path) 2687 return skb->ip_summed != CHECKSUM_PARTIAL; 2688 else 2689 return skb->ip_summed == CHECKSUM_NONE; 2690 } 2691 2692 /** 2693 * __skb_gso_segment - Perform segmentation on skb. 2694 * @skb: buffer to segment 2695 * @features: features for the output path (see dev->features) 2696 * @tx_path: whether it is called in TX path 2697 * 2698 * This function segments the given skb and returns a list of segments. 2699 * 2700 * It may return NULL if the skb requires no segmentation. This is 2701 * only possible when GSO is used for verifying header integrity. 2702 * 2703 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb. 2704 */ 2705 struct sk_buff *__skb_gso_segment(struct sk_buff *skb, 2706 netdev_features_t features, bool tx_path) 2707 { 2708 if (unlikely(skb_needs_check(skb, tx_path))) { 2709 int err; 2710 2711 skb_warn_bad_offload(skb); 2712 2713 err = skb_cow_head(skb, 0); 2714 if (err < 0) 2715 return ERR_PTR(err); 2716 } 2717 2718 /* Only report GSO partial support if it will enable us to 2719 * support segmentation on this frame without needing additional 2720 * work. 2721 */ 2722 if (features & NETIF_F_GSO_PARTIAL) { 2723 netdev_features_t partial_features = NETIF_F_GSO_ROBUST; 2724 struct net_device *dev = skb->dev; 2725 2726 partial_features |= dev->features & dev->gso_partial_features; 2727 if (!skb_gso_ok(skb, features | partial_features)) 2728 features &= ~NETIF_F_GSO_PARTIAL; 2729 } 2730 2731 BUILD_BUG_ON(SKB_SGO_CB_OFFSET + 2732 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb)); 2733 2734 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb); 2735 SKB_GSO_CB(skb)->encap_level = 0; 2736 2737 skb_reset_mac_header(skb); 2738 skb_reset_mac_len(skb); 2739 2740 return skb_mac_gso_segment(skb, features); 2741 } 2742 EXPORT_SYMBOL(__skb_gso_segment); 2743 2744 /* Take action when hardware reception checksum errors are detected. */ 2745 #ifdef CONFIG_BUG 2746 void netdev_rx_csum_fault(struct net_device *dev) 2747 { 2748 if (net_ratelimit()) { 2749 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>"); 2750 dump_stack(); 2751 } 2752 } 2753 EXPORT_SYMBOL(netdev_rx_csum_fault); 2754 #endif 2755 2756 /* Actually, we should eliminate this check as soon as we know, that: 2757 * 1. IOMMU is present and allows to map all the memory. 2758 * 2. No high memory really exists on this machine. 2759 */ 2760 2761 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) 2762 { 2763 #ifdef CONFIG_HIGHMEM 2764 int i; 2765 if (!(dev->features & NETIF_F_HIGHDMA)) { 2766 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2767 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2768 if (PageHighMem(skb_frag_page(frag))) 2769 return 1; 2770 } 2771 } 2772 2773 if (PCI_DMA_BUS_IS_PHYS) { 2774 struct device *pdev = dev->dev.parent; 2775 2776 if (!pdev) 2777 return 0; 2778 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2779 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2780 dma_addr_t addr = page_to_phys(skb_frag_page(frag)); 2781 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask) 2782 return 1; 2783 } 2784 } 2785 #endif 2786 return 0; 2787 } 2788 2789 /* If MPLS offload request, verify we are testing hardware MPLS features 2790 * instead of standard features for the netdev. 2791 */ 2792 #if IS_ENABLED(CONFIG_NET_MPLS_GSO) 2793 static netdev_features_t net_mpls_features(struct sk_buff *skb, 2794 netdev_features_t features, 2795 __be16 type) 2796 { 2797 if (eth_p_mpls(type)) 2798 features &= skb->dev->mpls_features; 2799 2800 return features; 2801 } 2802 #else 2803 static netdev_features_t net_mpls_features(struct sk_buff *skb, 2804 netdev_features_t features, 2805 __be16 type) 2806 { 2807 return features; 2808 } 2809 #endif 2810 2811 static netdev_features_t harmonize_features(struct sk_buff *skb, 2812 netdev_features_t features) 2813 { 2814 int tmp; 2815 __be16 type; 2816 2817 type = skb_network_protocol(skb, &tmp); 2818 features = net_mpls_features(skb, features, type); 2819 2820 if (skb->ip_summed != CHECKSUM_NONE && 2821 !can_checksum_protocol(features, type)) { 2822 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 2823 } else if (illegal_highdma(skb->dev, skb)) { 2824 features &= ~NETIF_F_SG; 2825 } 2826 2827 return features; 2828 } 2829 2830 netdev_features_t passthru_features_check(struct sk_buff *skb, 2831 struct net_device *dev, 2832 netdev_features_t features) 2833 { 2834 return features; 2835 } 2836 EXPORT_SYMBOL(passthru_features_check); 2837 2838 static netdev_features_t dflt_features_check(const struct sk_buff *skb, 2839 struct net_device *dev, 2840 netdev_features_t features) 2841 { 2842 return vlan_features_check(skb, features); 2843 } 2844 2845 static netdev_features_t gso_features_check(const struct sk_buff *skb, 2846 struct net_device *dev, 2847 netdev_features_t features) 2848 { 2849 u16 gso_segs = skb_shinfo(skb)->gso_segs; 2850 2851 if (gso_segs > dev->gso_max_segs) 2852 return features & ~NETIF_F_GSO_MASK; 2853 2854 /* Support for GSO partial features requires software 2855 * intervention before we can actually process the packets 2856 * so we need to strip support for any partial features now 2857 * and we can pull them back in after we have partially 2858 * segmented the frame. 2859 */ 2860 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)) 2861 features &= ~dev->gso_partial_features; 2862 2863 /* Make sure to clear the IPv4 ID mangling feature if the 2864 * IPv4 header has the potential to be fragmented. 2865 */ 2866 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { 2867 struct iphdr *iph = skb->encapsulation ? 2868 inner_ip_hdr(skb) : ip_hdr(skb); 2869 2870 if (!(iph->frag_off & htons(IP_DF))) 2871 features &= ~NETIF_F_TSO_MANGLEID; 2872 } 2873 2874 return features; 2875 } 2876 2877 netdev_features_t netif_skb_features(struct sk_buff *skb) 2878 { 2879 struct net_device *dev = skb->dev; 2880 netdev_features_t features = dev->features; 2881 2882 if (skb_is_gso(skb)) 2883 features = gso_features_check(skb, dev, features); 2884 2885 /* If encapsulation offload request, verify we are testing 2886 * hardware encapsulation features instead of standard 2887 * features for the netdev 2888 */ 2889 if (skb->encapsulation) 2890 features &= dev->hw_enc_features; 2891 2892 if (skb_vlan_tagged(skb)) 2893 features = netdev_intersect_features(features, 2894 dev->vlan_features | 2895 NETIF_F_HW_VLAN_CTAG_TX | 2896 NETIF_F_HW_VLAN_STAG_TX); 2897 2898 if (dev->netdev_ops->ndo_features_check) 2899 features &= dev->netdev_ops->ndo_features_check(skb, dev, 2900 features); 2901 else 2902 features &= dflt_features_check(skb, dev, features); 2903 2904 return harmonize_features(skb, features); 2905 } 2906 EXPORT_SYMBOL(netif_skb_features); 2907 2908 static int xmit_one(struct sk_buff *skb, struct net_device *dev, 2909 struct netdev_queue *txq, bool more) 2910 { 2911 unsigned int len; 2912 int rc; 2913 2914 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all)) 2915 dev_queue_xmit_nit(skb, dev); 2916 2917 len = skb->len; 2918 trace_net_dev_start_xmit(skb, dev); 2919 rc = netdev_start_xmit(skb, dev, txq, more); 2920 trace_net_dev_xmit(skb, rc, dev, len); 2921 2922 return rc; 2923 } 2924 2925 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev, 2926 struct netdev_queue *txq, int *ret) 2927 { 2928 struct sk_buff *skb = first; 2929 int rc = NETDEV_TX_OK; 2930 2931 while (skb) { 2932 struct sk_buff *next = skb->next; 2933 2934 skb->next = NULL; 2935 rc = xmit_one(skb, dev, txq, next != NULL); 2936 if (unlikely(!dev_xmit_complete(rc))) { 2937 skb->next = next; 2938 goto out; 2939 } 2940 2941 skb = next; 2942 if (netif_xmit_stopped(txq) && skb) { 2943 rc = NETDEV_TX_BUSY; 2944 break; 2945 } 2946 } 2947 2948 out: 2949 *ret = rc; 2950 return skb; 2951 } 2952 2953 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, 2954 netdev_features_t features) 2955 { 2956 if (skb_vlan_tag_present(skb) && 2957 !vlan_hw_offload_capable(features, skb->vlan_proto)) 2958 skb = __vlan_hwaccel_push_inside(skb); 2959 return skb; 2960 } 2961 2962 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev) 2963 { 2964 netdev_features_t features; 2965 2966 features = netif_skb_features(skb); 2967 skb = validate_xmit_vlan(skb, features); 2968 if (unlikely(!skb)) 2969 goto out_null; 2970 2971 if (netif_needs_gso(skb, features)) { 2972 struct sk_buff *segs; 2973 2974 segs = skb_gso_segment(skb, features); 2975 if (IS_ERR(segs)) { 2976 goto out_kfree_skb; 2977 } else if (segs) { 2978 consume_skb(skb); 2979 skb = segs; 2980 } 2981 } else { 2982 if (skb_needs_linearize(skb, features) && 2983 __skb_linearize(skb)) 2984 goto out_kfree_skb; 2985 2986 /* If packet is not checksummed and device does not 2987 * support checksumming for this protocol, complete 2988 * checksumming here. 2989 */ 2990 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2991 if (skb->encapsulation) 2992 skb_set_inner_transport_header(skb, 2993 skb_checksum_start_offset(skb)); 2994 else 2995 skb_set_transport_header(skb, 2996 skb_checksum_start_offset(skb)); 2997 if (!(features & NETIF_F_CSUM_MASK) && 2998 skb_checksum_help(skb)) 2999 goto out_kfree_skb; 3000 } 3001 } 3002 3003 return skb; 3004 3005 out_kfree_skb: 3006 kfree_skb(skb); 3007 out_null: 3008 atomic_long_inc(&dev->tx_dropped); 3009 return NULL; 3010 } 3011 3012 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev) 3013 { 3014 struct sk_buff *next, *head = NULL, *tail; 3015 3016 for (; skb != NULL; skb = next) { 3017 next = skb->next; 3018 skb->next = NULL; 3019 3020 /* in case skb wont be segmented, point to itself */ 3021 skb->prev = skb; 3022 3023 skb = validate_xmit_skb(skb, dev); 3024 if (!skb) 3025 continue; 3026 3027 if (!head) 3028 head = skb; 3029 else 3030 tail->next = skb; 3031 /* If skb was segmented, skb->prev points to 3032 * the last segment. If not, it still contains skb. 3033 */ 3034 tail = skb->prev; 3035 } 3036 return head; 3037 } 3038 EXPORT_SYMBOL_GPL(validate_xmit_skb_list); 3039 3040 static void qdisc_pkt_len_init(struct sk_buff *skb) 3041 { 3042 const struct skb_shared_info *shinfo = skb_shinfo(skb); 3043 3044 qdisc_skb_cb(skb)->pkt_len = skb->len; 3045 3046 /* To get more precise estimation of bytes sent on wire, 3047 * we add to pkt_len the headers size of all segments 3048 */ 3049 if (shinfo->gso_size) { 3050 unsigned int hdr_len; 3051 u16 gso_segs = shinfo->gso_segs; 3052 3053 /* mac layer + network layer */ 3054 hdr_len = skb_transport_header(skb) - skb_mac_header(skb); 3055 3056 /* + transport layer */ 3057 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) 3058 hdr_len += tcp_hdrlen(skb); 3059 else 3060 hdr_len += sizeof(struct udphdr); 3061 3062 if (shinfo->gso_type & SKB_GSO_DODGY) 3063 gso_segs = DIV_ROUND_UP(skb->len - hdr_len, 3064 shinfo->gso_size); 3065 3066 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; 3067 } 3068 } 3069 3070 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 3071 struct net_device *dev, 3072 struct netdev_queue *txq) 3073 { 3074 spinlock_t *root_lock = qdisc_lock(q); 3075 struct sk_buff *to_free = NULL; 3076 bool contended; 3077 int rc; 3078 3079 qdisc_calculate_pkt_len(skb, q); 3080 /* 3081 * Heuristic to force contended enqueues to serialize on a 3082 * separate lock before trying to get qdisc main lock. 3083 * This permits qdisc->running owner to get the lock more 3084 * often and dequeue packets faster. 3085 */ 3086 contended = qdisc_is_running(q); 3087 if (unlikely(contended)) 3088 spin_lock(&q->busylock); 3089 3090 spin_lock(root_lock); 3091 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { 3092 __qdisc_drop(skb, &to_free); 3093 rc = NET_XMIT_DROP; 3094 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && 3095 qdisc_run_begin(q)) { 3096 /* 3097 * This is a work-conserving queue; there are no old skbs 3098 * waiting to be sent out; and the qdisc is not running - 3099 * xmit the skb directly. 3100 */ 3101 3102 qdisc_bstats_update(q, skb); 3103 3104 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) { 3105 if (unlikely(contended)) { 3106 spin_unlock(&q->busylock); 3107 contended = false; 3108 } 3109 __qdisc_run(q); 3110 } else 3111 qdisc_run_end(q); 3112 3113 rc = NET_XMIT_SUCCESS; 3114 } else { 3115 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK; 3116 if (qdisc_run_begin(q)) { 3117 if (unlikely(contended)) { 3118 spin_unlock(&q->busylock); 3119 contended = false; 3120 } 3121 __qdisc_run(q); 3122 } 3123 } 3124 spin_unlock(root_lock); 3125 if (unlikely(to_free)) 3126 kfree_skb_list(to_free); 3127 if (unlikely(contended)) 3128 spin_unlock(&q->busylock); 3129 return rc; 3130 } 3131 3132 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO) 3133 static void skb_update_prio(struct sk_buff *skb) 3134 { 3135 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap); 3136 3137 if (!skb->priority && skb->sk && map) { 3138 unsigned int prioidx = 3139 sock_cgroup_prioidx(&skb->sk->sk_cgrp_data); 3140 3141 if (prioidx < map->priomap_len) 3142 skb->priority = map->priomap[prioidx]; 3143 } 3144 } 3145 #else 3146 #define skb_update_prio(skb) 3147 #endif 3148 3149 DEFINE_PER_CPU(int, xmit_recursion); 3150 EXPORT_SYMBOL(xmit_recursion); 3151 3152 /** 3153 * dev_loopback_xmit - loop back @skb 3154 * @net: network namespace this loopback is happening in 3155 * @sk: sk needed to be a netfilter okfn 3156 * @skb: buffer to transmit 3157 */ 3158 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb) 3159 { 3160 skb_reset_mac_header(skb); 3161 __skb_pull(skb, skb_network_offset(skb)); 3162 skb->pkt_type = PACKET_LOOPBACK; 3163 skb->ip_summed = CHECKSUM_UNNECESSARY; 3164 WARN_ON(!skb_dst(skb)); 3165 skb_dst_force(skb); 3166 netif_rx_ni(skb); 3167 return 0; 3168 } 3169 EXPORT_SYMBOL(dev_loopback_xmit); 3170 3171 #ifdef CONFIG_NET_EGRESS 3172 static struct sk_buff * 3173 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) 3174 { 3175 struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list); 3176 struct tcf_result cl_res; 3177 3178 if (!cl) 3179 return skb; 3180 3181 /* skb->tc_verd and qdisc_skb_cb(skb)->pkt_len were already set 3182 * earlier by the caller. 3183 */ 3184 qdisc_bstats_cpu_update(cl->q, skb); 3185 3186 switch (tc_classify(skb, cl, &cl_res, false)) { 3187 case TC_ACT_OK: 3188 case TC_ACT_RECLASSIFY: 3189 skb->tc_index = TC_H_MIN(cl_res.classid); 3190 break; 3191 case TC_ACT_SHOT: 3192 qdisc_qstats_cpu_drop(cl->q); 3193 *ret = NET_XMIT_DROP; 3194 kfree_skb(skb); 3195 return NULL; 3196 case TC_ACT_STOLEN: 3197 case TC_ACT_QUEUED: 3198 *ret = NET_XMIT_SUCCESS; 3199 consume_skb(skb); 3200 return NULL; 3201 case TC_ACT_REDIRECT: 3202 /* No need to push/pop skb's mac_header here on egress! */ 3203 skb_do_redirect(skb); 3204 *ret = NET_XMIT_SUCCESS; 3205 return NULL; 3206 default: 3207 break; 3208 } 3209 3210 return skb; 3211 } 3212 #endif /* CONFIG_NET_EGRESS */ 3213 3214 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb) 3215 { 3216 #ifdef CONFIG_XPS 3217 struct xps_dev_maps *dev_maps; 3218 struct xps_map *map; 3219 int queue_index = -1; 3220 3221 rcu_read_lock(); 3222 dev_maps = rcu_dereference(dev->xps_maps); 3223 if (dev_maps) { 3224 map = rcu_dereference( 3225 dev_maps->cpu_map[skb->sender_cpu - 1]); 3226 if (map) { 3227 if (map->len == 1) 3228 queue_index = map->queues[0]; 3229 else 3230 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb), 3231 map->len)]; 3232 if (unlikely(queue_index >= dev->real_num_tx_queues)) 3233 queue_index = -1; 3234 } 3235 } 3236 rcu_read_unlock(); 3237 3238 return queue_index; 3239 #else 3240 return -1; 3241 #endif 3242 } 3243 3244 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb) 3245 { 3246 struct sock *sk = skb->sk; 3247 int queue_index = sk_tx_queue_get(sk); 3248 3249 if (queue_index < 0 || skb->ooo_okay || 3250 queue_index >= dev->real_num_tx_queues) { 3251 int new_index = get_xps_queue(dev, skb); 3252 if (new_index < 0) 3253 new_index = skb_tx_hash(dev, skb); 3254 3255 if (queue_index != new_index && sk && 3256 sk_fullsock(sk) && 3257 rcu_access_pointer(sk->sk_dst_cache)) 3258 sk_tx_queue_set(sk, new_index); 3259 3260 queue_index = new_index; 3261 } 3262 3263 return queue_index; 3264 } 3265 3266 struct netdev_queue *netdev_pick_tx(struct net_device *dev, 3267 struct sk_buff *skb, 3268 void *accel_priv) 3269 { 3270 int queue_index = 0; 3271 3272 #ifdef CONFIG_XPS 3273 u32 sender_cpu = skb->sender_cpu - 1; 3274 3275 if (sender_cpu >= (u32)NR_CPUS) 3276 skb->sender_cpu = raw_smp_processor_id() + 1; 3277 #endif 3278 3279 if (dev->real_num_tx_queues != 1) { 3280 const struct net_device_ops *ops = dev->netdev_ops; 3281 if (ops->ndo_select_queue) 3282 queue_index = ops->ndo_select_queue(dev, skb, accel_priv, 3283 __netdev_pick_tx); 3284 else 3285 queue_index = __netdev_pick_tx(dev, skb); 3286 3287 if (!accel_priv) 3288 queue_index = netdev_cap_txqueue(dev, queue_index); 3289 } 3290 3291 skb_set_queue_mapping(skb, queue_index); 3292 return netdev_get_tx_queue(dev, queue_index); 3293 } 3294 3295 /** 3296 * __dev_queue_xmit - transmit a buffer 3297 * @skb: buffer to transmit 3298 * @accel_priv: private data used for L2 forwarding offload 3299 * 3300 * Queue a buffer for transmission to a network device. The caller must 3301 * have set the device and priority and built the buffer before calling 3302 * this function. The function can be called from an interrupt. 3303 * 3304 * A negative errno code is returned on a failure. A success does not 3305 * guarantee the frame will be transmitted as it may be dropped due 3306 * to congestion or traffic shaping. 3307 * 3308 * ----------------------------------------------------------------------------------- 3309 * I notice this method can also return errors from the queue disciplines, 3310 * including NET_XMIT_DROP, which is a positive value. So, errors can also 3311 * be positive. 3312 * 3313 * Regardless of the return value, the skb is consumed, so it is currently 3314 * difficult to retry a send to this method. (You can bump the ref count 3315 * before sending to hold a reference for retry if you are careful.) 3316 * 3317 * When calling this method, interrupts MUST be enabled. This is because 3318 * the BH enable code must have IRQs enabled so that it will not deadlock. 3319 * --BLG 3320 */ 3321 static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv) 3322 { 3323 struct net_device *dev = skb->dev; 3324 struct netdev_queue *txq; 3325 struct Qdisc *q; 3326 int rc = -ENOMEM; 3327 3328 skb_reset_mac_header(skb); 3329 3330 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP)) 3331 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED); 3332 3333 /* Disable soft irqs for various locks below. Also 3334 * stops preemption for RCU. 3335 */ 3336 rcu_read_lock_bh(); 3337 3338 skb_update_prio(skb); 3339 3340 qdisc_pkt_len_init(skb); 3341 #ifdef CONFIG_NET_CLS_ACT 3342 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS); 3343 # ifdef CONFIG_NET_EGRESS 3344 if (static_key_false(&egress_needed)) { 3345 skb = sch_handle_egress(skb, &rc, dev); 3346 if (!skb) 3347 goto out; 3348 } 3349 # endif 3350 #endif 3351 /* If device/qdisc don't need skb->dst, release it right now while 3352 * its hot in this cpu cache. 3353 */ 3354 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) 3355 skb_dst_drop(skb); 3356 else 3357 skb_dst_force(skb); 3358 3359 txq = netdev_pick_tx(dev, skb, accel_priv); 3360 q = rcu_dereference_bh(txq->qdisc); 3361 3362 trace_net_dev_queue(skb); 3363 if (q->enqueue) { 3364 rc = __dev_xmit_skb(skb, q, dev, txq); 3365 goto out; 3366 } 3367 3368 /* The device has no queue. Common case for software devices: 3369 loopback, all the sorts of tunnels... 3370 3371 Really, it is unlikely that netif_tx_lock protection is necessary 3372 here. (f.e. loopback and IP tunnels are clean ignoring statistics 3373 counters.) 3374 However, it is possible, that they rely on protection 3375 made by us here. 3376 3377 Check this and shot the lock. It is not prone from deadlocks. 3378 Either shot noqueue qdisc, it is even simpler 8) 3379 */ 3380 if (dev->flags & IFF_UP) { 3381 int cpu = smp_processor_id(); /* ok because BHs are off */ 3382 3383 if (txq->xmit_lock_owner != cpu) { 3384 if (unlikely(__this_cpu_read(xmit_recursion) > 3385 XMIT_RECURSION_LIMIT)) 3386 goto recursion_alert; 3387 3388 skb = validate_xmit_skb(skb, dev); 3389 if (!skb) 3390 goto out; 3391 3392 HARD_TX_LOCK(dev, txq, cpu); 3393 3394 if (!netif_xmit_stopped(txq)) { 3395 __this_cpu_inc(xmit_recursion); 3396 skb = dev_hard_start_xmit(skb, dev, txq, &rc); 3397 __this_cpu_dec(xmit_recursion); 3398 if (dev_xmit_complete(rc)) { 3399 HARD_TX_UNLOCK(dev, txq); 3400 goto out; 3401 } 3402 } 3403 HARD_TX_UNLOCK(dev, txq); 3404 net_crit_ratelimited("Virtual device %s asks to queue packet!\n", 3405 dev->name); 3406 } else { 3407 /* Recursion is detected! It is possible, 3408 * unfortunately 3409 */ 3410 recursion_alert: 3411 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", 3412 dev->name); 3413 } 3414 } 3415 3416 rc = -ENETDOWN; 3417 rcu_read_unlock_bh(); 3418 3419 atomic_long_inc(&dev->tx_dropped); 3420 kfree_skb_list(skb); 3421 return rc; 3422 out: 3423 rcu_read_unlock_bh(); 3424 return rc; 3425 } 3426 3427 int dev_queue_xmit(struct sk_buff *skb) 3428 { 3429 return __dev_queue_xmit(skb, NULL); 3430 } 3431 EXPORT_SYMBOL(dev_queue_xmit); 3432 3433 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv) 3434 { 3435 return __dev_queue_xmit(skb, accel_priv); 3436 } 3437 EXPORT_SYMBOL(dev_queue_xmit_accel); 3438 3439 3440 /*======================================================================= 3441 Receiver routines 3442 =======================================================================*/ 3443 3444 int netdev_max_backlog __read_mostly = 1000; 3445 EXPORT_SYMBOL(netdev_max_backlog); 3446 3447 int netdev_tstamp_prequeue __read_mostly = 1; 3448 int netdev_budget __read_mostly = 300; 3449 int weight_p __read_mostly = 64; /* old backlog weight */ 3450 3451 /* Called with irq disabled */ 3452 static inline void ____napi_schedule(struct softnet_data *sd, 3453 struct napi_struct *napi) 3454 { 3455 list_add_tail(&napi->poll_list, &sd->poll_list); 3456 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 3457 } 3458 3459 #ifdef CONFIG_RPS 3460 3461 /* One global table that all flow-based protocols share. */ 3462 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; 3463 EXPORT_SYMBOL(rps_sock_flow_table); 3464 u32 rps_cpu_mask __read_mostly; 3465 EXPORT_SYMBOL(rps_cpu_mask); 3466 3467 struct static_key rps_needed __read_mostly; 3468 EXPORT_SYMBOL(rps_needed); 3469 3470 static struct rps_dev_flow * 3471 set_rps_cpu(struct net_device *dev, struct sk_buff *skb, 3472 struct rps_dev_flow *rflow, u16 next_cpu) 3473 { 3474 if (next_cpu < nr_cpu_ids) { 3475 #ifdef CONFIG_RFS_ACCEL 3476 struct netdev_rx_queue *rxqueue; 3477 struct rps_dev_flow_table *flow_table; 3478 struct rps_dev_flow *old_rflow; 3479 u32 flow_id; 3480 u16 rxq_index; 3481 int rc; 3482 3483 /* Should we steer this flow to a different hardware queue? */ 3484 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || 3485 !(dev->features & NETIF_F_NTUPLE)) 3486 goto out; 3487 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); 3488 if (rxq_index == skb_get_rx_queue(skb)) 3489 goto out; 3490 3491 rxqueue = dev->_rx + rxq_index; 3492 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3493 if (!flow_table) 3494 goto out; 3495 flow_id = skb_get_hash(skb) & flow_table->mask; 3496 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, 3497 rxq_index, flow_id); 3498 if (rc < 0) 3499 goto out; 3500 old_rflow = rflow; 3501 rflow = &flow_table->flows[flow_id]; 3502 rflow->filter = rc; 3503 if (old_rflow->filter == rflow->filter) 3504 old_rflow->filter = RPS_NO_FILTER; 3505 out: 3506 #endif 3507 rflow->last_qtail = 3508 per_cpu(softnet_data, next_cpu).input_queue_head; 3509 } 3510 3511 rflow->cpu = next_cpu; 3512 return rflow; 3513 } 3514 3515 /* 3516 * get_rps_cpu is called from netif_receive_skb and returns the target 3517 * CPU from the RPS map of the receiving queue for a given skb. 3518 * rcu_read_lock must be held on entry. 3519 */ 3520 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, 3521 struct rps_dev_flow **rflowp) 3522 { 3523 const struct rps_sock_flow_table *sock_flow_table; 3524 struct netdev_rx_queue *rxqueue = dev->_rx; 3525 struct rps_dev_flow_table *flow_table; 3526 struct rps_map *map; 3527 int cpu = -1; 3528 u32 tcpu; 3529 u32 hash; 3530 3531 if (skb_rx_queue_recorded(skb)) { 3532 u16 index = skb_get_rx_queue(skb); 3533 3534 if (unlikely(index >= dev->real_num_rx_queues)) { 3535 WARN_ONCE(dev->real_num_rx_queues > 1, 3536 "%s received packet on queue %u, but number " 3537 "of RX queues is %u\n", 3538 dev->name, index, dev->real_num_rx_queues); 3539 goto done; 3540 } 3541 rxqueue += index; 3542 } 3543 3544 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */ 3545 3546 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3547 map = rcu_dereference(rxqueue->rps_map); 3548 if (!flow_table && !map) 3549 goto done; 3550 3551 skb_reset_network_header(skb); 3552 hash = skb_get_hash(skb); 3553 if (!hash) 3554 goto done; 3555 3556 sock_flow_table = rcu_dereference(rps_sock_flow_table); 3557 if (flow_table && sock_flow_table) { 3558 struct rps_dev_flow *rflow; 3559 u32 next_cpu; 3560 u32 ident; 3561 3562 /* First check into global flow table if there is a match */ 3563 ident = sock_flow_table->ents[hash & sock_flow_table->mask]; 3564 if ((ident ^ hash) & ~rps_cpu_mask) 3565 goto try_rps; 3566 3567 next_cpu = ident & rps_cpu_mask; 3568 3569 /* OK, now we know there is a match, 3570 * we can look at the local (per receive queue) flow table 3571 */ 3572 rflow = &flow_table->flows[hash & flow_table->mask]; 3573 tcpu = rflow->cpu; 3574 3575 /* 3576 * If the desired CPU (where last recvmsg was done) is 3577 * different from current CPU (one in the rx-queue flow 3578 * table entry), switch if one of the following holds: 3579 * - Current CPU is unset (>= nr_cpu_ids). 3580 * - Current CPU is offline. 3581 * - The current CPU's queue tail has advanced beyond the 3582 * last packet that was enqueued using this table entry. 3583 * This guarantees that all previous packets for the flow 3584 * have been dequeued, thus preserving in order delivery. 3585 */ 3586 if (unlikely(tcpu != next_cpu) && 3587 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) || 3588 ((int)(per_cpu(softnet_data, tcpu).input_queue_head - 3589 rflow->last_qtail)) >= 0)) { 3590 tcpu = next_cpu; 3591 rflow = set_rps_cpu(dev, skb, rflow, next_cpu); 3592 } 3593 3594 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { 3595 *rflowp = rflow; 3596 cpu = tcpu; 3597 goto done; 3598 } 3599 } 3600 3601 try_rps: 3602 3603 if (map) { 3604 tcpu = map->cpus[reciprocal_scale(hash, map->len)]; 3605 if (cpu_online(tcpu)) { 3606 cpu = tcpu; 3607 goto done; 3608 } 3609 } 3610 3611 done: 3612 return cpu; 3613 } 3614 3615 #ifdef CONFIG_RFS_ACCEL 3616 3617 /** 3618 * rps_may_expire_flow - check whether an RFS hardware filter may be removed 3619 * @dev: Device on which the filter was set 3620 * @rxq_index: RX queue index 3621 * @flow_id: Flow ID passed to ndo_rx_flow_steer() 3622 * @filter_id: Filter ID returned by ndo_rx_flow_steer() 3623 * 3624 * Drivers that implement ndo_rx_flow_steer() should periodically call 3625 * this function for each installed filter and remove the filters for 3626 * which it returns %true. 3627 */ 3628 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, 3629 u32 flow_id, u16 filter_id) 3630 { 3631 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; 3632 struct rps_dev_flow_table *flow_table; 3633 struct rps_dev_flow *rflow; 3634 bool expire = true; 3635 unsigned int cpu; 3636 3637 rcu_read_lock(); 3638 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3639 if (flow_table && flow_id <= flow_table->mask) { 3640 rflow = &flow_table->flows[flow_id]; 3641 cpu = ACCESS_ONCE(rflow->cpu); 3642 if (rflow->filter == filter_id && cpu < nr_cpu_ids && 3643 ((int)(per_cpu(softnet_data, cpu).input_queue_head - 3644 rflow->last_qtail) < 3645 (int)(10 * flow_table->mask))) 3646 expire = false; 3647 } 3648 rcu_read_unlock(); 3649 return expire; 3650 } 3651 EXPORT_SYMBOL(rps_may_expire_flow); 3652 3653 #endif /* CONFIG_RFS_ACCEL */ 3654 3655 /* Called from hardirq (IPI) context */ 3656 static void rps_trigger_softirq(void *data) 3657 { 3658 struct softnet_data *sd = data; 3659 3660 ____napi_schedule(sd, &sd->backlog); 3661 sd->received_rps++; 3662 } 3663 3664 #endif /* CONFIG_RPS */ 3665 3666 /* 3667 * Check if this softnet_data structure is another cpu one 3668 * If yes, queue it to our IPI list and return 1 3669 * If no, return 0 3670 */ 3671 static int rps_ipi_queued(struct softnet_data *sd) 3672 { 3673 #ifdef CONFIG_RPS 3674 struct softnet_data *mysd = this_cpu_ptr(&softnet_data); 3675 3676 if (sd != mysd) { 3677 sd->rps_ipi_next = mysd->rps_ipi_list; 3678 mysd->rps_ipi_list = sd; 3679 3680 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 3681 return 1; 3682 } 3683 #endif /* CONFIG_RPS */ 3684 return 0; 3685 } 3686 3687 #ifdef CONFIG_NET_FLOW_LIMIT 3688 int netdev_flow_limit_table_len __read_mostly = (1 << 12); 3689 #endif 3690 3691 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) 3692 { 3693 #ifdef CONFIG_NET_FLOW_LIMIT 3694 struct sd_flow_limit *fl; 3695 struct softnet_data *sd; 3696 unsigned int old_flow, new_flow; 3697 3698 if (qlen < (netdev_max_backlog >> 1)) 3699 return false; 3700 3701 sd = this_cpu_ptr(&softnet_data); 3702 3703 rcu_read_lock(); 3704 fl = rcu_dereference(sd->flow_limit); 3705 if (fl) { 3706 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1); 3707 old_flow = fl->history[fl->history_head]; 3708 fl->history[fl->history_head] = new_flow; 3709 3710 fl->history_head++; 3711 fl->history_head &= FLOW_LIMIT_HISTORY - 1; 3712 3713 if (likely(fl->buckets[old_flow])) 3714 fl->buckets[old_flow]--; 3715 3716 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { 3717 fl->count++; 3718 rcu_read_unlock(); 3719 return true; 3720 } 3721 } 3722 rcu_read_unlock(); 3723 #endif 3724 return false; 3725 } 3726 3727 /* 3728 * enqueue_to_backlog is called to queue an skb to a per CPU backlog 3729 * queue (may be a remote CPU queue). 3730 */ 3731 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, 3732 unsigned int *qtail) 3733 { 3734 struct softnet_data *sd; 3735 unsigned long flags; 3736 unsigned int qlen; 3737 3738 sd = &per_cpu(softnet_data, cpu); 3739 3740 local_irq_save(flags); 3741 3742 rps_lock(sd); 3743 if (!netif_running(skb->dev)) 3744 goto drop; 3745 qlen = skb_queue_len(&sd->input_pkt_queue); 3746 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) { 3747 if (qlen) { 3748 enqueue: 3749 __skb_queue_tail(&sd->input_pkt_queue, skb); 3750 input_queue_tail_incr_save(sd, qtail); 3751 rps_unlock(sd); 3752 local_irq_restore(flags); 3753 return NET_RX_SUCCESS; 3754 } 3755 3756 /* Schedule NAPI for backlog device 3757 * We can use non atomic operation since we own the queue lock 3758 */ 3759 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) { 3760 if (!rps_ipi_queued(sd)) 3761 ____napi_schedule(sd, &sd->backlog); 3762 } 3763 goto enqueue; 3764 } 3765 3766 drop: 3767 sd->dropped++; 3768 rps_unlock(sd); 3769 3770 local_irq_restore(flags); 3771 3772 atomic_long_inc(&skb->dev->rx_dropped); 3773 kfree_skb(skb); 3774 return NET_RX_DROP; 3775 } 3776 3777 static int netif_rx_internal(struct sk_buff *skb) 3778 { 3779 int ret; 3780 3781 net_timestamp_check(netdev_tstamp_prequeue, skb); 3782 3783 trace_netif_rx(skb); 3784 #ifdef CONFIG_RPS 3785 if (static_key_false(&rps_needed)) { 3786 struct rps_dev_flow voidflow, *rflow = &voidflow; 3787 int cpu; 3788 3789 preempt_disable(); 3790 rcu_read_lock(); 3791 3792 cpu = get_rps_cpu(skb->dev, skb, &rflow); 3793 if (cpu < 0) 3794 cpu = smp_processor_id(); 3795 3796 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 3797 3798 rcu_read_unlock(); 3799 preempt_enable(); 3800 } else 3801 #endif 3802 { 3803 unsigned int qtail; 3804 ret = enqueue_to_backlog(skb, get_cpu(), &qtail); 3805 put_cpu(); 3806 } 3807 return ret; 3808 } 3809 3810 /** 3811 * netif_rx - post buffer to the network code 3812 * @skb: buffer to post 3813 * 3814 * This function receives a packet from a device driver and queues it for 3815 * the upper (protocol) levels to process. It always succeeds. The buffer 3816 * may be dropped during processing for congestion control or by the 3817 * protocol layers. 3818 * 3819 * return values: 3820 * NET_RX_SUCCESS (no congestion) 3821 * NET_RX_DROP (packet was dropped) 3822 * 3823 */ 3824 3825 int netif_rx(struct sk_buff *skb) 3826 { 3827 trace_netif_rx_entry(skb); 3828 3829 return netif_rx_internal(skb); 3830 } 3831 EXPORT_SYMBOL(netif_rx); 3832 3833 int netif_rx_ni(struct sk_buff *skb) 3834 { 3835 int err; 3836 3837 trace_netif_rx_ni_entry(skb); 3838 3839 preempt_disable(); 3840 err = netif_rx_internal(skb); 3841 if (local_softirq_pending()) 3842 do_softirq(); 3843 preempt_enable(); 3844 3845 return err; 3846 } 3847 EXPORT_SYMBOL(netif_rx_ni); 3848 3849 static __latent_entropy void net_tx_action(struct softirq_action *h) 3850 { 3851 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 3852 3853 if (sd->completion_queue) { 3854 struct sk_buff *clist; 3855 3856 local_irq_disable(); 3857 clist = sd->completion_queue; 3858 sd->completion_queue = NULL; 3859 local_irq_enable(); 3860 3861 while (clist) { 3862 struct sk_buff *skb = clist; 3863 clist = clist->next; 3864 3865 WARN_ON(atomic_read(&skb->users)); 3866 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED)) 3867 trace_consume_skb(skb); 3868 else 3869 trace_kfree_skb(skb, net_tx_action); 3870 3871 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) 3872 __kfree_skb(skb); 3873 else 3874 __kfree_skb_defer(skb); 3875 } 3876 3877 __kfree_skb_flush(); 3878 } 3879 3880 if (sd->output_queue) { 3881 struct Qdisc *head; 3882 3883 local_irq_disable(); 3884 head = sd->output_queue; 3885 sd->output_queue = NULL; 3886 sd->output_queue_tailp = &sd->output_queue; 3887 local_irq_enable(); 3888 3889 while (head) { 3890 struct Qdisc *q = head; 3891 spinlock_t *root_lock; 3892 3893 head = head->next_sched; 3894 3895 root_lock = qdisc_lock(q); 3896 spin_lock(root_lock); 3897 /* We need to make sure head->next_sched is read 3898 * before clearing __QDISC_STATE_SCHED 3899 */ 3900 smp_mb__before_atomic(); 3901 clear_bit(__QDISC_STATE_SCHED, &q->state); 3902 qdisc_run(q); 3903 spin_unlock(root_lock); 3904 } 3905 } 3906 } 3907 3908 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE) 3909 /* This hook is defined here for ATM LANE */ 3910 int (*br_fdb_test_addr_hook)(struct net_device *dev, 3911 unsigned char *addr) __read_mostly; 3912 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); 3913 #endif 3914 3915 static inline struct sk_buff * 3916 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, 3917 struct net_device *orig_dev) 3918 { 3919 #ifdef CONFIG_NET_CLS_ACT 3920 struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list); 3921 struct tcf_result cl_res; 3922 3923 /* If there's at least one ingress present somewhere (so 3924 * we get here via enabled static key), remaining devices 3925 * that are not configured with an ingress qdisc will bail 3926 * out here. 3927 */ 3928 if (!cl) 3929 return skb; 3930 if (*pt_prev) { 3931 *ret = deliver_skb(skb, *pt_prev, orig_dev); 3932 *pt_prev = NULL; 3933 } 3934 3935 qdisc_skb_cb(skb)->pkt_len = skb->len; 3936 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS); 3937 qdisc_bstats_cpu_update(cl->q, skb); 3938 3939 switch (tc_classify(skb, cl, &cl_res, false)) { 3940 case TC_ACT_OK: 3941 case TC_ACT_RECLASSIFY: 3942 skb->tc_index = TC_H_MIN(cl_res.classid); 3943 break; 3944 case TC_ACT_SHOT: 3945 qdisc_qstats_cpu_drop(cl->q); 3946 kfree_skb(skb); 3947 return NULL; 3948 case TC_ACT_STOLEN: 3949 case TC_ACT_QUEUED: 3950 consume_skb(skb); 3951 return NULL; 3952 case TC_ACT_REDIRECT: 3953 /* skb_mac_header check was done by cls/act_bpf, so 3954 * we can safely push the L2 header back before 3955 * redirecting to another netdev 3956 */ 3957 __skb_push(skb, skb->mac_len); 3958 skb_do_redirect(skb); 3959 return NULL; 3960 default: 3961 break; 3962 } 3963 #endif /* CONFIG_NET_CLS_ACT */ 3964 return skb; 3965 } 3966 3967 /** 3968 * netdev_is_rx_handler_busy - check if receive handler is registered 3969 * @dev: device to check 3970 * 3971 * Check if a receive handler is already registered for a given device. 3972 * Return true if there one. 3973 * 3974 * The caller must hold the rtnl_mutex. 3975 */ 3976 bool netdev_is_rx_handler_busy(struct net_device *dev) 3977 { 3978 ASSERT_RTNL(); 3979 return dev && rtnl_dereference(dev->rx_handler); 3980 } 3981 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy); 3982 3983 /** 3984 * netdev_rx_handler_register - register receive handler 3985 * @dev: device to register a handler for 3986 * @rx_handler: receive handler to register 3987 * @rx_handler_data: data pointer that is used by rx handler 3988 * 3989 * Register a receive handler for a device. This handler will then be 3990 * called from __netif_receive_skb. A negative errno code is returned 3991 * on a failure. 3992 * 3993 * The caller must hold the rtnl_mutex. 3994 * 3995 * For a general description of rx_handler, see enum rx_handler_result. 3996 */ 3997 int netdev_rx_handler_register(struct net_device *dev, 3998 rx_handler_func_t *rx_handler, 3999 void *rx_handler_data) 4000 { 4001 ASSERT_RTNL(); 4002 4003 if (dev->rx_handler) 4004 return -EBUSY; 4005 4006 /* Note: rx_handler_data must be set before rx_handler */ 4007 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); 4008 rcu_assign_pointer(dev->rx_handler, rx_handler); 4009 4010 return 0; 4011 } 4012 EXPORT_SYMBOL_GPL(netdev_rx_handler_register); 4013 4014 /** 4015 * netdev_rx_handler_unregister - unregister receive handler 4016 * @dev: device to unregister a handler from 4017 * 4018 * Unregister a receive handler from a device. 4019 * 4020 * The caller must hold the rtnl_mutex. 4021 */ 4022 void netdev_rx_handler_unregister(struct net_device *dev) 4023 { 4024 4025 ASSERT_RTNL(); 4026 RCU_INIT_POINTER(dev->rx_handler, NULL); 4027 /* a reader seeing a non NULL rx_handler in a rcu_read_lock() 4028 * section has a guarantee to see a non NULL rx_handler_data 4029 * as well. 4030 */ 4031 synchronize_net(); 4032 RCU_INIT_POINTER(dev->rx_handler_data, NULL); 4033 } 4034 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); 4035 4036 /* 4037 * Limit the use of PFMEMALLOC reserves to those protocols that implement 4038 * the special handling of PFMEMALLOC skbs. 4039 */ 4040 static bool skb_pfmemalloc_protocol(struct sk_buff *skb) 4041 { 4042 switch (skb->protocol) { 4043 case htons(ETH_P_ARP): 4044 case htons(ETH_P_IP): 4045 case htons(ETH_P_IPV6): 4046 case htons(ETH_P_8021Q): 4047 case htons(ETH_P_8021AD): 4048 return true; 4049 default: 4050 return false; 4051 } 4052 } 4053 4054 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev, 4055 int *ret, struct net_device *orig_dev) 4056 { 4057 #ifdef CONFIG_NETFILTER_INGRESS 4058 if (nf_hook_ingress_active(skb)) { 4059 int ingress_retval; 4060 4061 if (*pt_prev) { 4062 *ret = deliver_skb(skb, *pt_prev, orig_dev); 4063 *pt_prev = NULL; 4064 } 4065 4066 rcu_read_lock(); 4067 ingress_retval = nf_hook_ingress(skb); 4068 rcu_read_unlock(); 4069 return ingress_retval; 4070 } 4071 #endif /* CONFIG_NETFILTER_INGRESS */ 4072 return 0; 4073 } 4074 4075 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc) 4076 { 4077 struct packet_type *ptype, *pt_prev; 4078 rx_handler_func_t *rx_handler; 4079 struct net_device *orig_dev; 4080 bool deliver_exact = false; 4081 int ret = NET_RX_DROP; 4082 __be16 type; 4083 4084 net_timestamp_check(!netdev_tstamp_prequeue, skb); 4085 4086 trace_netif_receive_skb(skb); 4087 4088 orig_dev = skb->dev; 4089 4090 skb_reset_network_header(skb); 4091 if (!skb_transport_header_was_set(skb)) 4092 skb_reset_transport_header(skb); 4093 skb_reset_mac_len(skb); 4094 4095 pt_prev = NULL; 4096 4097 another_round: 4098 skb->skb_iif = skb->dev->ifindex; 4099 4100 __this_cpu_inc(softnet_data.processed); 4101 4102 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 4103 skb->protocol == cpu_to_be16(ETH_P_8021AD)) { 4104 skb = skb_vlan_untag(skb); 4105 if (unlikely(!skb)) 4106 goto out; 4107 } 4108 4109 #ifdef CONFIG_NET_CLS_ACT 4110 if (skb->tc_verd & TC_NCLS) { 4111 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd); 4112 goto ncls; 4113 } 4114 #endif 4115 4116 if (pfmemalloc) 4117 goto skip_taps; 4118 4119 list_for_each_entry_rcu(ptype, &ptype_all, list) { 4120 if (pt_prev) 4121 ret = deliver_skb(skb, pt_prev, orig_dev); 4122 pt_prev = ptype; 4123 } 4124 4125 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { 4126 if (pt_prev) 4127 ret = deliver_skb(skb, pt_prev, orig_dev); 4128 pt_prev = ptype; 4129 } 4130 4131 skip_taps: 4132 #ifdef CONFIG_NET_INGRESS 4133 if (static_key_false(&ingress_needed)) { 4134 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev); 4135 if (!skb) 4136 goto out; 4137 4138 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0) 4139 goto out; 4140 } 4141 #endif 4142 #ifdef CONFIG_NET_CLS_ACT 4143 skb->tc_verd = 0; 4144 ncls: 4145 #endif 4146 if (pfmemalloc && !skb_pfmemalloc_protocol(skb)) 4147 goto drop; 4148 4149 if (skb_vlan_tag_present(skb)) { 4150 if (pt_prev) { 4151 ret = deliver_skb(skb, pt_prev, orig_dev); 4152 pt_prev = NULL; 4153 } 4154 if (vlan_do_receive(&skb)) 4155 goto another_round; 4156 else if (unlikely(!skb)) 4157 goto out; 4158 } 4159 4160 rx_handler = rcu_dereference(skb->dev->rx_handler); 4161 if (rx_handler) { 4162 if (pt_prev) { 4163 ret = deliver_skb(skb, pt_prev, orig_dev); 4164 pt_prev = NULL; 4165 } 4166 switch (rx_handler(&skb)) { 4167 case RX_HANDLER_CONSUMED: 4168 ret = NET_RX_SUCCESS; 4169 goto out; 4170 case RX_HANDLER_ANOTHER: 4171 goto another_round; 4172 case RX_HANDLER_EXACT: 4173 deliver_exact = true; 4174 case RX_HANDLER_PASS: 4175 break; 4176 default: 4177 BUG(); 4178 } 4179 } 4180 4181 if (unlikely(skb_vlan_tag_present(skb))) { 4182 if (skb_vlan_tag_get_id(skb)) 4183 skb->pkt_type = PACKET_OTHERHOST; 4184 /* Note: we might in the future use prio bits 4185 * and set skb->priority like in vlan_do_receive() 4186 * For the time being, just ignore Priority Code Point 4187 */ 4188 skb->vlan_tci = 0; 4189 } 4190 4191 type = skb->protocol; 4192 4193 /* deliver only exact match when indicated */ 4194 if (likely(!deliver_exact)) { 4195 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4196 &ptype_base[ntohs(type) & 4197 PTYPE_HASH_MASK]); 4198 } 4199 4200 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4201 &orig_dev->ptype_specific); 4202 4203 if (unlikely(skb->dev != orig_dev)) { 4204 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4205 &skb->dev->ptype_specific); 4206 } 4207 4208 if (pt_prev) { 4209 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) 4210 goto drop; 4211 else 4212 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 4213 } else { 4214 drop: 4215 if (!deliver_exact) 4216 atomic_long_inc(&skb->dev->rx_dropped); 4217 else 4218 atomic_long_inc(&skb->dev->rx_nohandler); 4219 kfree_skb(skb); 4220 /* Jamal, now you will not able to escape explaining 4221 * me how you were going to use this. :-) 4222 */ 4223 ret = NET_RX_DROP; 4224 } 4225 4226 out: 4227 return ret; 4228 } 4229 4230 static int __netif_receive_skb(struct sk_buff *skb) 4231 { 4232 int ret; 4233 4234 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) { 4235 unsigned long pflags = current->flags; 4236 4237 /* 4238 * PFMEMALLOC skbs are special, they should 4239 * - be delivered to SOCK_MEMALLOC sockets only 4240 * - stay away from userspace 4241 * - have bounded memory usage 4242 * 4243 * Use PF_MEMALLOC as this saves us from propagating the allocation 4244 * context down to all allocation sites. 4245 */ 4246 current->flags |= PF_MEMALLOC; 4247 ret = __netif_receive_skb_core(skb, true); 4248 tsk_restore_flags(current, pflags, PF_MEMALLOC); 4249 } else 4250 ret = __netif_receive_skb_core(skb, false); 4251 4252 return ret; 4253 } 4254 4255 static int netif_receive_skb_internal(struct sk_buff *skb) 4256 { 4257 int ret; 4258 4259 net_timestamp_check(netdev_tstamp_prequeue, skb); 4260 4261 if (skb_defer_rx_timestamp(skb)) 4262 return NET_RX_SUCCESS; 4263 4264 rcu_read_lock(); 4265 4266 #ifdef CONFIG_RPS 4267 if (static_key_false(&rps_needed)) { 4268 struct rps_dev_flow voidflow, *rflow = &voidflow; 4269 int cpu = get_rps_cpu(skb->dev, skb, &rflow); 4270 4271 if (cpu >= 0) { 4272 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 4273 rcu_read_unlock(); 4274 return ret; 4275 } 4276 } 4277 #endif 4278 ret = __netif_receive_skb(skb); 4279 rcu_read_unlock(); 4280 return ret; 4281 } 4282 4283 /** 4284 * netif_receive_skb - process receive buffer from network 4285 * @skb: buffer to process 4286 * 4287 * netif_receive_skb() is the main receive data processing function. 4288 * It always succeeds. The buffer may be dropped during processing 4289 * for congestion control or by the protocol layers. 4290 * 4291 * This function may only be called from softirq context and interrupts 4292 * should be enabled. 4293 * 4294 * Return values (usually ignored): 4295 * NET_RX_SUCCESS: no congestion 4296 * NET_RX_DROP: packet was dropped 4297 */ 4298 int netif_receive_skb(struct sk_buff *skb) 4299 { 4300 trace_netif_receive_skb_entry(skb); 4301 4302 return netif_receive_skb_internal(skb); 4303 } 4304 EXPORT_SYMBOL(netif_receive_skb); 4305 4306 DEFINE_PER_CPU(struct work_struct, flush_works); 4307 4308 /* Network device is going away, flush any packets still pending */ 4309 static void flush_backlog(struct work_struct *work) 4310 { 4311 struct sk_buff *skb, *tmp; 4312 struct softnet_data *sd; 4313 4314 local_bh_disable(); 4315 sd = this_cpu_ptr(&softnet_data); 4316 4317 local_irq_disable(); 4318 rps_lock(sd); 4319 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { 4320 if (skb->dev->reg_state == NETREG_UNREGISTERING) { 4321 __skb_unlink(skb, &sd->input_pkt_queue); 4322 kfree_skb(skb); 4323 input_queue_head_incr(sd); 4324 } 4325 } 4326 rps_unlock(sd); 4327 local_irq_enable(); 4328 4329 skb_queue_walk_safe(&sd->process_queue, skb, tmp) { 4330 if (skb->dev->reg_state == NETREG_UNREGISTERING) { 4331 __skb_unlink(skb, &sd->process_queue); 4332 kfree_skb(skb); 4333 input_queue_head_incr(sd); 4334 } 4335 } 4336 local_bh_enable(); 4337 } 4338 4339 static void flush_all_backlogs(void) 4340 { 4341 unsigned int cpu; 4342 4343 get_online_cpus(); 4344 4345 for_each_online_cpu(cpu) 4346 queue_work_on(cpu, system_highpri_wq, 4347 per_cpu_ptr(&flush_works, cpu)); 4348 4349 for_each_online_cpu(cpu) 4350 flush_work(per_cpu_ptr(&flush_works, cpu)); 4351 4352 put_online_cpus(); 4353 } 4354 4355 static int napi_gro_complete(struct sk_buff *skb) 4356 { 4357 struct packet_offload *ptype; 4358 __be16 type = skb->protocol; 4359 struct list_head *head = &offload_base; 4360 int err = -ENOENT; 4361 4362 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); 4363 4364 if (NAPI_GRO_CB(skb)->count == 1) { 4365 skb_shinfo(skb)->gso_size = 0; 4366 goto out; 4367 } 4368 4369 rcu_read_lock(); 4370 list_for_each_entry_rcu(ptype, head, list) { 4371 if (ptype->type != type || !ptype->callbacks.gro_complete) 4372 continue; 4373 4374 err = ptype->callbacks.gro_complete(skb, 0); 4375 break; 4376 } 4377 rcu_read_unlock(); 4378 4379 if (err) { 4380 WARN_ON(&ptype->list == head); 4381 kfree_skb(skb); 4382 return NET_RX_SUCCESS; 4383 } 4384 4385 out: 4386 return netif_receive_skb_internal(skb); 4387 } 4388 4389 /* napi->gro_list contains packets ordered by age. 4390 * youngest packets at the head of it. 4391 * Complete skbs in reverse order to reduce latencies. 4392 */ 4393 void napi_gro_flush(struct napi_struct *napi, bool flush_old) 4394 { 4395 struct sk_buff *skb, *prev = NULL; 4396 4397 /* scan list and build reverse chain */ 4398 for (skb = napi->gro_list; skb != NULL; skb = skb->next) { 4399 skb->prev = prev; 4400 prev = skb; 4401 } 4402 4403 for (skb = prev; skb; skb = prev) { 4404 skb->next = NULL; 4405 4406 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) 4407 return; 4408 4409 prev = skb->prev; 4410 napi_gro_complete(skb); 4411 napi->gro_count--; 4412 } 4413 4414 napi->gro_list = NULL; 4415 } 4416 EXPORT_SYMBOL(napi_gro_flush); 4417 4418 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb) 4419 { 4420 struct sk_buff *p; 4421 unsigned int maclen = skb->dev->hard_header_len; 4422 u32 hash = skb_get_hash_raw(skb); 4423 4424 for (p = napi->gro_list; p; p = p->next) { 4425 unsigned long diffs; 4426 4427 NAPI_GRO_CB(p)->flush = 0; 4428 4429 if (hash != skb_get_hash_raw(p)) { 4430 NAPI_GRO_CB(p)->same_flow = 0; 4431 continue; 4432 } 4433 4434 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; 4435 diffs |= p->vlan_tci ^ skb->vlan_tci; 4436 diffs |= skb_metadata_dst_cmp(p, skb); 4437 if (maclen == ETH_HLEN) 4438 diffs |= compare_ether_header(skb_mac_header(p), 4439 skb_mac_header(skb)); 4440 else if (!diffs) 4441 diffs = memcmp(skb_mac_header(p), 4442 skb_mac_header(skb), 4443 maclen); 4444 NAPI_GRO_CB(p)->same_flow = !diffs; 4445 } 4446 } 4447 4448 static void skb_gro_reset_offset(struct sk_buff *skb) 4449 { 4450 const struct skb_shared_info *pinfo = skb_shinfo(skb); 4451 const skb_frag_t *frag0 = &pinfo->frags[0]; 4452 4453 NAPI_GRO_CB(skb)->data_offset = 0; 4454 NAPI_GRO_CB(skb)->frag0 = NULL; 4455 NAPI_GRO_CB(skb)->frag0_len = 0; 4456 4457 if (skb_mac_header(skb) == skb_tail_pointer(skb) && 4458 pinfo->nr_frags && 4459 !PageHighMem(skb_frag_page(frag0))) { 4460 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); 4461 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0); 4462 } 4463 } 4464 4465 static void gro_pull_from_frag0(struct sk_buff *skb, int grow) 4466 { 4467 struct skb_shared_info *pinfo = skb_shinfo(skb); 4468 4469 BUG_ON(skb->end - skb->tail < grow); 4470 4471 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); 4472 4473 skb->data_len -= grow; 4474 skb->tail += grow; 4475 4476 pinfo->frags[0].page_offset += grow; 4477 skb_frag_size_sub(&pinfo->frags[0], grow); 4478 4479 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { 4480 skb_frag_unref(skb, 0); 4481 memmove(pinfo->frags, pinfo->frags + 1, 4482 --pinfo->nr_frags * sizeof(pinfo->frags[0])); 4483 } 4484 } 4485 4486 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 4487 { 4488 struct sk_buff **pp = NULL; 4489 struct packet_offload *ptype; 4490 __be16 type = skb->protocol; 4491 struct list_head *head = &offload_base; 4492 int same_flow; 4493 enum gro_result ret; 4494 int grow; 4495 4496 if (!(skb->dev->features & NETIF_F_GRO)) 4497 goto normal; 4498 4499 if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad) 4500 goto normal; 4501 4502 gro_list_prepare(napi, skb); 4503 4504 rcu_read_lock(); 4505 list_for_each_entry_rcu(ptype, head, list) { 4506 if (ptype->type != type || !ptype->callbacks.gro_receive) 4507 continue; 4508 4509 skb_set_network_header(skb, skb_gro_offset(skb)); 4510 skb_reset_mac_len(skb); 4511 NAPI_GRO_CB(skb)->same_flow = 0; 4512 NAPI_GRO_CB(skb)->flush = 0; 4513 NAPI_GRO_CB(skb)->free = 0; 4514 NAPI_GRO_CB(skb)->encap_mark = 0; 4515 NAPI_GRO_CB(skb)->recursion_counter = 0; 4516 NAPI_GRO_CB(skb)->is_fou = 0; 4517 NAPI_GRO_CB(skb)->is_atomic = 1; 4518 NAPI_GRO_CB(skb)->gro_remcsum_start = 0; 4519 4520 /* Setup for GRO checksum validation */ 4521 switch (skb->ip_summed) { 4522 case CHECKSUM_COMPLETE: 4523 NAPI_GRO_CB(skb)->csum = skb->csum; 4524 NAPI_GRO_CB(skb)->csum_valid = 1; 4525 NAPI_GRO_CB(skb)->csum_cnt = 0; 4526 break; 4527 case CHECKSUM_UNNECESSARY: 4528 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; 4529 NAPI_GRO_CB(skb)->csum_valid = 0; 4530 break; 4531 default: 4532 NAPI_GRO_CB(skb)->csum_cnt = 0; 4533 NAPI_GRO_CB(skb)->csum_valid = 0; 4534 } 4535 4536 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb); 4537 break; 4538 } 4539 rcu_read_unlock(); 4540 4541 if (&ptype->list == head) 4542 goto normal; 4543 4544 same_flow = NAPI_GRO_CB(skb)->same_flow; 4545 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; 4546 4547 if (pp) { 4548 struct sk_buff *nskb = *pp; 4549 4550 *pp = nskb->next; 4551 nskb->next = NULL; 4552 napi_gro_complete(nskb); 4553 napi->gro_count--; 4554 } 4555 4556 if (same_flow) 4557 goto ok; 4558 4559 if (NAPI_GRO_CB(skb)->flush) 4560 goto normal; 4561 4562 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) { 4563 struct sk_buff *nskb = napi->gro_list; 4564 4565 /* locate the end of the list to select the 'oldest' flow */ 4566 while (nskb->next) { 4567 pp = &nskb->next; 4568 nskb = *pp; 4569 } 4570 *pp = NULL; 4571 nskb->next = NULL; 4572 napi_gro_complete(nskb); 4573 } else { 4574 napi->gro_count++; 4575 } 4576 NAPI_GRO_CB(skb)->count = 1; 4577 NAPI_GRO_CB(skb)->age = jiffies; 4578 NAPI_GRO_CB(skb)->last = skb; 4579 skb_shinfo(skb)->gso_size = skb_gro_len(skb); 4580 skb->next = napi->gro_list; 4581 napi->gro_list = skb; 4582 ret = GRO_HELD; 4583 4584 pull: 4585 grow = skb_gro_offset(skb) - skb_headlen(skb); 4586 if (grow > 0) 4587 gro_pull_from_frag0(skb, grow); 4588 ok: 4589 return ret; 4590 4591 normal: 4592 ret = GRO_NORMAL; 4593 goto pull; 4594 } 4595 4596 struct packet_offload *gro_find_receive_by_type(__be16 type) 4597 { 4598 struct list_head *offload_head = &offload_base; 4599 struct packet_offload *ptype; 4600 4601 list_for_each_entry_rcu(ptype, offload_head, list) { 4602 if (ptype->type != type || !ptype->callbacks.gro_receive) 4603 continue; 4604 return ptype; 4605 } 4606 return NULL; 4607 } 4608 EXPORT_SYMBOL(gro_find_receive_by_type); 4609 4610 struct packet_offload *gro_find_complete_by_type(__be16 type) 4611 { 4612 struct list_head *offload_head = &offload_base; 4613 struct packet_offload *ptype; 4614 4615 list_for_each_entry_rcu(ptype, offload_head, list) { 4616 if (ptype->type != type || !ptype->callbacks.gro_complete) 4617 continue; 4618 return ptype; 4619 } 4620 return NULL; 4621 } 4622 EXPORT_SYMBOL(gro_find_complete_by_type); 4623 4624 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) 4625 { 4626 switch (ret) { 4627 case GRO_NORMAL: 4628 if (netif_receive_skb_internal(skb)) 4629 ret = GRO_DROP; 4630 break; 4631 4632 case GRO_DROP: 4633 kfree_skb(skb); 4634 break; 4635 4636 case GRO_MERGED_FREE: 4637 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) { 4638 skb_dst_drop(skb); 4639 kmem_cache_free(skbuff_head_cache, skb); 4640 } else { 4641 __kfree_skb(skb); 4642 } 4643 break; 4644 4645 case GRO_HELD: 4646 case GRO_MERGED: 4647 break; 4648 } 4649 4650 return ret; 4651 } 4652 4653 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 4654 { 4655 skb_mark_napi_id(skb, napi); 4656 trace_napi_gro_receive_entry(skb); 4657 4658 skb_gro_reset_offset(skb); 4659 4660 return napi_skb_finish(dev_gro_receive(napi, skb), skb); 4661 } 4662 EXPORT_SYMBOL(napi_gro_receive); 4663 4664 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) 4665 { 4666 if (unlikely(skb->pfmemalloc)) { 4667 consume_skb(skb); 4668 return; 4669 } 4670 __skb_pull(skb, skb_headlen(skb)); 4671 /* restore the reserve we had after netdev_alloc_skb_ip_align() */ 4672 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); 4673 skb->vlan_tci = 0; 4674 skb->dev = napi->dev; 4675 skb->skb_iif = 0; 4676 skb->encapsulation = 0; 4677 skb_shinfo(skb)->gso_type = 0; 4678 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 4679 4680 napi->skb = skb; 4681 } 4682 4683 struct sk_buff *napi_get_frags(struct napi_struct *napi) 4684 { 4685 struct sk_buff *skb = napi->skb; 4686 4687 if (!skb) { 4688 skb = napi_alloc_skb(napi, GRO_MAX_HEAD); 4689 if (skb) { 4690 napi->skb = skb; 4691 skb_mark_napi_id(skb, napi); 4692 } 4693 } 4694 return skb; 4695 } 4696 EXPORT_SYMBOL(napi_get_frags); 4697 4698 static gro_result_t napi_frags_finish(struct napi_struct *napi, 4699 struct sk_buff *skb, 4700 gro_result_t ret) 4701 { 4702 switch (ret) { 4703 case GRO_NORMAL: 4704 case GRO_HELD: 4705 __skb_push(skb, ETH_HLEN); 4706 skb->protocol = eth_type_trans(skb, skb->dev); 4707 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb)) 4708 ret = GRO_DROP; 4709 break; 4710 4711 case GRO_DROP: 4712 case GRO_MERGED_FREE: 4713 napi_reuse_skb(napi, skb); 4714 break; 4715 4716 case GRO_MERGED: 4717 break; 4718 } 4719 4720 return ret; 4721 } 4722 4723 /* Upper GRO stack assumes network header starts at gro_offset=0 4724 * Drivers could call both napi_gro_frags() and napi_gro_receive() 4725 * We copy ethernet header into skb->data to have a common layout. 4726 */ 4727 static struct sk_buff *napi_frags_skb(struct napi_struct *napi) 4728 { 4729 struct sk_buff *skb = napi->skb; 4730 const struct ethhdr *eth; 4731 unsigned int hlen = sizeof(*eth); 4732 4733 napi->skb = NULL; 4734 4735 skb_reset_mac_header(skb); 4736 skb_gro_reset_offset(skb); 4737 4738 eth = skb_gro_header_fast(skb, 0); 4739 if (unlikely(skb_gro_header_hard(skb, hlen))) { 4740 eth = skb_gro_header_slow(skb, hlen, 0); 4741 if (unlikely(!eth)) { 4742 net_warn_ratelimited("%s: dropping impossible skb from %s\n", 4743 __func__, napi->dev->name); 4744 napi_reuse_skb(napi, skb); 4745 return NULL; 4746 } 4747 } else { 4748 gro_pull_from_frag0(skb, hlen); 4749 NAPI_GRO_CB(skb)->frag0 += hlen; 4750 NAPI_GRO_CB(skb)->frag0_len -= hlen; 4751 } 4752 __skb_pull(skb, hlen); 4753 4754 /* 4755 * This works because the only protocols we care about don't require 4756 * special handling. 4757 * We'll fix it up properly in napi_frags_finish() 4758 */ 4759 skb->protocol = eth->h_proto; 4760 4761 return skb; 4762 } 4763 4764 gro_result_t napi_gro_frags(struct napi_struct *napi) 4765 { 4766 struct sk_buff *skb = napi_frags_skb(napi); 4767 4768 if (!skb) 4769 return GRO_DROP; 4770 4771 trace_napi_gro_frags_entry(skb); 4772 4773 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); 4774 } 4775 EXPORT_SYMBOL(napi_gro_frags); 4776 4777 /* Compute the checksum from gro_offset and return the folded value 4778 * after adding in any pseudo checksum. 4779 */ 4780 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb) 4781 { 4782 __wsum wsum; 4783 __sum16 sum; 4784 4785 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); 4786 4787 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ 4788 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); 4789 if (likely(!sum)) { 4790 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 4791 !skb->csum_complete_sw) 4792 netdev_rx_csum_fault(skb->dev); 4793 } 4794 4795 NAPI_GRO_CB(skb)->csum = wsum; 4796 NAPI_GRO_CB(skb)->csum_valid = 1; 4797 4798 return sum; 4799 } 4800 EXPORT_SYMBOL(__skb_gro_checksum_complete); 4801 4802 /* 4803 * net_rps_action_and_irq_enable sends any pending IPI's for rps. 4804 * Note: called with local irq disabled, but exits with local irq enabled. 4805 */ 4806 static void net_rps_action_and_irq_enable(struct softnet_data *sd) 4807 { 4808 #ifdef CONFIG_RPS 4809 struct softnet_data *remsd = sd->rps_ipi_list; 4810 4811 if (remsd) { 4812 sd->rps_ipi_list = NULL; 4813 4814 local_irq_enable(); 4815 4816 /* Send pending IPI's to kick RPS processing on remote cpus. */ 4817 while (remsd) { 4818 struct softnet_data *next = remsd->rps_ipi_next; 4819 4820 if (cpu_online(remsd->cpu)) 4821 smp_call_function_single_async(remsd->cpu, 4822 &remsd->csd); 4823 remsd = next; 4824 } 4825 } else 4826 #endif 4827 local_irq_enable(); 4828 } 4829 4830 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) 4831 { 4832 #ifdef CONFIG_RPS 4833 return sd->rps_ipi_list != NULL; 4834 #else 4835 return false; 4836 #endif 4837 } 4838 4839 static int process_backlog(struct napi_struct *napi, int quota) 4840 { 4841 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); 4842 bool again = true; 4843 int work = 0; 4844 4845 /* Check if we have pending ipi, its better to send them now, 4846 * not waiting net_rx_action() end. 4847 */ 4848 if (sd_has_rps_ipi_waiting(sd)) { 4849 local_irq_disable(); 4850 net_rps_action_and_irq_enable(sd); 4851 } 4852 4853 napi->weight = weight_p; 4854 while (again) { 4855 struct sk_buff *skb; 4856 4857 while ((skb = __skb_dequeue(&sd->process_queue))) { 4858 rcu_read_lock(); 4859 __netif_receive_skb(skb); 4860 rcu_read_unlock(); 4861 input_queue_head_incr(sd); 4862 if (++work >= quota) 4863 return work; 4864 4865 } 4866 4867 local_irq_disable(); 4868 rps_lock(sd); 4869 if (skb_queue_empty(&sd->input_pkt_queue)) { 4870 /* 4871 * Inline a custom version of __napi_complete(). 4872 * only current cpu owns and manipulates this napi, 4873 * and NAPI_STATE_SCHED is the only possible flag set 4874 * on backlog. 4875 * We can use a plain write instead of clear_bit(), 4876 * and we dont need an smp_mb() memory barrier. 4877 */ 4878 napi->state = 0; 4879 again = false; 4880 } else { 4881 skb_queue_splice_tail_init(&sd->input_pkt_queue, 4882 &sd->process_queue); 4883 } 4884 rps_unlock(sd); 4885 local_irq_enable(); 4886 } 4887 4888 return work; 4889 } 4890 4891 /** 4892 * __napi_schedule - schedule for receive 4893 * @n: entry to schedule 4894 * 4895 * The entry's receive function will be scheduled to run. 4896 * Consider using __napi_schedule_irqoff() if hard irqs are masked. 4897 */ 4898 void __napi_schedule(struct napi_struct *n) 4899 { 4900 unsigned long flags; 4901 4902 local_irq_save(flags); 4903 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 4904 local_irq_restore(flags); 4905 } 4906 EXPORT_SYMBOL(__napi_schedule); 4907 4908 /** 4909 * __napi_schedule_irqoff - schedule for receive 4910 * @n: entry to schedule 4911 * 4912 * Variant of __napi_schedule() assuming hard irqs are masked 4913 */ 4914 void __napi_schedule_irqoff(struct napi_struct *n) 4915 { 4916 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 4917 } 4918 EXPORT_SYMBOL(__napi_schedule_irqoff); 4919 4920 void __napi_complete(struct napi_struct *n) 4921 { 4922 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state)); 4923 4924 list_del_init(&n->poll_list); 4925 smp_mb__before_atomic(); 4926 clear_bit(NAPI_STATE_SCHED, &n->state); 4927 } 4928 EXPORT_SYMBOL(__napi_complete); 4929 4930 void napi_complete_done(struct napi_struct *n, int work_done) 4931 { 4932 unsigned long flags; 4933 4934 /* 4935 * don't let napi dequeue from the cpu poll list 4936 * just in case its running on a different cpu 4937 */ 4938 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state))) 4939 return; 4940 4941 if (n->gro_list) { 4942 unsigned long timeout = 0; 4943 4944 if (work_done) 4945 timeout = n->dev->gro_flush_timeout; 4946 4947 if (timeout) 4948 hrtimer_start(&n->timer, ns_to_ktime(timeout), 4949 HRTIMER_MODE_REL_PINNED); 4950 else 4951 napi_gro_flush(n, false); 4952 } 4953 if (likely(list_empty(&n->poll_list))) { 4954 WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state)); 4955 } else { 4956 /* If n->poll_list is not empty, we need to mask irqs */ 4957 local_irq_save(flags); 4958 __napi_complete(n); 4959 local_irq_restore(flags); 4960 } 4961 } 4962 EXPORT_SYMBOL(napi_complete_done); 4963 4964 /* must be called under rcu_read_lock(), as we dont take a reference */ 4965 static struct napi_struct *napi_by_id(unsigned int napi_id) 4966 { 4967 unsigned int hash = napi_id % HASH_SIZE(napi_hash); 4968 struct napi_struct *napi; 4969 4970 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node) 4971 if (napi->napi_id == napi_id) 4972 return napi; 4973 4974 return NULL; 4975 } 4976 4977 #if defined(CONFIG_NET_RX_BUSY_POLL) 4978 #define BUSY_POLL_BUDGET 8 4979 bool sk_busy_loop(struct sock *sk, int nonblock) 4980 { 4981 unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0; 4982 int (*busy_poll)(struct napi_struct *dev); 4983 struct napi_struct *napi; 4984 int rc = false; 4985 4986 rcu_read_lock(); 4987 4988 napi = napi_by_id(sk->sk_napi_id); 4989 if (!napi) 4990 goto out; 4991 4992 /* Note: ndo_busy_poll method is optional in linux-4.5 */ 4993 busy_poll = napi->dev->netdev_ops->ndo_busy_poll; 4994 4995 do { 4996 rc = 0; 4997 local_bh_disable(); 4998 if (busy_poll) { 4999 rc = busy_poll(napi); 5000 } else if (napi_schedule_prep(napi)) { 5001 void *have = netpoll_poll_lock(napi); 5002 5003 if (test_bit(NAPI_STATE_SCHED, &napi->state)) { 5004 rc = napi->poll(napi, BUSY_POLL_BUDGET); 5005 trace_napi_poll(napi, rc, BUSY_POLL_BUDGET); 5006 if (rc == BUSY_POLL_BUDGET) { 5007 napi_complete_done(napi, rc); 5008 napi_schedule(napi); 5009 } 5010 } 5011 netpoll_poll_unlock(have); 5012 } 5013 if (rc > 0) 5014 __NET_ADD_STATS(sock_net(sk), 5015 LINUX_MIB_BUSYPOLLRXPACKETS, rc); 5016 local_bh_enable(); 5017 5018 if (rc == LL_FLUSH_FAILED) 5019 break; /* permanent failure */ 5020 5021 cpu_relax(); 5022 } while (!nonblock && skb_queue_empty(&sk->sk_receive_queue) && 5023 !need_resched() && !busy_loop_timeout(end_time)); 5024 5025 rc = !skb_queue_empty(&sk->sk_receive_queue); 5026 out: 5027 rcu_read_unlock(); 5028 return rc; 5029 } 5030 EXPORT_SYMBOL(sk_busy_loop); 5031 5032 #endif /* CONFIG_NET_RX_BUSY_POLL */ 5033 5034 void napi_hash_add(struct napi_struct *napi) 5035 { 5036 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) || 5037 test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) 5038 return; 5039 5040 spin_lock(&napi_hash_lock); 5041 5042 /* 0..NR_CPUS+1 range is reserved for sender_cpu use */ 5043 do { 5044 if (unlikely(++napi_gen_id < NR_CPUS + 1)) 5045 napi_gen_id = NR_CPUS + 1; 5046 } while (napi_by_id(napi_gen_id)); 5047 napi->napi_id = napi_gen_id; 5048 5049 hlist_add_head_rcu(&napi->napi_hash_node, 5050 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); 5051 5052 spin_unlock(&napi_hash_lock); 5053 } 5054 EXPORT_SYMBOL_GPL(napi_hash_add); 5055 5056 /* Warning : caller is responsible to make sure rcu grace period 5057 * is respected before freeing memory containing @napi 5058 */ 5059 bool napi_hash_del(struct napi_struct *napi) 5060 { 5061 bool rcu_sync_needed = false; 5062 5063 spin_lock(&napi_hash_lock); 5064 5065 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) { 5066 rcu_sync_needed = true; 5067 hlist_del_rcu(&napi->napi_hash_node); 5068 } 5069 spin_unlock(&napi_hash_lock); 5070 return rcu_sync_needed; 5071 } 5072 EXPORT_SYMBOL_GPL(napi_hash_del); 5073 5074 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer) 5075 { 5076 struct napi_struct *napi; 5077 5078 napi = container_of(timer, struct napi_struct, timer); 5079 if (napi->gro_list) 5080 napi_schedule(napi); 5081 5082 return HRTIMER_NORESTART; 5083 } 5084 5085 void netif_napi_add(struct net_device *dev, struct napi_struct *napi, 5086 int (*poll)(struct napi_struct *, int), int weight) 5087 { 5088 INIT_LIST_HEAD(&napi->poll_list); 5089 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); 5090 napi->timer.function = napi_watchdog; 5091 napi->gro_count = 0; 5092 napi->gro_list = NULL; 5093 napi->skb = NULL; 5094 napi->poll = poll; 5095 if (weight > NAPI_POLL_WEIGHT) 5096 pr_err_once("netif_napi_add() called with weight %d on device %s\n", 5097 weight, dev->name); 5098 napi->weight = weight; 5099 list_add(&napi->dev_list, &dev->napi_list); 5100 napi->dev = dev; 5101 #ifdef CONFIG_NETPOLL 5102 spin_lock_init(&napi->poll_lock); 5103 napi->poll_owner = -1; 5104 #endif 5105 set_bit(NAPI_STATE_SCHED, &napi->state); 5106 napi_hash_add(napi); 5107 } 5108 EXPORT_SYMBOL(netif_napi_add); 5109 5110 void napi_disable(struct napi_struct *n) 5111 { 5112 might_sleep(); 5113 set_bit(NAPI_STATE_DISABLE, &n->state); 5114 5115 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state)) 5116 msleep(1); 5117 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state)) 5118 msleep(1); 5119 5120 hrtimer_cancel(&n->timer); 5121 5122 clear_bit(NAPI_STATE_DISABLE, &n->state); 5123 } 5124 EXPORT_SYMBOL(napi_disable); 5125 5126 /* Must be called in process context */ 5127 void netif_napi_del(struct napi_struct *napi) 5128 { 5129 might_sleep(); 5130 if (napi_hash_del(napi)) 5131 synchronize_net(); 5132 list_del_init(&napi->dev_list); 5133 napi_free_frags(napi); 5134 5135 kfree_skb_list(napi->gro_list); 5136 napi->gro_list = NULL; 5137 napi->gro_count = 0; 5138 } 5139 EXPORT_SYMBOL(netif_napi_del); 5140 5141 static int napi_poll(struct napi_struct *n, struct list_head *repoll) 5142 { 5143 void *have; 5144 int work, weight; 5145 5146 list_del_init(&n->poll_list); 5147 5148 have = netpoll_poll_lock(n); 5149 5150 weight = n->weight; 5151 5152 /* This NAPI_STATE_SCHED test is for avoiding a race 5153 * with netpoll's poll_napi(). Only the entity which 5154 * obtains the lock and sees NAPI_STATE_SCHED set will 5155 * actually make the ->poll() call. Therefore we avoid 5156 * accidentally calling ->poll() when NAPI is not scheduled. 5157 */ 5158 work = 0; 5159 if (test_bit(NAPI_STATE_SCHED, &n->state)) { 5160 work = n->poll(n, weight); 5161 trace_napi_poll(n, work, weight); 5162 } 5163 5164 WARN_ON_ONCE(work > weight); 5165 5166 if (likely(work < weight)) 5167 goto out_unlock; 5168 5169 /* Drivers must not modify the NAPI state if they 5170 * consume the entire weight. In such cases this code 5171 * still "owns" the NAPI instance and therefore can 5172 * move the instance around on the list at-will. 5173 */ 5174 if (unlikely(napi_disable_pending(n))) { 5175 napi_complete(n); 5176 goto out_unlock; 5177 } 5178 5179 if (n->gro_list) { 5180 /* flush too old packets 5181 * If HZ < 1000, flush all packets. 5182 */ 5183 napi_gro_flush(n, HZ >= 1000); 5184 } 5185 5186 /* Some drivers may have called napi_schedule 5187 * prior to exhausting their budget. 5188 */ 5189 if (unlikely(!list_empty(&n->poll_list))) { 5190 pr_warn_once("%s: Budget exhausted after napi rescheduled\n", 5191 n->dev ? n->dev->name : "backlog"); 5192 goto out_unlock; 5193 } 5194 5195 list_add_tail(&n->poll_list, repoll); 5196 5197 out_unlock: 5198 netpoll_poll_unlock(have); 5199 5200 return work; 5201 } 5202 5203 static __latent_entropy void net_rx_action(struct softirq_action *h) 5204 { 5205 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 5206 unsigned long time_limit = jiffies + 2; 5207 int budget = netdev_budget; 5208 LIST_HEAD(list); 5209 LIST_HEAD(repoll); 5210 5211 local_irq_disable(); 5212 list_splice_init(&sd->poll_list, &list); 5213 local_irq_enable(); 5214 5215 for (;;) { 5216 struct napi_struct *n; 5217 5218 if (list_empty(&list)) { 5219 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll)) 5220 return; 5221 break; 5222 } 5223 5224 n = list_first_entry(&list, struct napi_struct, poll_list); 5225 budget -= napi_poll(n, &repoll); 5226 5227 /* If softirq window is exhausted then punt. 5228 * Allow this to run for 2 jiffies since which will allow 5229 * an average latency of 1.5/HZ. 5230 */ 5231 if (unlikely(budget <= 0 || 5232 time_after_eq(jiffies, time_limit))) { 5233 sd->time_squeeze++; 5234 break; 5235 } 5236 } 5237 5238 __kfree_skb_flush(); 5239 local_irq_disable(); 5240 5241 list_splice_tail_init(&sd->poll_list, &list); 5242 list_splice_tail(&repoll, &list); 5243 list_splice(&list, &sd->poll_list); 5244 if (!list_empty(&sd->poll_list)) 5245 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 5246 5247 net_rps_action_and_irq_enable(sd); 5248 } 5249 5250 struct netdev_adjacent { 5251 struct net_device *dev; 5252 5253 /* upper master flag, there can only be one master device per list */ 5254 bool master; 5255 5256 /* counter for the number of times this device was added to us */ 5257 u16 ref_nr; 5258 5259 /* private field for the users */ 5260 void *private; 5261 5262 struct list_head list; 5263 struct rcu_head rcu; 5264 }; 5265 5266 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev, 5267 struct list_head *adj_list) 5268 { 5269 struct netdev_adjacent *adj; 5270 5271 list_for_each_entry(adj, adj_list, list) { 5272 if (adj->dev == adj_dev) 5273 return adj; 5274 } 5275 return NULL; 5276 } 5277 5278 /** 5279 * netdev_has_upper_dev - Check if device is linked to an upper device 5280 * @dev: device 5281 * @upper_dev: upper device to check 5282 * 5283 * Find out if a device is linked to specified upper device and return true 5284 * in case it is. Note that this checks only immediate upper device, 5285 * not through a complete stack of devices. The caller must hold the RTNL lock. 5286 */ 5287 bool netdev_has_upper_dev(struct net_device *dev, 5288 struct net_device *upper_dev) 5289 { 5290 ASSERT_RTNL(); 5291 5292 return __netdev_find_adj(upper_dev, &dev->all_adj_list.upper); 5293 } 5294 EXPORT_SYMBOL(netdev_has_upper_dev); 5295 5296 /** 5297 * netdev_has_any_upper_dev - Check if device is linked to some device 5298 * @dev: device 5299 * 5300 * Find out if a device is linked to an upper device and return true in case 5301 * it is. The caller must hold the RTNL lock. 5302 */ 5303 static bool netdev_has_any_upper_dev(struct net_device *dev) 5304 { 5305 ASSERT_RTNL(); 5306 5307 return !list_empty(&dev->all_adj_list.upper); 5308 } 5309 5310 /** 5311 * netdev_master_upper_dev_get - Get master upper device 5312 * @dev: device 5313 * 5314 * Find a master upper device and return pointer to it or NULL in case 5315 * it's not there. The caller must hold the RTNL lock. 5316 */ 5317 struct net_device *netdev_master_upper_dev_get(struct net_device *dev) 5318 { 5319 struct netdev_adjacent *upper; 5320 5321 ASSERT_RTNL(); 5322 5323 if (list_empty(&dev->adj_list.upper)) 5324 return NULL; 5325 5326 upper = list_first_entry(&dev->adj_list.upper, 5327 struct netdev_adjacent, list); 5328 if (likely(upper->master)) 5329 return upper->dev; 5330 return NULL; 5331 } 5332 EXPORT_SYMBOL(netdev_master_upper_dev_get); 5333 5334 void *netdev_adjacent_get_private(struct list_head *adj_list) 5335 { 5336 struct netdev_adjacent *adj; 5337 5338 adj = list_entry(adj_list, struct netdev_adjacent, list); 5339 5340 return adj->private; 5341 } 5342 EXPORT_SYMBOL(netdev_adjacent_get_private); 5343 5344 /** 5345 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list 5346 * @dev: device 5347 * @iter: list_head ** of the current position 5348 * 5349 * Gets the next device from the dev's upper list, starting from iter 5350 * position. The caller must hold RCU read lock. 5351 */ 5352 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev, 5353 struct list_head **iter) 5354 { 5355 struct netdev_adjacent *upper; 5356 5357 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 5358 5359 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5360 5361 if (&upper->list == &dev->adj_list.upper) 5362 return NULL; 5363 5364 *iter = &upper->list; 5365 5366 return upper->dev; 5367 } 5368 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu); 5369 5370 /** 5371 * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list 5372 * @dev: device 5373 * @iter: list_head ** of the current position 5374 * 5375 * Gets the next device from the dev's upper list, starting from iter 5376 * position. The caller must hold RCU read lock. 5377 */ 5378 struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev, 5379 struct list_head **iter) 5380 { 5381 struct netdev_adjacent *upper; 5382 5383 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 5384 5385 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5386 5387 if (&upper->list == &dev->all_adj_list.upper) 5388 return NULL; 5389 5390 *iter = &upper->list; 5391 5392 return upper->dev; 5393 } 5394 EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu); 5395 5396 /** 5397 * netdev_lower_get_next_private - Get the next ->private from the 5398 * lower neighbour list 5399 * @dev: device 5400 * @iter: list_head ** of the current position 5401 * 5402 * Gets the next netdev_adjacent->private from the dev's lower neighbour 5403 * list, starting from iter position. The caller must hold either hold the 5404 * RTNL lock or its own locking that guarantees that the neighbour lower 5405 * list will remain unchanged. 5406 */ 5407 void *netdev_lower_get_next_private(struct net_device *dev, 5408 struct list_head **iter) 5409 { 5410 struct netdev_adjacent *lower; 5411 5412 lower = list_entry(*iter, struct netdev_adjacent, list); 5413 5414 if (&lower->list == &dev->adj_list.lower) 5415 return NULL; 5416 5417 *iter = lower->list.next; 5418 5419 return lower->private; 5420 } 5421 EXPORT_SYMBOL(netdev_lower_get_next_private); 5422 5423 /** 5424 * netdev_lower_get_next_private_rcu - Get the next ->private from the 5425 * lower neighbour list, RCU 5426 * variant 5427 * @dev: device 5428 * @iter: list_head ** of the current position 5429 * 5430 * Gets the next netdev_adjacent->private from the dev's lower neighbour 5431 * list, starting from iter position. The caller must hold RCU read lock. 5432 */ 5433 void *netdev_lower_get_next_private_rcu(struct net_device *dev, 5434 struct list_head **iter) 5435 { 5436 struct netdev_adjacent *lower; 5437 5438 WARN_ON_ONCE(!rcu_read_lock_held()); 5439 5440 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5441 5442 if (&lower->list == &dev->adj_list.lower) 5443 return NULL; 5444 5445 *iter = &lower->list; 5446 5447 return lower->private; 5448 } 5449 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu); 5450 5451 /** 5452 * netdev_lower_get_next - Get the next device from the lower neighbour 5453 * list 5454 * @dev: device 5455 * @iter: list_head ** of the current position 5456 * 5457 * Gets the next netdev_adjacent from the dev's lower neighbour 5458 * list, starting from iter position. The caller must hold RTNL lock or 5459 * its own locking that guarantees that the neighbour lower 5460 * list will remain unchanged. 5461 */ 5462 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter) 5463 { 5464 struct netdev_adjacent *lower; 5465 5466 lower = list_entry(*iter, struct netdev_adjacent, list); 5467 5468 if (&lower->list == &dev->adj_list.lower) 5469 return NULL; 5470 5471 *iter = lower->list.next; 5472 5473 return lower->dev; 5474 } 5475 EXPORT_SYMBOL(netdev_lower_get_next); 5476 5477 /** 5478 * netdev_all_lower_get_next - Get the next device from all lower neighbour list 5479 * @dev: device 5480 * @iter: list_head ** of the current position 5481 * 5482 * Gets the next netdev_adjacent from the dev's all lower neighbour 5483 * list, starting from iter position. The caller must hold RTNL lock or 5484 * its own locking that guarantees that the neighbour all lower 5485 * list will remain unchanged. 5486 */ 5487 struct net_device *netdev_all_lower_get_next(struct net_device *dev, struct list_head **iter) 5488 { 5489 struct netdev_adjacent *lower; 5490 5491 lower = list_entry(*iter, struct netdev_adjacent, list); 5492 5493 if (&lower->list == &dev->all_adj_list.lower) 5494 return NULL; 5495 5496 *iter = lower->list.next; 5497 5498 return lower->dev; 5499 } 5500 EXPORT_SYMBOL(netdev_all_lower_get_next); 5501 5502 /** 5503 * netdev_all_lower_get_next_rcu - Get the next device from all 5504 * lower neighbour list, RCU variant 5505 * @dev: device 5506 * @iter: list_head ** of the current position 5507 * 5508 * Gets the next netdev_adjacent from the dev's all lower neighbour 5509 * list, starting from iter position. The caller must hold RCU read lock. 5510 */ 5511 struct net_device *netdev_all_lower_get_next_rcu(struct net_device *dev, 5512 struct list_head **iter) 5513 { 5514 struct netdev_adjacent *lower; 5515 5516 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5517 5518 if (&lower->list == &dev->all_adj_list.lower) 5519 return NULL; 5520 5521 *iter = &lower->list; 5522 5523 return lower->dev; 5524 } 5525 EXPORT_SYMBOL(netdev_all_lower_get_next_rcu); 5526 5527 /** 5528 * netdev_lower_get_first_private_rcu - Get the first ->private from the 5529 * lower neighbour list, RCU 5530 * variant 5531 * @dev: device 5532 * 5533 * Gets the first netdev_adjacent->private from the dev's lower neighbour 5534 * list. The caller must hold RCU read lock. 5535 */ 5536 void *netdev_lower_get_first_private_rcu(struct net_device *dev) 5537 { 5538 struct netdev_adjacent *lower; 5539 5540 lower = list_first_or_null_rcu(&dev->adj_list.lower, 5541 struct netdev_adjacent, list); 5542 if (lower) 5543 return lower->private; 5544 return NULL; 5545 } 5546 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu); 5547 5548 /** 5549 * netdev_master_upper_dev_get_rcu - Get master upper device 5550 * @dev: device 5551 * 5552 * Find a master upper device and return pointer to it or NULL in case 5553 * it's not there. The caller must hold the RCU read lock. 5554 */ 5555 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev) 5556 { 5557 struct netdev_adjacent *upper; 5558 5559 upper = list_first_or_null_rcu(&dev->adj_list.upper, 5560 struct netdev_adjacent, list); 5561 if (upper && likely(upper->master)) 5562 return upper->dev; 5563 return NULL; 5564 } 5565 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu); 5566 5567 static int netdev_adjacent_sysfs_add(struct net_device *dev, 5568 struct net_device *adj_dev, 5569 struct list_head *dev_list) 5570 { 5571 char linkname[IFNAMSIZ+7]; 5572 sprintf(linkname, dev_list == &dev->adj_list.upper ? 5573 "upper_%s" : "lower_%s", adj_dev->name); 5574 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), 5575 linkname); 5576 } 5577 static void netdev_adjacent_sysfs_del(struct net_device *dev, 5578 char *name, 5579 struct list_head *dev_list) 5580 { 5581 char linkname[IFNAMSIZ+7]; 5582 sprintf(linkname, dev_list == &dev->adj_list.upper ? 5583 "upper_%s" : "lower_%s", name); 5584 sysfs_remove_link(&(dev->dev.kobj), linkname); 5585 } 5586 5587 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev, 5588 struct net_device *adj_dev, 5589 struct list_head *dev_list) 5590 { 5591 return (dev_list == &dev->adj_list.upper || 5592 dev_list == &dev->adj_list.lower) && 5593 net_eq(dev_net(dev), dev_net(adj_dev)); 5594 } 5595 5596 static int __netdev_adjacent_dev_insert(struct net_device *dev, 5597 struct net_device *adj_dev, 5598 u16 ref_nr, 5599 struct list_head *dev_list, 5600 void *private, bool master) 5601 { 5602 struct netdev_adjacent *adj; 5603 int ret; 5604 5605 adj = __netdev_find_adj(adj_dev, dev_list); 5606 5607 if (adj) { 5608 adj->ref_nr += ref_nr; 5609 return 0; 5610 } 5611 5612 adj = kmalloc(sizeof(*adj), GFP_KERNEL); 5613 if (!adj) 5614 return -ENOMEM; 5615 5616 adj->dev = adj_dev; 5617 adj->master = master; 5618 adj->ref_nr = ref_nr; 5619 adj->private = private; 5620 dev_hold(adj_dev); 5621 5622 pr_debug("dev_hold for %s, because of link added from %s to %s\n", 5623 adj_dev->name, dev->name, adj_dev->name); 5624 5625 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) { 5626 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list); 5627 if (ret) 5628 goto free_adj; 5629 } 5630 5631 /* Ensure that master link is always the first item in list. */ 5632 if (master) { 5633 ret = sysfs_create_link(&(dev->dev.kobj), 5634 &(adj_dev->dev.kobj), "master"); 5635 if (ret) 5636 goto remove_symlinks; 5637 5638 list_add_rcu(&adj->list, dev_list); 5639 } else { 5640 list_add_tail_rcu(&adj->list, dev_list); 5641 } 5642 5643 return 0; 5644 5645 remove_symlinks: 5646 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 5647 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 5648 free_adj: 5649 kfree(adj); 5650 dev_put(adj_dev); 5651 5652 return ret; 5653 } 5654 5655 static void __netdev_adjacent_dev_remove(struct net_device *dev, 5656 struct net_device *adj_dev, 5657 u16 ref_nr, 5658 struct list_head *dev_list) 5659 { 5660 struct netdev_adjacent *adj; 5661 5662 adj = __netdev_find_adj(adj_dev, dev_list); 5663 5664 if (!adj) { 5665 pr_err("tried to remove device %s from %s\n", 5666 dev->name, adj_dev->name); 5667 BUG(); 5668 } 5669 5670 if (adj->ref_nr > ref_nr) { 5671 pr_debug("%s to %s ref_nr-%d = %d\n", dev->name, adj_dev->name, 5672 ref_nr, adj->ref_nr-ref_nr); 5673 adj->ref_nr -= ref_nr; 5674 return; 5675 } 5676 5677 if (adj->master) 5678 sysfs_remove_link(&(dev->dev.kobj), "master"); 5679 5680 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 5681 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 5682 5683 list_del_rcu(&adj->list); 5684 pr_debug("dev_put for %s, because link removed from %s to %s\n", 5685 adj_dev->name, dev->name, adj_dev->name); 5686 dev_put(adj_dev); 5687 kfree_rcu(adj, rcu); 5688 } 5689 5690 static int __netdev_adjacent_dev_link_lists(struct net_device *dev, 5691 struct net_device *upper_dev, 5692 u16 ref_nr, 5693 struct list_head *up_list, 5694 struct list_head *down_list, 5695 void *private, bool master) 5696 { 5697 int ret; 5698 5699 ret = __netdev_adjacent_dev_insert(dev, upper_dev, ref_nr, up_list, 5700 private, master); 5701 if (ret) 5702 return ret; 5703 5704 ret = __netdev_adjacent_dev_insert(upper_dev, dev, ref_nr, down_list, 5705 private, false); 5706 if (ret) { 5707 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list); 5708 return ret; 5709 } 5710 5711 return 0; 5712 } 5713 5714 static int __netdev_adjacent_dev_link(struct net_device *dev, 5715 struct net_device *upper_dev, 5716 u16 ref_nr) 5717 { 5718 return __netdev_adjacent_dev_link_lists(dev, upper_dev, ref_nr, 5719 &dev->all_adj_list.upper, 5720 &upper_dev->all_adj_list.lower, 5721 NULL, false); 5722 } 5723 5724 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev, 5725 struct net_device *upper_dev, 5726 u16 ref_nr, 5727 struct list_head *up_list, 5728 struct list_head *down_list) 5729 { 5730 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list); 5731 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list); 5732 } 5733 5734 static void __netdev_adjacent_dev_unlink(struct net_device *dev, 5735 struct net_device *upper_dev, 5736 u16 ref_nr) 5737 { 5738 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, ref_nr, 5739 &dev->all_adj_list.upper, 5740 &upper_dev->all_adj_list.lower); 5741 } 5742 5743 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev, 5744 struct net_device *upper_dev, 5745 void *private, bool master) 5746 { 5747 int ret = __netdev_adjacent_dev_link(dev, upper_dev, 1); 5748 5749 if (ret) 5750 return ret; 5751 5752 ret = __netdev_adjacent_dev_link_lists(dev, upper_dev, 1, 5753 &dev->adj_list.upper, 5754 &upper_dev->adj_list.lower, 5755 private, master); 5756 if (ret) { 5757 __netdev_adjacent_dev_unlink(dev, upper_dev, 1); 5758 return ret; 5759 } 5760 5761 return 0; 5762 } 5763 5764 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev, 5765 struct net_device *upper_dev) 5766 { 5767 __netdev_adjacent_dev_unlink(dev, upper_dev, 1); 5768 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1, 5769 &dev->adj_list.upper, 5770 &upper_dev->adj_list.lower); 5771 } 5772 5773 static int __netdev_upper_dev_link(struct net_device *dev, 5774 struct net_device *upper_dev, bool master, 5775 void *upper_priv, void *upper_info) 5776 { 5777 struct netdev_notifier_changeupper_info changeupper_info; 5778 struct netdev_adjacent *i, *j, *to_i, *to_j; 5779 int ret = 0; 5780 5781 ASSERT_RTNL(); 5782 5783 if (dev == upper_dev) 5784 return -EBUSY; 5785 5786 /* To prevent loops, check if dev is not upper device to upper_dev. */ 5787 if (__netdev_find_adj(dev, &upper_dev->all_adj_list.upper)) 5788 return -EBUSY; 5789 5790 if (__netdev_find_adj(upper_dev, &dev->adj_list.upper)) 5791 return -EEXIST; 5792 5793 if (master && netdev_master_upper_dev_get(dev)) 5794 return -EBUSY; 5795 5796 changeupper_info.upper_dev = upper_dev; 5797 changeupper_info.master = master; 5798 changeupper_info.linking = true; 5799 changeupper_info.upper_info = upper_info; 5800 5801 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev, 5802 &changeupper_info.info); 5803 ret = notifier_to_errno(ret); 5804 if (ret) 5805 return ret; 5806 5807 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv, 5808 master); 5809 if (ret) 5810 return ret; 5811 5812 /* Now that we linked these devs, make all the upper_dev's 5813 * all_adj_list.upper visible to every dev's all_adj_list.lower an 5814 * versa, and don't forget the devices itself. All of these 5815 * links are non-neighbours. 5816 */ 5817 list_for_each_entry(i, &dev->all_adj_list.lower, list) { 5818 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) { 5819 pr_debug("Interlinking %s with %s, non-neighbour\n", 5820 i->dev->name, j->dev->name); 5821 ret = __netdev_adjacent_dev_link(i->dev, j->dev, i->ref_nr); 5822 if (ret) 5823 goto rollback_mesh; 5824 } 5825 } 5826 5827 /* add dev to every upper_dev's upper device */ 5828 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) { 5829 pr_debug("linking %s's upper device %s with %s\n", 5830 upper_dev->name, i->dev->name, dev->name); 5831 ret = __netdev_adjacent_dev_link(dev, i->dev, i->ref_nr); 5832 if (ret) 5833 goto rollback_upper_mesh; 5834 } 5835 5836 /* add upper_dev to every dev's lower device */ 5837 list_for_each_entry(i, &dev->all_adj_list.lower, list) { 5838 pr_debug("linking %s's lower device %s with %s\n", dev->name, 5839 i->dev->name, upper_dev->name); 5840 ret = __netdev_adjacent_dev_link(i->dev, upper_dev, i->ref_nr); 5841 if (ret) 5842 goto rollback_lower_mesh; 5843 } 5844 5845 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev, 5846 &changeupper_info.info); 5847 ret = notifier_to_errno(ret); 5848 if (ret) 5849 goto rollback_lower_mesh; 5850 5851 return 0; 5852 5853 rollback_lower_mesh: 5854 to_i = i; 5855 list_for_each_entry(i, &dev->all_adj_list.lower, list) { 5856 if (i == to_i) 5857 break; 5858 __netdev_adjacent_dev_unlink(i->dev, upper_dev, i->ref_nr); 5859 } 5860 5861 i = NULL; 5862 5863 rollback_upper_mesh: 5864 to_i = i; 5865 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) { 5866 if (i == to_i) 5867 break; 5868 __netdev_adjacent_dev_unlink(dev, i->dev, i->ref_nr); 5869 } 5870 5871 i = j = NULL; 5872 5873 rollback_mesh: 5874 to_i = i; 5875 to_j = j; 5876 list_for_each_entry(i, &dev->all_adj_list.lower, list) { 5877 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) { 5878 if (i == to_i && j == to_j) 5879 break; 5880 __netdev_adjacent_dev_unlink(i->dev, j->dev, i->ref_nr); 5881 } 5882 if (i == to_i) 5883 break; 5884 } 5885 5886 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 5887 5888 return ret; 5889 } 5890 5891 /** 5892 * netdev_upper_dev_link - Add a link to the upper device 5893 * @dev: device 5894 * @upper_dev: new upper device 5895 * 5896 * Adds a link to device which is upper to this one. The caller must hold 5897 * the RTNL lock. On a failure a negative errno code is returned. 5898 * On success the reference counts are adjusted and the function 5899 * returns zero. 5900 */ 5901 int netdev_upper_dev_link(struct net_device *dev, 5902 struct net_device *upper_dev) 5903 { 5904 return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL); 5905 } 5906 EXPORT_SYMBOL(netdev_upper_dev_link); 5907 5908 /** 5909 * netdev_master_upper_dev_link - Add a master link to the upper device 5910 * @dev: device 5911 * @upper_dev: new upper device 5912 * @upper_priv: upper device private 5913 * @upper_info: upper info to be passed down via notifier 5914 * 5915 * Adds a link to device which is upper to this one. In this case, only 5916 * one master upper device can be linked, although other non-master devices 5917 * might be linked as well. The caller must hold the RTNL lock. 5918 * On a failure a negative errno code is returned. On success the reference 5919 * counts are adjusted and the function returns zero. 5920 */ 5921 int netdev_master_upper_dev_link(struct net_device *dev, 5922 struct net_device *upper_dev, 5923 void *upper_priv, void *upper_info) 5924 { 5925 return __netdev_upper_dev_link(dev, upper_dev, true, 5926 upper_priv, upper_info); 5927 } 5928 EXPORT_SYMBOL(netdev_master_upper_dev_link); 5929 5930 /** 5931 * netdev_upper_dev_unlink - Removes a link to upper device 5932 * @dev: device 5933 * @upper_dev: new upper device 5934 * 5935 * Removes a link to device which is upper to this one. The caller must hold 5936 * the RTNL lock. 5937 */ 5938 void netdev_upper_dev_unlink(struct net_device *dev, 5939 struct net_device *upper_dev) 5940 { 5941 struct netdev_notifier_changeupper_info changeupper_info; 5942 struct netdev_adjacent *i, *j; 5943 ASSERT_RTNL(); 5944 5945 changeupper_info.upper_dev = upper_dev; 5946 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev; 5947 changeupper_info.linking = false; 5948 5949 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev, 5950 &changeupper_info.info); 5951 5952 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 5953 5954 /* Here is the tricky part. We must remove all dev's lower 5955 * devices from all upper_dev's upper devices and vice 5956 * versa, to maintain the graph relationship. 5957 */ 5958 list_for_each_entry(i, &dev->all_adj_list.lower, list) 5959 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) 5960 __netdev_adjacent_dev_unlink(i->dev, j->dev, i->ref_nr); 5961 5962 /* remove also the devices itself from lower/upper device 5963 * list 5964 */ 5965 list_for_each_entry(i, &dev->all_adj_list.lower, list) 5966 __netdev_adjacent_dev_unlink(i->dev, upper_dev, i->ref_nr); 5967 5968 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) 5969 __netdev_adjacent_dev_unlink(dev, i->dev, i->ref_nr); 5970 5971 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev, 5972 &changeupper_info.info); 5973 } 5974 EXPORT_SYMBOL(netdev_upper_dev_unlink); 5975 5976 /** 5977 * netdev_bonding_info_change - Dispatch event about slave change 5978 * @dev: device 5979 * @bonding_info: info to dispatch 5980 * 5981 * Send NETDEV_BONDING_INFO to netdev notifiers with info. 5982 * The caller must hold the RTNL lock. 5983 */ 5984 void netdev_bonding_info_change(struct net_device *dev, 5985 struct netdev_bonding_info *bonding_info) 5986 { 5987 struct netdev_notifier_bonding_info info; 5988 5989 memcpy(&info.bonding_info, bonding_info, 5990 sizeof(struct netdev_bonding_info)); 5991 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev, 5992 &info.info); 5993 } 5994 EXPORT_SYMBOL(netdev_bonding_info_change); 5995 5996 static void netdev_adjacent_add_links(struct net_device *dev) 5997 { 5998 struct netdev_adjacent *iter; 5999 6000 struct net *net = dev_net(dev); 6001 6002 list_for_each_entry(iter, &dev->adj_list.upper, list) { 6003 if (!net_eq(net, dev_net(iter->dev))) 6004 continue; 6005 netdev_adjacent_sysfs_add(iter->dev, dev, 6006 &iter->dev->adj_list.lower); 6007 netdev_adjacent_sysfs_add(dev, iter->dev, 6008 &dev->adj_list.upper); 6009 } 6010 6011 list_for_each_entry(iter, &dev->adj_list.lower, list) { 6012 if (!net_eq(net, dev_net(iter->dev))) 6013 continue; 6014 netdev_adjacent_sysfs_add(iter->dev, dev, 6015 &iter->dev->adj_list.upper); 6016 netdev_adjacent_sysfs_add(dev, iter->dev, 6017 &dev->adj_list.lower); 6018 } 6019 } 6020 6021 static void netdev_adjacent_del_links(struct net_device *dev) 6022 { 6023 struct netdev_adjacent *iter; 6024 6025 struct net *net = dev_net(dev); 6026 6027 list_for_each_entry(iter, &dev->adj_list.upper, list) { 6028 if (!net_eq(net, dev_net(iter->dev))) 6029 continue; 6030 netdev_adjacent_sysfs_del(iter->dev, dev->name, 6031 &iter->dev->adj_list.lower); 6032 netdev_adjacent_sysfs_del(dev, iter->dev->name, 6033 &dev->adj_list.upper); 6034 } 6035 6036 list_for_each_entry(iter, &dev->adj_list.lower, list) { 6037 if (!net_eq(net, dev_net(iter->dev))) 6038 continue; 6039 netdev_adjacent_sysfs_del(iter->dev, dev->name, 6040 &iter->dev->adj_list.upper); 6041 netdev_adjacent_sysfs_del(dev, iter->dev->name, 6042 &dev->adj_list.lower); 6043 } 6044 } 6045 6046 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname) 6047 { 6048 struct netdev_adjacent *iter; 6049 6050 struct net *net = dev_net(dev); 6051 6052 list_for_each_entry(iter, &dev->adj_list.upper, list) { 6053 if (!net_eq(net, dev_net(iter->dev))) 6054 continue; 6055 netdev_adjacent_sysfs_del(iter->dev, oldname, 6056 &iter->dev->adj_list.lower); 6057 netdev_adjacent_sysfs_add(iter->dev, dev, 6058 &iter->dev->adj_list.lower); 6059 } 6060 6061 list_for_each_entry(iter, &dev->adj_list.lower, list) { 6062 if (!net_eq(net, dev_net(iter->dev))) 6063 continue; 6064 netdev_adjacent_sysfs_del(iter->dev, oldname, 6065 &iter->dev->adj_list.upper); 6066 netdev_adjacent_sysfs_add(iter->dev, dev, 6067 &iter->dev->adj_list.upper); 6068 } 6069 } 6070 6071 void *netdev_lower_dev_get_private(struct net_device *dev, 6072 struct net_device *lower_dev) 6073 { 6074 struct netdev_adjacent *lower; 6075 6076 if (!lower_dev) 6077 return NULL; 6078 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower); 6079 if (!lower) 6080 return NULL; 6081 6082 return lower->private; 6083 } 6084 EXPORT_SYMBOL(netdev_lower_dev_get_private); 6085 6086 6087 int dev_get_nest_level(struct net_device *dev) 6088 { 6089 struct net_device *lower = NULL; 6090 struct list_head *iter; 6091 int max_nest = -1; 6092 int nest; 6093 6094 ASSERT_RTNL(); 6095 6096 netdev_for_each_lower_dev(dev, lower, iter) { 6097 nest = dev_get_nest_level(lower); 6098 if (max_nest < nest) 6099 max_nest = nest; 6100 } 6101 6102 return max_nest + 1; 6103 } 6104 EXPORT_SYMBOL(dev_get_nest_level); 6105 6106 /** 6107 * netdev_lower_change - Dispatch event about lower device state change 6108 * @lower_dev: device 6109 * @lower_state_info: state to dispatch 6110 * 6111 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info. 6112 * The caller must hold the RTNL lock. 6113 */ 6114 void netdev_lower_state_changed(struct net_device *lower_dev, 6115 void *lower_state_info) 6116 { 6117 struct netdev_notifier_changelowerstate_info changelowerstate_info; 6118 6119 ASSERT_RTNL(); 6120 changelowerstate_info.lower_state_info = lower_state_info; 6121 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev, 6122 &changelowerstate_info.info); 6123 } 6124 EXPORT_SYMBOL(netdev_lower_state_changed); 6125 6126 int netdev_default_l2upper_neigh_construct(struct net_device *dev, 6127 struct neighbour *n) 6128 { 6129 struct net_device *lower_dev, *stop_dev; 6130 struct list_head *iter; 6131 int err; 6132 6133 netdev_for_each_lower_dev(dev, lower_dev, iter) { 6134 if (!lower_dev->netdev_ops->ndo_neigh_construct) 6135 continue; 6136 err = lower_dev->netdev_ops->ndo_neigh_construct(lower_dev, n); 6137 if (err) { 6138 stop_dev = lower_dev; 6139 goto rollback; 6140 } 6141 } 6142 return 0; 6143 6144 rollback: 6145 netdev_for_each_lower_dev(dev, lower_dev, iter) { 6146 if (lower_dev == stop_dev) 6147 break; 6148 if (!lower_dev->netdev_ops->ndo_neigh_destroy) 6149 continue; 6150 lower_dev->netdev_ops->ndo_neigh_destroy(lower_dev, n); 6151 } 6152 return err; 6153 } 6154 EXPORT_SYMBOL_GPL(netdev_default_l2upper_neigh_construct); 6155 6156 void netdev_default_l2upper_neigh_destroy(struct net_device *dev, 6157 struct neighbour *n) 6158 { 6159 struct net_device *lower_dev; 6160 struct list_head *iter; 6161 6162 netdev_for_each_lower_dev(dev, lower_dev, iter) { 6163 if (!lower_dev->netdev_ops->ndo_neigh_destroy) 6164 continue; 6165 lower_dev->netdev_ops->ndo_neigh_destroy(lower_dev, n); 6166 } 6167 } 6168 EXPORT_SYMBOL_GPL(netdev_default_l2upper_neigh_destroy); 6169 6170 static void dev_change_rx_flags(struct net_device *dev, int flags) 6171 { 6172 const struct net_device_ops *ops = dev->netdev_ops; 6173 6174 if (ops->ndo_change_rx_flags) 6175 ops->ndo_change_rx_flags(dev, flags); 6176 } 6177 6178 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify) 6179 { 6180 unsigned int old_flags = dev->flags; 6181 kuid_t uid; 6182 kgid_t gid; 6183 6184 ASSERT_RTNL(); 6185 6186 dev->flags |= IFF_PROMISC; 6187 dev->promiscuity += inc; 6188 if (dev->promiscuity == 0) { 6189 /* 6190 * Avoid overflow. 6191 * If inc causes overflow, untouch promisc and return error. 6192 */ 6193 if (inc < 0) 6194 dev->flags &= ~IFF_PROMISC; 6195 else { 6196 dev->promiscuity -= inc; 6197 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n", 6198 dev->name); 6199 return -EOVERFLOW; 6200 } 6201 } 6202 if (dev->flags != old_flags) { 6203 pr_info("device %s %s promiscuous mode\n", 6204 dev->name, 6205 dev->flags & IFF_PROMISC ? "entered" : "left"); 6206 if (audit_enabled) { 6207 current_uid_gid(&uid, &gid); 6208 audit_log(current->audit_context, GFP_ATOMIC, 6209 AUDIT_ANOM_PROMISCUOUS, 6210 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u", 6211 dev->name, (dev->flags & IFF_PROMISC), 6212 (old_flags & IFF_PROMISC), 6213 from_kuid(&init_user_ns, audit_get_loginuid(current)), 6214 from_kuid(&init_user_ns, uid), 6215 from_kgid(&init_user_ns, gid), 6216 audit_get_sessionid(current)); 6217 } 6218 6219 dev_change_rx_flags(dev, IFF_PROMISC); 6220 } 6221 if (notify) 6222 __dev_notify_flags(dev, old_flags, IFF_PROMISC); 6223 return 0; 6224 } 6225 6226 /** 6227 * dev_set_promiscuity - update promiscuity count on a device 6228 * @dev: device 6229 * @inc: modifier 6230 * 6231 * Add or remove promiscuity from a device. While the count in the device 6232 * remains above zero the interface remains promiscuous. Once it hits zero 6233 * the device reverts back to normal filtering operation. A negative inc 6234 * value is used to drop promiscuity on the device. 6235 * Return 0 if successful or a negative errno code on error. 6236 */ 6237 int dev_set_promiscuity(struct net_device *dev, int inc) 6238 { 6239 unsigned int old_flags = dev->flags; 6240 int err; 6241 6242 err = __dev_set_promiscuity(dev, inc, true); 6243 if (err < 0) 6244 return err; 6245 if (dev->flags != old_flags) 6246 dev_set_rx_mode(dev); 6247 return err; 6248 } 6249 EXPORT_SYMBOL(dev_set_promiscuity); 6250 6251 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify) 6252 { 6253 unsigned int old_flags = dev->flags, old_gflags = dev->gflags; 6254 6255 ASSERT_RTNL(); 6256 6257 dev->flags |= IFF_ALLMULTI; 6258 dev->allmulti += inc; 6259 if (dev->allmulti == 0) { 6260 /* 6261 * Avoid overflow. 6262 * If inc causes overflow, untouch allmulti and return error. 6263 */ 6264 if (inc < 0) 6265 dev->flags &= ~IFF_ALLMULTI; 6266 else { 6267 dev->allmulti -= inc; 6268 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n", 6269 dev->name); 6270 return -EOVERFLOW; 6271 } 6272 } 6273 if (dev->flags ^ old_flags) { 6274 dev_change_rx_flags(dev, IFF_ALLMULTI); 6275 dev_set_rx_mode(dev); 6276 if (notify) 6277 __dev_notify_flags(dev, old_flags, 6278 dev->gflags ^ old_gflags); 6279 } 6280 return 0; 6281 } 6282 6283 /** 6284 * dev_set_allmulti - update allmulti count on a device 6285 * @dev: device 6286 * @inc: modifier 6287 * 6288 * Add or remove reception of all multicast frames to a device. While the 6289 * count in the device remains above zero the interface remains listening 6290 * to all interfaces. Once it hits zero the device reverts back to normal 6291 * filtering operation. A negative @inc value is used to drop the counter 6292 * when releasing a resource needing all multicasts. 6293 * Return 0 if successful or a negative errno code on error. 6294 */ 6295 6296 int dev_set_allmulti(struct net_device *dev, int inc) 6297 { 6298 return __dev_set_allmulti(dev, inc, true); 6299 } 6300 EXPORT_SYMBOL(dev_set_allmulti); 6301 6302 /* 6303 * Upload unicast and multicast address lists to device and 6304 * configure RX filtering. When the device doesn't support unicast 6305 * filtering it is put in promiscuous mode while unicast addresses 6306 * are present. 6307 */ 6308 void __dev_set_rx_mode(struct net_device *dev) 6309 { 6310 const struct net_device_ops *ops = dev->netdev_ops; 6311 6312 /* dev_open will call this function so the list will stay sane. */ 6313 if (!(dev->flags&IFF_UP)) 6314 return; 6315 6316 if (!netif_device_present(dev)) 6317 return; 6318 6319 if (!(dev->priv_flags & IFF_UNICAST_FLT)) { 6320 /* Unicast addresses changes may only happen under the rtnl, 6321 * therefore calling __dev_set_promiscuity here is safe. 6322 */ 6323 if (!netdev_uc_empty(dev) && !dev->uc_promisc) { 6324 __dev_set_promiscuity(dev, 1, false); 6325 dev->uc_promisc = true; 6326 } else if (netdev_uc_empty(dev) && dev->uc_promisc) { 6327 __dev_set_promiscuity(dev, -1, false); 6328 dev->uc_promisc = false; 6329 } 6330 } 6331 6332 if (ops->ndo_set_rx_mode) 6333 ops->ndo_set_rx_mode(dev); 6334 } 6335 6336 void dev_set_rx_mode(struct net_device *dev) 6337 { 6338 netif_addr_lock_bh(dev); 6339 __dev_set_rx_mode(dev); 6340 netif_addr_unlock_bh(dev); 6341 } 6342 6343 /** 6344 * dev_get_flags - get flags reported to userspace 6345 * @dev: device 6346 * 6347 * Get the combination of flag bits exported through APIs to userspace. 6348 */ 6349 unsigned int dev_get_flags(const struct net_device *dev) 6350 { 6351 unsigned int flags; 6352 6353 flags = (dev->flags & ~(IFF_PROMISC | 6354 IFF_ALLMULTI | 6355 IFF_RUNNING | 6356 IFF_LOWER_UP | 6357 IFF_DORMANT)) | 6358 (dev->gflags & (IFF_PROMISC | 6359 IFF_ALLMULTI)); 6360 6361 if (netif_running(dev)) { 6362 if (netif_oper_up(dev)) 6363 flags |= IFF_RUNNING; 6364 if (netif_carrier_ok(dev)) 6365 flags |= IFF_LOWER_UP; 6366 if (netif_dormant(dev)) 6367 flags |= IFF_DORMANT; 6368 } 6369 6370 return flags; 6371 } 6372 EXPORT_SYMBOL(dev_get_flags); 6373 6374 int __dev_change_flags(struct net_device *dev, unsigned int flags) 6375 { 6376 unsigned int old_flags = dev->flags; 6377 int ret; 6378 6379 ASSERT_RTNL(); 6380 6381 /* 6382 * Set the flags on our device. 6383 */ 6384 6385 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | 6386 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL | 6387 IFF_AUTOMEDIA)) | 6388 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | 6389 IFF_ALLMULTI)); 6390 6391 /* 6392 * Load in the correct multicast list now the flags have changed. 6393 */ 6394 6395 if ((old_flags ^ flags) & IFF_MULTICAST) 6396 dev_change_rx_flags(dev, IFF_MULTICAST); 6397 6398 dev_set_rx_mode(dev); 6399 6400 /* 6401 * Have we downed the interface. We handle IFF_UP ourselves 6402 * according to user attempts to set it, rather than blindly 6403 * setting it. 6404 */ 6405 6406 ret = 0; 6407 if ((old_flags ^ flags) & IFF_UP) 6408 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev); 6409 6410 if ((flags ^ dev->gflags) & IFF_PROMISC) { 6411 int inc = (flags & IFF_PROMISC) ? 1 : -1; 6412 unsigned int old_flags = dev->flags; 6413 6414 dev->gflags ^= IFF_PROMISC; 6415 6416 if (__dev_set_promiscuity(dev, inc, false) >= 0) 6417 if (dev->flags != old_flags) 6418 dev_set_rx_mode(dev); 6419 } 6420 6421 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI 6422 is important. Some (broken) drivers set IFF_PROMISC, when 6423 IFF_ALLMULTI is requested not asking us and not reporting. 6424 */ 6425 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { 6426 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; 6427 6428 dev->gflags ^= IFF_ALLMULTI; 6429 __dev_set_allmulti(dev, inc, false); 6430 } 6431 6432 return ret; 6433 } 6434 6435 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, 6436 unsigned int gchanges) 6437 { 6438 unsigned int changes = dev->flags ^ old_flags; 6439 6440 if (gchanges) 6441 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC); 6442 6443 if (changes & IFF_UP) { 6444 if (dev->flags & IFF_UP) 6445 call_netdevice_notifiers(NETDEV_UP, dev); 6446 else 6447 call_netdevice_notifiers(NETDEV_DOWN, dev); 6448 } 6449 6450 if (dev->flags & IFF_UP && 6451 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) { 6452 struct netdev_notifier_change_info change_info; 6453 6454 change_info.flags_changed = changes; 6455 call_netdevice_notifiers_info(NETDEV_CHANGE, dev, 6456 &change_info.info); 6457 } 6458 } 6459 6460 /** 6461 * dev_change_flags - change device settings 6462 * @dev: device 6463 * @flags: device state flags 6464 * 6465 * Change settings on device based state flags. The flags are 6466 * in the userspace exported format. 6467 */ 6468 int dev_change_flags(struct net_device *dev, unsigned int flags) 6469 { 6470 int ret; 6471 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; 6472 6473 ret = __dev_change_flags(dev, flags); 6474 if (ret < 0) 6475 return ret; 6476 6477 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); 6478 __dev_notify_flags(dev, old_flags, changes); 6479 return ret; 6480 } 6481 EXPORT_SYMBOL(dev_change_flags); 6482 6483 static int __dev_set_mtu(struct net_device *dev, int new_mtu) 6484 { 6485 const struct net_device_ops *ops = dev->netdev_ops; 6486 6487 if (ops->ndo_change_mtu) 6488 return ops->ndo_change_mtu(dev, new_mtu); 6489 6490 dev->mtu = new_mtu; 6491 return 0; 6492 } 6493 6494 /** 6495 * dev_set_mtu - Change maximum transfer unit 6496 * @dev: device 6497 * @new_mtu: new transfer unit 6498 * 6499 * Change the maximum transfer size of the network device. 6500 */ 6501 int dev_set_mtu(struct net_device *dev, int new_mtu) 6502 { 6503 int err, orig_mtu; 6504 6505 if (new_mtu == dev->mtu) 6506 return 0; 6507 6508 /* MTU must be positive. */ 6509 if (new_mtu < 0) 6510 return -EINVAL; 6511 6512 if (!netif_device_present(dev)) 6513 return -ENODEV; 6514 6515 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev); 6516 err = notifier_to_errno(err); 6517 if (err) 6518 return err; 6519 6520 orig_mtu = dev->mtu; 6521 err = __dev_set_mtu(dev, new_mtu); 6522 6523 if (!err) { 6524 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); 6525 err = notifier_to_errno(err); 6526 if (err) { 6527 /* setting mtu back and notifying everyone again, 6528 * so that they have a chance to revert changes. 6529 */ 6530 __dev_set_mtu(dev, orig_mtu); 6531 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); 6532 } 6533 } 6534 return err; 6535 } 6536 EXPORT_SYMBOL(dev_set_mtu); 6537 6538 /** 6539 * dev_set_group - Change group this device belongs to 6540 * @dev: device 6541 * @new_group: group this device should belong to 6542 */ 6543 void dev_set_group(struct net_device *dev, int new_group) 6544 { 6545 dev->group = new_group; 6546 } 6547 EXPORT_SYMBOL(dev_set_group); 6548 6549 /** 6550 * dev_set_mac_address - Change Media Access Control Address 6551 * @dev: device 6552 * @sa: new address 6553 * 6554 * Change the hardware (MAC) address of the device 6555 */ 6556 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa) 6557 { 6558 const struct net_device_ops *ops = dev->netdev_ops; 6559 int err; 6560 6561 if (!ops->ndo_set_mac_address) 6562 return -EOPNOTSUPP; 6563 if (sa->sa_family != dev->type) 6564 return -EINVAL; 6565 if (!netif_device_present(dev)) 6566 return -ENODEV; 6567 err = ops->ndo_set_mac_address(dev, sa); 6568 if (err) 6569 return err; 6570 dev->addr_assign_type = NET_ADDR_SET; 6571 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 6572 add_device_randomness(dev->dev_addr, dev->addr_len); 6573 return 0; 6574 } 6575 EXPORT_SYMBOL(dev_set_mac_address); 6576 6577 /** 6578 * dev_change_carrier - Change device carrier 6579 * @dev: device 6580 * @new_carrier: new value 6581 * 6582 * Change device carrier 6583 */ 6584 int dev_change_carrier(struct net_device *dev, bool new_carrier) 6585 { 6586 const struct net_device_ops *ops = dev->netdev_ops; 6587 6588 if (!ops->ndo_change_carrier) 6589 return -EOPNOTSUPP; 6590 if (!netif_device_present(dev)) 6591 return -ENODEV; 6592 return ops->ndo_change_carrier(dev, new_carrier); 6593 } 6594 EXPORT_SYMBOL(dev_change_carrier); 6595 6596 /** 6597 * dev_get_phys_port_id - Get device physical port ID 6598 * @dev: device 6599 * @ppid: port ID 6600 * 6601 * Get device physical port ID 6602 */ 6603 int dev_get_phys_port_id(struct net_device *dev, 6604 struct netdev_phys_item_id *ppid) 6605 { 6606 const struct net_device_ops *ops = dev->netdev_ops; 6607 6608 if (!ops->ndo_get_phys_port_id) 6609 return -EOPNOTSUPP; 6610 return ops->ndo_get_phys_port_id(dev, ppid); 6611 } 6612 EXPORT_SYMBOL(dev_get_phys_port_id); 6613 6614 /** 6615 * dev_get_phys_port_name - Get device physical port name 6616 * @dev: device 6617 * @name: port name 6618 * @len: limit of bytes to copy to name 6619 * 6620 * Get device physical port name 6621 */ 6622 int dev_get_phys_port_name(struct net_device *dev, 6623 char *name, size_t len) 6624 { 6625 const struct net_device_ops *ops = dev->netdev_ops; 6626 6627 if (!ops->ndo_get_phys_port_name) 6628 return -EOPNOTSUPP; 6629 return ops->ndo_get_phys_port_name(dev, name, len); 6630 } 6631 EXPORT_SYMBOL(dev_get_phys_port_name); 6632 6633 /** 6634 * dev_change_proto_down - update protocol port state information 6635 * @dev: device 6636 * @proto_down: new value 6637 * 6638 * This info can be used by switch drivers to set the phys state of the 6639 * port. 6640 */ 6641 int dev_change_proto_down(struct net_device *dev, bool proto_down) 6642 { 6643 const struct net_device_ops *ops = dev->netdev_ops; 6644 6645 if (!ops->ndo_change_proto_down) 6646 return -EOPNOTSUPP; 6647 if (!netif_device_present(dev)) 6648 return -ENODEV; 6649 return ops->ndo_change_proto_down(dev, proto_down); 6650 } 6651 EXPORT_SYMBOL(dev_change_proto_down); 6652 6653 /** 6654 * dev_change_xdp_fd - set or clear a bpf program for a device rx path 6655 * @dev: device 6656 * @fd: new program fd or negative value to clear 6657 * 6658 * Set or clear a bpf program for a device 6659 */ 6660 int dev_change_xdp_fd(struct net_device *dev, int fd) 6661 { 6662 const struct net_device_ops *ops = dev->netdev_ops; 6663 struct bpf_prog *prog = NULL; 6664 struct netdev_xdp xdp = {}; 6665 int err; 6666 6667 if (!ops->ndo_xdp) 6668 return -EOPNOTSUPP; 6669 if (fd >= 0) { 6670 prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_XDP); 6671 if (IS_ERR(prog)) 6672 return PTR_ERR(prog); 6673 } 6674 6675 xdp.command = XDP_SETUP_PROG; 6676 xdp.prog = prog; 6677 err = ops->ndo_xdp(dev, &xdp); 6678 if (err < 0 && prog) 6679 bpf_prog_put(prog); 6680 6681 return err; 6682 } 6683 EXPORT_SYMBOL(dev_change_xdp_fd); 6684 6685 /** 6686 * dev_new_index - allocate an ifindex 6687 * @net: the applicable net namespace 6688 * 6689 * Returns a suitable unique value for a new device interface 6690 * number. The caller must hold the rtnl semaphore or the 6691 * dev_base_lock to be sure it remains unique. 6692 */ 6693 static int dev_new_index(struct net *net) 6694 { 6695 int ifindex = net->ifindex; 6696 for (;;) { 6697 if (++ifindex <= 0) 6698 ifindex = 1; 6699 if (!__dev_get_by_index(net, ifindex)) 6700 return net->ifindex = ifindex; 6701 } 6702 } 6703 6704 /* Delayed registration/unregisteration */ 6705 static LIST_HEAD(net_todo_list); 6706 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq); 6707 6708 static void net_set_todo(struct net_device *dev) 6709 { 6710 list_add_tail(&dev->todo_list, &net_todo_list); 6711 dev_net(dev)->dev_unreg_count++; 6712 } 6713 6714 static void rollback_registered_many(struct list_head *head) 6715 { 6716 struct net_device *dev, *tmp; 6717 LIST_HEAD(close_head); 6718 6719 BUG_ON(dev_boot_phase); 6720 ASSERT_RTNL(); 6721 6722 list_for_each_entry_safe(dev, tmp, head, unreg_list) { 6723 /* Some devices call without registering 6724 * for initialization unwind. Remove those 6725 * devices and proceed with the remaining. 6726 */ 6727 if (dev->reg_state == NETREG_UNINITIALIZED) { 6728 pr_debug("unregister_netdevice: device %s/%p never was registered\n", 6729 dev->name, dev); 6730 6731 WARN_ON(1); 6732 list_del(&dev->unreg_list); 6733 continue; 6734 } 6735 dev->dismantle = true; 6736 BUG_ON(dev->reg_state != NETREG_REGISTERED); 6737 } 6738 6739 /* If device is running, close it first. */ 6740 list_for_each_entry(dev, head, unreg_list) 6741 list_add_tail(&dev->close_list, &close_head); 6742 dev_close_many(&close_head, true); 6743 6744 list_for_each_entry(dev, head, unreg_list) { 6745 /* And unlink it from device chain. */ 6746 unlist_netdevice(dev); 6747 6748 dev->reg_state = NETREG_UNREGISTERING; 6749 } 6750 flush_all_backlogs(); 6751 6752 synchronize_net(); 6753 6754 list_for_each_entry(dev, head, unreg_list) { 6755 struct sk_buff *skb = NULL; 6756 6757 /* Shutdown queueing discipline. */ 6758 dev_shutdown(dev); 6759 6760 6761 /* Notify protocols, that we are about to destroy 6762 this device. They should clean all the things. 6763 */ 6764 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 6765 6766 if (!dev->rtnl_link_ops || 6767 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 6768 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 6769 GFP_KERNEL); 6770 6771 /* 6772 * Flush the unicast and multicast chains 6773 */ 6774 dev_uc_flush(dev); 6775 dev_mc_flush(dev); 6776 6777 if (dev->netdev_ops->ndo_uninit) 6778 dev->netdev_ops->ndo_uninit(dev); 6779 6780 if (skb) 6781 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL); 6782 6783 /* Notifier chain MUST detach us all upper devices. */ 6784 WARN_ON(netdev_has_any_upper_dev(dev)); 6785 6786 /* Remove entries from kobject tree */ 6787 netdev_unregister_kobject(dev); 6788 #ifdef CONFIG_XPS 6789 /* Remove XPS queueing entries */ 6790 netif_reset_xps_queues_gt(dev, 0); 6791 #endif 6792 } 6793 6794 synchronize_net(); 6795 6796 list_for_each_entry(dev, head, unreg_list) 6797 dev_put(dev); 6798 } 6799 6800 static void rollback_registered(struct net_device *dev) 6801 { 6802 LIST_HEAD(single); 6803 6804 list_add(&dev->unreg_list, &single); 6805 rollback_registered_many(&single); 6806 list_del(&single); 6807 } 6808 6809 static netdev_features_t netdev_sync_upper_features(struct net_device *lower, 6810 struct net_device *upper, netdev_features_t features) 6811 { 6812 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 6813 netdev_features_t feature; 6814 int feature_bit; 6815 6816 for_each_netdev_feature(&upper_disables, feature_bit) { 6817 feature = __NETIF_F_BIT(feature_bit); 6818 if (!(upper->wanted_features & feature) 6819 && (features & feature)) { 6820 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n", 6821 &feature, upper->name); 6822 features &= ~feature; 6823 } 6824 } 6825 6826 return features; 6827 } 6828 6829 static void netdev_sync_lower_features(struct net_device *upper, 6830 struct net_device *lower, netdev_features_t features) 6831 { 6832 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 6833 netdev_features_t feature; 6834 int feature_bit; 6835 6836 for_each_netdev_feature(&upper_disables, feature_bit) { 6837 feature = __NETIF_F_BIT(feature_bit); 6838 if (!(features & feature) && (lower->features & feature)) { 6839 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n", 6840 &feature, lower->name); 6841 lower->wanted_features &= ~feature; 6842 netdev_update_features(lower); 6843 6844 if (unlikely(lower->features & feature)) 6845 netdev_WARN(upper, "failed to disable %pNF on %s!\n", 6846 &feature, lower->name); 6847 } 6848 } 6849 } 6850 6851 static netdev_features_t netdev_fix_features(struct net_device *dev, 6852 netdev_features_t features) 6853 { 6854 /* Fix illegal checksum combinations */ 6855 if ((features & NETIF_F_HW_CSUM) && 6856 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { 6857 netdev_warn(dev, "mixed HW and IP checksum settings.\n"); 6858 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); 6859 } 6860 6861 /* TSO requires that SG is present as well. */ 6862 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) { 6863 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n"); 6864 features &= ~NETIF_F_ALL_TSO; 6865 } 6866 6867 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) && 6868 !(features & NETIF_F_IP_CSUM)) { 6869 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n"); 6870 features &= ~NETIF_F_TSO; 6871 features &= ~NETIF_F_TSO_ECN; 6872 } 6873 6874 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) && 6875 !(features & NETIF_F_IPV6_CSUM)) { 6876 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n"); 6877 features &= ~NETIF_F_TSO6; 6878 } 6879 6880 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */ 6881 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO)) 6882 features &= ~NETIF_F_TSO_MANGLEID; 6883 6884 /* TSO ECN requires that TSO is present as well. */ 6885 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN) 6886 features &= ~NETIF_F_TSO_ECN; 6887 6888 /* Software GSO depends on SG. */ 6889 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) { 6890 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n"); 6891 features &= ~NETIF_F_GSO; 6892 } 6893 6894 /* UFO needs SG and checksumming */ 6895 if (features & NETIF_F_UFO) { 6896 /* maybe split UFO into V4 and V6? */ 6897 if (!(features & NETIF_F_HW_CSUM) && 6898 ((features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) != 6899 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) { 6900 netdev_dbg(dev, 6901 "Dropping NETIF_F_UFO since no checksum offload features.\n"); 6902 features &= ~NETIF_F_UFO; 6903 } 6904 6905 if (!(features & NETIF_F_SG)) { 6906 netdev_dbg(dev, 6907 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n"); 6908 features &= ~NETIF_F_UFO; 6909 } 6910 } 6911 6912 /* GSO partial features require GSO partial be set */ 6913 if ((features & dev->gso_partial_features) && 6914 !(features & NETIF_F_GSO_PARTIAL)) { 6915 netdev_dbg(dev, 6916 "Dropping partially supported GSO features since no GSO partial.\n"); 6917 features &= ~dev->gso_partial_features; 6918 } 6919 6920 #ifdef CONFIG_NET_RX_BUSY_POLL 6921 if (dev->netdev_ops->ndo_busy_poll) 6922 features |= NETIF_F_BUSY_POLL; 6923 else 6924 #endif 6925 features &= ~NETIF_F_BUSY_POLL; 6926 6927 return features; 6928 } 6929 6930 int __netdev_update_features(struct net_device *dev) 6931 { 6932 struct net_device *upper, *lower; 6933 netdev_features_t features; 6934 struct list_head *iter; 6935 int err = -1; 6936 6937 ASSERT_RTNL(); 6938 6939 features = netdev_get_wanted_features(dev); 6940 6941 if (dev->netdev_ops->ndo_fix_features) 6942 features = dev->netdev_ops->ndo_fix_features(dev, features); 6943 6944 /* driver might be less strict about feature dependencies */ 6945 features = netdev_fix_features(dev, features); 6946 6947 /* some features can't be enabled if they're off an an upper device */ 6948 netdev_for_each_upper_dev_rcu(dev, upper, iter) 6949 features = netdev_sync_upper_features(dev, upper, features); 6950 6951 if (dev->features == features) 6952 goto sync_lower; 6953 6954 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", 6955 &dev->features, &features); 6956 6957 if (dev->netdev_ops->ndo_set_features) 6958 err = dev->netdev_ops->ndo_set_features(dev, features); 6959 else 6960 err = 0; 6961 6962 if (unlikely(err < 0)) { 6963 netdev_err(dev, 6964 "set_features() failed (%d); wanted %pNF, left %pNF\n", 6965 err, &features, &dev->features); 6966 /* return non-0 since some features might have changed and 6967 * it's better to fire a spurious notification than miss it 6968 */ 6969 return -1; 6970 } 6971 6972 sync_lower: 6973 /* some features must be disabled on lower devices when disabled 6974 * on an upper device (think: bonding master or bridge) 6975 */ 6976 netdev_for_each_lower_dev(dev, lower, iter) 6977 netdev_sync_lower_features(dev, lower, features); 6978 6979 if (!err) 6980 dev->features = features; 6981 6982 return err < 0 ? 0 : 1; 6983 } 6984 6985 /** 6986 * netdev_update_features - recalculate device features 6987 * @dev: the device to check 6988 * 6989 * Recalculate dev->features set and send notifications if it 6990 * has changed. Should be called after driver or hardware dependent 6991 * conditions might have changed that influence the features. 6992 */ 6993 void netdev_update_features(struct net_device *dev) 6994 { 6995 if (__netdev_update_features(dev)) 6996 netdev_features_change(dev); 6997 } 6998 EXPORT_SYMBOL(netdev_update_features); 6999 7000 /** 7001 * netdev_change_features - recalculate device features 7002 * @dev: the device to check 7003 * 7004 * Recalculate dev->features set and send notifications even 7005 * if they have not changed. Should be called instead of 7006 * netdev_update_features() if also dev->vlan_features might 7007 * have changed to allow the changes to be propagated to stacked 7008 * VLAN devices. 7009 */ 7010 void netdev_change_features(struct net_device *dev) 7011 { 7012 __netdev_update_features(dev); 7013 netdev_features_change(dev); 7014 } 7015 EXPORT_SYMBOL(netdev_change_features); 7016 7017 /** 7018 * netif_stacked_transfer_operstate - transfer operstate 7019 * @rootdev: the root or lower level device to transfer state from 7020 * @dev: the device to transfer operstate to 7021 * 7022 * Transfer operational state from root to device. This is normally 7023 * called when a stacking relationship exists between the root 7024 * device and the device(a leaf device). 7025 */ 7026 void netif_stacked_transfer_operstate(const struct net_device *rootdev, 7027 struct net_device *dev) 7028 { 7029 if (rootdev->operstate == IF_OPER_DORMANT) 7030 netif_dormant_on(dev); 7031 else 7032 netif_dormant_off(dev); 7033 7034 if (netif_carrier_ok(rootdev)) { 7035 if (!netif_carrier_ok(dev)) 7036 netif_carrier_on(dev); 7037 } else { 7038 if (netif_carrier_ok(dev)) 7039 netif_carrier_off(dev); 7040 } 7041 } 7042 EXPORT_SYMBOL(netif_stacked_transfer_operstate); 7043 7044 #ifdef CONFIG_SYSFS 7045 static int netif_alloc_rx_queues(struct net_device *dev) 7046 { 7047 unsigned int i, count = dev->num_rx_queues; 7048 struct netdev_rx_queue *rx; 7049 size_t sz = count * sizeof(*rx); 7050 7051 BUG_ON(count < 1); 7052 7053 rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 7054 if (!rx) { 7055 rx = vzalloc(sz); 7056 if (!rx) 7057 return -ENOMEM; 7058 } 7059 dev->_rx = rx; 7060 7061 for (i = 0; i < count; i++) 7062 rx[i].dev = dev; 7063 return 0; 7064 } 7065 #endif 7066 7067 static void netdev_init_one_queue(struct net_device *dev, 7068 struct netdev_queue *queue, void *_unused) 7069 { 7070 /* Initialize queue lock */ 7071 spin_lock_init(&queue->_xmit_lock); 7072 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); 7073 queue->xmit_lock_owner = -1; 7074 netdev_queue_numa_node_write(queue, NUMA_NO_NODE); 7075 queue->dev = dev; 7076 #ifdef CONFIG_BQL 7077 dql_init(&queue->dql, HZ); 7078 #endif 7079 } 7080 7081 static void netif_free_tx_queues(struct net_device *dev) 7082 { 7083 kvfree(dev->_tx); 7084 } 7085 7086 static int netif_alloc_netdev_queues(struct net_device *dev) 7087 { 7088 unsigned int count = dev->num_tx_queues; 7089 struct netdev_queue *tx; 7090 size_t sz = count * sizeof(*tx); 7091 7092 if (count < 1 || count > 0xffff) 7093 return -EINVAL; 7094 7095 tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 7096 if (!tx) { 7097 tx = vzalloc(sz); 7098 if (!tx) 7099 return -ENOMEM; 7100 } 7101 dev->_tx = tx; 7102 7103 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); 7104 spin_lock_init(&dev->tx_global_lock); 7105 7106 return 0; 7107 } 7108 7109 void netif_tx_stop_all_queues(struct net_device *dev) 7110 { 7111 unsigned int i; 7112 7113 for (i = 0; i < dev->num_tx_queues; i++) { 7114 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 7115 netif_tx_stop_queue(txq); 7116 } 7117 } 7118 EXPORT_SYMBOL(netif_tx_stop_all_queues); 7119 7120 /** 7121 * register_netdevice - register a network device 7122 * @dev: device to register 7123 * 7124 * Take a completed network device structure and add it to the kernel 7125 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 7126 * chain. 0 is returned on success. A negative errno code is returned 7127 * on a failure to set up the device, or if the name is a duplicate. 7128 * 7129 * Callers must hold the rtnl semaphore. You may want 7130 * register_netdev() instead of this. 7131 * 7132 * BUGS: 7133 * The locking appears insufficient to guarantee two parallel registers 7134 * will not get the same name. 7135 */ 7136 7137 int register_netdevice(struct net_device *dev) 7138 { 7139 int ret; 7140 struct net *net = dev_net(dev); 7141 7142 BUG_ON(dev_boot_phase); 7143 ASSERT_RTNL(); 7144 7145 might_sleep(); 7146 7147 /* When net_device's are persistent, this will be fatal. */ 7148 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); 7149 BUG_ON(!net); 7150 7151 spin_lock_init(&dev->addr_list_lock); 7152 netdev_set_addr_lockdep_class(dev); 7153 7154 ret = dev_get_valid_name(net, dev, dev->name); 7155 if (ret < 0) 7156 goto out; 7157 7158 /* Init, if this function is available */ 7159 if (dev->netdev_ops->ndo_init) { 7160 ret = dev->netdev_ops->ndo_init(dev); 7161 if (ret) { 7162 if (ret > 0) 7163 ret = -EIO; 7164 goto out; 7165 } 7166 } 7167 7168 if (((dev->hw_features | dev->features) & 7169 NETIF_F_HW_VLAN_CTAG_FILTER) && 7170 (!dev->netdev_ops->ndo_vlan_rx_add_vid || 7171 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { 7172 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n"); 7173 ret = -EINVAL; 7174 goto err_uninit; 7175 } 7176 7177 ret = -EBUSY; 7178 if (!dev->ifindex) 7179 dev->ifindex = dev_new_index(net); 7180 else if (__dev_get_by_index(net, dev->ifindex)) 7181 goto err_uninit; 7182 7183 /* Transfer changeable features to wanted_features and enable 7184 * software offloads (GSO and GRO). 7185 */ 7186 dev->hw_features |= NETIF_F_SOFT_FEATURES; 7187 dev->features |= NETIF_F_SOFT_FEATURES; 7188 dev->wanted_features = dev->features & dev->hw_features; 7189 7190 if (!(dev->flags & IFF_LOOPBACK)) 7191 dev->hw_features |= NETIF_F_NOCACHE_COPY; 7192 7193 /* If IPv4 TCP segmentation offload is supported we should also 7194 * allow the device to enable segmenting the frame with the option 7195 * of ignoring a static IP ID value. This doesn't enable the 7196 * feature itself but allows the user to enable it later. 7197 */ 7198 if (dev->hw_features & NETIF_F_TSO) 7199 dev->hw_features |= NETIF_F_TSO_MANGLEID; 7200 if (dev->vlan_features & NETIF_F_TSO) 7201 dev->vlan_features |= NETIF_F_TSO_MANGLEID; 7202 if (dev->mpls_features & NETIF_F_TSO) 7203 dev->mpls_features |= NETIF_F_TSO_MANGLEID; 7204 if (dev->hw_enc_features & NETIF_F_TSO) 7205 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 7206 7207 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. 7208 */ 7209 dev->vlan_features |= NETIF_F_HIGHDMA; 7210 7211 /* Make NETIF_F_SG inheritable to tunnel devices. 7212 */ 7213 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL; 7214 7215 /* Make NETIF_F_SG inheritable to MPLS. 7216 */ 7217 dev->mpls_features |= NETIF_F_SG; 7218 7219 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); 7220 ret = notifier_to_errno(ret); 7221 if (ret) 7222 goto err_uninit; 7223 7224 ret = netdev_register_kobject(dev); 7225 if (ret) 7226 goto err_uninit; 7227 dev->reg_state = NETREG_REGISTERED; 7228 7229 __netdev_update_features(dev); 7230 7231 /* 7232 * Default initial state at registry is that the 7233 * device is present. 7234 */ 7235 7236 set_bit(__LINK_STATE_PRESENT, &dev->state); 7237 7238 linkwatch_init_dev(dev); 7239 7240 dev_init_scheduler(dev); 7241 dev_hold(dev); 7242 list_netdevice(dev); 7243 add_device_randomness(dev->dev_addr, dev->addr_len); 7244 7245 /* If the device has permanent device address, driver should 7246 * set dev_addr and also addr_assign_type should be set to 7247 * NET_ADDR_PERM (default value). 7248 */ 7249 if (dev->addr_assign_type == NET_ADDR_PERM) 7250 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); 7251 7252 /* Notify protocols, that a new device appeared. */ 7253 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); 7254 ret = notifier_to_errno(ret); 7255 if (ret) { 7256 rollback_registered(dev); 7257 dev->reg_state = NETREG_UNREGISTERED; 7258 } 7259 /* 7260 * Prevent userspace races by waiting until the network 7261 * device is fully setup before sending notifications. 7262 */ 7263 if (!dev->rtnl_link_ops || 7264 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 7265 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); 7266 7267 out: 7268 return ret; 7269 7270 err_uninit: 7271 if (dev->netdev_ops->ndo_uninit) 7272 dev->netdev_ops->ndo_uninit(dev); 7273 goto out; 7274 } 7275 EXPORT_SYMBOL(register_netdevice); 7276 7277 /** 7278 * init_dummy_netdev - init a dummy network device for NAPI 7279 * @dev: device to init 7280 * 7281 * This takes a network device structure and initialize the minimum 7282 * amount of fields so it can be used to schedule NAPI polls without 7283 * registering a full blown interface. This is to be used by drivers 7284 * that need to tie several hardware interfaces to a single NAPI 7285 * poll scheduler due to HW limitations. 7286 */ 7287 int init_dummy_netdev(struct net_device *dev) 7288 { 7289 /* Clear everything. Note we don't initialize spinlocks 7290 * are they aren't supposed to be taken by any of the 7291 * NAPI code and this dummy netdev is supposed to be 7292 * only ever used for NAPI polls 7293 */ 7294 memset(dev, 0, sizeof(struct net_device)); 7295 7296 /* make sure we BUG if trying to hit standard 7297 * register/unregister code path 7298 */ 7299 dev->reg_state = NETREG_DUMMY; 7300 7301 /* NAPI wants this */ 7302 INIT_LIST_HEAD(&dev->napi_list); 7303 7304 /* a dummy interface is started by default */ 7305 set_bit(__LINK_STATE_PRESENT, &dev->state); 7306 set_bit(__LINK_STATE_START, &dev->state); 7307 7308 /* Note : We dont allocate pcpu_refcnt for dummy devices, 7309 * because users of this 'device' dont need to change 7310 * its refcount. 7311 */ 7312 7313 return 0; 7314 } 7315 EXPORT_SYMBOL_GPL(init_dummy_netdev); 7316 7317 7318 /** 7319 * register_netdev - register a network device 7320 * @dev: device to register 7321 * 7322 * Take a completed network device structure and add it to the kernel 7323 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 7324 * chain. 0 is returned on success. A negative errno code is returned 7325 * on a failure to set up the device, or if the name is a duplicate. 7326 * 7327 * This is a wrapper around register_netdevice that takes the rtnl semaphore 7328 * and expands the device name if you passed a format string to 7329 * alloc_netdev. 7330 */ 7331 int register_netdev(struct net_device *dev) 7332 { 7333 int err; 7334 7335 rtnl_lock(); 7336 err = register_netdevice(dev); 7337 rtnl_unlock(); 7338 return err; 7339 } 7340 EXPORT_SYMBOL(register_netdev); 7341 7342 int netdev_refcnt_read(const struct net_device *dev) 7343 { 7344 int i, refcnt = 0; 7345 7346 for_each_possible_cpu(i) 7347 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); 7348 return refcnt; 7349 } 7350 EXPORT_SYMBOL(netdev_refcnt_read); 7351 7352 /** 7353 * netdev_wait_allrefs - wait until all references are gone. 7354 * @dev: target net_device 7355 * 7356 * This is called when unregistering network devices. 7357 * 7358 * Any protocol or device that holds a reference should register 7359 * for netdevice notification, and cleanup and put back the 7360 * reference if they receive an UNREGISTER event. 7361 * We can get stuck here if buggy protocols don't correctly 7362 * call dev_put. 7363 */ 7364 static void netdev_wait_allrefs(struct net_device *dev) 7365 { 7366 unsigned long rebroadcast_time, warning_time; 7367 int refcnt; 7368 7369 linkwatch_forget_dev(dev); 7370 7371 rebroadcast_time = warning_time = jiffies; 7372 refcnt = netdev_refcnt_read(dev); 7373 7374 while (refcnt != 0) { 7375 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { 7376 rtnl_lock(); 7377 7378 /* Rebroadcast unregister notification */ 7379 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 7380 7381 __rtnl_unlock(); 7382 rcu_barrier(); 7383 rtnl_lock(); 7384 7385 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 7386 if (test_bit(__LINK_STATE_LINKWATCH_PENDING, 7387 &dev->state)) { 7388 /* We must not have linkwatch events 7389 * pending on unregister. If this 7390 * happens, we simply run the queue 7391 * unscheduled, resulting in a noop 7392 * for this device. 7393 */ 7394 linkwatch_run_queue(); 7395 } 7396 7397 __rtnl_unlock(); 7398 7399 rebroadcast_time = jiffies; 7400 } 7401 7402 msleep(250); 7403 7404 refcnt = netdev_refcnt_read(dev); 7405 7406 if (time_after(jiffies, warning_time + 10 * HZ)) { 7407 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", 7408 dev->name, refcnt); 7409 warning_time = jiffies; 7410 } 7411 } 7412 } 7413 7414 /* The sequence is: 7415 * 7416 * rtnl_lock(); 7417 * ... 7418 * register_netdevice(x1); 7419 * register_netdevice(x2); 7420 * ... 7421 * unregister_netdevice(y1); 7422 * unregister_netdevice(y2); 7423 * ... 7424 * rtnl_unlock(); 7425 * free_netdev(y1); 7426 * free_netdev(y2); 7427 * 7428 * We are invoked by rtnl_unlock(). 7429 * This allows us to deal with problems: 7430 * 1) We can delete sysfs objects which invoke hotplug 7431 * without deadlocking with linkwatch via keventd. 7432 * 2) Since we run with the RTNL semaphore not held, we can sleep 7433 * safely in order to wait for the netdev refcnt to drop to zero. 7434 * 7435 * We must not return until all unregister events added during 7436 * the interval the lock was held have been completed. 7437 */ 7438 void netdev_run_todo(void) 7439 { 7440 struct list_head list; 7441 7442 /* Snapshot list, allow later requests */ 7443 list_replace_init(&net_todo_list, &list); 7444 7445 __rtnl_unlock(); 7446 7447 7448 /* Wait for rcu callbacks to finish before next phase */ 7449 if (!list_empty(&list)) 7450 rcu_barrier(); 7451 7452 while (!list_empty(&list)) { 7453 struct net_device *dev 7454 = list_first_entry(&list, struct net_device, todo_list); 7455 list_del(&dev->todo_list); 7456 7457 rtnl_lock(); 7458 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 7459 __rtnl_unlock(); 7460 7461 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { 7462 pr_err("network todo '%s' but state %d\n", 7463 dev->name, dev->reg_state); 7464 dump_stack(); 7465 continue; 7466 } 7467 7468 dev->reg_state = NETREG_UNREGISTERED; 7469 7470 netdev_wait_allrefs(dev); 7471 7472 /* paranoia */ 7473 BUG_ON(netdev_refcnt_read(dev)); 7474 BUG_ON(!list_empty(&dev->ptype_all)); 7475 BUG_ON(!list_empty(&dev->ptype_specific)); 7476 WARN_ON(rcu_access_pointer(dev->ip_ptr)); 7477 WARN_ON(rcu_access_pointer(dev->ip6_ptr)); 7478 WARN_ON(dev->dn_ptr); 7479 7480 if (dev->destructor) 7481 dev->destructor(dev); 7482 7483 /* Report a network device has been unregistered */ 7484 rtnl_lock(); 7485 dev_net(dev)->dev_unreg_count--; 7486 __rtnl_unlock(); 7487 wake_up(&netdev_unregistering_wq); 7488 7489 /* Free network device */ 7490 kobject_put(&dev->dev.kobj); 7491 } 7492 } 7493 7494 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has 7495 * all the same fields in the same order as net_device_stats, with only 7496 * the type differing, but rtnl_link_stats64 may have additional fields 7497 * at the end for newer counters. 7498 */ 7499 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64, 7500 const struct net_device_stats *netdev_stats) 7501 { 7502 #if BITS_PER_LONG == 64 7503 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats)); 7504 memcpy(stats64, netdev_stats, sizeof(*stats64)); 7505 /* zero out counters that only exist in rtnl_link_stats64 */ 7506 memset((char *)stats64 + sizeof(*netdev_stats), 0, 7507 sizeof(*stats64) - sizeof(*netdev_stats)); 7508 #else 7509 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long); 7510 const unsigned long *src = (const unsigned long *)netdev_stats; 7511 u64 *dst = (u64 *)stats64; 7512 7513 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64)); 7514 for (i = 0; i < n; i++) 7515 dst[i] = src[i]; 7516 /* zero out counters that only exist in rtnl_link_stats64 */ 7517 memset((char *)stats64 + n * sizeof(u64), 0, 7518 sizeof(*stats64) - n * sizeof(u64)); 7519 #endif 7520 } 7521 EXPORT_SYMBOL(netdev_stats_to_stats64); 7522 7523 /** 7524 * dev_get_stats - get network device statistics 7525 * @dev: device to get statistics from 7526 * @storage: place to store stats 7527 * 7528 * Get network statistics from device. Return @storage. 7529 * The device driver may provide its own method by setting 7530 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats; 7531 * otherwise the internal statistics structure is used. 7532 */ 7533 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, 7534 struct rtnl_link_stats64 *storage) 7535 { 7536 const struct net_device_ops *ops = dev->netdev_ops; 7537 7538 if (ops->ndo_get_stats64) { 7539 memset(storage, 0, sizeof(*storage)); 7540 ops->ndo_get_stats64(dev, storage); 7541 } else if (ops->ndo_get_stats) { 7542 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); 7543 } else { 7544 netdev_stats_to_stats64(storage, &dev->stats); 7545 } 7546 storage->rx_dropped += atomic_long_read(&dev->rx_dropped); 7547 storage->tx_dropped += atomic_long_read(&dev->tx_dropped); 7548 storage->rx_nohandler += atomic_long_read(&dev->rx_nohandler); 7549 return storage; 7550 } 7551 EXPORT_SYMBOL(dev_get_stats); 7552 7553 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) 7554 { 7555 struct netdev_queue *queue = dev_ingress_queue(dev); 7556 7557 #ifdef CONFIG_NET_CLS_ACT 7558 if (queue) 7559 return queue; 7560 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 7561 if (!queue) 7562 return NULL; 7563 netdev_init_one_queue(dev, queue, NULL); 7564 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); 7565 queue->qdisc_sleeping = &noop_qdisc; 7566 rcu_assign_pointer(dev->ingress_queue, queue); 7567 #endif 7568 return queue; 7569 } 7570 7571 static const struct ethtool_ops default_ethtool_ops; 7572 7573 void netdev_set_default_ethtool_ops(struct net_device *dev, 7574 const struct ethtool_ops *ops) 7575 { 7576 if (dev->ethtool_ops == &default_ethtool_ops) 7577 dev->ethtool_ops = ops; 7578 } 7579 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops); 7580 7581 void netdev_freemem(struct net_device *dev) 7582 { 7583 char *addr = (char *)dev - dev->padded; 7584 7585 kvfree(addr); 7586 } 7587 7588 /** 7589 * alloc_netdev_mqs - allocate network device 7590 * @sizeof_priv: size of private data to allocate space for 7591 * @name: device name format string 7592 * @name_assign_type: origin of device name 7593 * @setup: callback to initialize device 7594 * @txqs: the number of TX subqueues to allocate 7595 * @rxqs: the number of RX subqueues to allocate 7596 * 7597 * Allocates a struct net_device with private data area for driver use 7598 * and performs basic initialization. Also allocates subqueue structs 7599 * for each queue on the device. 7600 */ 7601 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, 7602 unsigned char name_assign_type, 7603 void (*setup)(struct net_device *), 7604 unsigned int txqs, unsigned int rxqs) 7605 { 7606 struct net_device *dev; 7607 size_t alloc_size; 7608 struct net_device *p; 7609 7610 BUG_ON(strlen(name) >= sizeof(dev->name)); 7611 7612 if (txqs < 1) { 7613 pr_err("alloc_netdev: Unable to allocate device with zero queues\n"); 7614 return NULL; 7615 } 7616 7617 #ifdef CONFIG_SYSFS 7618 if (rxqs < 1) { 7619 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n"); 7620 return NULL; 7621 } 7622 #endif 7623 7624 alloc_size = sizeof(struct net_device); 7625 if (sizeof_priv) { 7626 /* ensure 32-byte alignment of private area */ 7627 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN); 7628 alloc_size += sizeof_priv; 7629 } 7630 /* ensure 32-byte alignment of whole construct */ 7631 alloc_size += NETDEV_ALIGN - 1; 7632 7633 p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 7634 if (!p) 7635 p = vzalloc(alloc_size); 7636 if (!p) 7637 return NULL; 7638 7639 dev = PTR_ALIGN(p, NETDEV_ALIGN); 7640 dev->padded = (char *)dev - (char *)p; 7641 7642 dev->pcpu_refcnt = alloc_percpu(int); 7643 if (!dev->pcpu_refcnt) 7644 goto free_dev; 7645 7646 if (dev_addr_init(dev)) 7647 goto free_pcpu; 7648 7649 dev_mc_init(dev); 7650 dev_uc_init(dev); 7651 7652 dev_net_set(dev, &init_net); 7653 7654 dev->gso_max_size = GSO_MAX_SIZE; 7655 dev->gso_max_segs = GSO_MAX_SEGS; 7656 7657 INIT_LIST_HEAD(&dev->napi_list); 7658 INIT_LIST_HEAD(&dev->unreg_list); 7659 INIT_LIST_HEAD(&dev->close_list); 7660 INIT_LIST_HEAD(&dev->link_watch_list); 7661 INIT_LIST_HEAD(&dev->adj_list.upper); 7662 INIT_LIST_HEAD(&dev->adj_list.lower); 7663 INIT_LIST_HEAD(&dev->all_adj_list.upper); 7664 INIT_LIST_HEAD(&dev->all_adj_list.lower); 7665 INIT_LIST_HEAD(&dev->ptype_all); 7666 INIT_LIST_HEAD(&dev->ptype_specific); 7667 #ifdef CONFIG_NET_SCHED 7668 hash_init(dev->qdisc_hash); 7669 #endif 7670 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; 7671 setup(dev); 7672 7673 if (!dev->tx_queue_len) { 7674 dev->priv_flags |= IFF_NO_QUEUE; 7675 dev->tx_queue_len = 1; 7676 } 7677 7678 dev->num_tx_queues = txqs; 7679 dev->real_num_tx_queues = txqs; 7680 if (netif_alloc_netdev_queues(dev)) 7681 goto free_all; 7682 7683 #ifdef CONFIG_SYSFS 7684 dev->num_rx_queues = rxqs; 7685 dev->real_num_rx_queues = rxqs; 7686 if (netif_alloc_rx_queues(dev)) 7687 goto free_all; 7688 #endif 7689 7690 strcpy(dev->name, name); 7691 dev->name_assign_type = name_assign_type; 7692 dev->group = INIT_NETDEV_GROUP; 7693 if (!dev->ethtool_ops) 7694 dev->ethtool_ops = &default_ethtool_ops; 7695 7696 nf_hook_ingress_init(dev); 7697 7698 return dev; 7699 7700 free_all: 7701 free_netdev(dev); 7702 return NULL; 7703 7704 free_pcpu: 7705 free_percpu(dev->pcpu_refcnt); 7706 free_dev: 7707 netdev_freemem(dev); 7708 return NULL; 7709 } 7710 EXPORT_SYMBOL(alloc_netdev_mqs); 7711 7712 /** 7713 * free_netdev - free network device 7714 * @dev: device 7715 * 7716 * This function does the last stage of destroying an allocated device 7717 * interface. The reference to the device object is released. 7718 * If this is the last reference then it will be freed. 7719 * Must be called in process context. 7720 */ 7721 void free_netdev(struct net_device *dev) 7722 { 7723 struct napi_struct *p, *n; 7724 7725 might_sleep(); 7726 netif_free_tx_queues(dev); 7727 #ifdef CONFIG_SYSFS 7728 kvfree(dev->_rx); 7729 #endif 7730 7731 kfree(rcu_dereference_protected(dev->ingress_queue, 1)); 7732 7733 /* Flush device addresses */ 7734 dev_addr_flush(dev); 7735 7736 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) 7737 netif_napi_del(p); 7738 7739 free_percpu(dev->pcpu_refcnt); 7740 dev->pcpu_refcnt = NULL; 7741 7742 /* Compatibility with error handling in drivers */ 7743 if (dev->reg_state == NETREG_UNINITIALIZED) { 7744 netdev_freemem(dev); 7745 return; 7746 } 7747 7748 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); 7749 dev->reg_state = NETREG_RELEASED; 7750 7751 /* will free via device release */ 7752 put_device(&dev->dev); 7753 } 7754 EXPORT_SYMBOL(free_netdev); 7755 7756 /** 7757 * synchronize_net - Synchronize with packet receive processing 7758 * 7759 * Wait for packets currently being received to be done. 7760 * Does not block later packets from starting. 7761 */ 7762 void synchronize_net(void) 7763 { 7764 might_sleep(); 7765 if (rtnl_is_locked()) 7766 synchronize_rcu_expedited(); 7767 else 7768 synchronize_rcu(); 7769 } 7770 EXPORT_SYMBOL(synchronize_net); 7771 7772 /** 7773 * unregister_netdevice_queue - remove device from the kernel 7774 * @dev: device 7775 * @head: list 7776 * 7777 * This function shuts down a device interface and removes it 7778 * from the kernel tables. 7779 * If head not NULL, device is queued to be unregistered later. 7780 * 7781 * Callers must hold the rtnl semaphore. You may want 7782 * unregister_netdev() instead of this. 7783 */ 7784 7785 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) 7786 { 7787 ASSERT_RTNL(); 7788 7789 if (head) { 7790 list_move_tail(&dev->unreg_list, head); 7791 } else { 7792 rollback_registered(dev); 7793 /* Finish processing unregister after unlock */ 7794 net_set_todo(dev); 7795 } 7796 } 7797 EXPORT_SYMBOL(unregister_netdevice_queue); 7798 7799 /** 7800 * unregister_netdevice_many - unregister many devices 7801 * @head: list of devices 7802 * 7803 * Note: As most callers use a stack allocated list_head, 7804 * we force a list_del() to make sure stack wont be corrupted later. 7805 */ 7806 void unregister_netdevice_many(struct list_head *head) 7807 { 7808 struct net_device *dev; 7809 7810 if (!list_empty(head)) { 7811 rollback_registered_many(head); 7812 list_for_each_entry(dev, head, unreg_list) 7813 net_set_todo(dev); 7814 list_del(head); 7815 } 7816 } 7817 EXPORT_SYMBOL(unregister_netdevice_many); 7818 7819 /** 7820 * unregister_netdev - remove device from the kernel 7821 * @dev: device 7822 * 7823 * This function shuts down a device interface and removes it 7824 * from the kernel tables. 7825 * 7826 * This is just a wrapper for unregister_netdevice that takes 7827 * the rtnl semaphore. In general you want to use this and not 7828 * unregister_netdevice. 7829 */ 7830 void unregister_netdev(struct net_device *dev) 7831 { 7832 rtnl_lock(); 7833 unregister_netdevice(dev); 7834 rtnl_unlock(); 7835 } 7836 EXPORT_SYMBOL(unregister_netdev); 7837 7838 /** 7839 * dev_change_net_namespace - move device to different nethost namespace 7840 * @dev: device 7841 * @net: network namespace 7842 * @pat: If not NULL name pattern to try if the current device name 7843 * is already taken in the destination network namespace. 7844 * 7845 * This function shuts down a device interface and moves it 7846 * to a new network namespace. On success 0 is returned, on 7847 * a failure a netagive errno code is returned. 7848 * 7849 * Callers must hold the rtnl semaphore. 7850 */ 7851 7852 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat) 7853 { 7854 int err; 7855 7856 ASSERT_RTNL(); 7857 7858 /* Don't allow namespace local devices to be moved. */ 7859 err = -EINVAL; 7860 if (dev->features & NETIF_F_NETNS_LOCAL) 7861 goto out; 7862 7863 /* Ensure the device has been registrered */ 7864 if (dev->reg_state != NETREG_REGISTERED) 7865 goto out; 7866 7867 /* Get out if there is nothing todo */ 7868 err = 0; 7869 if (net_eq(dev_net(dev), net)) 7870 goto out; 7871 7872 /* Pick the destination device name, and ensure 7873 * we can use it in the destination network namespace. 7874 */ 7875 err = -EEXIST; 7876 if (__dev_get_by_name(net, dev->name)) { 7877 /* We get here if we can't use the current device name */ 7878 if (!pat) 7879 goto out; 7880 if (dev_get_valid_name(net, dev, pat) < 0) 7881 goto out; 7882 } 7883 7884 /* 7885 * And now a mini version of register_netdevice unregister_netdevice. 7886 */ 7887 7888 /* If device is running close it first. */ 7889 dev_close(dev); 7890 7891 /* And unlink it from device chain */ 7892 err = -ENODEV; 7893 unlist_netdevice(dev); 7894 7895 synchronize_net(); 7896 7897 /* Shutdown queueing discipline. */ 7898 dev_shutdown(dev); 7899 7900 /* Notify protocols, that we are about to destroy 7901 this device. They should clean all the things. 7902 7903 Note that dev->reg_state stays at NETREG_REGISTERED. 7904 This is wanted because this way 8021q and macvlan know 7905 the device is just moving and can keep their slaves up. 7906 */ 7907 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 7908 rcu_barrier(); 7909 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 7910 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL); 7911 7912 /* 7913 * Flush the unicast and multicast chains 7914 */ 7915 dev_uc_flush(dev); 7916 dev_mc_flush(dev); 7917 7918 /* Send a netdev-removed uevent to the old namespace */ 7919 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); 7920 netdev_adjacent_del_links(dev); 7921 7922 /* Actually switch the network namespace */ 7923 dev_net_set(dev, net); 7924 7925 /* If there is an ifindex conflict assign a new one */ 7926 if (__dev_get_by_index(net, dev->ifindex)) 7927 dev->ifindex = dev_new_index(net); 7928 7929 /* Send a netdev-add uevent to the new namespace */ 7930 kobject_uevent(&dev->dev.kobj, KOBJ_ADD); 7931 netdev_adjacent_add_links(dev); 7932 7933 /* Fixup kobjects */ 7934 err = device_rename(&dev->dev, dev->name); 7935 WARN_ON(err); 7936 7937 /* Add the device back in the hashes */ 7938 list_netdevice(dev); 7939 7940 /* Notify protocols, that a new device appeared. */ 7941 call_netdevice_notifiers(NETDEV_REGISTER, dev); 7942 7943 /* 7944 * Prevent userspace races by waiting until the network 7945 * device is fully setup before sending notifications. 7946 */ 7947 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); 7948 7949 synchronize_net(); 7950 err = 0; 7951 out: 7952 return err; 7953 } 7954 EXPORT_SYMBOL_GPL(dev_change_net_namespace); 7955 7956 static int dev_cpu_callback(struct notifier_block *nfb, 7957 unsigned long action, 7958 void *ocpu) 7959 { 7960 struct sk_buff **list_skb; 7961 struct sk_buff *skb; 7962 unsigned int cpu, oldcpu = (unsigned long)ocpu; 7963 struct softnet_data *sd, *oldsd; 7964 7965 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN) 7966 return NOTIFY_OK; 7967 7968 local_irq_disable(); 7969 cpu = smp_processor_id(); 7970 sd = &per_cpu(softnet_data, cpu); 7971 oldsd = &per_cpu(softnet_data, oldcpu); 7972 7973 /* Find end of our completion_queue. */ 7974 list_skb = &sd->completion_queue; 7975 while (*list_skb) 7976 list_skb = &(*list_skb)->next; 7977 /* Append completion queue from offline CPU. */ 7978 *list_skb = oldsd->completion_queue; 7979 oldsd->completion_queue = NULL; 7980 7981 /* Append output queue from offline CPU. */ 7982 if (oldsd->output_queue) { 7983 *sd->output_queue_tailp = oldsd->output_queue; 7984 sd->output_queue_tailp = oldsd->output_queue_tailp; 7985 oldsd->output_queue = NULL; 7986 oldsd->output_queue_tailp = &oldsd->output_queue; 7987 } 7988 /* Append NAPI poll list from offline CPU, with one exception : 7989 * process_backlog() must be called by cpu owning percpu backlog. 7990 * We properly handle process_queue & input_pkt_queue later. 7991 */ 7992 while (!list_empty(&oldsd->poll_list)) { 7993 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, 7994 struct napi_struct, 7995 poll_list); 7996 7997 list_del_init(&napi->poll_list); 7998 if (napi->poll == process_backlog) 7999 napi->state = 0; 8000 else 8001 ____napi_schedule(sd, napi); 8002 } 8003 8004 raise_softirq_irqoff(NET_TX_SOFTIRQ); 8005 local_irq_enable(); 8006 8007 /* Process offline CPU's input_pkt_queue */ 8008 while ((skb = __skb_dequeue(&oldsd->process_queue))) { 8009 netif_rx_ni(skb); 8010 input_queue_head_incr(oldsd); 8011 } 8012 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { 8013 netif_rx_ni(skb); 8014 input_queue_head_incr(oldsd); 8015 } 8016 8017 return NOTIFY_OK; 8018 } 8019 8020 8021 /** 8022 * netdev_increment_features - increment feature set by one 8023 * @all: current feature set 8024 * @one: new feature set 8025 * @mask: mask feature set 8026 * 8027 * Computes a new feature set after adding a device with feature set 8028 * @one to the master device with current feature set @all. Will not 8029 * enable anything that is off in @mask. Returns the new feature set. 8030 */ 8031 netdev_features_t netdev_increment_features(netdev_features_t all, 8032 netdev_features_t one, netdev_features_t mask) 8033 { 8034 if (mask & NETIF_F_HW_CSUM) 8035 mask |= NETIF_F_CSUM_MASK; 8036 mask |= NETIF_F_VLAN_CHALLENGED; 8037 8038 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask; 8039 all &= one | ~NETIF_F_ALL_FOR_ALL; 8040 8041 /* If one device supports hw checksumming, set for all. */ 8042 if (all & NETIF_F_HW_CSUM) 8043 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM); 8044 8045 return all; 8046 } 8047 EXPORT_SYMBOL(netdev_increment_features); 8048 8049 static struct hlist_head * __net_init netdev_create_hash(void) 8050 { 8051 int i; 8052 struct hlist_head *hash; 8053 8054 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL); 8055 if (hash != NULL) 8056 for (i = 0; i < NETDEV_HASHENTRIES; i++) 8057 INIT_HLIST_HEAD(&hash[i]); 8058 8059 return hash; 8060 } 8061 8062 /* Initialize per network namespace state */ 8063 static int __net_init netdev_init(struct net *net) 8064 { 8065 if (net != &init_net) 8066 INIT_LIST_HEAD(&net->dev_base_head); 8067 8068 net->dev_name_head = netdev_create_hash(); 8069 if (net->dev_name_head == NULL) 8070 goto err_name; 8071 8072 net->dev_index_head = netdev_create_hash(); 8073 if (net->dev_index_head == NULL) 8074 goto err_idx; 8075 8076 return 0; 8077 8078 err_idx: 8079 kfree(net->dev_name_head); 8080 err_name: 8081 return -ENOMEM; 8082 } 8083 8084 /** 8085 * netdev_drivername - network driver for the device 8086 * @dev: network device 8087 * 8088 * Determine network driver for device. 8089 */ 8090 const char *netdev_drivername(const struct net_device *dev) 8091 { 8092 const struct device_driver *driver; 8093 const struct device *parent; 8094 const char *empty = ""; 8095 8096 parent = dev->dev.parent; 8097 if (!parent) 8098 return empty; 8099 8100 driver = parent->driver; 8101 if (driver && driver->name) 8102 return driver->name; 8103 return empty; 8104 } 8105 8106 static void __netdev_printk(const char *level, const struct net_device *dev, 8107 struct va_format *vaf) 8108 { 8109 if (dev && dev->dev.parent) { 8110 dev_printk_emit(level[1] - '0', 8111 dev->dev.parent, 8112 "%s %s %s%s: %pV", 8113 dev_driver_string(dev->dev.parent), 8114 dev_name(dev->dev.parent), 8115 netdev_name(dev), netdev_reg_state(dev), 8116 vaf); 8117 } else if (dev) { 8118 printk("%s%s%s: %pV", 8119 level, netdev_name(dev), netdev_reg_state(dev), vaf); 8120 } else { 8121 printk("%s(NULL net_device): %pV", level, vaf); 8122 } 8123 } 8124 8125 void netdev_printk(const char *level, const struct net_device *dev, 8126 const char *format, ...) 8127 { 8128 struct va_format vaf; 8129 va_list args; 8130 8131 va_start(args, format); 8132 8133 vaf.fmt = format; 8134 vaf.va = &args; 8135 8136 __netdev_printk(level, dev, &vaf); 8137 8138 va_end(args); 8139 } 8140 EXPORT_SYMBOL(netdev_printk); 8141 8142 #define define_netdev_printk_level(func, level) \ 8143 void func(const struct net_device *dev, const char *fmt, ...) \ 8144 { \ 8145 struct va_format vaf; \ 8146 va_list args; \ 8147 \ 8148 va_start(args, fmt); \ 8149 \ 8150 vaf.fmt = fmt; \ 8151 vaf.va = &args; \ 8152 \ 8153 __netdev_printk(level, dev, &vaf); \ 8154 \ 8155 va_end(args); \ 8156 } \ 8157 EXPORT_SYMBOL(func); 8158 8159 define_netdev_printk_level(netdev_emerg, KERN_EMERG); 8160 define_netdev_printk_level(netdev_alert, KERN_ALERT); 8161 define_netdev_printk_level(netdev_crit, KERN_CRIT); 8162 define_netdev_printk_level(netdev_err, KERN_ERR); 8163 define_netdev_printk_level(netdev_warn, KERN_WARNING); 8164 define_netdev_printk_level(netdev_notice, KERN_NOTICE); 8165 define_netdev_printk_level(netdev_info, KERN_INFO); 8166 8167 static void __net_exit netdev_exit(struct net *net) 8168 { 8169 kfree(net->dev_name_head); 8170 kfree(net->dev_index_head); 8171 } 8172 8173 static struct pernet_operations __net_initdata netdev_net_ops = { 8174 .init = netdev_init, 8175 .exit = netdev_exit, 8176 }; 8177 8178 static void __net_exit default_device_exit(struct net *net) 8179 { 8180 struct net_device *dev, *aux; 8181 /* 8182 * Push all migratable network devices back to the 8183 * initial network namespace 8184 */ 8185 rtnl_lock(); 8186 for_each_netdev_safe(net, dev, aux) { 8187 int err; 8188 char fb_name[IFNAMSIZ]; 8189 8190 /* Ignore unmoveable devices (i.e. loopback) */ 8191 if (dev->features & NETIF_F_NETNS_LOCAL) 8192 continue; 8193 8194 /* Leave virtual devices for the generic cleanup */ 8195 if (dev->rtnl_link_ops) 8196 continue; 8197 8198 /* Push remaining network devices to init_net */ 8199 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); 8200 err = dev_change_net_namespace(dev, &init_net, fb_name); 8201 if (err) { 8202 pr_emerg("%s: failed to move %s to init_net: %d\n", 8203 __func__, dev->name, err); 8204 BUG(); 8205 } 8206 } 8207 rtnl_unlock(); 8208 } 8209 8210 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list) 8211 { 8212 /* Return with the rtnl_lock held when there are no network 8213 * devices unregistering in any network namespace in net_list. 8214 */ 8215 struct net *net; 8216 bool unregistering; 8217 DEFINE_WAIT_FUNC(wait, woken_wake_function); 8218 8219 add_wait_queue(&netdev_unregistering_wq, &wait); 8220 for (;;) { 8221 unregistering = false; 8222 rtnl_lock(); 8223 list_for_each_entry(net, net_list, exit_list) { 8224 if (net->dev_unreg_count > 0) { 8225 unregistering = true; 8226 break; 8227 } 8228 } 8229 if (!unregistering) 8230 break; 8231 __rtnl_unlock(); 8232 8233 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); 8234 } 8235 remove_wait_queue(&netdev_unregistering_wq, &wait); 8236 } 8237 8238 static void __net_exit default_device_exit_batch(struct list_head *net_list) 8239 { 8240 /* At exit all network devices most be removed from a network 8241 * namespace. Do this in the reverse order of registration. 8242 * Do this across as many network namespaces as possible to 8243 * improve batching efficiency. 8244 */ 8245 struct net_device *dev; 8246 struct net *net; 8247 LIST_HEAD(dev_kill_list); 8248 8249 /* To prevent network device cleanup code from dereferencing 8250 * loopback devices or network devices that have been freed 8251 * wait here for all pending unregistrations to complete, 8252 * before unregistring the loopback device and allowing the 8253 * network namespace be freed. 8254 * 8255 * The netdev todo list containing all network devices 8256 * unregistrations that happen in default_device_exit_batch 8257 * will run in the rtnl_unlock() at the end of 8258 * default_device_exit_batch. 8259 */ 8260 rtnl_lock_unregistering(net_list); 8261 list_for_each_entry(net, net_list, exit_list) { 8262 for_each_netdev_reverse(net, dev) { 8263 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) 8264 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); 8265 else 8266 unregister_netdevice_queue(dev, &dev_kill_list); 8267 } 8268 } 8269 unregister_netdevice_many(&dev_kill_list); 8270 rtnl_unlock(); 8271 } 8272 8273 static struct pernet_operations __net_initdata default_device_ops = { 8274 .exit = default_device_exit, 8275 .exit_batch = default_device_exit_batch, 8276 }; 8277 8278 /* 8279 * Initialize the DEV module. At boot time this walks the device list and 8280 * unhooks any devices that fail to initialise (normally hardware not 8281 * present) and leaves us with a valid list of present and active devices. 8282 * 8283 */ 8284 8285 /* 8286 * This is called single threaded during boot, so no need 8287 * to take the rtnl semaphore. 8288 */ 8289 static int __init net_dev_init(void) 8290 { 8291 int i, rc = -ENOMEM; 8292 8293 BUG_ON(!dev_boot_phase); 8294 8295 if (dev_proc_init()) 8296 goto out; 8297 8298 if (netdev_kobject_init()) 8299 goto out; 8300 8301 INIT_LIST_HEAD(&ptype_all); 8302 for (i = 0; i < PTYPE_HASH_SIZE; i++) 8303 INIT_LIST_HEAD(&ptype_base[i]); 8304 8305 INIT_LIST_HEAD(&offload_base); 8306 8307 if (register_pernet_subsys(&netdev_net_ops)) 8308 goto out; 8309 8310 /* 8311 * Initialise the packet receive queues. 8312 */ 8313 8314 for_each_possible_cpu(i) { 8315 struct work_struct *flush = per_cpu_ptr(&flush_works, i); 8316 struct softnet_data *sd = &per_cpu(softnet_data, i); 8317 8318 INIT_WORK(flush, flush_backlog); 8319 8320 skb_queue_head_init(&sd->input_pkt_queue); 8321 skb_queue_head_init(&sd->process_queue); 8322 INIT_LIST_HEAD(&sd->poll_list); 8323 sd->output_queue_tailp = &sd->output_queue; 8324 #ifdef CONFIG_RPS 8325 sd->csd.func = rps_trigger_softirq; 8326 sd->csd.info = sd; 8327 sd->cpu = i; 8328 #endif 8329 8330 sd->backlog.poll = process_backlog; 8331 sd->backlog.weight = weight_p; 8332 } 8333 8334 dev_boot_phase = 0; 8335 8336 /* The loopback device is special if any other network devices 8337 * is present in a network namespace the loopback device must 8338 * be present. Since we now dynamically allocate and free the 8339 * loopback device ensure this invariant is maintained by 8340 * keeping the loopback device as the first device on the 8341 * list of network devices. Ensuring the loopback devices 8342 * is the first device that appears and the last network device 8343 * that disappears. 8344 */ 8345 if (register_pernet_device(&loopback_net_ops)) 8346 goto out; 8347 8348 if (register_pernet_device(&default_device_ops)) 8349 goto out; 8350 8351 open_softirq(NET_TX_SOFTIRQ, net_tx_action); 8352 open_softirq(NET_RX_SOFTIRQ, net_rx_action); 8353 8354 hotcpu_notifier(dev_cpu_callback, 0); 8355 dst_subsys_init(); 8356 rc = 0; 8357 out: 8358 return rc; 8359 } 8360 8361 subsys_initcall(net_dev_init); 8362