1 /* 2 * NET3 Protocol independent device support routines. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 * 9 * Derived from the non IP parts of dev.c 1.0.19 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Mark Evans, <evansmp@uhura.aston.ac.uk> 13 * 14 * Additional Authors: 15 * Florian la Roche <rzsfl@rz.uni-sb.de> 16 * Alan Cox <gw4pts@gw4pts.ampr.org> 17 * David Hinds <dahinds@users.sourceforge.net> 18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 19 * Adam Sulmicki <adam@cfar.umd.edu> 20 * Pekka Riikonen <priikone@poesidon.pspt.fi> 21 * 22 * Changes: 23 * D.J. Barrow : Fixed bug where dev->refcnt gets set 24 * to 2 if register_netdev gets called 25 * before net_dev_init & also removed a 26 * few lines of code in the process. 27 * Alan Cox : device private ioctl copies fields back. 28 * Alan Cox : Transmit queue code does relevant 29 * stunts to keep the queue safe. 30 * Alan Cox : Fixed double lock. 31 * Alan Cox : Fixed promisc NULL pointer trap 32 * ???????? : Support the full private ioctl range 33 * Alan Cox : Moved ioctl permission check into 34 * drivers 35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI 36 * Alan Cox : 100 backlog just doesn't cut it when 37 * you start doing multicast video 8) 38 * Alan Cox : Rewrote net_bh and list manager. 39 * Alan Cox : Fix ETH_P_ALL echoback lengths. 40 * Alan Cox : Took out transmit every packet pass 41 * Saved a few bytes in the ioctl handler 42 * Alan Cox : Network driver sets packet type before 43 * calling netif_rx. Saves a function 44 * call a packet. 45 * Alan Cox : Hashed net_bh() 46 * Richard Kooijman: Timestamp fixes. 47 * Alan Cox : Wrong field in SIOCGIFDSTADDR 48 * Alan Cox : Device lock protection. 49 * Alan Cox : Fixed nasty side effect of device close 50 * changes. 51 * Rudi Cilibrasi : Pass the right thing to 52 * set_mac_address() 53 * Dave Miller : 32bit quantity for the device lock to 54 * make it work out on a Sparc. 55 * Bjorn Ekwall : Added KERNELD hack. 56 * Alan Cox : Cleaned up the backlog initialise. 57 * Craig Metz : SIOCGIFCONF fix if space for under 58 * 1 device. 59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there 60 * is no device open function. 61 * Andi Kleen : Fix error reporting for SIOCGIFCONF 62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF 63 * Cyrus Durgin : Cleaned for KMOD 64 * Adam Sulmicki : Bug Fix : Network Device Unload 65 * A network device unload needs to purge 66 * the backlog queue. 67 * Paul Rusty Russell : SIOCSIFNAME 68 * Pekka Riikonen : Netdev boot-time settings code 69 * Andrew Morton : Make unregister_netdevice wait 70 * indefinitely on dev->refcnt 71 * J Hadi Salim : - Backlog queue sampling 72 * - netif_rx() feedback 73 */ 74 75 #include <linux/uaccess.h> 76 #include <linux/bitops.h> 77 #include <linux/capability.h> 78 #include <linux/cpu.h> 79 #include <linux/types.h> 80 #include <linux/kernel.h> 81 #include <linux/hash.h> 82 #include <linux/slab.h> 83 #include <linux/sched.h> 84 #include <linux/mutex.h> 85 #include <linux/string.h> 86 #include <linux/mm.h> 87 #include <linux/socket.h> 88 #include <linux/sockios.h> 89 #include <linux/errno.h> 90 #include <linux/interrupt.h> 91 #include <linux/if_ether.h> 92 #include <linux/netdevice.h> 93 #include <linux/etherdevice.h> 94 #include <linux/ethtool.h> 95 #include <linux/notifier.h> 96 #include <linux/skbuff.h> 97 #include <linux/bpf.h> 98 #include <net/net_namespace.h> 99 #include <net/sock.h> 100 #include <net/busy_poll.h> 101 #include <linux/rtnetlink.h> 102 #include <linux/stat.h> 103 #include <net/dst.h> 104 #include <net/dst_metadata.h> 105 #include <net/pkt_sched.h> 106 #include <net/checksum.h> 107 #include <net/xfrm.h> 108 #include <linux/highmem.h> 109 #include <linux/init.h> 110 #include <linux/module.h> 111 #include <linux/netpoll.h> 112 #include <linux/rcupdate.h> 113 #include <linux/delay.h> 114 #include <net/iw_handler.h> 115 #include <asm/current.h> 116 #include <linux/audit.h> 117 #include <linux/dmaengine.h> 118 #include <linux/err.h> 119 #include <linux/ctype.h> 120 #include <linux/if_arp.h> 121 #include <linux/if_vlan.h> 122 #include <linux/ip.h> 123 #include <net/ip.h> 124 #include <net/mpls.h> 125 #include <linux/ipv6.h> 126 #include <linux/in.h> 127 #include <linux/jhash.h> 128 #include <linux/random.h> 129 #include <trace/events/napi.h> 130 #include <trace/events/net.h> 131 #include <trace/events/skb.h> 132 #include <linux/pci.h> 133 #include <linux/inetdevice.h> 134 #include <linux/cpu_rmap.h> 135 #include <linux/static_key.h> 136 #include <linux/hashtable.h> 137 #include <linux/vmalloc.h> 138 #include <linux/if_macvlan.h> 139 #include <linux/errqueue.h> 140 #include <linux/hrtimer.h> 141 #include <linux/netfilter_ingress.h> 142 #include <linux/crash_dump.h> 143 144 #include "net-sysfs.h" 145 146 /* Instead of increasing this, you should create a hash table. */ 147 #define MAX_GRO_SKBS 8 148 149 /* This should be increased if a protocol with a bigger head is added. */ 150 #define GRO_MAX_HEAD (MAX_HEADER + 128) 151 152 static DEFINE_SPINLOCK(ptype_lock); 153 static DEFINE_SPINLOCK(offload_lock); 154 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; 155 struct list_head ptype_all __read_mostly; /* Taps */ 156 static struct list_head offload_base __read_mostly; 157 158 static int netif_rx_internal(struct sk_buff *skb); 159 static int call_netdevice_notifiers_info(unsigned long val, 160 struct net_device *dev, 161 struct netdev_notifier_info *info); 162 163 /* 164 * The @dev_base_head list is protected by @dev_base_lock and the rtnl 165 * semaphore. 166 * 167 * Pure readers hold dev_base_lock for reading, or rcu_read_lock() 168 * 169 * Writers must hold the rtnl semaphore while they loop through the 170 * dev_base_head list, and hold dev_base_lock for writing when they do the 171 * actual updates. This allows pure readers to access the list even 172 * while a writer is preparing to update it. 173 * 174 * To put it another way, dev_base_lock is held for writing only to 175 * protect against pure readers; the rtnl semaphore provides the 176 * protection against other writers. 177 * 178 * See, for example usages, register_netdevice() and 179 * unregister_netdevice(), which must be called with the rtnl 180 * semaphore held. 181 */ 182 DEFINE_RWLOCK(dev_base_lock); 183 EXPORT_SYMBOL(dev_base_lock); 184 185 /* protects napi_hash addition/deletion and napi_gen_id */ 186 static DEFINE_SPINLOCK(napi_hash_lock); 187 188 static unsigned int napi_gen_id = NR_CPUS; 189 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8); 190 191 static seqcount_t devnet_rename_seq; 192 193 static inline void dev_base_seq_inc(struct net *net) 194 { 195 while (++net->dev_base_seq == 0); 196 } 197 198 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name) 199 { 200 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ)); 201 202 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; 203 } 204 205 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) 206 { 207 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; 208 } 209 210 static inline void rps_lock(struct softnet_data *sd) 211 { 212 #ifdef CONFIG_RPS 213 spin_lock(&sd->input_pkt_queue.lock); 214 #endif 215 } 216 217 static inline void rps_unlock(struct softnet_data *sd) 218 { 219 #ifdef CONFIG_RPS 220 spin_unlock(&sd->input_pkt_queue.lock); 221 #endif 222 } 223 224 /* Device list insertion */ 225 static void list_netdevice(struct net_device *dev) 226 { 227 struct net *net = dev_net(dev); 228 229 ASSERT_RTNL(); 230 231 write_lock_bh(&dev_base_lock); 232 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); 233 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); 234 hlist_add_head_rcu(&dev->index_hlist, 235 dev_index_hash(net, dev->ifindex)); 236 write_unlock_bh(&dev_base_lock); 237 238 dev_base_seq_inc(net); 239 } 240 241 /* Device list removal 242 * caller must respect a RCU grace period before freeing/reusing dev 243 */ 244 static void unlist_netdevice(struct net_device *dev) 245 { 246 ASSERT_RTNL(); 247 248 /* Unlink dev from the device chain */ 249 write_lock_bh(&dev_base_lock); 250 list_del_rcu(&dev->dev_list); 251 hlist_del_rcu(&dev->name_hlist); 252 hlist_del_rcu(&dev->index_hlist); 253 write_unlock_bh(&dev_base_lock); 254 255 dev_base_seq_inc(dev_net(dev)); 256 } 257 258 /* 259 * Our notifier list 260 */ 261 262 static RAW_NOTIFIER_HEAD(netdev_chain); 263 264 /* 265 * Device drivers call our routines to queue packets here. We empty the 266 * queue in the local softnet handler. 267 */ 268 269 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data); 270 EXPORT_PER_CPU_SYMBOL(softnet_data); 271 272 #ifdef CONFIG_LOCKDEP 273 /* 274 * register_netdevice() inits txq->_xmit_lock and sets lockdep class 275 * according to dev->type 276 */ 277 static const unsigned short netdev_lock_type[] = 278 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25, 279 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET, 280 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM, 281 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, 282 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, 283 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, 284 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, 285 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, 286 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, 287 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, 288 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, 289 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, 290 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM, 291 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE, 292 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE}; 293 294 static const char *const netdev_lock_name[] = 295 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", 296 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET", 297 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM", 298 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", 299 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", 300 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", 301 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", 302 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", 303 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", 304 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", 305 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", 306 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", 307 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM", 308 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE", 309 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"}; 310 311 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; 312 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; 313 314 static inline unsigned short netdev_lock_pos(unsigned short dev_type) 315 { 316 int i; 317 318 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++) 319 if (netdev_lock_type[i] == dev_type) 320 return i; 321 /* the last key is used by default */ 322 return ARRAY_SIZE(netdev_lock_type) - 1; 323 } 324 325 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 326 unsigned short dev_type) 327 { 328 int i; 329 330 i = netdev_lock_pos(dev_type); 331 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i], 332 netdev_lock_name[i]); 333 } 334 335 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 336 { 337 int i; 338 339 i = netdev_lock_pos(dev->type); 340 lockdep_set_class_and_name(&dev->addr_list_lock, 341 &netdev_addr_lock_key[i], 342 netdev_lock_name[i]); 343 } 344 #else 345 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 346 unsigned short dev_type) 347 { 348 } 349 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 350 { 351 } 352 #endif 353 354 /******************************************************************************* 355 356 Protocol management and registration routines 357 358 *******************************************************************************/ 359 360 /* 361 * Add a protocol ID to the list. Now that the input handler is 362 * smarter we can dispense with all the messy stuff that used to be 363 * here. 364 * 365 * BEWARE!!! Protocol handlers, mangling input packets, 366 * MUST BE last in hash buckets and checking protocol handlers 367 * MUST start from promiscuous ptype_all chain in net_bh. 368 * It is true now, do not change it. 369 * Explanation follows: if protocol handler, mangling packet, will 370 * be the first on list, it is not able to sense, that packet 371 * is cloned and should be copied-on-write, so that it will 372 * change it and subsequent readers will get broken packet. 373 * --ANK (980803) 374 */ 375 376 static inline struct list_head *ptype_head(const struct packet_type *pt) 377 { 378 if (pt->type == htons(ETH_P_ALL)) 379 return pt->dev ? &pt->dev->ptype_all : &ptype_all; 380 else 381 return pt->dev ? &pt->dev->ptype_specific : 382 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; 383 } 384 385 /** 386 * dev_add_pack - add packet handler 387 * @pt: packet type declaration 388 * 389 * Add a protocol handler to the networking stack. The passed &packet_type 390 * is linked into kernel lists and may not be freed until it has been 391 * removed from the kernel lists. 392 * 393 * This call does not sleep therefore it can not 394 * guarantee all CPU's that are in middle of receiving packets 395 * will see the new packet type (until the next received packet). 396 */ 397 398 void dev_add_pack(struct packet_type *pt) 399 { 400 struct list_head *head = ptype_head(pt); 401 402 spin_lock(&ptype_lock); 403 list_add_rcu(&pt->list, head); 404 spin_unlock(&ptype_lock); 405 } 406 EXPORT_SYMBOL(dev_add_pack); 407 408 /** 409 * __dev_remove_pack - remove packet handler 410 * @pt: packet type declaration 411 * 412 * Remove a protocol handler that was previously added to the kernel 413 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 414 * from the kernel lists and can be freed or reused once this function 415 * returns. 416 * 417 * The packet type might still be in use by receivers 418 * and must not be freed until after all the CPU's have gone 419 * through a quiescent state. 420 */ 421 void __dev_remove_pack(struct packet_type *pt) 422 { 423 struct list_head *head = ptype_head(pt); 424 struct packet_type *pt1; 425 426 spin_lock(&ptype_lock); 427 428 list_for_each_entry(pt1, head, list) { 429 if (pt == pt1) { 430 list_del_rcu(&pt->list); 431 goto out; 432 } 433 } 434 435 pr_warn("dev_remove_pack: %p not found\n", pt); 436 out: 437 spin_unlock(&ptype_lock); 438 } 439 EXPORT_SYMBOL(__dev_remove_pack); 440 441 /** 442 * dev_remove_pack - remove packet handler 443 * @pt: packet type declaration 444 * 445 * Remove a protocol handler that was previously added to the kernel 446 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 447 * from the kernel lists and can be freed or reused once this function 448 * returns. 449 * 450 * This call sleeps to guarantee that no CPU is looking at the packet 451 * type after return. 452 */ 453 void dev_remove_pack(struct packet_type *pt) 454 { 455 __dev_remove_pack(pt); 456 457 synchronize_net(); 458 } 459 EXPORT_SYMBOL(dev_remove_pack); 460 461 462 /** 463 * dev_add_offload - register offload handlers 464 * @po: protocol offload declaration 465 * 466 * Add protocol offload handlers to the networking stack. The passed 467 * &proto_offload is linked into kernel lists and may not be freed until 468 * it has been removed from the kernel lists. 469 * 470 * This call does not sleep therefore it can not 471 * guarantee all CPU's that are in middle of receiving packets 472 * will see the new offload handlers (until the next received packet). 473 */ 474 void dev_add_offload(struct packet_offload *po) 475 { 476 struct packet_offload *elem; 477 478 spin_lock(&offload_lock); 479 list_for_each_entry(elem, &offload_base, list) { 480 if (po->priority < elem->priority) 481 break; 482 } 483 list_add_rcu(&po->list, elem->list.prev); 484 spin_unlock(&offload_lock); 485 } 486 EXPORT_SYMBOL(dev_add_offload); 487 488 /** 489 * __dev_remove_offload - remove offload handler 490 * @po: packet offload declaration 491 * 492 * Remove a protocol offload handler that was previously added to the 493 * kernel offload handlers by dev_add_offload(). The passed &offload_type 494 * is removed from the kernel lists and can be freed or reused once this 495 * function returns. 496 * 497 * The packet type might still be in use by receivers 498 * and must not be freed until after all the CPU's have gone 499 * through a quiescent state. 500 */ 501 static void __dev_remove_offload(struct packet_offload *po) 502 { 503 struct list_head *head = &offload_base; 504 struct packet_offload *po1; 505 506 spin_lock(&offload_lock); 507 508 list_for_each_entry(po1, head, list) { 509 if (po == po1) { 510 list_del_rcu(&po->list); 511 goto out; 512 } 513 } 514 515 pr_warn("dev_remove_offload: %p not found\n", po); 516 out: 517 spin_unlock(&offload_lock); 518 } 519 520 /** 521 * dev_remove_offload - remove packet offload handler 522 * @po: packet offload declaration 523 * 524 * Remove a packet offload handler that was previously added to the kernel 525 * offload handlers by dev_add_offload(). The passed &offload_type is 526 * removed from the kernel lists and can be freed or reused once this 527 * function returns. 528 * 529 * This call sleeps to guarantee that no CPU is looking at the packet 530 * type after return. 531 */ 532 void dev_remove_offload(struct packet_offload *po) 533 { 534 __dev_remove_offload(po); 535 536 synchronize_net(); 537 } 538 EXPORT_SYMBOL(dev_remove_offload); 539 540 /****************************************************************************** 541 542 Device Boot-time Settings Routines 543 544 *******************************************************************************/ 545 546 /* Boot time configuration table */ 547 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX]; 548 549 /** 550 * netdev_boot_setup_add - add new setup entry 551 * @name: name of the device 552 * @map: configured settings for the device 553 * 554 * Adds new setup entry to the dev_boot_setup list. The function 555 * returns 0 on error and 1 on success. This is a generic routine to 556 * all netdevices. 557 */ 558 static int netdev_boot_setup_add(char *name, struct ifmap *map) 559 { 560 struct netdev_boot_setup *s; 561 int i; 562 563 s = dev_boot_setup; 564 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 565 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') { 566 memset(s[i].name, 0, sizeof(s[i].name)); 567 strlcpy(s[i].name, name, IFNAMSIZ); 568 memcpy(&s[i].map, map, sizeof(s[i].map)); 569 break; 570 } 571 } 572 573 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1; 574 } 575 576 /** 577 * netdev_boot_setup_check - check boot time settings 578 * @dev: the netdevice 579 * 580 * Check boot time settings for the device. 581 * The found settings are set for the device to be used 582 * later in the device probing. 583 * Returns 0 if no settings found, 1 if they are. 584 */ 585 int netdev_boot_setup_check(struct net_device *dev) 586 { 587 struct netdev_boot_setup *s = dev_boot_setup; 588 int i; 589 590 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 591 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' && 592 !strcmp(dev->name, s[i].name)) { 593 dev->irq = s[i].map.irq; 594 dev->base_addr = s[i].map.base_addr; 595 dev->mem_start = s[i].map.mem_start; 596 dev->mem_end = s[i].map.mem_end; 597 return 1; 598 } 599 } 600 return 0; 601 } 602 EXPORT_SYMBOL(netdev_boot_setup_check); 603 604 605 /** 606 * netdev_boot_base - get address from boot time settings 607 * @prefix: prefix for network device 608 * @unit: id for network device 609 * 610 * Check boot time settings for the base address of device. 611 * The found settings are set for the device to be used 612 * later in the device probing. 613 * Returns 0 if no settings found. 614 */ 615 unsigned long netdev_boot_base(const char *prefix, int unit) 616 { 617 const struct netdev_boot_setup *s = dev_boot_setup; 618 char name[IFNAMSIZ]; 619 int i; 620 621 sprintf(name, "%s%d", prefix, unit); 622 623 /* 624 * If device already registered then return base of 1 625 * to indicate not to probe for this interface 626 */ 627 if (__dev_get_by_name(&init_net, name)) 628 return 1; 629 630 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) 631 if (!strcmp(name, s[i].name)) 632 return s[i].map.base_addr; 633 return 0; 634 } 635 636 /* 637 * Saves at boot time configured settings for any netdevice. 638 */ 639 int __init netdev_boot_setup(char *str) 640 { 641 int ints[5]; 642 struct ifmap map; 643 644 str = get_options(str, ARRAY_SIZE(ints), ints); 645 if (!str || !*str) 646 return 0; 647 648 /* Save settings */ 649 memset(&map, 0, sizeof(map)); 650 if (ints[0] > 0) 651 map.irq = ints[1]; 652 if (ints[0] > 1) 653 map.base_addr = ints[2]; 654 if (ints[0] > 2) 655 map.mem_start = ints[3]; 656 if (ints[0] > 3) 657 map.mem_end = ints[4]; 658 659 /* Add new entry to the list */ 660 return netdev_boot_setup_add(str, &map); 661 } 662 663 __setup("netdev=", netdev_boot_setup); 664 665 /******************************************************************************* 666 667 Device Interface Subroutines 668 669 *******************************************************************************/ 670 671 /** 672 * dev_get_iflink - get 'iflink' value of a interface 673 * @dev: targeted interface 674 * 675 * Indicates the ifindex the interface is linked to. 676 * Physical interfaces have the same 'ifindex' and 'iflink' values. 677 */ 678 679 int dev_get_iflink(const struct net_device *dev) 680 { 681 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) 682 return dev->netdev_ops->ndo_get_iflink(dev); 683 684 return dev->ifindex; 685 } 686 EXPORT_SYMBOL(dev_get_iflink); 687 688 /** 689 * dev_fill_metadata_dst - Retrieve tunnel egress information. 690 * @dev: targeted interface 691 * @skb: The packet. 692 * 693 * For better visibility of tunnel traffic OVS needs to retrieve 694 * egress tunnel information for a packet. Following API allows 695 * user to get this info. 696 */ 697 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb) 698 { 699 struct ip_tunnel_info *info; 700 701 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst) 702 return -EINVAL; 703 704 info = skb_tunnel_info_unclone(skb); 705 if (!info) 706 return -ENOMEM; 707 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX))) 708 return -EINVAL; 709 710 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb); 711 } 712 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst); 713 714 /** 715 * __dev_get_by_name - find a device by its name 716 * @net: the applicable net namespace 717 * @name: name to find 718 * 719 * Find an interface by name. Must be called under RTNL semaphore 720 * or @dev_base_lock. If the name is found a pointer to the device 721 * is returned. If the name is not found then %NULL is returned. The 722 * reference counters are not incremented so the caller must be 723 * careful with locks. 724 */ 725 726 struct net_device *__dev_get_by_name(struct net *net, const char *name) 727 { 728 struct net_device *dev; 729 struct hlist_head *head = dev_name_hash(net, name); 730 731 hlist_for_each_entry(dev, head, name_hlist) 732 if (!strncmp(dev->name, name, IFNAMSIZ)) 733 return dev; 734 735 return NULL; 736 } 737 EXPORT_SYMBOL(__dev_get_by_name); 738 739 /** 740 * dev_get_by_name_rcu - find a device by its name 741 * @net: the applicable net namespace 742 * @name: name to find 743 * 744 * Find an interface by name. 745 * If the name is found a pointer to the device is returned. 746 * If the name is not found then %NULL is returned. 747 * The reference counters are not incremented so the caller must be 748 * careful with locks. The caller must hold RCU lock. 749 */ 750 751 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name) 752 { 753 struct net_device *dev; 754 struct hlist_head *head = dev_name_hash(net, name); 755 756 hlist_for_each_entry_rcu(dev, head, name_hlist) 757 if (!strncmp(dev->name, name, IFNAMSIZ)) 758 return dev; 759 760 return NULL; 761 } 762 EXPORT_SYMBOL(dev_get_by_name_rcu); 763 764 /** 765 * dev_get_by_name - find a device by its name 766 * @net: the applicable net namespace 767 * @name: name to find 768 * 769 * Find an interface by name. This can be called from any 770 * context and does its own locking. The returned handle has 771 * the usage count incremented and the caller must use dev_put() to 772 * release it when it is no longer needed. %NULL is returned if no 773 * matching device is found. 774 */ 775 776 struct net_device *dev_get_by_name(struct net *net, const char *name) 777 { 778 struct net_device *dev; 779 780 rcu_read_lock(); 781 dev = dev_get_by_name_rcu(net, name); 782 if (dev) 783 dev_hold(dev); 784 rcu_read_unlock(); 785 return dev; 786 } 787 EXPORT_SYMBOL(dev_get_by_name); 788 789 /** 790 * __dev_get_by_index - find a device by its ifindex 791 * @net: the applicable net namespace 792 * @ifindex: index of device 793 * 794 * Search for an interface by index. Returns %NULL if the device 795 * is not found or a pointer to the device. The device has not 796 * had its reference counter increased so the caller must be careful 797 * about locking. The caller must hold either the RTNL semaphore 798 * or @dev_base_lock. 799 */ 800 801 struct net_device *__dev_get_by_index(struct net *net, int ifindex) 802 { 803 struct net_device *dev; 804 struct hlist_head *head = dev_index_hash(net, ifindex); 805 806 hlist_for_each_entry(dev, head, index_hlist) 807 if (dev->ifindex == ifindex) 808 return dev; 809 810 return NULL; 811 } 812 EXPORT_SYMBOL(__dev_get_by_index); 813 814 /** 815 * dev_get_by_index_rcu - find a device by its ifindex 816 * @net: the applicable net namespace 817 * @ifindex: index of device 818 * 819 * Search for an interface by index. Returns %NULL if the device 820 * is not found or a pointer to the device. The device has not 821 * had its reference counter increased so the caller must be careful 822 * about locking. The caller must hold RCU lock. 823 */ 824 825 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex) 826 { 827 struct net_device *dev; 828 struct hlist_head *head = dev_index_hash(net, ifindex); 829 830 hlist_for_each_entry_rcu(dev, head, index_hlist) 831 if (dev->ifindex == ifindex) 832 return dev; 833 834 return NULL; 835 } 836 EXPORT_SYMBOL(dev_get_by_index_rcu); 837 838 839 /** 840 * dev_get_by_index - find a device by its ifindex 841 * @net: the applicable net namespace 842 * @ifindex: index of device 843 * 844 * Search for an interface by index. Returns NULL if the device 845 * is not found or a pointer to the device. The device returned has 846 * had a reference added and the pointer is safe until the user calls 847 * dev_put to indicate they have finished with it. 848 */ 849 850 struct net_device *dev_get_by_index(struct net *net, int ifindex) 851 { 852 struct net_device *dev; 853 854 rcu_read_lock(); 855 dev = dev_get_by_index_rcu(net, ifindex); 856 if (dev) 857 dev_hold(dev); 858 rcu_read_unlock(); 859 return dev; 860 } 861 EXPORT_SYMBOL(dev_get_by_index); 862 863 /** 864 * netdev_get_name - get a netdevice name, knowing its ifindex. 865 * @net: network namespace 866 * @name: a pointer to the buffer where the name will be stored. 867 * @ifindex: the ifindex of the interface to get the name from. 868 * 869 * The use of raw_seqcount_begin() and cond_resched() before 870 * retrying is required as we want to give the writers a chance 871 * to complete when CONFIG_PREEMPT is not set. 872 */ 873 int netdev_get_name(struct net *net, char *name, int ifindex) 874 { 875 struct net_device *dev; 876 unsigned int seq; 877 878 retry: 879 seq = raw_seqcount_begin(&devnet_rename_seq); 880 rcu_read_lock(); 881 dev = dev_get_by_index_rcu(net, ifindex); 882 if (!dev) { 883 rcu_read_unlock(); 884 return -ENODEV; 885 } 886 887 strcpy(name, dev->name); 888 rcu_read_unlock(); 889 if (read_seqcount_retry(&devnet_rename_seq, seq)) { 890 cond_resched(); 891 goto retry; 892 } 893 894 return 0; 895 } 896 897 /** 898 * dev_getbyhwaddr_rcu - find a device by its hardware address 899 * @net: the applicable net namespace 900 * @type: media type of device 901 * @ha: hardware address 902 * 903 * Search for an interface by MAC address. Returns NULL if the device 904 * is not found or a pointer to the device. 905 * The caller must hold RCU or RTNL. 906 * The returned device has not had its ref count increased 907 * and the caller must therefore be careful about locking 908 * 909 */ 910 911 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type, 912 const char *ha) 913 { 914 struct net_device *dev; 915 916 for_each_netdev_rcu(net, dev) 917 if (dev->type == type && 918 !memcmp(dev->dev_addr, ha, dev->addr_len)) 919 return dev; 920 921 return NULL; 922 } 923 EXPORT_SYMBOL(dev_getbyhwaddr_rcu); 924 925 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type) 926 { 927 struct net_device *dev; 928 929 ASSERT_RTNL(); 930 for_each_netdev(net, dev) 931 if (dev->type == type) 932 return dev; 933 934 return NULL; 935 } 936 EXPORT_SYMBOL(__dev_getfirstbyhwtype); 937 938 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) 939 { 940 struct net_device *dev, *ret = NULL; 941 942 rcu_read_lock(); 943 for_each_netdev_rcu(net, dev) 944 if (dev->type == type) { 945 dev_hold(dev); 946 ret = dev; 947 break; 948 } 949 rcu_read_unlock(); 950 return ret; 951 } 952 EXPORT_SYMBOL(dev_getfirstbyhwtype); 953 954 /** 955 * __dev_get_by_flags - find any device with given flags 956 * @net: the applicable net namespace 957 * @if_flags: IFF_* values 958 * @mask: bitmask of bits in if_flags to check 959 * 960 * Search for any interface with the given flags. Returns NULL if a device 961 * is not found or a pointer to the device. Must be called inside 962 * rtnl_lock(), and result refcount is unchanged. 963 */ 964 965 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags, 966 unsigned short mask) 967 { 968 struct net_device *dev, *ret; 969 970 ASSERT_RTNL(); 971 972 ret = NULL; 973 for_each_netdev(net, dev) { 974 if (((dev->flags ^ if_flags) & mask) == 0) { 975 ret = dev; 976 break; 977 } 978 } 979 return ret; 980 } 981 EXPORT_SYMBOL(__dev_get_by_flags); 982 983 /** 984 * dev_valid_name - check if name is okay for network device 985 * @name: name string 986 * 987 * Network device names need to be valid file names to 988 * to allow sysfs to work. We also disallow any kind of 989 * whitespace. 990 */ 991 bool dev_valid_name(const char *name) 992 { 993 if (*name == '\0') 994 return false; 995 if (strlen(name) >= IFNAMSIZ) 996 return false; 997 if (!strcmp(name, ".") || !strcmp(name, "..")) 998 return false; 999 1000 while (*name) { 1001 if (*name == '/' || *name == ':' || isspace(*name)) 1002 return false; 1003 name++; 1004 } 1005 return true; 1006 } 1007 EXPORT_SYMBOL(dev_valid_name); 1008 1009 /** 1010 * __dev_alloc_name - allocate a name for a device 1011 * @net: network namespace to allocate the device name in 1012 * @name: name format string 1013 * @buf: scratch buffer and result name string 1014 * 1015 * Passed a format string - eg "lt%d" it will try and find a suitable 1016 * id. It scans list of devices to build up a free map, then chooses 1017 * the first empty slot. The caller must hold the dev_base or rtnl lock 1018 * while allocating the name and adding the device in order to avoid 1019 * duplicates. 1020 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1021 * Returns the number of the unit assigned or a negative errno code. 1022 */ 1023 1024 static int __dev_alloc_name(struct net *net, const char *name, char *buf) 1025 { 1026 int i = 0; 1027 const char *p; 1028 const int max_netdevices = 8*PAGE_SIZE; 1029 unsigned long *inuse; 1030 struct net_device *d; 1031 1032 p = strnchr(name, IFNAMSIZ-1, '%'); 1033 if (p) { 1034 /* 1035 * Verify the string as this thing may have come from 1036 * the user. There must be either one "%d" and no other "%" 1037 * characters. 1038 */ 1039 if (p[1] != 'd' || strchr(p + 2, '%')) 1040 return -EINVAL; 1041 1042 /* Use one page as a bit array of possible slots */ 1043 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC); 1044 if (!inuse) 1045 return -ENOMEM; 1046 1047 for_each_netdev(net, d) { 1048 if (!sscanf(d->name, name, &i)) 1049 continue; 1050 if (i < 0 || i >= max_netdevices) 1051 continue; 1052 1053 /* avoid cases where sscanf is not exact inverse of printf */ 1054 snprintf(buf, IFNAMSIZ, name, i); 1055 if (!strncmp(buf, d->name, IFNAMSIZ)) 1056 set_bit(i, inuse); 1057 } 1058 1059 i = find_first_zero_bit(inuse, max_netdevices); 1060 free_page((unsigned long) inuse); 1061 } 1062 1063 if (buf != name) 1064 snprintf(buf, IFNAMSIZ, name, i); 1065 if (!__dev_get_by_name(net, buf)) 1066 return i; 1067 1068 /* It is possible to run out of possible slots 1069 * when the name is long and there isn't enough space left 1070 * for the digits, or if all bits are used. 1071 */ 1072 return -ENFILE; 1073 } 1074 1075 /** 1076 * dev_alloc_name - allocate a name for a device 1077 * @dev: device 1078 * @name: name format string 1079 * 1080 * Passed a format string - eg "lt%d" it will try and find a suitable 1081 * id. It scans list of devices to build up a free map, then chooses 1082 * the first empty slot. The caller must hold the dev_base or rtnl lock 1083 * while allocating the name and adding the device in order to avoid 1084 * duplicates. 1085 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1086 * Returns the number of the unit assigned or a negative errno code. 1087 */ 1088 1089 int dev_alloc_name(struct net_device *dev, const char *name) 1090 { 1091 char buf[IFNAMSIZ]; 1092 struct net *net; 1093 int ret; 1094 1095 BUG_ON(!dev_net(dev)); 1096 net = dev_net(dev); 1097 ret = __dev_alloc_name(net, name, buf); 1098 if (ret >= 0) 1099 strlcpy(dev->name, buf, IFNAMSIZ); 1100 return ret; 1101 } 1102 EXPORT_SYMBOL(dev_alloc_name); 1103 1104 static int dev_alloc_name_ns(struct net *net, 1105 struct net_device *dev, 1106 const char *name) 1107 { 1108 char buf[IFNAMSIZ]; 1109 int ret; 1110 1111 ret = __dev_alloc_name(net, name, buf); 1112 if (ret >= 0) 1113 strlcpy(dev->name, buf, IFNAMSIZ); 1114 return ret; 1115 } 1116 1117 static int dev_get_valid_name(struct net *net, 1118 struct net_device *dev, 1119 const char *name) 1120 { 1121 BUG_ON(!net); 1122 1123 if (!dev_valid_name(name)) 1124 return -EINVAL; 1125 1126 if (strchr(name, '%')) 1127 return dev_alloc_name_ns(net, dev, name); 1128 else if (__dev_get_by_name(net, name)) 1129 return -EEXIST; 1130 else if (dev->name != name) 1131 strlcpy(dev->name, name, IFNAMSIZ); 1132 1133 return 0; 1134 } 1135 1136 /** 1137 * dev_change_name - change name of a device 1138 * @dev: device 1139 * @newname: name (or format string) must be at least IFNAMSIZ 1140 * 1141 * Change name of a device, can pass format strings "eth%d". 1142 * for wildcarding. 1143 */ 1144 int dev_change_name(struct net_device *dev, const char *newname) 1145 { 1146 unsigned char old_assign_type; 1147 char oldname[IFNAMSIZ]; 1148 int err = 0; 1149 int ret; 1150 struct net *net; 1151 1152 ASSERT_RTNL(); 1153 BUG_ON(!dev_net(dev)); 1154 1155 net = dev_net(dev); 1156 if (dev->flags & IFF_UP) 1157 return -EBUSY; 1158 1159 write_seqcount_begin(&devnet_rename_seq); 1160 1161 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) { 1162 write_seqcount_end(&devnet_rename_seq); 1163 return 0; 1164 } 1165 1166 memcpy(oldname, dev->name, IFNAMSIZ); 1167 1168 err = dev_get_valid_name(net, dev, newname); 1169 if (err < 0) { 1170 write_seqcount_end(&devnet_rename_seq); 1171 return err; 1172 } 1173 1174 if (oldname[0] && !strchr(oldname, '%')) 1175 netdev_info(dev, "renamed from %s\n", oldname); 1176 1177 old_assign_type = dev->name_assign_type; 1178 dev->name_assign_type = NET_NAME_RENAMED; 1179 1180 rollback: 1181 ret = device_rename(&dev->dev, dev->name); 1182 if (ret) { 1183 memcpy(dev->name, oldname, IFNAMSIZ); 1184 dev->name_assign_type = old_assign_type; 1185 write_seqcount_end(&devnet_rename_seq); 1186 return ret; 1187 } 1188 1189 write_seqcount_end(&devnet_rename_seq); 1190 1191 netdev_adjacent_rename_links(dev, oldname); 1192 1193 write_lock_bh(&dev_base_lock); 1194 hlist_del_rcu(&dev->name_hlist); 1195 write_unlock_bh(&dev_base_lock); 1196 1197 synchronize_rcu(); 1198 1199 write_lock_bh(&dev_base_lock); 1200 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); 1201 write_unlock_bh(&dev_base_lock); 1202 1203 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); 1204 ret = notifier_to_errno(ret); 1205 1206 if (ret) { 1207 /* err >= 0 after dev_alloc_name() or stores the first errno */ 1208 if (err >= 0) { 1209 err = ret; 1210 write_seqcount_begin(&devnet_rename_seq); 1211 memcpy(dev->name, oldname, IFNAMSIZ); 1212 memcpy(oldname, newname, IFNAMSIZ); 1213 dev->name_assign_type = old_assign_type; 1214 old_assign_type = NET_NAME_RENAMED; 1215 goto rollback; 1216 } else { 1217 pr_err("%s: name change rollback failed: %d\n", 1218 dev->name, ret); 1219 } 1220 } 1221 1222 return err; 1223 } 1224 1225 /** 1226 * dev_set_alias - change ifalias of a device 1227 * @dev: device 1228 * @alias: name up to IFALIASZ 1229 * @len: limit of bytes to copy from info 1230 * 1231 * Set ifalias for a device, 1232 */ 1233 int dev_set_alias(struct net_device *dev, const char *alias, size_t len) 1234 { 1235 char *new_ifalias; 1236 1237 ASSERT_RTNL(); 1238 1239 if (len >= IFALIASZ) 1240 return -EINVAL; 1241 1242 if (!len) { 1243 kfree(dev->ifalias); 1244 dev->ifalias = NULL; 1245 return 0; 1246 } 1247 1248 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL); 1249 if (!new_ifalias) 1250 return -ENOMEM; 1251 dev->ifalias = new_ifalias; 1252 1253 strlcpy(dev->ifalias, alias, len+1); 1254 return len; 1255 } 1256 1257 1258 /** 1259 * netdev_features_change - device changes features 1260 * @dev: device to cause notification 1261 * 1262 * Called to indicate a device has changed features. 1263 */ 1264 void netdev_features_change(struct net_device *dev) 1265 { 1266 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); 1267 } 1268 EXPORT_SYMBOL(netdev_features_change); 1269 1270 /** 1271 * netdev_state_change - device changes state 1272 * @dev: device to cause notification 1273 * 1274 * Called to indicate a device has changed state. This function calls 1275 * the notifier chains for netdev_chain and sends a NEWLINK message 1276 * to the routing socket. 1277 */ 1278 void netdev_state_change(struct net_device *dev) 1279 { 1280 if (dev->flags & IFF_UP) { 1281 struct netdev_notifier_change_info change_info; 1282 1283 change_info.flags_changed = 0; 1284 call_netdevice_notifiers_info(NETDEV_CHANGE, dev, 1285 &change_info.info); 1286 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL); 1287 } 1288 } 1289 EXPORT_SYMBOL(netdev_state_change); 1290 1291 /** 1292 * netdev_notify_peers - notify network peers about existence of @dev 1293 * @dev: network device 1294 * 1295 * Generate traffic such that interested network peers are aware of 1296 * @dev, such as by generating a gratuitous ARP. This may be used when 1297 * a device wants to inform the rest of the network about some sort of 1298 * reconfiguration such as a failover event or virtual machine 1299 * migration. 1300 */ 1301 void netdev_notify_peers(struct net_device *dev) 1302 { 1303 rtnl_lock(); 1304 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev); 1305 rtnl_unlock(); 1306 } 1307 EXPORT_SYMBOL(netdev_notify_peers); 1308 1309 static int __dev_open(struct net_device *dev) 1310 { 1311 const struct net_device_ops *ops = dev->netdev_ops; 1312 int ret; 1313 1314 ASSERT_RTNL(); 1315 1316 if (!netif_device_present(dev)) 1317 return -ENODEV; 1318 1319 /* Block netpoll from trying to do any rx path servicing. 1320 * If we don't do this there is a chance ndo_poll_controller 1321 * or ndo_poll may be running while we open the device 1322 */ 1323 netpoll_poll_disable(dev); 1324 1325 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev); 1326 ret = notifier_to_errno(ret); 1327 if (ret) 1328 return ret; 1329 1330 set_bit(__LINK_STATE_START, &dev->state); 1331 1332 if (ops->ndo_validate_addr) 1333 ret = ops->ndo_validate_addr(dev); 1334 1335 if (!ret && ops->ndo_open) 1336 ret = ops->ndo_open(dev); 1337 1338 netpoll_poll_enable(dev); 1339 1340 if (ret) 1341 clear_bit(__LINK_STATE_START, &dev->state); 1342 else { 1343 dev->flags |= IFF_UP; 1344 dev_set_rx_mode(dev); 1345 dev_activate(dev); 1346 add_device_randomness(dev->dev_addr, dev->addr_len); 1347 } 1348 1349 return ret; 1350 } 1351 1352 /** 1353 * dev_open - prepare an interface for use. 1354 * @dev: device to open 1355 * 1356 * Takes a device from down to up state. The device's private open 1357 * function is invoked and then the multicast lists are loaded. Finally 1358 * the device is moved into the up state and a %NETDEV_UP message is 1359 * sent to the netdev notifier chain. 1360 * 1361 * Calling this function on an active interface is a nop. On a failure 1362 * a negative errno code is returned. 1363 */ 1364 int dev_open(struct net_device *dev) 1365 { 1366 int ret; 1367 1368 if (dev->flags & IFF_UP) 1369 return 0; 1370 1371 ret = __dev_open(dev); 1372 if (ret < 0) 1373 return ret; 1374 1375 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); 1376 call_netdevice_notifiers(NETDEV_UP, dev); 1377 1378 return ret; 1379 } 1380 EXPORT_SYMBOL(dev_open); 1381 1382 static int __dev_close_many(struct list_head *head) 1383 { 1384 struct net_device *dev; 1385 1386 ASSERT_RTNL(); 1387 might_sleep(); 1388 1389 list_for_each_entry(dev, head, close_list) { 1390 /* Temporarily disable netpoll until the interface is down */ 1391 netpoll_poll_disable(dev); 1392 1393 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); 1394 1395 clear_bit(__LINK_STATE_START, &dev->state); 1396 1397 /* Synchronize to scheduled poll. We cannot touch poll list, it 1398 * can be even on different cpu. So just clear netif_running(). 1399 * 1400 * dev->stop() will invoke napi_disable() on all of it's 1401 * napi_struct instances on this device. 1402 */ 1403 smp_mb__after_atomic(); /* Commit netif_running(). */ 1404 } 1405 1406 dev_deactivate_many(head); 1407 1408 list_for_each_entry(dev, head, close_list) { 1409 const struct net_device_ops *ops = dev->netdev_ops; 1410 1411 /* 1412 * Call the device specific close. This cannot fail. 1413 * Only if device is UP 1414 * 1415 * We allow it to be called even after a DETACH hot-plug 1416 * event. 1417 */ 1418 if (ops->ndo_stop) 1419 ops->ndo_stop(dev); 1420 1421 dev->flags &= ~IFF_UP; 1422 netpoll_poll_enable(dev); 1423 } 1424 1425 return 0; 1426 } 1427 1428 static int __dev_close(struct net_device *dev) 1429 { 1430 int retval; 1431 LIST_HEAD(single); 1432 1433 list_add(&dev->close_list, &single); 1434 retval = __dev_close_many(&single); 1435 list_del(&single); 1436 1437 return retval; 1438 } 1439 1440 int dev_close_many(struct list_head *head, bool unlink) 1441 { 1442 struct net_device *dev, *tmp; 1443 1444 /* Remove the devices that don't need to be closed */ 1445 list_for_each_entry_safe(dev, tmp, head, close_list) 1446 if (!(dev->flags & IFF_UP)) 1447 list_del_init(&dev->close_list); 1448 1449 __dev_close_many(head); 1450 1451 list_for_each_entry_safe(dev, tmp, head, close_list) { 1452 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); 1453 call_netdevice_notifiers(NETDEV_DOWN, dev); 1454 if (unlink) 1455 list_del_init(&dev->close_list); 1456 } 1457 1458 return 0; 1459 } 1460 EXPORT_SYMBOL(dev_close_many); 1461 1462 /** 1463 * dev_close - shutdown an interface. 1464 * @dev: device to shutdown 1465 * 1466 * This function moves an active device into down state. A 1467 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device 1468 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier 1469 * chain. 1470 */ 1471 int dev_close(struct net_device *dev) 1472 { 1473 if (dev->flags & IFF_UP) { 1474 LIST_HEAD(single); 1475 1476 list_add(&dev->close_list, &single); 1477 dev_close_many(&single, true); 1478 list_del(&single); 1479 } 1480 return 0; 1481 } 1482 EXPORT_SYMBOL(dev_close); 1483 1484 1485 /** 1486 * dev_disable_lro - disable Large Receive Offload on a device 1487 * @dev: device 1488 * 1489 * Disable Large Receive Offload (LRO) on a net device. Must be 1490 * called under RTNL. This is needed if received packets may be 1491 * forwarded to another interface. 1492 */ 1493 void dev_disable_lro(struct net_device *dev) 1494 { 1495 struct net_device *lower_dev; 1496 struct list_head *iter; 1497 1498 dev->wanted_features &= ~NETIF_F_LRO; 1499 netdev_update_features(dev); 1500 1501 if (unlikely(dev->features & NETIF_F_LRO)) 1502 netdev_WARN(dev, "failed to disable LRO!\n"); 1503 1504 netdev_for_each_lower_dev(dev, lower_dev, iter) 1505 dev_disable_lro(lower_dev); 1506 } 1507 EXPORT_SYMBOL(dev_disable_lro); 1508 1509 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val, 1510 struct net_device *dev) 1511 { 1512 struct netdev_notifier_info info; 1513 1514 netdev_notifier_info_init(&info, dev); 1515 return nb->notifier_call(nb, val, &info); 1516 } 1517 1518 static int dev_boot_phase = 1; 1519 1520 /** 1521 * register_netdevice_notifier - register a network notifier block 1522 * @nb: notifier 1523 * 1524 * Register a notifier to be called when network device events occur. 1525 * The notifier passed is linked into the kernel structures and must 1526 * not be reused until it has been unregistered. A negative errno code 1527 * is returned on a failure. 1528 * 1529 * When registered all registration and up events are replayed 1530 * to the new notifier to allow device to have a race free 1531 * view of the network device list. 1532 */ 1533 1534 int register_netdevice_notifier(struct notifier_block *nb) 1535 { 1536 struct net_device *dev; 1537 struct net_device *last; 1538 struct net *net; 1539 int err; 1540 1541 rtnl_lock(); 1542 err = raw_notifier_chain_register(&netdev_chain, nb); 1543 if (err) 1544 goto unlock; 1545 if (dev_boot_phase) 1546 goto unlock; 1547 for_each_net(net) { 1548 for_each_netdev(net, dev) { 1549 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev); 1550 err = notifier_to_errno(err); 1551 if (err) 1552 goto rollback; 1553 1554 if (!(dev->flags & IFF_UP)) 1555 continue; 1556 1557 call_netdevice_notifier(nb, NETDEV_UP, dev); 1558 } 1559 } 1560 1561 unlock: 1562 rtnl_unlock(); 1563 return err; 1564 1565 rollback: 1566 last = dev; 1567 for_each_net(net) { 1568 for_each_netdev(net, dev) { 1569 if (dev == last) 1570 goto outroll; 1571 1572 if (dev->flags & IFF_UP) { 1573 call_netdevice_notifier(nb, NETDEV_GOING_DOWN, 1574 dev); 1575 call_netdevice_notifier(nb, NETDEV_DOWN, dev); 1576 } 1577 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); 1578 } 1579 } 1580 1581 outroll: 1582 raw_notifier_chain_unregister(&netdev_chain, nb); 1583 goto unlock; 1584 } 1585 EXPORT_SYMBOL(register_netdevice_notifier); 1586 1587 /** 1588 * unregister_netdevice_notifier - unregister a network notifier block 1589 * @nb: notifier 1590 * 1591 * Unregister a notifier previously registered by 1592 * register_netdevice_notifier(). The notifier is unlinked into the 1593 * kernel structures and may then be reused. A negative errno code 1594 * is returned on a failure. 1595 * 1596 * After unregistering unregister and down device events are synthesized 1597 * for all devices on the device list to the removed notifier to remove 1598 * the need for special case cleanup code. 1599 */ 1600 1601 int unregister_netdevice_notifier(struct notifier_block *nb) 1602 { 1603 struct net_device *dev; 1604 struct net *net; 1605 int err; 1606 1607 rtnl_lock(); 1608 err = raw_notifier_chain_unregister(&netdev_chain, nb); 1609 if (err) 1610 goto unlock; 1611 1612 for_each_net(net) { 1613 for_each_netdev(net, dev) { 1614 if (dev->flags & IFF_UP) { 1615 call_netdevice_notifier(nb, NETDEV_GOING_DOWN, 1616 dev); 1617 call_netdevice_notifier(nb, NETDEV_DOWN, dev); 1618 } 1619 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); 1620 } 1621 } 1622 unlock: 1623 rtnl_unlock(); 1624 return err; 1625 } 1626 EXPORT_SYMBOL(unregister_netdevice_notifier); 1627 1628 /** 1629 * call_netdevice_notifiers_info - call all network notifier blocks 1630 * @val: value passed unmodified to notifier function 1631 * @dev: net_device pointer passed unmodified to notifier function 1632 * @info: notifier information data 1633 * 1634 * Call all network notifier blocks. Parameters and return value 1635 * are as for raw_notifier_call_chain(). 1636 */ 1637 1638 static int call_netdevice_notifiers_info(unsigned long val, 1639 struct net_device *dev, 1640 struct netdev_notifier_info *info) 1641 { 1642 ASSERT_RTNL(); 1643 netdev_notifier_info_init(info, dev); 1644 return raw_notifier_call_chain(&netdev_chain, val, info); 1645 } 1646 1647 /** 1648 * call_netdevice_notifiers - call all network notifier blocks 1649 * @val: value passed unmodified to notifier function 1650 * @dev: net_device pointer passed unmodified to notifier function 1651 * 1652 * Call all network notifier blocks. Parameters and return value 1653 * are as for raw_notifier_call_chain(). 1654 */ 1655 1656 int call_netdevice_notifiers(unsigned long val, struct net_device *dev) 1657 { 1658 struct netdev_notifier_info info; 1659 1660 return call_netdevice_notifiers_info(val, dev, &info); 1661 } 1662 EXPORT_SYMBOL(call_netdevice_notifiers); 1663 1664 #ifdef CONFIG_NET_INGRESS 1665 static struct static_key ingress_needed __read_mostly; 1666 1667 void net_inc_ingress_queue(void) 1668 { 1669 static_key_slow_inc(&ingress_needed); 1670 } 1671 EXPORT_SYMBOL_GPL(net_inc_ingress_queue); 1672 1673 void net_dec_ingress_queue(void) 1674 { 1675 static_key_slow_dec(&ingress_needed); 1676 } 1677 EXPORT_SYMBOL_GPL(net_dec_ingress_queue); 1678 #endif 1679 1680 #ifdef CONFIG_NET_EGRESS 1681 static struct static_key egress_needed __read_mostly; 1682 1683 void net_inc_egress_queue(void) 1684 { 1685 static_key_slow_inc(&egress_needed); 1686 } 1687 EXPORT_SYMBOL_GPL(net_inc_egress_queue); 1688 1689 void net_dec_egress_queue(void) 1690 { 1691 static_key_slow_dec(&egress_needed); 1692 } 1693 EXPORT_SYMBOL_GPL(net_dec_egress_queue); 1694 #endif 1695 1696 static struct static_key netstamp_needed __read_mostly; 1697 #ifdef HAVE_JUMP_LABEL 1698 /* We are not allowed to call static_key_slow_dec() from irq context 1699 * If net_disable_timestamp() is called from irq context, defer the 1700 * static_key_slow_dec() calls. 1701 */ 1702 static atomic_t netstamp_needed_deferred; 1703 #endif 1704 1705 void net_enable_timestamp(void) 1706 { 1707 #ifdef HAVE_JUMP_LABEL 1708 int deferred = atomic_xchg(&netstamp_needed_deferred, 0); 1709 1710 if (deferred) { 1711 while (--deferred) 1712 static_key_slow_dec(&netstamp_needed); 1713 return; 1714 } 1715 #endif 1716 static_key_slow_inc(&netstamp_needed); 1717 } 1718 EXPORT_SYMBOL(net_enable_timestamp); 1719 1720 void net_disable_timestamp(void) 1721 { 1722 #ifdef HAVE_JUMP_LABEL 1723 if (in_interrupt()) { 1724 atomic_inc(&netstamp_needed_deferred); 1725 return; 1726 } 1727 #endif 1728 static_key_slow_dec(&netstamp_needed); 1729 } 1730 EXPORT_SYMBOL(net_disable_timestamp); 1731 1732 static inline void net_timestamp_set(struct sk_buff *skb) 1733 { 1734 skb->tstamp = 0; 1735 if (static_key_false(&netstamp_needed)) 1736 __net_timestamp(skb); 1737 } 1738 1739 #define net_timestamp_check(COND, SKB) \ 1740 if (static_key_false(&netstamp_needed)) { \ 1741 if ((COND) && !(SKB)->tstamp) \ 1742 __net_timestamp(SKB); \ 1743 } \ 1744 1745 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb) 1746 { 1747 unsigned int len; 1748 1749 if (!(dev->flags & IFF_UP)) 1750 return false; 1751 1752 len = dev->mtu + dev->hard_header_len + VLAN_HLEN; 1753 if (skb->len <= len) 1754 return true; 1755 1756 /* if TSO is enabled, we don't care about the length as the packet 1757 * could be forwarded without being segmented before 1758 */ 1759 if (skb_is_gso(skb)) 1760 return true; 1761 1762 return false; 1763 } 1764 EXPORT_SYMBOL_GPL(is_skb_forwardable); 1765 1766 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 1767 { 1768 int ret = ____dev_forward_skb(dev, skb); 1769 1770 if (likely(!ret)) { 1771 skb->protocol = eth_type_trans(skb, dev); 1772 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); 1773 } 1774 1775 return ret; 1776 } 1777 EXPORT_SYMBOL_GPL(__dev_forward_skb); 1778 1779 /** 1780 * dev_forward_skb - loopback an skb to another netif 1781 * 1782 * @dev: destination network device 1783 * @skb: buffer to forward 1784 * 1785 * return values: 1786 * NET_RX_SUCCESS (no congestion) 1787 * NET_RX_DROP (packet was dropped, but freed) 1788 * 1789 * dev_forward_skb can be used for injecting an skb from the 1790 * start_xmit function of one device into the receive queue 1791 * of another device. 1792 * 1793 * The receiving device may be in another namespace, so 1794 * we have to clear all information in the skb that could 1795 * impact namespace isolation. 1796 */ 1797 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 1798 { 1799 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb); 1800 } 1801 EXPORT_SYMBOL_GPL(dev_forward_skb); 1802 1803 static inline int deliver_skb(struct sk_buff *skb, 1804 struct packet_type *pt_prev, 1805 struct net_device *orig_dev) 1806 { 1807 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) 1808 return -ENOMEM; 1809 atomic_inc(&skb->users); 1810 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 1811 } 1812 1813 static inline void deliver_ptype_list_skb(struct sk_buff *skb, 1814 struct packet_type **pt, 1815 struct net_device *orig_dev, 1816 __be16 type, 1817 struct list_head *ptype_list) 1818 { 1819 struct packet_type *ptype, *pt_prev = *pt; 1820 1821 list_for_each_entry_rcu(ptype, ptype_list, list) { 1822 if (ptype->type != type) 1823 continue; 1824 if (pt_prev) 1825 deliver_skb(skb, pt_prev, orig_dev); 1826 pt_prev = ptype; 1827 } 1828 *pt = pt_prev; 1829 } 1830 1831 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) 1832 { 1833 if (!ptype->af_packet_priv || !skb->sk) 1834 return false; 1835 1836 if (ptype->id_match) 1837 return ptype->id_match(ptype, skb->sk); 1838 else if ((struct sock *)ptype->af_packet_priv == skb->sk) 1839 return true; 1840 1841 return false; 1842 } 1843 1844 /* 1845 * Support routine. Sends outgoing frames to any network 1846 * taps currently in use. 1847 */ 1848 1849 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) 1850 { 1851 struct packet_type *ptype; 1852 struct sk_buff *skb2 = NULL; 1853 struct packet_type *pt_prev = NULL; 1854 struct list_head *ptype_list = &ptype_all; 1855 1856 rcu_read_lock(); 1857 again: 1858 list_for_each_entry_rcu(ptype, ptype_list, list) { 1859 /* Never send packets back to the socket 1860 * they originated from - MvS (miquels@drinkel.ow.org) 1861 */ 1862 if (skb_loop_sk(ptype, skb)) 1863 continue; 1864 1865 if (pt_prev) { 1866 deliver_skb(skb2, pt_prev, skb->dev); 1867 pt_prev = ptype; 1868 continue; 1869 } 1870 1871 /* need to clone skb, done only once */ 1872 skb2 = skb_clone(skb, GFP_ATOMIC); 1873 if (!skb2) 1874 goto out_unlock; 1875 1876 net_timestamp_set(skb2); 1877 1878 /* skb->nh should be correctly 1879 * set by sender, so that the second statement is 1880 * just protection against buggy protocols. 1881 */ 1882 skb_reset_mac_header(skb2); 1883 1884 if (skb_network_header(skb2) < skb2->data || 1885 skb_network_header(skb2) > skb_tail_pointer(skb2)) { 1886 net_crit_ratelimited("protocol %04x is buggy, dev %s\n", 1887 ntohs(skb2->protocol), 1888 dev->name); 1889 skb_reset_network_header(skb2); 1890 } 1891 1892 skb2->transport_header = skb2->network_header; 1893 skb2->pkt_type = PACKET_OUTGOING; 1894 pt_prev = ptype; 1895 } 1896 1897 if (ptype_list == &ptype_all) { 1898 ptype_list = &dev->ptype_all; 1899 goto again; 1900 } 1901 out_unlock: 1902 if (pt_prev) 1903 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); 1904 rcu_read_unlock(); 1905 } 1906 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit); 1907 1908 /** 1909 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change 1910 * @dev: Network device 1911 * @txq: number of queues available 1912 * 1913 * If real_num_tx_queues is changed the tc mappings may no longer be 1914 * valid. To resolve this verify the tc mapping remains valid and if 1915 * not NULL the mapping. With no priorities mapping to this 1916 * offset/count pair it will no longer be used. In the worst case TC0 1917 * is invalid nothing can be done so disable priority mappings. If is 1918 * expected that drivers will fix this mapping if they can before 1919 * calling netif_set_real_num_tx_queues. 1920 */ 1921 static void netif_setup_tc(struct net_device *dev, unsigned int txq) 1922 { 1923 int i; 1924 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 1925 1926 /* If TC0 is invalidated disable TC mapping */ 1927 if (tc->offset + tc->count > txq) { 1928 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n"); 1929 dev->num_tc = 0; 1930 return; 1931 } 1932 1933 /* Invalidated prio to tc mappings set to TC0 */ 1934 for (i = 1; i < TC_BITMASK + 1; i++) { 1935 int q = netdev_get_prio_tc_map(dev, i); 1936 1937 tc = &dev->tc_to_txq[q]; 1938 if (tc->offset + tc->count > txq) { 1939 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n", 1940 i, q); 1941 netdev_set_prio_tc_map(dev, i, 0); 1942 } 1943 } 1944 } 1945 1946 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq) 1947 { 1948 if (dev->num_tc) { 1949 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 1950 int i; 1951 1952 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) { 1953 if ((txq - tc->offset) < tc->count) 1954 return i; 1955 } 1956 1957 return -1; 1958 } 1959 1960 return 0; 1961 } 1962 1963 #ifdef CONFIG_XPS 1964 static DEFINE_MUTEX(xps_map_mutex); 1965 #define xmap_dereference(P) \ 1966 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex)) 1967 1968 static bool remove_xps_queue(struct xps_dev_maps *dev_maps, 1969 int tci, u16 index) 1970 { 1971 struct xps_map *map = NULL; 1972 int pos; 1973 1974 if (dev_maps) 1975 map = xmap_dereference(dev_maps->cpu_map[tci]); 1976 if (!map) 1977 return false; 1978 1979 for (pos = map->len; pos--;) { 1980 if (map->queues[pos] != index) 1981 continue; 1982 1983 if (map->len > 1) { 1984 map->queues[pos] = map->queues[--map->len]; 1985 break; 1986 } 1987 1988 RCU_INIT_POINTER(dev_maps->cpu_map[tci], NULL); 1989 kfree_rcu(map, rcu); 1990 return false; 1991 } 1992 1993 return true; 1994 } 1995 1996 static bool remove_xps_queue_cpu(struct net_device *dev, 1997 struct xps_dev_maps *dev_maps, 1998 int cpu, u16 offset, u16 count) 1999 { 2000 int num_tc = dev->num_tc ? : 1; 2001 bool active = false; 2002 int tci; 2003 2004 for (tci = cpu * num_tc; num_tc--; tci++) { 2005 int i, j; 2006 2007 for (i = count, j = offset; i--; j++) { 2008 if (!remove_xps_queue(dev_maps, cpu, j)) 2009 break; 2010 } 2011 2012 active |= i < 0; 2013 } 2014 2015 return active; 2016 } 2017 2018 static void netif_reset_xps_queues(struct net_device *dev, u16 offset, 2019 u16 count) 2020 { 2021 struct xps_dev_maps *dev_maps; 2022 int cpu, i; 2023 bool active = false; 2024 2025 mutex_lock(&xps_map_mutex); 2026 dev_maps = xmap_dereference(dev->xps_maps); 2027 2028 if (!dev_maps) 2029 goto out_no_maps; 2030 2031 for_each_possible_cpu(cpu) 2032 active |= remove_xps_queue_cpu(dev, dev_maps, cpu, 2033 offset, count); 2034 2035 if (!active) { 2036 RCU_INIT_POINTER(dev->xps_maps, NULL); 2037 kfree_rcu(dev_maps, rcu); 2038 } 2039 2040 for (i = offset + (count - 1); count--; i--) 2041 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i), 2042 NUMA_NO_NODE); 2043 2044 out_no_maps: 2045 mutex_unlock(&xps_map_mutex); 2046 } 2047 2048 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index) 2049 { 2050 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index); 2051 } 2052 2053 static struct xps_map *expand_xps_map(struct xps_map *map, 2054 int cpu, u16 index) 2055 { 2056 struct xps_map *new_map; 2057 int alloc_len = XPS_MIN_MAP_ALLOC; 2058 int i, pos; 2059 2060 for (pos = 0; map && pos < map->len; pos++) { 2061 if (map->queues[pos] != index) 2062 continue; 2063 return map; 2064 } 2065 2066 /* Need to add queue to this CPU's existing map */ 2067 if (map) { 2068 if (pos < map->alloc_len) 2069 return map; 2070 2071 alloc_len = map->alloc_len * 2; 2072 } 2073 2074 /* Need to allocate new map to store queue on this CPU's map */ 2075 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL, 2076 cpu_to_node(cpu)); 2077 if (!new_map) 2078 return NULL; 2079 2080 for (i = 0; i < pos; i++) 2081 new_map->queues[i] = map->queues[i]; 2082 new_map->alloc_len = alloc_len; 2083 new_map->len = pos; 2084 2085 return new_map; 2086 } 2087 2088 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask, 2089 u16 index) 2090 { 2091 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL; 2092 int i, cpu, tci, numa_node_id = -2; 2093 int maps_sz, num_tc = 1, tc = 0; 2094 struct xps_map *map, *new_map; 2095 bool active = false; 2096 2097 if (dev->num_tc) { 2098 num_tc = dev->num_tc; 2099 tc = netdev_txq_to_tc(dev, index); 2100 if (tc < 0) 2101 return -EINVAL; 2102 } 2103 2104 maps_sz = XPS_DEV_MAPS_SIZE(num_tc); 2105 if (maps_sz < L1_CACHE_BYTES) 2106 maps_sz = L1_CACHE_BYTES; 2107 2108 mutex_lock(&xps_map_mutex); 2109 2110 dev_maps = xmap_dereference(dev->xps_maps); 2111 2112 /* allocate memory for queue storage */ 2113 for_each_cpu_and(cpu, cpu_online_mask, mask) { 2114 if (!new_dev_maps) 2115 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL); 2116 if (!new_dev_maps) { 2117 mutex_unlock(&xps_map_mutex); 2118 return -ENOMEM; 2119 } 2120 2121 tci = cpu * num_tc + tc; 2122 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[tci]) : 2123 NULL; 2124 2125 map = expand_xps_map(map, cpu, index); 2126 if (!map) 2127 goto error; 2128 2129 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map); 2130 } 2131 2132 if (!new_dev_maps) 2133 goto out_no_new_maps; 2134 2135 for_each_possible_cpu(cpu) { 2136 /* copy maps belonging to foreign traffic classes */ 2137 for (i = tc, tci = cpu * num_tc; dev_maps && i--; tci++) { 2138 /* fill in the new device map from the old device map */ 2139 map = xmap_dereference(dev_maps->cpu_map[tci]); 2140 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map); 2141 } 2142 2143 /* We need to explicitly update tci as prevous loop 2144 * could break out early if dev_maps is NULL. 2145 */ 2146 tci = cpu * num_tc + tc; 2147 2148 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) { 2149 /* add queue to CPU maps */ 2150 int pos = 0; 2151 2152 map = xmap_dereference(new_dev_maps->cpu_map[tci]); 2153 while ((pos < map->len) && (map->queues[pos] != index)) 2154 pos++; 2155 2156 if (pos == map->len) 2157 map->queues[map->len++] = index; 2158 #ifdef CONFIG_NUMA 2159 if (numa_node_id == -2) 2160 numa_node_id = cpu_to_node(cpu); 2161 else if (numa_node_id != cpu_to_node(cpu)) 2162 numa_node_id = -1; 2163 #endif 2164 } else if (dev_maps) { 2165 /* fill in the new device map from the old device map */ 2166 map = xmap_dereference(dev_maps->cpu_map[tci]); 2167 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map); 2168 } 2169 2170 /* copy maps belonging to foreign traffic classes */ 2171 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) { 2172 /* fill in the new device map from the old device map */ 2173 map = xmap_dereference(dev_maps->cpu_map[tci]); 2174 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map); 2175 } 2176 } 2177 2178 rcu_assign_pointer(dev->xps_maps, new_dev_maps); 2179 2180 /* Cleanup old maps */ 2181 if (!dev_maps) 2182 goto out_no_old_maps; 2183 2184 for_each_possible_cpu(cpu) { 2185 for (i = num_tc, tci = cpu * num_tc; i--; tci++) { 2186 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]); 2187 map = xmap_dereference(dev_maps->cpu_map[tci]); 2188 if (map && map != new_map) 2189 kfree_rcu(map, rcu); 2190 } 2191 } 2192 2193 kfree_rcu(dev_maps, rcu); 2194 2195 out_no_old_maps: 2196 dev_maps = new_dev_maps; 2197 active = true; 2198 2199 out_no_new_maps: 2200 /* update Tx queue numa node */ 2201 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index), 2202 (numa_node_id >= 0) ? numa_node_id : 2203 NUMA_NO_NODE); 2204 2205 if (!dev_maps) 2206 goto out_no_maps; 2207 2208 /* removes queue from unused CPUs */ 2209 for_each_possible_cpu(cpu) { 2210 for (i = tc, tci = cpu * num_tc; i--; tci++) 2211 active |= remove_xps_queue(dev_maps, tci, index); 2212 if (!cpumask_test_cpu(cpu, mask) || !cpu_online(cpu)) 2213 active |= remove_xps_queue(dev_maps, tci, index); 2214 for (i = num_tc - tc, tci++; --i; tci++) 2215 active |= remove_xps_queue(dev_maps, tci, index); 2216 } 2217 2218 /* free map if not active */ 2219 if (!active) { 2220 RCU_INIT_POINTER(dev->xps_maps, NULL); 2221 kfree_rcu(dev_maps, rcu); 2222 } 2223 2224 out_no_maps: 2225 mutex_unlock(&xps_map_mutex); 2226 2227 return 0; 2228 error: 2229 /* remove any maps that we added */ 2230 for_each_possible_cpu(cpu) { 2231 for (i = num_tc, tci = cpu * num_tc; i--; tci++) { 2232 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]); 2233 map = dev_maps ? 2234 xmap_dereference(dev_maps->cpu_map[tci]) : 2235 NULL; 2236 if (new_map && new_map != map) 2237 kfree(new_map); 2238 } 2239 } 2240 2241 mutex_unlock(&xps_map_mutex); 2242 2243 kfree(new_dev_maps); 2244 return -ENOMEM; 2245 } 2246 EXPORT_SYMBOL(netif_set_xps_queue); 2247 2248 #endif 2249 void netdev_reset_tc(struct net_device *dev) 2250 { 2251 #ifdef CONFIG_XPS 2252 netif_reset_xps_queues_gt(dev, 0); 2253 #endif 2254 dev->num_tc = 0; 2255 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq)); 2256 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map)); 2257 } 2258 EXPORT_SYMBOL(netdev_reset_tc); 2259 2260 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset) 2261 { 2262 if (tc >= dev->num_tc) 2263 return -EINVAL; 2264 2265 #ifdef CONFIG_XPS 2266 netif_reset_xps_queues(dev, offset, count); 2267 #endif 2268 dev->tc_to_txq[tc].count = count; 2269 dev->tc_to_txq[tc].offset = offset; 2270 return 0; 2271 } 2272 EXPORT_SYMBOL(netdev_set_tc_queue); 2273 2274 int netdev_set_num_tc(struct net_device *dev, u8 num_tc) 2275 { 2276 if (num_tc > TC_MAX_QUEUE) 2277 return -EINVAL; 2278 2279 #ifdef CONFIG_XPS 2280 netif_reset_xps_queues_gt(dev, 0); 2281 #endif 2282 dev->num_tc = num_tc; 2283 return 0; 2284 } 2285 EXPORT_SYMBOL(netdev_set_num_tc); 2286 2287 /* 2288 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues 2289 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed. 2290 */ 2291 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) 2292 { 2293 int rc; 2294 2295 if (txq < 1 || txq > dev->num_tx_queues) 2296 return -EINVAL; 2297 2298 if (dev->reg_state == NETREG_REGISTERED || 2299 dev->reg_state == NETREG_UNREGISTERING) { 2300 ASSERT_RTNL(); 2301 2302 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, 2303 txq); 2304 if (rc) 2305 return rc; 2306 2307 if (dev->num_tc) 2308 netif_setup_tc(dev, txq); 2309 2310 if (txq < dev->real_num_tx_queues) { 2311 qdisc_reset_all_tx_gt(dev, txq); 2312 #ifdef CONFIG_XPS 2313 netif_reset_xps_queues_gt(dev, txq); 2314 #endif 2315 } 2316 } 2317 2318 dev->real_num_tx_queues = txq; 2319 return 0; 2320 } 2321 EXPORT_SYMBOL(netif_set_real_num_tx_queues); 2322 2323 #ifdef CONFIG_SYSFS 2324 /** 2325 * netif_set_real_num_rx_queues - set actual number of RX queues used 2326 * @dev: Network device 2327 * @rxq: Actual number of RX queues 2328 * 2329 * This must be called either with the rtnl_lock held or before 2330 * registration of the net device. Returns 0 on success, or a 2331 * negative error code. If called before registration, it always 2332 * succeeds. 2333 */ 2334 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) 2335 { 2336 int rc; 2337 2338 if (rxq < 1 || rxq > dev->num_rx_queues) 2339 return -EINVAL; 2340 2341 if (dev->reg_state == NETREG_REGISTERED) { 2342 ASSERT_RTNL(); 2343 2344 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, 2345 rxq); 2346 if (rc) 2347 return rc; 2348 } 2349 2350 dev->real_num_rx_queues = rxq; 2351 return 0; 2352 } 2353 EXPORT_SYMBOL(netif_set_real_num_rx_queues); 2354 #endif 2355 2356 /** 2357 * netif_get_num_default_rss_queues - default number of RSS queues 2358 * 2359 * This routine should set an upper limit on the number of RSS queues 2360 * used by default by multiqueue devices. 2361 */ 2362 int netif_get_num_default_rss_queues(void) 2363 { 2364 return is_kdump_kernel() ? 2365 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus()); 2366 } 2367 EXPORT_SYMBOL(netif_get_num_default_rss_queues); 2368 2369 static void __netif_reschedule(struct Qdisc *q) 2370 { 2371 struct softnet_data *sd; 2372 unsigned long flags; 2373 2374 local_irq_save(flags); 2375 sd = this_cpu_ptr(&softnet_data); 2376 q->next_sched = NULL; 2377 *sd->output_queue_tailp = q; 2378 sd->output_queue_tailp = &q->next_sched; 2379 raise_softirq_irqoff(NET_TX_SOFTIRQ); 2380 local_irq_restore(flags); 2381 } 2382 2383 void __netif_schedule(struct Qdisc *q) 2384 { 2385 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) 2386 __netif_reschedule(q); 2387 } 2388 EXPORT_SYMBOL(__netif_schedule); 2389 2390 struct dev_kfree_skb_cb { 2391 enum skb_free_reason reason; 2392 }; 2393 2394 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb) 2395 { 2396 return (struct dev_kfree_skb_cb *)skb->cb; 2397 } 2398 2399 void netif_schedule_queue(struct netdev_queue *txq) 2400 { 2401 rcu_read_lock(); 2402 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) { 2403 struct Qdisc *q = rcu_dereference(txq->qdisc); 2404 2405 __netif_schedule(q); 2406 } 2407 rcu_read_unlock(); 2408 } 2409 EXPORT_SYMBOL(netif_schedule_queue); 2410 2411 /** 2412 * netif_wake_subqueue - allow sending packets on subqueue 2413 * @dev: network device 2414 * @queue_index: sub queue index 2415 * 2416 * Resume individual transmit queue of a device with multiple transmit queues. 2417 */ 2418 void netif_wake_subqueue(struct net_device *dev, u16 queue_index) 2419 { 2420 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index); 2421 2422 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) { 2423 struct Qdisc *q; 2424 2425 rcu_read_lock(); 2426 q = rcu_dereference(txq->qdisc); 2427 __netif_schedule(q); 2428 rcu_read_unlock(); 2429 } 2430 } 2431 EXPORT_SYMBOL(netif_wake_subqueue); 2432 2433 void netif_tx_wake_queue(struct netdev_queue *dev_queue) 2434 { 2435 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { 2436 struct Qdisc *q; 2437 2438 rcu_read_lock(); 2439 q = rcu_dereference(dev_queue->qdisc); 2440 __netif_schedule(q); 2441 rcu_read_unlock(); 2442 } 2443 } 2444 EXPORT_SYMBOL(netif_tx_wake_queue); 2445 2446 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason) 2447 { 2448 unsigned long flags; 2449 2450 if (likely(atomic_read(&skb->users) == 1)) { 2451 smp_rmb(); 2452 atomic_set(&skb->users, 0); 2453 } else if (likely(!atomic_dec_and_test(&skb->users))) { 2454 return; 2455 } 2456 get_kfree_skb_cb(skb)->reason = reason; 2457 local_irq_save(flags); 2458 skb->next = __this_cpu_read(softnet_data.completion_queue); 2459 __this_cpu_write(softnet_data.completion_queue, skb); 2460 raise_softirq_irqoff(NET_TX_SOFTIRQ); 2461 local_irq_restore(flags); 2462 } 2463 EXPORT_SYMBOL(__dev_kfree_skb_irq); 2464 2465 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason) 2466 { 2467 if (in_irq() || irqs_disabled()) 2468 __dev_kfree_skb_irq(skb, reason); 2469 else 2470 dev_kfree_skb(skb); 2471 } 2472 EXPORT_SYMBOL(__dev_kfree_skb_any); 2473 2474 2475 /** 2476 * netif_device_detach - mark device as removed 2477 * @dev: network device 2478 * 2479 * Mark device as removed from system and therefore no longer available. 2480 */ 2481 void netif_device_detach(struct net_device *dev) 2482 { 2483 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && 2484 netif_running(dev)) { 2485 netif_tx_stop_all_queues(dev); 2486 } 2487 } 2488 EXPORT_SYMBOL(netif_device_detach); 2489 2490 /** 2491 * netif_device_attach - mark device as attached 2492 * @dev: network device 2493 * 2494 * Mark device as attached from system and restart if needed. 2495 */ 2496 void netif_device_attach(struct net_device *dev) 2497 { 2498 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && 2499 netif_running(dev)) { 2500 netif_tx_wake_all_queues(dev); 2501 __netdev_watchdog_up(dev); 2502 } 2503 } 2504 EXPORT_SYMBOL(netif_device_attach); 2505 2506 /* 2507 * Returns a Tx hash based on the given packet descriptor a Tx queues' number 2508 * to be used as a distribution range. 2509 */ 2510 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb, 2511 unsigned int num_tx_queues) 2512 { 2513 u32 hash; 2514 u16 qoffset = 0; 2515 u16 qcount = num_tx_queues; 2516 2517 if (skb_rx_queue_recorded(skb)) { 2518 hash = skb_get_rx_queue(skb); 2519 while (unlikely(hash >= num_tx_queues)) 2520 hash -= num_tx_queues; 2521 return hash; 2522 } 2523 2524 if (dev->num_tc) { 2525 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); 2526 qoffset = dev->tc_to_txq[tc].offset; 2527 qcount = dev->tc_to_txq[tc].count; 2528 } 2529 2530 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset; 2531 } 2532 EXPORT_SYMBOL(__skb_tx_hash); 2533 2534 static void skb_warn_bad_offload(const struct sk_buff *skb) 2535 { 2536 static const netdev_features_t null_features; 2537 struct net_device *dev = skb->dev; 2538 const char *name = ""; 2539 2540 if (!net_ratelimit()) 2541 return; 2542 2543 if (dev) { 2544 if (dev->dev.parent) 2545 name = dev_driver_string(dev->dev.parent); 2546 else 2547 name = netdev_name(dev); 2548 } 2549 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d " 2550 "gso_type=%d ip_summed=%d\n", 2551 name, dev ? &dev->features : &null_features, 2552 skb->sk ? &skb->sk->sk_route_caps : &null_features, 2553 skb->len, skb->data_len, skb_shinfo(skb)->gso_size, 2554 skb_shinfo(skb)->gso_type, skb->ip_summed); 2555 } 2556 2557 /* 2558 * Invalidate hardware checksum when packet is to be mangled, and 2559 * complete checksum manually on outgoing path. 2560 */ 2561 int skb_checksum_help(struct sk_buff *skb) 2562 { 2563 __wsum csum; 2564 int ret = 0, offset; 2565 2566 if (skb->ip_summed == CHECKSUM_COMPLETE) 2567 goto out_set_summed; 2568 2569 if (unlikely(skb_shinfo(skb)->gso_size)) { 2570 skb_warn_bad_offload(skb); 2571 return -EINVAL; 2572 } 2573 2574 /* Before computing a checksum, we should make sure no frag could 2575 * be modified by an external entity : checksum could be wrong. 2576 */ 2577 if (skb_has_shared_frag(skb)) { 2578 ret = __skb_linearize(skb); 2579 if (ret) 2580 goto out; 2581 } 2582 2583 offset = skb_checksum_start_offset(skb); 2584 BUG_ON(offset >= skb_headlen(skb)); 2585 csum = skb_checksum(skb, offset, skb->len - offset, 0); 2586 2587 offset += skb->csum_offset; 2588 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb)); 2589 2590 if (skb_cloned(skb) && 2591 !skb_clone_writable(skb, offset + sizeof(__sum16))) { 2592 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2593 if (ret) 2594 goto out; 2595 } 2596 2597 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0; 2598 out_set_summed: 2599 skb->ip_summed = CHECKSUM_NONE; 2600 out: 2601 return ret; 2602 } 2603 EXPORT_SYMBOL(skb_checksum_help); 2604 2605 __be16 skb_network_protocol(struct sk_buff *skb, int *depth) 2606 { 2607 __be16 type = skb->protocol; 2608 2609 /* Tunnel gso handlers can set protocol to ethernet. */ 2610 if (type == htons(ETH_P_TEB)) { 2611 struct ethhdr *eth; 2612 2613 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr)))) 2614 return 0; 2615 2616 eth = (struct ethhdr *)skb_mac_header(skb); 2617 type = eth->h_proto; 2618 } 2619 2620 return __vlan_get_protocol(skb, type, depth); 2621 } 2622 2623 /** 2624 * skb_mac_gso_segment - mac layer segmentation handler. 2625 * @skb: buffer to segment 2626 * @features: features for the output path (see dev->features) 2627 */ 2628 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, 2629 netdev_features_t features) 2630 { 2631 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); 2632 struct packet_offload *ptype; 2633 int vlan_depth = skb->mac_len; 2634 __be16 type = skb_network_protocol(skb, &vlan_depth); 2635 2636 if (unlikely(!type)) 2637 return ERR_PTR(-EINVAL); 2638 2639 __skb_pull(skb, vlan_depth); 2640 2641 rcu_read_lock(); 2642 list_for_each_entry_rcu(ptype, &offload_base, list) { 2643 if (ptype->type == type && ptype->callbacks.gso_segment) { 2644 segs = ptype->callbacks.gso_segment(skb, features); 2645 break; 2646 } 2647 } 2648 rcu_read_unlock(); 2649 2650 __skb_push(skb, skb->data - skb_mac_header(skb)); 2651 2652 return segs; 2653 } 2654 EXPORT_SYMBOL(skb_mac_gso_segment); 2655 2656 2657 /* openvswitch calls this on rx path, so we need a different check. 2658 */ 2659 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) 2660 { 2661 if (tx_path) 2662 return skb->ip_summed != CHECKSUM_PARTIAL; 2663 else 2664 return skb->ip_summed == CHECKSUM_NONE; 2665 } 2666 2667 /** 2668 * __skb_gso_segment - Perform segmentation on skb. 2669 * @skb: buffer to segment 2670 * @features: features for the output path (see dev->features) 2671 * @tx_path: whether it is called in TX path 2672 * 2673 * This function segments the given skb and returns a list of segments. 2674 * 2675 * It may return NULL if the skb requires no segmentation. This is 2676 * only possible when GSO is used for verifying header integrity. 2677 * 2678 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb. 2679 */ 2680 struct sk_buff *__skb_gso_segment(struct sk_buff *skb, 2681 netdev_features_t features, bool tx_path) 2682 { 2683 if (unlikely(skb_needs_check(skb, tx_path))) { 2684 int err; 2685 2686 skb_warn_bad_offload(skb); 2687 2688 err = skb_cow_head(skb, 0); 2689 if (err < 0) 2690 return ERR_PTR(err); 2691 } 2692 2693 /* Only report GSO partial support if it will enable us to 2694 * support segmentation on this frame without needing additional 2695 * work. 2696 */ 2697 if (features & NETIF_F_GSO_PARTIAL) { 2698 netdev_features_t partial_features = NETIF_F_GSO_ROBUST; 2699 struct net_device *dev = skb->dev; 2700 2701 partial_features |= dev->features & dev->gso_partial_features; 2702 if (!skb_gso_ok(skb, features | partial_features)) 2703 features &= ~NETIF_F_GSO_PARTIAL; 2704 } 2705 2706 BUILD_BUG_ON(SKB_SGO_CB_OFFSET + 2707 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb)); 2708 2709 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb); 2710 SKB_GSO_CB(skb)->encap_level = 0; 2711 2712 skb_reset_mac_header(skb); 2713 skb_reset_mac_len(skb); 2714 2715 return skb_mac_gso_segment(skb, features); 2716 } 2717 EXPORT_SYMBOL(__skb_gso_segment); 2718 2719 /* Take action when hardware reception checksum errors are detected. */ 2720 #ifdef CONFIG_BUG 2721 void netdev_rx_csum_fault(struct net_device *dev) 2722 { 2723 if (net_ratelimit()) { 2724 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>"); 2725 dump_stack(); 2726 } 2727 } 2728 EXPORT_SYMBOL(netdev_rx_csum_fault); 2729 #endif 2730 2731 /* Actually, we should eliminate this check as soon as we know, that: 2732 * 1. IOMMU is present and allows to map all the memory. 2733 * 2. No high memory really exists on this machine. 2734 */ 2735 2736 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) 2737 { 2738 #ifdef CONFIG_HIGHMEM 2739 int i; 2740 if (!(dev->features & NETIF_F_HIGHDMA)) { 2741 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2742 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2743 if (PageHighMem(skb_frag_page(frag))) 2744 return 1; 2745 } 2746 } 2747 2748 if (PCI_DMA_BUS_IS_PHYS) { 2749 struct device *pdev = dev->dev.parent; 2750 2751 if (!pdev) 2752 return 0; 2753 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2754 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2755 dma_addr_t addr = page_to_phys(skb_frag_page(frag)); 2756 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask) 2757 return 1; 2758 } 2759 } 2760 #endif 2761 return 0; 2762 } 2763 2764 /* If MPLS offload request, verify we are testing hardware MPLS features 2765 * instead of standard features for the netdev. 2766 */ 2767 #if IS_ENABLED(CONFIG_NET_MPLS_GSO) 2768 static netdev_features_t net_mpls_features(struct sk_buff *skb, 2769 netdev_features_t features, 2770 __be16 type) 2771 { 2772 if (eth_p_mpls(type)) 2773 features &= skb->dev->mpls_features; 2774 2775 return features; 2776 } 2777 #else 2778 static netdev_features_t net_mpls_features(struct sk_buff *skb, 2779 netdev_features_t features, 2780 __be16 type) 2781 { 2782 return features; 2783 } 2784 #endif 2785 2786 static netdev_features_t harmonize_features(struct sk_buff *skb, 2787 netdev_features_t features) 2788 { 2789 int tmp; 2790 __be16 type; 2791 2792 type = skb_network_protocol(skb, &tmp); 2793 features = net_mpls_features(skb, features, type); 2794 2795 if (skb->ip_summed != CHECKSUM_NONE && 2796 !can_checksum_protocol(features, type)) { 2797 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 2798 } else if (illegal_highdma(skb->dev, skb)) { 2799 features &= ~NETIF_F_SG; 2800 } 2801 2802 return features; 2803 } 2804 2805 netdev_features_t passthru_features_check(struct sk_buff *skb, 2806 struct net_device *dev, 2807 netdev_features_t features) 2808 { 2809 return features; 2810 } 2811 EXPORT_SYMBOL(passthru_features_check); 2812 2813 static netdev_features_t dflt_features_check(const struct sk_buff *skb, 2814 struct net_device *dev, 2815 netdev_features_t features) 2816 { 2817 return vlan_features_check(skb, features); 2818 } 2819 2820 static netdev_features_t gso_features_check(const struct sk_buff *skb, 2821 struct net_device *dev, 2822 netdev_features_t features) 2823 { 2824 u16 gso_segs = skb_shinfo(skb)->gso_segs; 2825 2826 if (gso_segs > dev->gso_max_segs) 2827 return features & ~NETIF_F_GSO_MASK; 2828 2829 /* Support for GSO partial features requires software 2830 * intervention before we can actually process the packets 2831 * so we need to strip support for any partial features now 2832 * and we can pull them back in after we have partially 2833 * segmented the frame. 2834 */ 2835 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)) 2836 features &= ~dev->gso_partial_features; 2837 2838 /* Make sure to clear the IPv4 ID mangling feature if the 2839 * IPv4 header has the potential to be fragmented. 2840 */ 2841 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { 2842 struct iphdr *iph = skb->encapsulation ? 2843 inner_ip_hdr(skb) : ip_hdr(skb); 2844 2845 if (!(iph->frag_off & htons(IP_DF))) 2846 features &= ~NETIF_F_TSO_MANGLEID; 2847 } 2848 2849 return features; 2850 } 2851 2852 netdev_features_t netif_skb_features(struct sk_buff *skb) 2853 { 2854 struct net_device *dev = skb->dev; 2855 netdev_features_t features = dev->features; 2856 2857 if (skb_is_gso(skb)) 2858 features = gso_features_check(skb, dev, features); 2859 2860 /* If encapsulation offload request, verify we are testing 2861 * hardware encapsulation features instead of standard 2862 * features for the netdev 2863 */ 2864 if (skb->encapsulation) 2865 features &= dev->hw_enc_features; 2866 2867 if (skb_vlan_tagged(skb)) 2868 features = netdev_intersect_features(features, 2869 dev->vlan_features | 2870 NETIF_F_HW_VLAN_CTAG_TX | 2871 NETIF_F_HW_VLAN_STAG_TX); 2872 2873 if (dev->netdev_ops->ndo_features_check) 2874 features &= dev->netdev_ops->ndo_features_check(skb, dev, 2875 features); 2876 else 2877 features &= dflt_features_check(skb, dev, features); 2878 2879 return harmonize_features(skb, features); 2880 } 2881 EXPORT_SYMBOL(netif_skb_features); 2882 2883 static int xmit_one(struct sk_buff *skb, struct net_device *dev, 2884 struct netdev_queue *txq, bool more) 2885 { 2886 unsigned int len; 2887 int rc; 2888 2889 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all)) 2890 dev_queue_xmit_nit(skb, dev); 2891 2892 len = skb->len; 2893 trace_net_dev_start_xmit(skb, dev); 2894 rc = netdev_start_xmit(skb, dev, txq, more); 2895 trace_net_dev_xmit(skb, rc, dev, len); 2896 2897 return rc; 2898 } 2899 2900 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev, 2901 struct netdev_queue *txq, int *ret) 2902 { 2903 struct sk_buff *skb = first; 2904 int rc = NETDEV_TX_OK; 2905 2906 while (skb) { 2907 struct sk_buff *next = skb->next; 2908 2909 skb->next = NULL; 2910 rc = xmit_one(skb, dev, txq, next != NULL); 2911 if (unlikely(!dev_xmit_complete(rc))) { 2912 skb->next = next; 2913 goto out; 2914 } 2915 2916 skb = next; 2917 if (netif_xmit_stopped(txq) && skb) { 2918 rc = NETDEV_TX_BUSY; 2919 break; 2920 } 2921 } 2922 2923 out: 2924 *ret = rc; 2925 return skb; 2926 } 2927 2928 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, 2929 netdev_features_t features) 2930 { 2931 if (skb_vlan_tag_present(skb) && 2932 !vlan_hw_offload_capable(features, skb->vlan_proto)) 2933 skb = __vlan_hwaccel_push_inside(skb); 2934 return skb; 2935 } 2936 2937 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev) 2938 { 2939 netdev_features_t features; 2940 2941 features = netif_skb_features(skb); 2942 skb = validate_xmit_vlan(skb, features); 2943 if (unlikely(!skb)) 2944 goto out_null; 2945 2946 if (netif_needs_gso(skb, features)) { 2947 struct sk_buff *segs; 2948 2949 segs = skb_gso_segment(skb, features); 2950 if (IS_ERR(segs)) { 2951 goto out_kfree_skb; 2952 } else if (segs) { 2953 consume_skb(skb); 2954 skb = segs; 2955 } 2956 } else { 2957 if (skb_needs_linearize(skb, features) && 2958 __skb_linearize(skb)) 2959 goto out_kfree_skb; 2960 2961 /* If packet is not checksummed and device does not 2962 * support checksumming for this protocol, complete 2963 * checksumming here. 2964 */ 2965 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2966 if (skb->encapsulation) 2967 skb_set_inner_transport_header(skb, 2968 skb_checksum_start_offset(skb)); 2969 else 2970 skb_set_transport_header(skb, 2971 skb_checksum_start_offset(skb)); 2972 if (!(features & NETIF_F_CSUM_MASK) && 2973 skb_checksum_help(skb)) 2974 goto out_kfree_skb; 2975 } 2976 } 2977 2978 return skb; 2979 2980 out_kfree_skb: 2981 kfree_skb(skb); 2982 out_null: 2983 atomic_long_inc(&dev->tx_dropped); 2984 return NULL; 2985 } 2986 2987 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev) 2988 { 2989 struct sk_buff *next, *head = NULL, *tail; 2990 2991 for (; skb != NULL; skb = next) { 2992 next = skb->next; 2993 skb->next = NULL; 2994 2995 /* in case skb wont be segmented, point to itself */ 2996 skb->prev = skb; 2997 2998 skb = validate_xmit_skb(skb, dev); 2999 if (!skb) 3000 continue; 3001 3002 if (!head) 3003 head = skb; 3004 else 3005 tail->next = skb; 3006 /* If skb was segmented, skb->prev points to 3007 * the last segment. If not, it still contains skb. 3008 */ 3009 tail = skb->prev; 3010 } 3011 return head; 3012 } 3013 EXPORT_SYMBOL_GPL(validate_xmit_skb_list); 3014 3015 static void qdisc_pkt_len_init(struct sk_buff *skb) 3016 { 3017 const struct skb_shared_info *shinfo = skb_shinfo(skb); 3018 3019 qdisc_skb_cb(skb)->pkt_len = skb->len; 3020 3021 /* To get more precise estimation of bytes sent on wire, 3022 * we add to pkt_len the headers size of all segments 3023 */ 3024 if (shinfo->gso_size) { 3025 unsigned int hdr_len; 3026 u16 gso_segs = shinfo->gso_segs; 3027 3028 /* mac layer + network layer */ 3029 hdr_len = skb_transport_header(skb) - skb_mac_header(skb); 3030 3031 /* + transport layer */ 3032 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) 3033 hdr_len += tcp_hdrlen(skb); 3034 else 3035 hdr_len += sizeof(struct udphdr); 3036 3037 if (shinfo->gso_type & SKB_GSO_DODGY) 3038 gso_segs = DIV_ROUND_UP(skb->len - hdr_len, 3039 shinfo->gso_size); 3040 3041 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; 3042 } 3043 } 3044 3045 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 3046 struct net_device *dev, 3047 struct netdev_queue *txq) 3048 { 3049 spinlock_t *root_lock = qdisc_lock(q); 3050 struct sk_buff *to_free = NULL; 3051 bool contended; 3052 int rc; 3053 3054 qdisc_calculate_pkt_len(skb, q); 3055 /* 3056 * Heuristic to force contended enqueues to serialize on a 3057 * separate lock before trying to get qdisc main lock. 3058 * This permits qdisc->running owner to get the lock more 3059 * often and dequeue packets faster. 3060 */ 3061 contended = qdisc_is_running(q); 3062 if (unlikely(contended)) 3063 spin_lock(&q->busylock); 3064 3065 spin_lock(root_lock); 3066 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { 3067 __qdisc_drop(skb, &to_free); 3068 rc = NET_XMIT_DROP; 3069 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && 3070 qdisc_run_begin(q)) { 3071 /* 3072 * This is a work-conserving queue; there are no old skbs 3073 * waiting to be sent out; and the qdisc is not running - 3074 * xmit the skb directly. 3075 */ 3076 3077 qdisc_bstats_update(q, skb); 3078 3079 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) { 3080 if (unlikely(contended)) { 3081 spin_unlock(&q->busylock); 3082 contended = false; 3083 } 3084 __qdisc_run(q); 3085 } else 3086 qdisc_run_end(q); 3087 3088 rc = NET_XMIT_SUCCESS; 3089 } else { 3090 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK; 3091 if (qdisc_run_begin(q)) { 3092 if (unlikely(contended)) { 3093 spin_unlock(&q->busylock); 3094 contended = false; 3095 } 3096 __qdisc_run(q); 3097 } 3098 } 3099 spin_unlock(root_lock); 3100 if (unlikely(to_free)) 3101 kfree_skb_list(to_free); 3102 if (unlikely(contended)) 3103 spin_unlock(&q->busylock); 3104 return rc; 3105 } 3106 3107 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO) 3108 static void skb_update_prio(struct sk_buff *skb) 3109 { 3110 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap); 3111 3112 if (!skb->priority && skb->sk && map) { 3113 unsigned int prioidx = 3114 sock_cgroup_prioidx(&skb->sk->sk_cgrp_data); 3115 3116 if (prioidx < map->priomap_len) 3117 skb->priority = map->priomap[prioidx]; 3118 } 3119 } 3120 #else 3121 #define skb_update_prio(skb) 3122 #endif 3123 3124 DEFINE_PER_CPU(int, xmit_recursion); 3125 EXPORT_SYMBOL(xmit_recursion); 3126 3127 /** 3128 * dev_loopback_xmit - loop back @skb 3129 * @net: network namespace this loopback is happening in 3130 * @sk: sk needed to be a netfilter okfn 3131 * @skb: buffer to transmit 3132 */ 3133 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb) 3134 { 3135 skb_reset_mac_header(skb); 3136 __skb_pull(skb, skb_network_offset(skb)); 3137 skb->pkt_type = PACKET_LOOPBACK; 3138 skb->ip_summed = CHECKSUM_UNNECESSARY; 3139 WARN_ON(!skb_dst(skb)); 3140 skb_dst_force(skb); 3141 netif_rx_ni(skb); 3142 return 0; 3143 } 3144 EXPORT_SYMBOL(dev_loopback_xmit); 3145 3146 #ifdef CONFIG_NET_EGRESS 3147 static struct sk_buff * 3148 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) 3149 { 3150 struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list); 3151 struct tcf_result cl_res; 3152 3153 if (!cl) 3154 return skb; 3155 3156 /* skb->tc_verd and qdisc_skb_cb(skb)->pkt_len were already set 3157 * earlier by the caller. 3158 */ 3159 qdisc_bstats_cpu_update(cl->q, skb); 3160 3161 switch (tc_classify(skb, cl, &cl_res, false)) { 3162 case TC_ACT_OK: 3163 case TC_ACT_RECLASSIFY: 3164 skb->tc_index = TC_H_MIN(cl_res.classid); 3165 break; 3166 case TC_ACT_SHOT: 3167 qdisc_qstats_cpu_drop(cl->q); 3168 *ret = NET_XMIT_DROP; 3169 kfree_skb(skb); 3170 return NULL; 3171 case TC_ACT_STOLEN: 3172 case TC_ACT_QUEUED: 3173 *ret = NET_XMIT_SUCCESS; 3174 consume_skb(skb); 3175 return NULL; 3176 case TC_ACT_REDIRECT: 3177 /* No need to push/pop skb's mac_header here on egress! */ 3178 skb_do_redirect(skb); 3179 *ret = NET_XMIT_SUCCESS; 3180 return NULL; 3181 default: 3182 break; 3183 } 3184 3185 return skb; 3186 } 3187 #endif /* CONFIG_NET_EGRESS */ 3188 3189 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb) 3190 { 3191 #ifdef CONFIG_XPS 3192 struct xps_dev_maps *dev_maps; 3193 struct xps_map *map; 3194 int queue_index = -1; 3195 3196 rcu_read_lock(); 3197 dev_maps = rcu_dereference(dev->xps_maps); 3198 if (dev_maps) { 3199 unsigned int tci = skb->sender_cpu - 1; 3200 3201 if (dev->num_tc) { 3202 tci *= dev->num_tc; 3203 tci += netdev_get_prio_tc_map(dev, skb->priority); 3204 } 3205 3206 map = rcu_dereference(dev_maps->cpu_map[tci]); 3207 if (map) { 3208 if (map->len == 1) 3209 queue_index = map->queues[0]; 3210 else 3211 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb), 3212 map->len)]; 3213 if (unlikely(queue_index >= dev->real_num_tx_queues)) 3214 queue_index = -1; 3215 } 3216 } 3217 rcu_read_unlock(); 3218 3219 return queue_index; 3220 #else 3221 return -1; 3222 #endif 3223 } 3224 3225 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb) 3226 { 3227 struct sock *sk = skb->sk; 3228 int queue_index = sk_tx_queue_get(sk); 3229 3230 if (queue_index < 0 || skb->ooo_okay || 3231 queue_index >= dev->real_num_tx_queues) { 3232 int new_index = get_xps_queue(dev, skb); 3233 if (new_index < 0) 3234 new_index = skb_tx_hash(dev, skb); 3235 3236 if (queue_index != new_index && sk && 3237 sk_fullsock(sk) && 3238 rcu_access_pointer(sk->sk_dst_cache)) 3239 sk_tx_queue_set(sk, new_index); 3240 3241 queue_index = new_index; 3242 } 3243 3244 return queue_index; 3245 } 3246 3247 struct netdev_queue *netdev_pick_tx(struct net_device *dev, 3248 struct sk_buff *skb, 3249 void *accel_priv) 3250 { 3251 int queue_index = 0; 3252 3253 #ifdef CONFIG_XPS 3254 u32 sender_cpu = skb->sender_cpu - 1; 3255 3256 if (sender_cpu >= (u32)NR_CPUS) 3257 skb->sender_cpu = raw_smp_processor_id() + 1; 3258 #endif 3259 3260 if (dev->real_num_tx_queues != 1) { 3261 const struct net_device_ops *ops = dev->netdev_ops; 3262 if (ops->ndo_select_queue) 3263 queue_index = ops->ndo_select_queue(dev, skb, accel_priv, 3264 __netdev_pick_tx); 3265 else 3266 queue_index = __netdev_pick_tx(dev, skb); 3267 3268 if (!accel_priv) 3269 queue_index = netdev_cap_txqueue(dev, queue_index); 3270 } 3271 3272 skb_set_queue_mapping(skb, queue_index); 3273 return netdev_get_tx_queue(dev, queue_index); 3274 } 3275 3276 /** 3277 * __dev_queue_xmit - transmit a buffer 3278 * @skb: buffer to transmit 3279 * @accel_priv: private data used for L2 forwarding offload 3280 * 3281 * Queue a buffer for transmission to a network device. The caller must 3282 * have set the device and priority and built the buffer before calling 3283 * this function. The function can be called from an interrupt. 3284 * 3285 * A negative errno code is returned on a failure. A success does not 3286 * guarantee the frame will be transmitted as it may be dropped due 3287 * to congestion or traffic shaping. 3288 * 3289 * ----------------------------------------------------------------------------------- 3290 * I notice this method can also return errors from the queue disciplines, 3291 * including NET_XMIT_DROP, which is a positive value. So, errors can also 3292 * be positive. 3293 * 3294 * Regardless of the return value, the skb is consumed, so it is currently 3295 * difficult to retry a send to this method. (You can bump the ref count 3296 * before sending to hold a reference for retry if you are careful.) 3297 * 3298 * When calling this method, interrupts MUST be enabled. This is because 3299 * the BH enable code must have IRQs enabled so that it will not deadlock. 3300 * --BLG 3301 */ 3302 static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv) 3303 { 3304 struct net_device *dev = skb->dev; 3305 struct netdev_queue *txq; 3306 struct Qdisc *q; 3307 int rc = -ENOMEM; 3308 3309 skb_reset_mac_header(skb); 3310 3311 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP)) 3312 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED); 3313 3314 /* Disable soft irqs for various locks below. Also 3315 * stops preemption for RCU. 3316 */ 3317 rcu_read_lock_bh(); 3318 3319 skb_update_prio(skb); 3320 3321 qdisc_pkt_len_init(skb); 3322 #ifdef CONFIG_NET_CLS_ACT 3323 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS); 3324 # ifdef CONFIG_NET_EGRESS 3325 if (static_key_false(&egress_needed)) { 3326 skb = sch_handle_egress(skb, &rc, dev); 3327 if (!skb) 3328 goto out; 3329 } 3330 # endif 3331 #endif 3332 /* If device/qdisc don't need skb->dst, release it right now while 3333 * its hot in this cpu cache. 3334 */ 3335 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) 3336 skb_dst_drop(skb); 3337 else 3338 skb_dst_force(skb); 3339 3340 txq = netdev_pick_tx(dev, skb, accel_priv); 3341 q = rcu_dereference_bh(txq->qdisc); 3342 3343 trace_net_dev_queue(skb); 3344 if (q->enqueue) { 3345 rc = __dev_xmit_skb(skb, q, dev, txq); 3346 goto out; 3347 } 3348 3349 /* The device has no queue. Common case for software devices: 3350 loopback, all the sorts of tunnels... 3351 3352 Really, it is unlikely that netif_tx_lock protection is necessary 3353 here. (f.e. loopback and IP tunnels are clean ignoring statistics 3354 counters.) 3355 However, it is possible, that they rely on protection 3356 made by us here. 3357 3358 Check this and shot the lock. It is not prone from deadlocks. 3359 Either shot noqueue qdisc, it is even simpler 8) 3360 */ 3361 if (dev->flags & IFF_UP) { 3362 int cpu = smp_processor_id(); /* ok because BHs are off */ 3363 3364 if (txq->xmit_lock_owner != cpu) { 3365 if (unlikely(__this_cpu_read(xmit_recursion) > 3366 XMIT_RECURSION_LIMIT)) 3367 goto recursion_alert; 3368 3369 skb = validate_xmit_skb(skb, dev); 3370 if (!skb) 3371 goto out; 3372 3373 HARD_TX_LOCK(dev, txq, cpu); 3374 3375 if (!netif_xmit_stopped(txq)) { 3376 __this_cpu_inc(xmit_recursion); 3377 skb = dev_hard_start_xmit(skb, dev, txq, &rc); 3378 __this_cpu_dec(xmit_recursion); 3379 if (dev_xmit_complete(rc)) { 3380 HARD_TX_UNLOCK(dev, txq); 3381 goto out; 3382 } 3383 } 3384 HARD_TX_UNLOCK(dev, txq); 3385 net_crit_ratelimited("Virtual device %s asks to queue packet!\n", 3386 dev->name); 3387 } else { 3388 /* Recursion is detected! It is possible, 3389 * unfortunately 3390 */ 3391 recursion_alert: 3392 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", 3393 dev->name); 3394 } 3395 } 3396 3397 rc = -ENETDOWN; 3398 rcu_read_unlock_bh(); 3399 3400 atomic_long_inc(&dev->tx_dropped); 3401 kfree_skb_list(skb); 3402 return rc; 3403 out: 3404 rcu_read_unlock_bh(); 3405 return rc; 3406 } 3407 3408 int dev_queue_xmit(struct sk_buff *skb) 3409 { 3410 return __dev_queue_xmit(skb, NULL); 3411 } 3412 EXPORT_SYMBOL(dev_queue_xmit); 3413 3414 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv) 3415 { 3416 return __dev_queue_xmit(skb, accel_priv); 3417 } 3418 EXPORT_SYMBOL(dev_queue_xmit_accel); 3419 3420 3421 /*======================================================================= 3422 Receiver routines 3423 =======================================================================*/ 3424 3425 int netdev_max_backlog __read_mostly = 1000; 3426 EXPORT_SYMBOL(netdev_max_backlog); 3427 3428 int netdev_tstamp_prequeue __read_mostly = 1; 3429 int netdev_budget __read_mostly = 300; 3430 int weight_p __read_mostly = 64; /* old backlog weight */ 3431 3432 /* Called with irq disabled */ 3433 static inline void ____napi_schedule(struct softnet_data *sd, 3434 struct napi_struct *napi) 3435 { 3436 list_add_tail(&napi->poll_list, &sd->poll_list); 3437 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 3438 } 3439 3440 #ifdef CONFIG_RPS 3441 3442 /* One global table that all flow-based protocols share. */ 3443 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; 3444 EXPORT_SYMBOL(rps_sock_flow_table); 3445 u32 rps_cpu_mask __read_mostly; 3446 EXPORT_SYMBOL(rps_cpu_mask); 3447 3448 struct static_key rps_needed __read_mostly; 3449 EXPORT_SYMBOL(rps_needed); 3450 struct static_key rfs_needed __read_mostly; 3451 EXPORT_SYMBOL(rfs_needed); 3452 3453 static struct rps_dev_flow * 3454 set_rps_cpu(struct net_device *dev, struct sk_buff *skb, 3455 struct rps_dev_flow *rflow, u16 next_cpu) 3456 { 3457 if (next_cpu < nr_cpu_ids) { 3458 #ifdef CONFIG_RFS_ACCEL 3459 struct netdev_rx_queue *rxqueue; 3460 struct rps_dev_flow_table *flow_table; 3461 struct rps_dev_flow *old_rflow; 3462 u32 flow_id; 3463 u16 rxq_index; 3464 int rc; 3465 3466 /* Should we steer this flow to a different hardware queue? */ 3467 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || 3468 !(dev->features & NETIF_F_NTUPLE)) 3469 goto out; 3470 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); 3471 if (rxq_index == skb_get_rx_queue(skb)) 3472 goto out; 3473 3474 rxqueue = dev->_rx + rxq_index; 3475 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3476 if (!flow_table) 3477 goto out; 3478 flow_id = skb_get_hash(skb) & flow_table->mask; 3479 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, 3480 rxq_index, flow_id); 3481 if (rc < 0) 3482 goto out; 3483 old_rflow = rflow; 3484 rflow = &flow_table->flows[flow_id]; 3485 rflow->filter = rc; 3486 if (old_rflow->filter == rflow->filter) 3487 old_rflow->filter = RPS_NO_FILTER; 3488 out: 3489 #endif 3490 rflow->last_qtail = 3491 per_cpu(softnet_data, next_cpu).input_queue_head; 3492 } 3493 3494 rflow->cpu = next_cpu; 3495 return rflow; 3496 } 3497 3498 /* 3499 * get_rps_cpu is called from netif_receive_skb and returns the target 3500 * CPU from the RPS map of the receiving queue for a given skb. 3501 * rcu_read_lock must be held on entry. 3502 */ 3503 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, 3504 struct rps_dev_flow **rflowp) 3505 { 3506 const struct rps_sock_flow_table *sock_flow_table; 3507 struct netdev_rx_queue *rxqueue = dev->_rx; 3508 struct rps_dev_flow_table *flow_table; 3509 struct rps_map *map; 3510 int cpu = -1; 3511 u32 tcpu; 3512 u32 hash; 3513 3514 if (skb_rx_queue_recorded(skb)) { 3515 u16 index = skb_get_rx_queue(skb); 3516 3517 if (unlikely(index >= dev->real_num_rx_queues)) { 3518 WARN_ONCE(dev->real_num_rx_queues > 1, 3519 "%s received packet on queue %u, but number " 3520 "of RX queues is %u\n", 3521 dev->name, index, dev->real_num_rx_queues); 3522 goto done; 3523 } 3524 rxqueue += index; 3525 } 3526 3527 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */ 3528 3529 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3530 map = rcu_dereference(rxqueue->rps_map); 3531 if (!flow_table && !map) 3532 goto done; 3533 3534 skb_reset_network_header(skb); 3535 hash = skb_get_hash(skb); 3536 if (!hash) 3537 goto done; 3538 3539 sock_flow_table = rcu_dereference(rps_sock_flow_table); 3540 if (flow_table && sock_flow_table) { 3541 struct rps_dev_flow *rflow; 3542 u32 next_cpu; 3543 u32 ident; 3544 3545 /* First check into global flow table if there is a match */ 3546 ident = sock_flow_table->ents[hash & sock_flow_table->mask]; 3547 if ((ident ^ hash) & ~rps_cpu_mask) 3548 goto try_rps; 3549 3550 next_cpu = ident & rps_cpu_mask; 3551 3552 /* OK, now we know there is a match, 3553 * we can look at the local (per receive queue) flow table 3554 */ 3555 rflow = &flow_table->flows[hash & flow_table->mask]; 3556 tcpu = rflow->cpu; 3557 3558 /* 3559 * If the desired CPU (where last recvmsg was done) is 3560 * different from current CPU (one in the rx-queue flow 3561 * table entry), switch if one of the following holds: 3562 * - Current CPU is unset (>= nr_cpu_ids). 3563 * - Current CPU is offline. 3564 * - The current CPU's queue tail has advanced beyond the 3565 * last packet that was enqueued using this table entry. 3566 * This guarantees that all previous packets for the flow 3567 * have been dequeued, thus preserving in order delivery. 3568 */ 3569 if (unlikely(tcpu != next_cpu) && 3570 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) || 3571 ((int)(per_cpu(softnet_data, tcpu).input_queue_head - 3572 rflow->last_qtail)) >= 0)) { 3573 tcpu = next_cpu; 3574 rflow = set_rps_cpu(dev, skb, rflow, next_cpu); 3575 } 3576 3577 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { 3578 *rflowp = rflow; 3579 cpu = tcpu; 3580 goto done; 3581 } 3582 } 3583 3584 try_rps: 3585 3586 if (map) { 3587 tcpu = map->cpus[reciprocal_scale(hash, map->len)]; 3588 if (cpu_online(tcpu)) { 3589 cpu = tcpu; 3590 goto done; 3591 } 3592 } 3593 3594 done: 3595 return cpu; 3596 } 3597 3598 #ifdef CONFIG_RFS_ACCEL 3599 3600 /** 3601 * rps_may_expire_flow - check whether an RFS hardware filter may be removed 3602 * @dev: Device on which the filter was set 3603 * @rxq_index: RX queue index 3604 * @flow_id: Flow ID passed to ndo_rx_flow_steer() 3605 * @filter_id: Filter ID returned by ndo_rx_flow_steer() 3606 * 3607 * Drivers that implement ndo_rx_flow_steer() should periodically call 3608 * this function for each installed filter and remove the filters for 3609 * which it returns %true. 3610 */ 3611 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, 3612 u32 flow_id, u16 filter_id) 3613 { 3614 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; 3615 struct rps_dev_flow_table *flow_table; 3616 struct rps_dev_flow *rflow; 3617 bool expire = true; 3618 unsigned int cpu; 3619 3620 rcu_read_lock(); 3621 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3622 if (flow_table && flow_id <= flow_table->mask) { 3623 rflow = &flow_table->flows[flow_id]; 3624 cpu = ACCESS_ONCE(rflow->cpu); 3625 if (rflow->filter == filter_id && cpu < nr_cpu_ids && 3626 ((int)(per_cpu(softnet_data, cpu).input_queue_head - 3627 rflow->last_qtail) < 3628 (int)(10 * flow_table->mask))) 3629 expire = false; 3630 } 3631 rcu_read_unlock(); 3632 return expire; 3633 } 3634 EXPORT_SYMBOL(rps_may_expire_flow); 3635 3636 #endif /* CONFIG_RFS_ACCEL */ 3637 3638 /* Called from hardirq (IPI) context */ 3639 static void rps_trigger_softirq(void *data) 3640 { 3641 struct softnet_data *sd = data; 3642 3643 ____napi_schedule(sd, &sd->backlog); 3644 sd->received_rps++; 3645 } 3646 3647 #endif /* CONFIG_RPS */ 3648 3649 /* 3650 * Check if this softnet_data structure is another cpu one 3651 * If yes, queue it to our IPI list and return 1 3652 * If no, return 0 3653 */ 3654 static int rps_ipi_queued(struct softnet_data *sd) 3655 { 3656 #ifdef CONFIG_RPS 3657 struct softnet_data *mysd = this_cpu_ptr(&softnet_data); 3658 3659 if (sd != mysd) { 3660 sd->rps_ipi_next = mysd->rps_ipi_list; 3661 mysd->rps_ipi_list = sd; 3662 3663 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 3664 return 1; 3665 } 3666 #endif /* CONFIG_RPS */ 3667 return 0; 3668 } 3669 3670 #ifdef CONFIG_NET_FLOW_LIMIT 3671 int netdev_flow_limit_table_len __read_mostly = (1 << 12); 3672 #endif 3673 3674 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) 3675 { 3676 #ifdef CONFIG_NET_FLOW_LIMIT 3677 struct sd_flow_limit *fl; 3678 struct softnet_data *sd; 3679 unsigned int old_flow, new_flow; 3680 3681 if (qlen < (netdev_max_backlog >> 1)) 3682 return false; 3683 3684 sd = this_cpu_ptr(&softnet_data); 3685 3686 rcu_read_lock(); 3687 fl = rcu_dereference(sd->flow_limit); 3688 if (fl) { 3689 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1); 3690 old_flow = fl->history[fl->history_head]; 3691 fl->history[fl->history_head] = new_flow; 3692 3693 fl->history_head++; 3694 fl->history_head &= FLOW_LIMIT_HISTORY - 1; 3695 3696 if (likely(fl->buckets[old_flow])) 3697 fl->buckets[old_flow]--; 3698 3699 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { 3700 fl->count++; 3701 rcu_read_unlock(); 3702 return true; 3703 } 3704 } 3705 rcu_read_unlock(); 3706 #endif 3707 return false; 3708 } 3709 3710 /* 3711 * enqueue_to_backlog is called to queue an skb to a per CPU backlog 3712 * queue (may be a remote CPU queue). 3713 */ 3714 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, 3715 unsigned int *qtail) 3716 { 3717 struct softnet_data *sd; 3718 unsigned long flags; 3719 unsigned int qlen; 3720 3721 sd = &per_cpu(softnet_data, cpu); 3722 3723 local_irq_save(flags); 3724 3725 rps_lock(sd); 3726 if (!netif_running(skb->dev)) 3727 goto drop; 3728 qlen = skb_queue_len(&sd->input_pkt_queue); 3729 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) { 3730 if (qlen) { 3731 enqueue: 3732 __skb_queue_tail(&sd->input_pkt_queue, skb); 3733 input_queue_tail_incr_save(sd, qtail); 3734 rps_unlock(sd); 3735 local_irq_restore(flags); 3736 return NET_RX_SUCCESS; 3737 } 3738 3739 /* Schedule NAPI for backlog device 3740 * We can use non atomic operation since we own the queue lock 3741 */ 3742 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) { 3743 if (!rps_ipi_queued(sd)) 3744 ____napi_schedule(sd, &sd->backlog); 3745 } 3746 goto enqueue; 3747 } 3748 3749 drop: 3750 sd->dropped++; 3751 rps_unlock(sd); 3752 3753 local_irq_restore(flags); 3754 3755 atomic_long_inc(&skb->dev->rx_dropped); 3756 kfree_skb(skb); 3757 return NET_RX_DROP; 3758 } 3759 3760 static int netif_rx_internal(struct sk_buff *skb) 3761 { 3762 int ret; 3763 3764 net_timestamp_check(netdev_tstamp_prequeue, skb); 3765 3766 trace_netif_rx(skb); 3767 #ifdef CONFIG_RPS 3768 if (static_key_false(&rps_needed)) { 3769 struct rps_dev_flow voidflow, *rflow = &voidflow; 3770 int cpu; 3771 3772 preempt_disable(); 3773 rcu_read_lock(); 3774 3775 cpu = get_rps_cpu(skb->dev, skb, &rflow); 3776 if (cpu < 0) 3777 cpu = smp_processor_id(); 3778 3779 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 3780 3781 rcu_read_unlock(); 3782 preempt_enable(); 3783 } else 3784 #endif 3785 { 3786 unsigned int qtail; 3787 ret = enqueue_to_backlog(skb, get_cpu(), &qtail); 3788 put_cpu(); 3789 } 3790 return ret; 3791 } 3792 3793 /** 3794 * netif_rx - post buffer to the network code 3795 * @skb: buffer to post 3796 * 3797 * This function receives a packet from a device driver and queues it for 3798 * the upper (protocol) levels to process. It always succeeds. The buffer 3799 * may be dropped during processing for congestion control or by the 3800 * protocol layers. 3801 * 3802 * return values: 3803 * NET_RX_SUCCESS (no congestion) 3804 * NET_RX_DROP (packet was dropped) 3805 * 3806 */ 3807 3808 int netif_rx(struct sk_buff *skb) 3809 { 3810 trace_netif_rx_entry(skb); 3811 3812 return netif_rx_internal(skb); 3813 } 3814 EXPORT_SYMBOL(netif_rx); 3815 3816 int netif_rx_ni(struct sk_buff *skb) 3817 { 3818 int err; 3819 3820 trace_netif_rx_ni_entry(skb); 3821 3822 preempt_disable(); 3823 err = netif_rx_internal(skb); 3824 if (local_softirq_pending()) 3825 do_softirq(); 3826 preempt_enable(); 3827 3828 return err; 3829 } 3830 EXPORT_SYMBOL(netif_rx_ni); 3831 3832 static __latent_entropy void net_tx_action(struct softirq_action *h) 3833 { 3834 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 3835 3836 if (sd->completion_queue) { 3837 struct sk_buff *clist; 3838 3839 local_irq_disable(); 3840 clist = sd->completion_queue; 3841 sd->completion_queue = NULL; 3842 local_irq_enable(); 3843 3844 while (clist) { 3845 struct sk_buff *skb = clist; 3846 clist = clist->next; 3847 3848 WARN_ON(atomic_read(&skb->users)); 3849 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED)) 3850 trace_consume_skb(skb); 3851 else 3852 trace_kfree_skb(skb, net_tx_action); 3853 3854 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) 3855 __kfree_skb(skb); 3856 else 3857 __kfree_skb_defer(skb); 3858 } 3859 3860 __kfree_skb_flush(); 3861 } 3862 3863 if (sd->output_queue) { 3864 struct Qdisc *head; 3865 3866 local_irq_disable(); 3867 head = sd->output_queue; 3868 sd->output_queue = NULL; 3869 sd->output_queue_tailp = &sd->output_queue; 3870 local_irq_enable(); 3871 3872 while (head) { 3873 struct Qdisc *q = head; 3874 spinlock_t *root_lock; 3875 3876 head = head->next_sched; 3877 3878 root_lock = qdisc_lock(q); 3879 spin_lock(root_lock); 3880 /* We need to make sure head->next_sched is read 3881 * before clearing __QDISC_STATE_SCHED 3882 */ 3883 smp_mb__before_atomic(); 3884 clear_bit(__QDISC_STATE_SCHED, &q->state); 3885 qdisc_run(q); 3886 spin_unlock(root_lock); 3887 } 3888 } 3889 } 3890 3891 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE) 3892 /* This hook is defined here for ATM LANE */ 3893 int (*br_fdb_test_addr_hook)(struct net_device *dev, 3894 unsigned char *addr) __read_mostly; 3895 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); 3896 #endif 3897 3898 static inline struct sk_buff * 3899 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, 3900 struct net_device *orig_dev) 3901 { 3902 #ifdef CONFIG_NET_CLS_ACT 3903 struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list); 3904 struct tcf_result cl_res; 3905 3906 /* If there's at least one ingress present somewhere (so 3907 * we get here via enabled static key), remaining devices 3908 * that are not configured with an ingress qdisc will bail 3909 * out here. 3910 */ 3911 if (!cl) 3912 return skb; 3913 if (*pt_prev) { 3914 *ret = deliver_skb(skb, *pt_prev, orig_dev); 3915 *pt_prev = NULL; 3916 } 3917 3918 qdisc_skb_cb(skb)->pkt_len = skb->len; 3919 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS); 3920 qdisc_bstats_cpu_update(cl->q, skb); 3921 3922 switch (tc_classify(skb, cl, &cl_res, false)) { 3923 case TC_ACT_OK: 3924 case TC_ACT_RECLASSIFY: 3925 skb->tc_index = TC_H_MIN(cl_res.classid); 3926 break; 3927 case TC_ACT_SHOT: 3928 qdisc_qstats_cpu_drop(cl->q); 3929 kfree_skb(skb); 3930 return NULL; 3931 case TC_ACT_STOLEN: 3932 case TC_ACT_QUEUED: 3933 consume_skb(skb); 3934 return NULL; 3935 case TC_ACT_REDIRECT: 3936 /* skb_mac_header check was done by cls/act_bpf, so 3937 * we can safely push the L2 header back before 3938 * redirecting to another netdev 3939 */ 3940 __skb_push(skb, skb->mac_len); 3941 skb_do_redirect(skb); 3942 return NULL; 3943 default: 3944 break; 3945 } 3946 #endif /* CONFIG_NET_CLS_ACT */ 3947 return skb; 3948 } 3949 3950 /** 3951 * netdev_is_rx_handler_busy - check if receive handler is registered 3952 * @dev: device to check 3953 * 3954 * Check if a receive handler is already registered for a given device. 3955 * Return true if there one. 3956 * 3957 * The caller must hold the rtnl_mutex. 3958 */ 3959 bool netdev_is_rx_handler_busy(struct net_device *dev) 3960 { 3961 ASSERT_RTNL(); 3962 return dev && rtnl_dereference(dev->rx_handler); 3963 } 3964 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy); 3965 3966 /** 3967 * netdev_rx_handler_register - register receive handler 3968 * @dev: device to register a handler for 3969 * @rx_handler: receive handler to register 3970 * @rx_handler_data: data pointer that is used by rx handler 3971 * 3972 * Register a receive handler for a device. This handler will then be 3973 * called from __netif_receive_skb. A negative errno code is returned 3974 * on a failure. 3975 * 3976 * The caller must hold the rtnl_mutex. 3977 * 3978 * For a general description of rx_handler, see enum rx_handler_result. 3979 */ 3980 int netdev_rx_handler_register(struct net_device *dev, 3981 rx_handler_func_t *rx_handler, 3982 void *rx_handler_data) 3983 { 3984 ASSERT_RTNL(); 3985 3986 if (dev->rx_handler) 3987 return -EBUSY; 3988 3989 /* Note: rx_handler_data must be set before rx_handler */ 3990 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); 3991 rcu_assign_pointer(dev->rx_handler, rx_handler); 3992 3993 return 0; 3994 } 3995 EXPORT_SYMBOL_GPL(netdev_rx_handler_register); 3996 3997 /** 3998 * netdev_rx_handler_unregister - unregister receive handler 3999 * @dev: device to unregister a handler from 4000 * 4001 * Unregister a receive handler from a device. 4002 * 4003 * The caller must hold the rtnl_mutex. 4004 */ 4005 void netdev_rx_handler_unregister(struct net_device *dev) 4006 { 4007 4008 ASSERT_RTNL(); 4009 RCU_INIT_POINTER(dev->rx_handler, NULL); 4010 /* a reader seeing a non NULL rx_handler in a rcu_read_lock() 4011 * section has a guarantee to see a non NULL rx_handler_data 4012 * as well. 4013 */ 4014 synchronize_net(); 4015 RCU_INIT_POINTER(dev->rx_handler_data, NULL); 4016 } 4017 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); 4018 4019 /* 4020 * Limit the use of PFMEMALLOC reserves to those protocols that implement 4021 * the special handling of PFMEMALLOC skbs. 4022 */ 4023 static bool skb_pfmemalloc_protocol(struct sk_buff *skb) 4024 { 4025 switch (skb->protocol) { 4026 case htons(ETH_P_ARP): 4027 case htons(ETH_P_IP): 4028 case htons(ETH_P_IPV6): 4029 case htons(ETH_P_8021Q): 4030 case htons(ETH_P_8021AD): 4031 return true; 4032 default: 4033 return false; 4034 } 4035 } 4036 4037 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev, 4038 int *ret, struct net_device *orig_dev) 4039 { 4040 #ifdef CONFIG_NETFILTER_INGRESS 4041 if (nf_hook_ingress_active(skb)) { 4042 int ingress_retval; 4043 4044 if (*pt_prev) { 4045 *ret = deliver_skb(skb, *pt_prev, orig_dev); 4046 *pt_prev = NULL; 4047 } 4048 4049 rcu_read_lock(); 4050 ingress_retval = nf_hook_ingress(skb); 4051 rcu_read_unlock(); 4052 return ingress_retval; 4053 } 4054 #endif /* CONFIG_NETFILTER_INGRESS */ 4055 return 0; 4056 } 4057 4058 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc) 4059 { 4060 struct packet_type *ptype, *pt_prev; 4061 rx_handler_func_t *rx_handler; 4062 struct net_device *orig_dev; 4063 bool deliver_exact = false; 4064 int ret = NET_RX_DROP; 4065 __be16 type; 4066 4067 net_timestamp_check(!netdev_tstamp_prequeue, skb); 4068 4069 trace_netif_receive_skb(skb); 4070 4071 orig_dev = skb->dev; 4072 4073 skb_reset_network_header(skb); 4074 if (!skb_transport_header_was_set(skb)) 4075 skb_reset_transport_header(skb); 4076 skb_reset_mac_len(skb); 4077 4078 pt_prev = NULL; 4079 4080 another_round: 4081 skb->skb_iif = skb->dev->ifindex; 4082 4083 __this_cpu_inc(softnet_data.processed); 4084 4085 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 4086 skb->protocol == cpu_to_be16(ETH_P_8021AD)) { 4087 skb = skb_vlan_untag(skb); 4088 if (unlikely(!skb)) 4089 goto out; 4090 } 4091 4092 #ifdef CONFIG_NET_CLS_ACT 4093 if (skb->tc_verd & TC_NCLS) { 4094 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd); 4095 goto ncls; 4096 } 4097 #endif 4098 4099 if (pfmemalloc) 4100 goto skip_taps; 4101 4102 list_for_each_entry_rcu(ptype, &ptype_all, list) { 4103 if (pt_prev) 4104 ret = deliver_skb(skb, pt_prev, orig_dev); 4105 pt_prev = ptype; 4106 } 4107 4108 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { 4109 if (pt_prev) 4110 ret = deliver_skb(skb, pt_prev, orig_dev); 4111 pt_prev = ptype; 4112 } 4113 4114 skip_taps: 4115 #ifdef CONFIG_NET_INGRESS 4116 if (static_key_false(&ingress_needed)) { 4117 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev); 4118 if (!skb) 4119 goto out; 4120 4121 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0) 4122 goto out; 4123 } 4124 #endif 4125 #ifdef CONFIG_NET_CLS_ACT 4126 skb->tc_verd = 0; 4127 ncls: 4128 #endif 4129 if (pfmemalloc && !skb_pfmemalloc_protocol(skb)) 4130 goto drop; 4131 4132 if (skb_vlan_tag_present(skb)) { 4133 if (pt_prev) { 4134 ret = deliver_skb(skb, pt_prev, orig_dev); 4135 pt_prev = NULL; 4136 } 4137 if (vlan_do_receive(&skb)) 4138 goto another_round; 4139 else if (unlikely(!skb)) 4140 goto out; 4141 } 4142 4143 rx_handler = rcu_dereference(skb->dev->rx_handler); 4144 if (rx_handler) { 4145 if (pt_prev) { 4146 ret = deliver_skb(skb, pt_prev, orig_dev); 4147 pt_prev = NULL; 4148 } 4149 switch (rx_handler(&skb)) { 4150 case RX_HANDLER_CONSUMED: 4151 ret = NET_RX_SUCCESS; 4152 goto out; 4153 case RX_HANDLER_ANOTHER: 4154 goto another_round; 4155 case RX_HANDLER_EXACT: 4156 deliver_exact = true; 4157 case RX_HANDLER_PASS: 4158 break; 4159 default: 4160 BUG(); 4161 } 4162 } 4163 4164 if (unlikely(skb_vlan_tag_present(skb))) { 4165 if (skb_vlan_tag_get_id(skb)) 4166 skb->pkt_type = PACKET_OTHERHOST; 4167 /* Note: we might in the future use prio bits 4168 * and set skb->priority like in vlan_do_receive() 4169 * For the time being, just ignore Priority Code Point 4170 */ 4171 skb->vlan_tci = 0; 4172 } 4173 4174 type = skb->protocol; 4175 4176 /* deliver only exact match when indicated */ 4177 if (likely(!deliver_exact)) { 4178 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4179 &ptype_base[ntohs(type) & 4180 PTYPE_HASH_MASK]); 4181 } 4182 4183 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4184 &orig_dev->ptype_specific); 4185 4186 if (unlikely(skb->dev != orig_dev)) { 4187 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4188 &skb->dev->ptype_specific); 4189 } 4190 4191 if (pt_prev) { 4192 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) 4193 goto drop; 4194 else 4195 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 4196 } else { 4197 drop: 4198 if (!deliver_exact) 4199 atomic_long_inc(&skb->dev->rx_dropped); 4200 else 4201 atomic_long_inc(&skb->dev->rx_nohandler); 4202 kfree_skb(skb); 4203 /* Jamal, now you will not able to escape explaining 4204 * me how you were going to use this. :-) 4205 */ 4206 ret = NET_RX_DROP; 4207 } 4208 4209 out: 4210 return ret; 4211 } 4212 4213 static int __netif_receive_skb(struct sk_buff *skb) 4214 { 4215 int ret; 4216 4217 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) { 4218 unsigned long pflags = current->flags; 4219 4220 /* 4221 * PFMEMALLOC skbs are special, they should 4222 * - be delivered to SOCK_MEMALLOC sockets only 4223 * - stay away from userspace 4224 * - have bounded memory usage 4225 * 4226 * Use PF_MEMALLOC as this saves us from propagating the allocation 4227 * context down to all allocation sites. 4228 */ 4229 current->flags |= PF_MEMALLOC; 4230 ret = __netif_receive_skb_core(skb, true); 4231 tsk_restore_flags(current, pflags, PF_MEMALLOC); 4232 } else 4233 ret = __netif_receive_skb_core(skb, false); 4234 4235 return ret; 4236 } 4237 4238 static int netif_receive_skb_internal(struct sk_buff *skb) 4239 { 4240 int ret; 4241 4242 net_timestamp_check(netdev_tstamp_prequeue, skb); 4243 4244 if (skb_defer_rx_timestamp(skb)) 4245 return NET_RX_SUCCESS; 4246 4247 rcu_read_lock(); 4248 4249 #ifdef CONFIG_RPS 4250 if (static_key_false(&rps_needed)) { 4251 struct rps_dev_flow voidflow, *rflow = &voidflow; 4252 int cpu = get_rps_cpu(skb->dev, skb, &rflow); 4253 4254 if (cpu >= 0) { 4255 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 4256 rcu_read_unlock(); 4257 return ret; 4258 } 4259 } 4260 #endif 4261 ret = __netif_receive_skb(skb); 4262 rcu_read_unlock(); 4263 return ret; 4264 } 4265 4266 /** 4267 * netif_receive_skb - process receive buffer from network 4268 * @skb: buffer to process 4269 * 4270 * netif_receive_skb() is the main receive data processing function. 4271 * It always succeeds. The buffer may be dropped during processing 4272 * for congestion control or by the protocol layers. 4273 * 4274 * This function may only be called from softirq context and interrupts 4275 * should be enabled. 4276 * 4277 * Return values (usually ignored): 4278 * NET_RX_SUCCESS: no congestion 4279 * NET_RX_DROP: packet was dropped 4280 */ 4281 int netif_receive_skb(struct sk_buff *skb) 4282 { 4283 trace_netif_receive_skb_entry(skb); 4284 4285 return netif_receive_skb_internal(skb); 4286 } 4287 EXPORT_SYMBOL(netif_receive_skb); 4288 4289 DEFINE_PER_CPU(struct work_struct, flush_works); 4290 4291 /* Network device is going away, flush any packets still pending */ 4292 static void flush_backlog(struct work_struct *work) 4293 { 4294 struct sk_buff *skb, *tmp; 4295 struct softnet_data *sd; 4296 4297 local_bh_disable(); 4298 sd = this_cpu_ptr(&softnet_data); 4299 4300 local_irq_disable(); 4301 rps_lock(sd); 4302 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { 4303 if (skb->dev->reg_state == NETREG_UNREGISTERING) { 4304 __skb_unlink(skb, &sd->input_pkt_queue); 4305 kfree_skb(skb); 4306 input_queue_head_incr(sd); 4307 } 4308 } 4309 rps_unlock(sd); 4310 local_irq_enable(); 4311 4312 skb_queue_walk_safe(&sd->process_queue, skb, tmp) { 4313 if (skb->dev->reg_state == NETREG_UNREGISTERING) { 4314 __skb_unlink(skb, &sd->process_queue); 4315 kfree_skb(skb); 4316 input_queue_head_incr(sd); 4317 } 4318 } 4319 local_bh_enable(); 4320 } 4321 4322 static void flush_all_backlogs(void) 4323 { 4324 unsigned int cpu; 4325 4326 get_online_cpus(); 4327 4328 for_each_online_cpu(cpu) 4329 queue_work_on(cpu, system_highpri_wq, 4330 per_cpu_ptr(&flush_works, cpu)); 4331 4332 for_each_online_cpu(cpu) 4333 flush_work(per_cpu_ptr(&flush_works, cpu)); 4334 4335 put_online_cpus(); 4336 } 4337 4338 static int napi_gro_complete(struct sk_buff *skb) 4339 { 4340 struct packet_offload *ptype; 4341 __be16 type = skb->protocol; 4342 struct list_head *head = &offload_base; 4343 int err = -ENOENT; 4344 4345 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); 4346 4347 if (NAPI_GRO_CB(skb)->count == 1) { 4348 skb_shinfo(skb)->gso_size = 0; 4349 goto out; 4350 } 4351 4352 rcu_read_lock(); 4353 list_for_each_entry_rcu(ptype, head, list) { 4354 if (ptype->type != type || !ptype->callbacks.gro_complete) 4355 continue; 4356 4357 err = ptype->callbacks.gro_complete(skb, 0); 4358 break; 4359 } 4360 rcu_read_unlock(); 4361 4362 if (err) { 4363 WARN_ON(&ptype->list == head); 4364 kfree_skb(skb); 4365 return NET_RX_SUCCESS; 4366 } 4367 4368 out: 4369 return netif_receive_skb_internal(skb); 4370 } 4371 4372 /* napi->gro_list contains packets ordered by age. 4373 * youngest packets at the head of it. 4374 * Complete skbs in reverse order to reduce latencies. 4375 */ 4376 void napi_gro_flush(struct napi_struct *napi, bool flush_old) 4377 { 4378 struct sk_buff *skb, *prev = NULL; 4379 4380 /* scan list and build reverse chain */ 4381 for (skb = napi->gro_list; skb != NULL; skb = skb->next) { 4382 skb->prev = prev; 4383 prev = skb; 4384 } 4385 4386 for (skb = prev; skb; skb = prev) { 4387 skb->next = NULL; 4388 4389 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) 4390 return; 4391 4392 prev = skb->prev; 4393 napi_gro_complete(skb); 4394 napi->gro_count--; 4395 } 4396 4397 napi->gro_list = NULL; 4398 } 4399 EXPORT_SYMBOL(napi_gro_flush); 4400 4401 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb) 4402 { 4403 struct sk_buff *p; 4404 unsigned int maclen = skb->dev->hard_header_len; 4405 u32 hash = skb_get_hash_raw(skb); 4406 4407 for (p = napi->gro_list; p; p = p->next) { 4408 unsigned long diffs; 4409 4410 NAPI_GRO_CB(p)->flush = 0; 4411 4412 if (hash != skb_get_hash_raw(p)) { 4413 NAPI_GRO_CB(p)->same_flow = 0; 4414 continue; 4415 } 4416 4417 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; 4418 diffs |= p->vlan_tci ^ skb->vlan_tci; 4419 diffs |= skb_metadata_dst_cmp(p, skb); 4420 if (maclen == ETH_HLEN) 4421 diffs |= compare_ether_header(skb_mac_header(p), 4422 skb_mac_header(skb)); 4423 else if (!diffs) 4424 diffs = memcmp(skb_mac_header(p), 4425 skb_mac_header(skb), 4426 maclen); 4427 NAPI_GRO_CB(p)->same_flow = !diffs; 4428 } 4429 } 4430 4431 static void skb_gro_reset_offset(struct sk_buff *skb) 4432 { 4433 const struct skb_shared_info *pinfo = skb_shinfo(skb); 4434 const skb_frag_t *frag0 = &pinfo->frags[0]; 4435 4436 NAPI_GRO_CB(skb)->data_offset = 0; 4437 NAPI_GRO_CB(skb)->frag0 = NULL; 4438 NAPI_GRO_CB(skb)->frag0_len = 0; 4439 4440 if (skb_mac_header(skb) == skb_tail_pointer(skb) && 4441 pinfo->nr_frags && 4442 !PageHighMem(skb_frag_page(frag0))) { 4443 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); 4444 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0); 4445 } 4446 } 4447 4448 static void gro_pull_from_frag0(struct sk_buff *skb, int grow) 4449 { 4450 struct skb_shared_info *pinfo = skb_shinfo(skb); 4451 4452 BUG_ON(skb->end - skb->tail < grow); 4453 4454 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); 4455 4456 skb->data_len -= grow; 4457 skb->tail += grow; 4458 4459 pinfo->frags[0].page_offset += grow; 4460 skb_frag_size_sub(&pinfo->frags[0], grow); 4461 4462 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { 4463 skb_frag_unref(skb, 0); 4464 memmove(pinfo->frags, pinfo->frags + 1, 4465 --pinfo->nr_frags * sizeof(pinfo->frags[0])); 4466 } 4467 } 4468 4469 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 4470 { 4471 struct sk_buff **pp = NULL; 4472 struct packet_offload *ptype; 4473 __be16 type = skb->protocol; 4474 struct list_head *head = &offload_base; 4475 int same_flow; 4476 enum gro_result ret; 4477 int grow; 4478 4479 if (!(skb->dev->features & NETIF_F_GRO)) 4480 goto normal; 4481 4482 if (skb->csum_bad) 4483 goto normal; 4484 4485 gro_list_prepare(napi, skb); 4486 4487 rcu_read_lock(); 4488 list_for_each_entry_rcu(ptype, head, list) { 4489 if (ptype->type != type || !ptype->callbacks.gro_receive) 4490 continue; 4491 4492 skb_set_network_header(skb, skb_gro_offset(skb)); 4493 skb_reset_mac_len(skb); 4494 NAPI_GRO_CB(skb)->same_flow = 0; 4495 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb); 4496 NAPI_GRO_CB(skb)->free = 0; 4497 NAPI_GRO_CB(skb)->encap_mark = 0; 4498 NAPI_GRO_CB(skb)->recursion_counter = 0; 4499 NAPI_GRO_CB(skb)->is_fou = 0; 4500 NAPI_GRO_CB(skb)->is_atomic = 1; 4501 NAPI_GRO_CB(skb)->gro_remcsum_start = 0; 4502 4503 /* Setup for GRO checksum validation */ 4504 switch (skb->ip_summed) { 4505 case CHECKSUM_COMPLETE: 4506 NAPI_GRO_CB(skb)->csum = skb->csum; 4507 NAPI_GRO_CB(skb)->csum_valid = 1; 4508 NAPI_GRO_CB(skb)->csum_cnt = 0; 4509 break; 4510 case CHECKSUM_UNNECESSARY: 4511 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; 4512 NAPI_GRO_CB(skb)->csum_valid = 0; 4513 break; 4514 default: 4515 NAPI_GRO_CB(skb)->csum_cnt = 0; 4516 NAPI_GRO_CB(skb)->csum_valid = 0; 4517 } 4518 4519 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb); 4520 break; 4521 } 4522 rcu_read_unlock(); 4523 4524 if (&ptype->list == head) 4525 goto normal; 4526 4527 same_flow = NAPI_GRO_CB(skb)->same_flow; 4528 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; 4529 4530 if (pp) { 4531 struct sk_buff *nskb = *pp; 4532 4533 *pp = nskb->next; 4534 nskb->next = NULL; 4535 napi_gro_complete(nskb); 4536 napi->gro_count--; 4537 } 4538 4539 if (same_flow) 4540 goto ok; 4541 4542 if (NAPI_GRO_CB(skb)->flush) 4543 goto normal; 4544 4545 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) { 4546 struct sk_buff *nskb = napi->gro_list; 4547 4548 /* locate the end of the list to select the 'oldest' flow */ 4549 while (nskb->next) { 4550 pp = &nskb->next; 4551 nskb = *pp; 4552 } 4553 *pp = NULL; 4554 nskb->next = NULL; 4555 napi_gro_complete(nskb); 4556 } else { 4557 napi->gro_count++; 4558 } 4559 NAPI_GRO_CB(skb)->count = 1; 4560 NAPI_GRO_CB(skb)->age = jiffies; 4561 NAPI_GRO_CB(skb)->last = skb; 4562 skb_shinfo(skb)->gso_size = skb_gro_len(skb); 4563 skb->next = napi->gro_list; 4564 napi->gro_list = skb; 4565 ret = GRO_HELD; 4566 4567 pull: 4568 grow = skb_gro_offset(skb) - skb_headlen(skb); 4569 if (grow > 0) 4570 gro_pull_from_frag0(skb, grow); 4571 ok: 4572 return ret; 4573 4574 normal: 4575 ret = GRO_NORMAL; 4576 goto pull; 4577 } 4578 4579 struct packet_offload *gro_find_receive_by_type(__be16 type) 4580 { 4581 struct list_head *offload_head = &offload_base; 4582 struct packet_offload *ptype; 4583 4584 list_for_each_entry_rcu(ptype, offload_head, list) { 4585 if (ptype->type != type || !ptype->callbacks.gro_receive) 4586 continue; 4587 return ptype; 4588 } 4589 return NULL; 4590 } 4591 EXPORT_SYMBOL(gro_find_receive_by_type); 4592 4593 struct packet_offload *gro_find_complete_by_type(__be16 type) 4594 { 4595 struct list_head *offload_head = &offload_base; 4596 struct packet_offload *ptype; 4597 4598 list_for_each_entry_rcu(ptype, offload_head, list) { 4599 if (ptype->type != type || !ptype->callbacks.gro_complete) 4600 continue; 4601 return ptype; 4602 } 4603 return NULL; 4604 } 4605 EXPORT_SYMBOL(gro_find_complete_by_type); 4606 4607 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) 4608 { 4609 switch (ret) { 4610 case GRO_NORMAL: 4611 if (netif_receive_skb_internal(skb)) 4612 ret = GRO_DROP; 4613 break; 4614 4615 case GRO_DROP: 4616 kfree_skb(skb); 4617 break; 4618 4619 case GRO_MERGED_FREE: 4620 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) { 4621 skb_dst_drop(skb); 4622 kmem_cache_free(skbuff_head_cache, skb); 4623 } else { 4624 __kfree_skb(skb); 4625 } 4626 break; 4627 4628 case GRO_HELD: 4629 case GRO_MERGED: 4630 break; 4631 } 4632 4633 return ret; 4634 } 4635 4636 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 4637 { 4638 skb_mark_napi_id(skb, napi); 4639 trace_napi_gro_receive_entry(skb); 4640 4641 skb_gro_reset_offset(skb); 4642 4643 return napi_skb_finish(dev_gro_receive(napi, skb), skb); 4644 } 4645 EXPORT_SYMBOL(napi_gro_receive); 4646 4647 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) 4648 { 4649 if (unlikely(skb->pfmemalloc)) { 4650 consume_skb(skb); 4651 return; 4652 } 4653 __skb_pull(skb, skb_headlen(skb)); 4654 /* restore the reserve we had after netdev_alloc_skb_ip_align() */ 4655 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); 4656 skb->vlan_tci = 0; 4657 skb->dev = napi->dev; 4658 skb->skb_iif = 0; 4659 skb->encapsulation = 0; 4660 skb_shinfo(skb)->gso_type = 0; 4661 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 4662 4663 napi->skb = skb; 4664 } 4665 4666 struct sk_buff *napi_get_frags(struct napi_struct *napi) 4667 { 4668 struct sk_buff *skb = napi->skb; 4669 4670 if (!skb) { 4671 skb = napi_alloc_skb(napi, GRO_MAX_HEAD); 4672 if (skb) { 4673 napi->skb = skb; 4674 skb_mark_napi_id(skb, napi); 4675 } 4676 } 4677 return skb; 4678 } 4679 EXPORT_SYMBOL(napi_get_frags); 4680 4681 static gro_result_t napi_frags_finish(struct napi_struct *napi, 4682 struct sk_buff *skb, 4683 gro_result_t ret) 4684 { 4685 switch (ret) { 4686 case GRO_NORMAL: 4687 case GRO_HELD: 4688 __skb_push(skb, ETH_HLEN); 4689 skb->protocol = eth_type_trans(skb, skb->dev); 4690 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb)) 4691 ret = GRO_DROP; 4692 break; 4693 4694 case GRO_DROP: 4695 case GRO_MERGED_FREE: 4696 napi_reuse_skb(napi, skb); 4697 break; 4698 4699 case GRO_MERGED: 4700 break; 4701 } 4702 4703 return ret; 4704 } 4705 4706 /* Upper GRO stack assumes network header starts at gro_offset=0 4707 * Drivers could call both napi_gro_frags() and napi_gro_receive() 4708 * We copy ethernet header into skb->data to have a common layout. 4709 */ 4710 static struct sk_buff *napi_frags_skb(struct napi_struct *napi) 4711 { 4712 struct sk_buff *skb = napi->skb; 4713 const struct ethhdr *eth; 4714 unsigned int hlen = sizeof(*eth); 4715 4716 napi->skb = NULL; 4717 4718 skb_reset_mac_header(skb); 4719 skb_gro_reset_offset(skb); 4720 4721 eth = skb_gro_header_fast(skb, 0); 4722 if (unlikely(skb_gro_header_hard(skb, hlen))) { 4723 eth = skb_gro_header_slow(skb, hlen, 0); 4724 if (unlikely(!eth)) { 4725 net_warn_ratelimited("%s: dropping impossible skb from %s\n", 4726 __func__, napi->dev->name); 4727 napi_reuse_skb(napi, skb); 4728 return NULL; 4729 } 4730 } else { 4731 gro_pull_from_frag0(skb, hlen); 4732 NAPI_GRO_CB(skb)->frag0 += hlen; 4733 NAPI_GRO_CB(skb)->frag0_len -= hlen; 4734 } 4735 __skb_pull(skb, hlen); 4736 4737 /* 4738 * This works because the only protocols we care about don't require 4739 * special handling. 4740 * We'll fix it up properly in napi_frags_finish() 4741 */ 4742 skb->protocol = eth->h_proto; 4743 4744 return skb; 4745 } 4746 4747 gro_result_t napi_gro_frags(struct napi_struct *napi) 4748 { 4749 struct sk_buff *skb = napi_frags_skb(napi); 4750 4751 if (!skb) 4752 return GRO_DROP; 4753 4754 trace_napi_gro_frags_entry(skb); 4755 4756 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); 4757 } 4758 EXPORT_SYMBOL(napi_gro_frags); 4759 4760 /* Compute the checksum from gro_offset and return the folded value 4761 * after adding in any pseudo checksum. 4762 */ 4763 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb) 4764 { 4765 __wsum wsum; 4766 __sum16 sum; 4767 4768 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); 4769 4770 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ 4771 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); 4772 if (likely(!sum)) { 4773 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 4774 !skb->csum_complete_sw) 4775 netdev_rx_csum_fault(skb->dev); 4776 } 4777 4778 NAPI_GRO_CB(skb)->csum = wsum; 4779 NAPI_GRO_CB(skb)->csum_valid = 1; 4780 4781 return sum; 4782 } 4783 EXPORT_SYMBOL(__skb_gro_checksum_complete); 4784 4785 /* 4786 * net_rps_action_and_irq_enable sends any pending IPI's for rps. 4787 * Note: called with local irq disabled, but exits with local irq enabled. 4788 */ 4789 static void net_rps_action_and_irq_enable(struct softnet_data *sd) 4790 { 4791 #ifdef CONFIG_RPS 4792 struct softnet_data *remsd = sd->rps_ipi_list; 4793 4794 if (remsd) { 4795 sd->rps_ipi_list = NULL; 4796 4797 local_irq_enable(); 4798 4799 /* Send pending IPI's to kick RPS processing on remote cpus. */ 4800 while (remsd) { 4801 struct softnet_data *next = remsd->rps_ipi_next; 4802 4803 if (cpu_online(remsd->cpu)) 4804 smp_call_function_single_async(remsd->cpu, 4805 &remsd->csd); 4806 remsd = next; 4807 } 4808 } else 4809 #endif 4810 local_irq_enable(); 4811 } 4812 4813 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) 4814 { 4815 #ifdef CONFIG_RPS 4816 return sd->rps_ipi_list != NULL; 4817 #else 4818 return false; 4819 #endif 4820 } 4821 4822 static int process_backlog(struct napi_struct *napi, int quota) 4823 { 4824 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); 4825 bool again = true; 4826 int work = 0; 4827 4828 /* Check if we have pending ipi, its better to send them now, 4829 * not waiting net_rx_action() end. 4830 */ 4831 if (sd_has_rps_ipi_waiting(sd)) { 4832 local_irq_disable(); 4833 net_rps_action_and_irq_enable(sd); 4834 } 4835 4836 napi->weight = weight_p; 4837 while (again) { 4838 struct sk_buff *skb; 4839 4840 while ((skb = __skb_dequeue(&sd->process_queue))) { 4841 rcu_read_lock(); 4842 __netif_receive_skb(skb); 4843 rcu_read_unlock(); 4844 input_queue_head_incr(sd); 4845 if (++work >= quota) 4846 return work; 4847 4848 } 4849 4850 local_irq_disable(); 4851 rps_lock(sd); 4852 if (skb_queue_empty(&sd->input_pkt_queue)) { 4853 /* 4854 * Inline a custom version of __napi_complete(). 4855 * only current cpu owns and manipulates this napi, 4856 * and NAPI_STATE_SCHED is the only possible flag set 4857 * on backlog. 4858 * We can use a plain write instead of clear_bit(), 4859 * and we dont need an smp_mb() memory barrier. 4860 */ 4861 napi->state = 0; 4862 again = false; 4863 } else { 4864 skb_queue_splice_tail_init(&sd->input_pkt_queue, 4865 &sd->process_queue); 4866 } 4867 rps_unlock(sd); 4868 local_irq_enable(); 4869 } 4870 4871 return work; 4872 } 4873 4874 /** 4875 * __napi_schedule - schedule for receive 4876 * @n: entry to schedule 4877 * 4878 * The entry's receive function will be scheduled to run. 4879 * Consider using __napi_schedule_irqoff() if hard irqs are masked. 4880 */ 4881 void __napi_schedule(struct napi_struct *n) 4882 { 4883 unsigned long flags; 4884 4885 local_irq_save(flags); 4886 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 4887 local_irq_restore(flags); 4888 } 4889 EXPORT_SYMBOL(__napi_schedule); 4890 4891 /** 4892 * __napi_schedule_irqoff - schedule for receive 4893 * @n: entry to schedule 4894 * 4895 * Variant of __napi_schedule() assuming hard irqs are masked 4896 */ 4897 void __napi_schedule_irqoff(struct napi_struct *n) 4898 { 4899 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 4900 } 4901 EXPORT_SYMBOL(__napi_schedule_irqoff); 4902 4903 bool __napi_complete(struct napi_struct *n) 4904 { 4905 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state)); 4906 4907 /* Some drivers call us directly, instead of calling 4908 * napi_complete_done(). 4909 */ 4910 if (unlikely(test_bit(NAPI_STATE_IN_BUSY_POLL, &n->state))) 4911 return false; 4912 4913 list_del_init(&n->poll_list); 4914 smp_mb__before_atomic(); 4915 clear_bit(NAPI_STATE_SCHED, &n->state); 4916 return true; 4917 } 4918 EXPORT_SYMBOL(__napi_complete); 4919 4920 bool napi_complete_done(struct napi_struct *n, int work_done) 4921 { 4922 unsigned long flags; 4923 4924 /* 4925 * 1) Don't let napi dequeue from the cpu poll list 4926 * just in case its running on a different cpu. 4927 * 2) If we are busy polling, do nothing here, we have 4928 * the guarantee we will be called later. 4929 */ 4930 if (unlikely(n->state & (NAPIF_STATE_NPSVC | 4931 NAPIF_STATE_IN_BUSY_POLL))) 4932 return false; 4933 4934 if (n->gro_list) { 4935 unsigned long timeout = 0; 4936 4937 if (work_done) 4938 timeout = n->dev->gro_flush_timeout; 4939 4940 if (timeout) 4941 hrtimer_start(&n->timer, ns_to_ktime(timeout), 4942 HRTIMER_MODE_REL_PINNED); 4943 else 4944 napi_gro_flush(n, false); 4945 } 4946 if (likely(list_empty(&n->poll_list))) { 4947 WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state)); 4948 } else { 4949 /* If n->poll_list is not empty, we need to mask irqs */ 4950 local_irq_save(flags); 4951 __napi_complete(n); 4952 local_irq_restore(flags); 4953 } 4954 return true; 4955 } 4956 EXPORT_SYMBOL(napi_complete_done); 4957 4958 /* must be called under rcu_read_lock(), as we dont take a reference */ 4959 static struct napi_struct *napi_by_id(unsigned int napi_id) 4960 { 4961 unsigned int hash = napi_id % HASH_SIZE(napi_hash); 4962 struct napi_struct *napi; 4963 4964 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node) 4965 if (napi->napi_id == napi_id) 4966 return napi; 4967 4968 return NULL; 4969 } 4970 4971 #if defined(CONFIG_NET_RX_BUSY_POLL) 4972 4973 #define BUSY_POLL_BUDGET 8 4974 4975 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock) 4976 { 4977 int rc; 4978 4979 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state); 4980 4981 local_bh_disable(); 4982 4983 /* All we really want here is to re-enable device interrupts. 4984 * Ideally, a new ndo_busy_poll_stop() could avoid another round. 4985 */ 4986 rc = napi->poll(napi, BUSY_POLL_BUDGET); 4987 netpoll_poll_unlock(have_poll_lock); 4988 if (rc == BUSY_POLL_BUDGET) 4989 __napi_schedule(napi); 4990 local_bh_enable(); 4991 if (local_softirq_pending()) 4992 do_softirq(); 4993 } 4994 4995 bool sk_busy_loop(struct sock *sk, int nonblock) 4996 { 4997 unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0; 4998 int (*napi_poll)(struct napi_struct *napi, int budget); 4999 int (*busy_poll)(struct napi_struct *dev); 5000 void *have_poll_lock = NULL; 5001 struct napi_struct *napi; 5002 int rc; 5003 5004 restart: 5005 rc = false; 5006 napi_poll = NULL; 5007 5008 rcu_read_lock(); 5009 5010 napi = napi_by_id(sk->sk_napi_id); 5011 if (!napi) 5012 goto out; 5013 5014 /* Note: ndo_busy_poll method is optional in linux-4.5 */ 5015 busy_poll = napi->dev->netdev_ops->ndo_busy_poll; 5016 5017 preempt_disable(); 5018 for (;;) { 5019 rc = 0; 5020 local_bh_disable(); 5021 if (busy_poll) { 5022 rc = busy_poll(napi); 5023 goto count; 5024 } 5025 if (!napi_poll) { 5026 unsigned long val = READ_ONCE(napi->state); 5027 5028 /* If multiple threads are competing for this napi, 5029 * we avoid dirtying napi->state as much as we can. 5030 */ 5031 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED | 5032 NAPIF_STATE_IN_BUSY_POLL)) 5033 goto count; 5034 if (cmpxchg(&napi->state, val, 5035 val | NAPIF_STATE_IN_BUSY_POLL | 5036 NAPIF_STATE_SCHED) != val) 5037 goto count; 5038 have_poll_lock = netpoll_poll_lock(napi); 5039 napi_poll = napi->poll; 5040 } 5041 rc = napi_poll(napi, BUSY_POLL_BUDGET); 5042 trace_napi_poll(napi, rc, BUSY_POLL_BUDGET); 5043 count: 5044 if (rc > 0) 5045 __NET_ADD_STATS(sock_net(sk), 5046 LINUX_MIB_BUSYPOLLRXPACKETS, rc); 5047 local_bh_enable(); 5048 5049 if (rc == LL_FLUSH_FAILED) 5050 break; /* permanent failure */ 5051 5052 if (nonblock || !skb_queue_empty(&sk->sk_receive_queue) || 5053 busy_loop_timeout(end_time)) 5054 break; 5055 5056 if (unlikely(need_resched())) { 5057 if (napi_poll) 5058 busy_poll_stop(napi, have_poll_lock); 5059 preempt_enable(); 5060 rcu_read_unlock(); 5061 cond_resched(); 5062 rc = !skb_queue_empty(&sk->sk_receive_queue); 5063 if (rc || busy_loop_timeout(end_time)) 5064 return rc; 5065 goto restart; 5066 } 5067 cpu_relax(); 5068 } 5069 if (napi_poll) 5070 busy_poll_stop(napi, have_poll_lock); 5071 preempt_enable(); 5072 rc = !skb_queue_empty(&sk->sk_receive_queue); 5073 out: 5074 rcu_read_unlock(); 5075 return rc; 5076 } 5077 EXPORT_SYMBOL(sk_busy_loop); 5078 5079 #endif /* CONFIG_NET_RX_BUSY_POLL */ 5080 5081 static void napi_hash_add(struct napi_struct *napi) 5082 { 5083 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) || 5084 test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) 5085 return; 5086 5087 spin_lock(&napi_hash_lock); 5088 5089 /* 0..NR_CPUS+1 range is reserved for sender_cpu use */ 5090 do { 5091 if (unlikely(++napi_gen_id < NR_CPUS + 1)) 5092 napi_gen_id = NR_CPUS + 1; 5093 } while (napi_by_id(napi_gen_id)); 5094 napi->napi_id = napi_gen_id; 5095 5096 hlist_add_head_rcu(&napi->napi_hash_node, 5097 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); 5098 5099 spin_unlock(&napi_hash_lock); 5100 } 5101 5102 /* Warning : caller is responsible to make sure rcu grace period 5103 * is respected before freeing memory containing @napi 5104 */ 5105 bool napi_hash_del(struct napi_struct *napi) 5106 { 5107 bool rcu_sync_needed = false; 5108 5109 spin_lock(&napi_hash_lock); 5110 5111 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) { 5112 rcu_sync_needed = true; 5113 hlist_del_rcu(&napi->napi_hash_node); 5114 } 5115 spin_unlock(&napi_hash_lock); 5116 return rcu_sync_needed; 5117 } 5118 EXPORT_SYMBOL_GPL(napi_hash_del); 5119 5120 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer) 5121 { 5122 struct napi_struct *napi; 5123 5124 napi = container_of(timer, struct napi_struct, timer); 5125 if (napi->gro_list) 5126 napi_schedule(napi); 5127 5128 return HRTIMER_NORESTART; 5129 } 5130 5131 void netif_napi_add(struct net_device *dev, struct napi_struct *napi, 5132 int (*poll)(struct napi_struct *, int), int weight) 5133 { 5134 INIT_LIST_HEAD(&napi->poll_list); 5135 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); 5136 napi->timer.function = napi_watchdog; 5137 napi->gro_count = 0; 5138 napi->gro_list = NULL; 5139 napi->skb = NULL; 5140 napi->poll = poll; 5141 if (weight > NAPI_POLL_WEIGHT) 5142 pr_err_once("netif_napi_add() called with weight %d on device %s\n", 5143 weight, dev->name); 5144 napi->weight = weight; 5145 list_add(&napi->dev_list, &dev->napi_list); 5146 napi->dev = dev; 5147 #ifdef CONFIG_NETPOLL 5148 napi->poll_owner = -1; 5149 #endif 5150 set_bit(NAPI_STATE_SCHED, &napi->state); 5151 napi_hash_add(napi); 5152 } 5153 EXPORT_SYMBOL(netif_napi_add); 5154 5155 void napi_disable(struct napi_struct *n) 5156 { 5157 might_sleep(); 5158 set_bit(NAPI_STATE_DISABLE, &n->state); 5159 5160 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state)) 5161 msleep(1); 5162 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state)) 5163 msleep(1); 5164 5165 hrtimer_cancel(&n->timer); 5166 5167 clear_bit(NAPI_STATE_DISABLE, &n->state); 5168 } 5169 EXPORT_SYMBOL(napi_disable); 5170 5171 /* Must be called in process context */ 5172 void netif_napi_del(struct napi_struct *napi) 5173 { 5174 might_sleep(); 5175 if (napi_hash_del(napi)) 5176 synchronize_net(); 5177 list_del_init(&napi->dev_list); 5178 napi_free_frags(napi); 5179 5180 kfree_skb_list(napi->gro_list); 5181 napi->gro_list = NULL; 5182 napi->gro_count = 0; 5183 } 5184 EXPORT_SYMBOL(netif_napi_del); 5185 5186 static int napi_poll(struct napi_struct *n, struct list_head *repoll) 5187 { 5188 void *have; 5189 int work, weight; 5190 5191 list_del_init(&n->poll_list); 5192 5193 have = netpoll_poll_lock(n); 5194 5195 weight = n->weight; 5196 5197 /* This NAPI_STATE_SCHED test is for avoiding a race 5198 * with netpoll's poll_napi(). Only the entity which 5199 * obtains the lock and sees NAPI_STATE_SCHED set will 5200 * actually make the ->poll() call. Therefore we avoid 5201 * accidentally calling ->poll() when NAPI is not scheduled. 5202 */ 5203 work = 0; 5204 if (test_bit(NAPI_STATE_SCHED, &n->state)) { 5205 work = n->poll(n, weight); 5206 trace_napi_poll(n, work, weight); 5207 } 5208 5209 WARN_ON_ONCE(work > weight); 5210 5211 if (likely(work < weight)) 5212 goto out_unlock; 5213 5214 /* Drivers must not modify the NAPI state if they 5215 * consume the entire weight. In such cases this code 5216 * still "owns" the NAPI instance and therefore can 5217 * move the instance around on the list at-will. 5218 */ 5219 if (unlikely(napi_disable_pending(n))) { 5220 napi_complete(n); 5221 goto out_unlock; 5222 } 5223 5224 if (n->gro_list) { 5225 /* flush too old packets 5226 * If HZ < 1000, flush all packets. 5227 */ 5228 napi_gro_flush(n, HZ >= 1000); 5229 } 5230 5231 /* Some drivers may have called napi_schedule 5232 * prior to exhausting their budget. 5233 */ 5234 if (unlikely(!list_empty(&n->poll_list))) { 5235 pr_warn_once("%s: Budget exhausted after napi rescheduled\n", 5236 n->dev ? n->dev->name : "backlog"); 5237 goto out_unlock; 5238 } 5239 5240 list_add_tail(&n->poll_list, repoll); 5241 5242 out_unlock: 5243 netpoll_poll_unlock(have); 5244 5245 return work; 5246 } 5247 5248 static __latent_entropy void net_rx_action(struct softirq_action *h) 5249 { 5250 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 5251 unsigned long time_limit = jiffies + 2; 5252 int budget = netdev_budget; 5253 LIST_HEAD(list); 5254 LIST_HEAD(repoll); 5255 5256 local_irq_disable(); 5257 list_splice_init(&sd->poll_list, &list); 5258 local_irq_enable(); 5259 5260 for (;;) { 5261 struct napi_struct *n; 5262 5263 if (list_empty(&list)) { 5264 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll)) 5265 goto out; 5266 break; 5267 } 5268 5269 n = list_first_entry(&list, struct napi_struct, poll_list); 5270 budget -= napi_poll(n, &repoll); 5271 5272 /* If softirq window is exhausted then punt. 5273 * Allow this to run for 2 jiffies since which will allow 5274 * an average latency of 1.5/HZ. 5275 */ 5276 if (unlikely(budget <= 0 || 5277 time_after_eq(jiffies, time_limit))) { 5278 sd->time_squeeze++; 5279 break; 5280 } 5281 } 5282 5283 local_irq_disable(); 5284 5285 list_splice_tail_init(&sd->poll_list, &list); 5286 list_splice_tail(&repoll, &list); 5287 list_splice(&list, &sd->poll_list); 5288 if (!list_empty(&sd->poll_list)) 5289 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 5290 5291 net_rps_action_and_irq_enable(sd); 5292 out: 5293 __kfree_skb_flush(); 5294 } 5295 5296 struct netdev_adjacent { 5297 struct net_device *dev; 5298 5299 /* upper master flag, there can only be one master device per list */ 5300 bool master; 5301 5302 /* counter for the number of times this device was added to us */ 5303 u16 ref_nr; 5304 5305 /* private field for the users */ 5306 void *private; 5307 5308 struct list_head list; 5309 struct rcu_head rcu; 5310 }; 5311 5312 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev, 5313 struct list_head *adj_list) 5314 { 5315 struct netdev_adjacent *adj; 5316 5317 list_for_each_entry(adj, adj_list, list) { 5318 if (adj->dev == adj_dev) 5319 return adj; 5320 } 5321 return NULL; 5322 } 5323 5324 static int __netdev_has_upper_dev(struct net_device *upper_dev, void *data) 5325 { 5326 struct net_device *dev = data; 5327 5328 return upper_dev == dev; 5329 } 5330 5331 /** 5332 * netdev_has_upper_dev - Check if device is linked to an upper device 5333 * @dev: device 5334 * @upper_dev: upper device to check 5335 * 5336 * Find out if a device is linked to specified upper device and return true 5337 * in case it is. Note that this checks only immediate upper device, 5338 * not through a complete stack of devices. The caller must hold the RTNL lock. 5339 */ 5340 bool netdev_has_upper_dev(struct net_device *dev, 5341 struct net_device *upper_dev) 5342 { 5343 ASSERT_RTNL(); 5344 5345 return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev, 5346 upper_dev); 5347 } 5348 EXPORT_SYMBOL(netdev_has_upper_dev); 5349 5350 /** 5351 * netdev_has_upper_dev_all - Check if device is linked to an upper device 5352 * @dev: device 5353 * @upper_dev: upper device to check 5354 * 5355 * Find out if a device is linked to specified upper device and return true 5356 * in case it is. Note that this checks the entire upper device chain. 5357 * The caller must hold rcu lock. 5358 */ 5359 5360 bool netdev_has_upper_dev_all_rcu(struct net_device *dev, 5361 struct net_device *upper_dev) 5362 { 5363 return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev, 5364 upper_dev); 5365 } 5366 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu); 5367 5368 /** 5369 * netdev_has_any_upper_dev - Check if device is linked to some device 5370 * @dev: device 5371 * 5372 * Find out if a device is linked to an upper device and return true in case 5373 * it is. The caller must hold the RTNL lock. 5374 */ 5375 static bool netdev_has_any_upper_dev(struct net_device *dev) 5376 { 5377 ASSERT_RTNL(); 5378 5379 return !list_empty(&dev->adj_list.upper); 5380 } 5381 5382 /** 5383 * netdev_master_upper_dev_get - Get master upper device 5384 * @dev: device 5385 * 5386 * Find a master upper device and return pointer to it or NULL in case 5387 * it's not there. The caller must hold the RTNL lock. 5388 */ 5389 struct net_device *netdev_master_upper_dev_get(struct net_device *dev) 5390 { 5391 struct netdev_adjacent *upper; 5392 5393 ASSERT_RTNL(); 5394 5395 if (list_empty(&dev->adj_list.upper)) 5396 return NULL; 5397 5398 upper = list_first_entry(&dev->adj_list.upper, 5399 struct netdev_adjacent, list); 5400 if (likely(upper->master)) 5401 return upper->dev; 5402 return NULL; 5403 } 5404 EXPORT_SYMBOL(netdev_master_upper_dev_get); 5405 5406 /** 5407 * netdev_has_any_lower_dev - Check if device is linked to some device 5408 * @dev: device 5409 * 5410 * Find out if a device is linked to a lower device and return true in case 5411 * it is. The caller must hold the RTNL lock. 5412 */ 5413 static bool netdev_has_any_lower_dev(struct net_device *dev) 5414 { 5415 ASSERT_RTNL(); 5416 5417 return !list_empty(&dev->adj_list.lower); 5418 } 5419 5420 void *netdev_adjacent_get_private(struct list_head *adj_list) 5421 { 5422 struct netdev_adjacent *adj; 5423 5424 adj = list_entry(adj_list, struct netdev_adjacent, list); 5425 5426 return adj->private; 5427 } 5428 EXPORT_SYMBOL(netdev_adjacent_get_private); 5429 5430 /** 5431 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list 5432 * @dev: device 5433 * @iter: list_head ** of the current position 5434 * 5435 * Gets the next device from the dev's upper list, starting from iter 5436 * position. The caller must hold RCU read lock. 5437 */ 5438 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev, 5439 struct list_head **iter) 5440 { 5441 struct netdev_adjacent *upper; 5442 5443 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 5444 5445 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5446 5447 if (&upper->list == &dev->adj_list.upper) 5448 return NULL; 5449 5450 *iter = &upper->list; 5451 5452 return upper->dev; 5453 } 5454 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu); 5455 5456 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev, 5457 struct list_head **iter) 5458 { 5459 struct netdev_adjacent *upper; 5460 5461 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 5462 5463 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5464 5465 if (&upper->list == &dev->adj_list.upper) 5466 return NULL; 5467 5468 *iter = &upper->list; 5469 5470 return upper->dev; 5471 } 5472 5473 int netdev_walk_all_upper_dev_rcu(struct net_device *dev, 5474 int (*fn)(struct net_device *dev, 5475 void *data), 5476 void *data) 5477 { 5478 struct net_device *udev; 5479 struct list_head *iter; 5480 int ret; 5481 5482 for (iter = &dev->adj_list.upper, 5483 udev = netdev_next_upper_dev_rcu(dev, &iter); 5484 udev; 5485 udev = netdev_next_upper_dev_rcu(dev, &iter)) { 5486 /* first is the upper device itself */ 5487 ret = fn(udev, data); 5488 if (ret) 5489 return ret; 5490 5491 /* then look at all of its upper devices */ 5492 ret = netdev_walk_all_upper_dev_rcu(udev, fn, data); 5493 if (ret) 5494 return ret; 5495 } 5496 5497 return 0; 5498 } 5499 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu); 5500 5501 /** 5502 * netdev_lower_get_next_private - Get the next ->private from the 5503 * lower neighbour list 5504 * @dev: device 5505 * @iter: list_head ** of the current position 5506 * 5507 * Gets the next netdev_adjacent->private from the dev's lower neighbour 5508 * list, starting from iter position. The caller must hold either hold the 5509 * RTNL lock or its own locking that guarantees that the neighbour lower 5510 * list will remain unchanged. 5511 */ 5512 void *netdev_lower_get_next_private(struct net_device *dev, 5513 struct list_head **iter) 5514 { 5515 struct netdev_adjacent *lower; 5516 5517 lower = list_entry(*iter, struct netdev_adjacent, list); 5518 5519 if (&lower->list == &dev->adj_list.lower) 5520 return NULL; 5521 5522 *iter = lower->list.next; 5523 5524 return lower->private; 5525 } 5526 EXPORT_SYMBOL(netdev_lower_get_next_private); 5527 5528 /** 5529 * netdev_lower_get_next_private_rcu - Get the next ->private from the 5530 * lower neighbour list, RCU 5531 * variant 5532 * @dev: device 5533 * @iter: list_head ** of the current position 5534 * 5535 * Gets the next netdev_adjacent->private from the dev's lower neighbour 5536 * list, starting from iter position. The caller must hold RCU read lock. 5537 */ 5538 void *netdev_lower_get_next_private_rcu(struct net_device *dev, 5539 struct list_head **iter) 5540 { 5541 struct netdev_adjacent *lower; 5542 5543 WARN_ON_ONCE(!rcu_read_lock_held()); 5544 5545 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5546 5547 if (&lower->list == &dev->adj_list.lower) 5548 return NULL; 5549 5550 *iter = &lower->list; 5551 5552 return lower->private; 5553 } 5554 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu); 5555 5556 /** 5557 * netdev_lower_get_next - Get the next device from the lower neighbour 5558 * list 5559 * @dev: device 5560 * @iter: list_head ** of the current position 5561 * 5562 * Gets the next netdev_adjacent from the dev's lower neighbour 5563 * list, starting from iter position. The caller must hold RTNL lock or 5564 * its own locking that guarantees that the neighbour lower 5565 * list will remain unchanged. 5566 */ 5567 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter) 5568 { 5569 struct netdev_adjacent *lower; 5570 5571 lower = list_entry(*iter, struct netdev_adjacent, list); 5572 5573 if (&lower->list == &dev->adj_list.lower) 5574 return NULL; 5575 5576 *iter = lower->list.next; 5577 5578 return lower->dev; 5579 } 5580 EXPORT_SYMBOL(netdev_lower_get_next); 5581 5582 static struct net_device *netdev_next_lower_dev(struct net_device *dev, 5583 struct list_head **iter) 5584 { 5585 struct netdev_adjacent *lower; 5586 5587 lower = list_entry((*iter)->next, struct netdev_adjacent, list); 5588 5589 if (&lower->list == &dev->adj_list.lower) 5590 return NULL; 5591 5592 *iter = &lower->list; 5593 5594 return lower->dev; 5595 } 5596 5597 int netdev_walk_all_lower_dev(struct net_device *dev, 5598 int (*fn)(struct net_device *dev, 5599 void *data), 5600 void *data) 5601 { 5602 struct net_device *ldev; 5603 struct list_head *iter; 5604 int ret; 5605 5606 for (iter = &dev->adj_list.lower, 5607 ldev = netdev_next_lower_dev(dev, &iter); 5608 ldev; 5609 ldev = netdev_next_lower_dev(dev, &iter)) { 5610 /* first is the lower device itself */ 5611 ret = fn(ldev, data); 5612 if (ret) 5613 return ret; 5614 5615 /* then look at all of its lower devices */ 5616 ret = netdev_walk_all_lower_dev(ldev, fn, data); 5617 if (ret) 5618 return ret; 5619 } 5620 5621 return 0; 5622 } 5623 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev); 5624 5625 static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev, 5626 struct list_head **iter) 5627 { 5628 struct netdev_adjacent *lower; 5629 5630 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5631 if (&lower->list == &dev->adj_list.lower) 5632 return NULL; 5633 5634 *iter = &lower->list; 5635 5636 return lower->dev; 5637 } 5638 5639 int netdev_walk_all_lower_dev_rcu(struct net_device *dev, 5640 int (*fn)(struct net_device *dev, 5641 void *data), 5642 void *data) 5643 { 5644 struct net_device *ldev; 5645 struct list_head *iter; 5646 int ret; 5647 5648 for (iter = &dev->adj_list.lower, 5649 ldev = netdev_next_lower_dev_rcu(dev, &iter); 5650 ldev; 5651 ldev = netdev_next_lower_dev_rcu(dev, &iter)) { 5652 /* first is the lower device itself */ 5653 ret = fn(ldev, data); 5654 if (ret) 5655 return ret; 5656 5657 /* then look at all of its lower devices */ 5658 ret = netdev_walk_all_lower_dev_rcu(ldev, fn, data); 5659 if (ret) 5660 return ret; 5661 } 5662 5663 return 0; 5664 } 5665 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu); 5666 5667 /** 5668 * netdev_lower_get_first_private_rcu - Get the first ->private from the 5669 * lower neighbour list, RCU 5670 * variant 5671 * @dev: device 5672 * 5673 * Gets the first netdev_adjacent->private from the dev's lower neighbour 5674 * list. The caller must hold RCU read lock. 5675 */ 5676 void *netdev_lower_get_first_private_rcu(struct net_device *dev) 5677 { 5678 struct netdev_adjacent *lower; 5679 5680 lower = list_first_or_null_rcu(&dev->adj_list.lower, 5681 struct netdev_adjacent, list); 5682 if (lower) 5683 return lower->private; 5684 return NULL; 5685 } 5686 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu); 5687 5688 /** 5689 * netdev_master_upper_dev_get_rcu - Get master upper device 5690 * @dev: device 5691 * 5692 * Find a master upper device and return pointer to it or NULL in case 5693 * it's not there. The caller must hold the RCU read lock. 5694 */ 5695 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev) 5696 { 5697 struct netdev_adjacent *upper; 5698 5699 upper = list_first_or_null_rcu(&dev->adj_list.upper, 5700 struct netdev_adjacent, list); 5701 if (upper && likely(upper->master)) 5702 return upper->dev; 5703 return NULL; 5704 } 5705 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu); 5706 5707 static int netdev_adjacent_sysfs_add(struct net_device *dev, 5708 struct net_device *adj_dev, 5709 struct list_head *dev_list) 5710 { 5711 char linkname[IFNAMSIZ+7]; 5712 sprintf(linkname, dev_list == &dev->adj_list.upper ? 5713 "upper_%s" : "lower_%s", adj_dev->name); 5714 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), 5715 linkname); 5716 } 5717 static void netdev_adjacent_sysfs_del(struct net_device *dev, 5718 char *name, 5719 struct list_head *dev_list) 5720 { 5721 char linkname[IFNAMSIZ+7]; 5722 sprintf(linkname, dev_list == &dev->adj_list.upper ? 5723 "upper_%s" : "lower_%s", name); 5724 sysfs_remove_link(&(dev->dev.kobj), linkname); 5725 } 5726 5727 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev, 5728 struct net_device *adj_dev, 5729 struct list_head *dev_list) 5730 { 5731 return (dev_list == &dev->adj_list.upper || 5732 dev_list == &dev->adj_list.lower) && 5733 net_eq(dev_net(dev), dev_net(adj_dev)); 5734 } 5735 5736 static int __netdev_adjacent_dev_insert(struct net_device *dev, 5737 struct net_device *adj_dev, 5738 struct list_head *dev_list, 5739 void *private, bool master) 5740 { 5741 struct netdev_adjacent *adj; 5742 int ret; 5743 5744 adj = __netdev_find_adj(adj_dev, dev_list); 5745 5746 if (adj) { 5747 adj->ref_nr += 1; 5748 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n", 5749 dev->name, adj_dev->name, adj->ref_nr); 5750 5751 return 0; 5752 } 5753 5754 adj = kmalloc(sizeof(*adj), GFP_KERNEL); 5755 if (!adj) 5756 return -ENOMEM; 5757 5758 adj->dev = adj_dev; 5759 adj->master = master; 5760 adj->ref_nr = 1; 5761 adj->private = private; 5762 dev_hold(adj_dev); 5763 5764 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n", 5765 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name); 5766 5767 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) { 5768 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list); 5769 if (ret) 5770 goto free_adj; 5771 } 5772 5773 /* Ensure that master link is always the first item in list. */ 5774 if (master) { 5775 ret = sysfs_create_link(&(dev->dev.kobj), 5776 &(adj_dev->dev.kobj), "master"); 5777 if (ret) 5778 goto remove_symlinks; 5779 5780 list_add_rcu(&adj->list, dev_list); 5781 } else { 5782 list_add_tail_rcu(&adj->list, dev_list); 5783 } 5784 5785 return 0; 5786 5787 remove_symlinks: 5788 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 5789 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 5790 free_adj: 5791 kfree(adj); 5792 dev_put(adj_dev); 5793 5794 return ret; 5795 } 5796 5797 static void __netdev_adjacent_dev_remove(struct net_device *dev, 5798 struct net_device *adj_dev, 5799 u16 ref_nr, 5800 struct list_head *dev_list) 5801 { 5802 struct netdev_adjacent *adj; 5803 5804 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n", 5805 dev->name, adj_dev->name, ref_nr); 5806 5807 adj = __netdev_find_adj(adj_dev, dev_list); 5808 5809 if (!adj) { 5810 pr_err("Adjacency does not exist for device %s from %s\n", 5811 dev->name, adj_dev->name); 5812 WARN_ON(1); 5813 return; 5814 } 5815 5816 if (adj->ref_nr > ref_nr) { 5817 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n", 5818 dev->name, adj_dev->name, ref_nr, 5819 adj->ref_nr - ref_nr); 5820 adj->ref_nr -= ref_nr; 5821 return; 5822 } 5823 5824 if (adj->master) 5825 sysfs_remove_link(&(dev->dev.kobj), "master"); 5826 5827 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 5828 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 5829 5830 list_del_rcu(&adj->list); 5831 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n", 5832 adj_dev->name, dev->name, adj_dev->name); 5833 dev_put(adj_dev); 5834 kfree_rcu(adj, rcu); 5835 } 5836 5837 static int __netdev_adjacent_dev_link_lists(struct net_device *dev, 5838 struct net_device *upper_dev, 5839 struct list_head *up_list, 5840 struct list_head *down_list, 5841 void *private, bool master) 5842 { 5843 int ret; 5844 5845 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, 5846 private, master); 5847 if (ret) 5848 return ret; 5849 5850 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, 5851 private, false); 5852 if (ret) { 5853 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list); 5854 return ret; 5855 } 5856 5857 return 0; 5858 } 5859 5860 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev, 5861 struct net_device *upper_dev, 5862 u16 ref_nr, 5863 struct list_head *up_list, 5864 struct list_head *down_list) 5865 { 5866 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list); 5867 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list); 5868 } 5869 5870 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev, 5871 struct net_device *upper_dev, 5872 void *private, bool master) 5873 { 5874 return __netdev_adjacent_dev_link_lists(dev, upper_dev, 5875 &dev->adj_list.upper, 5876 &upper_dev->adj_list.lower, 5877 private, master); 5878 } 5879 5880 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev, 5881 struct net_device *upper_dev) 5882 { 5883 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1, 5884 &dev->adj_list.upper, 5885 &upper_dev->adj_list.lower); 5886 } 5887 5888 static int __netdev_upper_dev_link(struct net_device *dev, 5889 struct net_device *upper_dev, bool master, 5890 void *upper_priv, void *upper_info) 5891 { 5892 struct netdev_notifier_changeupper_info changeupper_info; 5893 int ret = 0; 5894 5895 ASSERT_RTNL(); 5896 5897 if (dev == upper_dev) 5898 return -EBUSY; 5899 5900 /* To prevent loops, check if dev is not upper device to upper_dev. */ 5901 if (netdev_has_upper_dev(upper_dev, dev)) 5902 return -EBUSY; 5903 5904 if (netdev_has_upper_dev(dev, upper_dev)) 5905 return -EEXIST; 5906 5907 if (master && netdev_master_upper_dev_get(dev)) 5908 return -EBUSY; 5909 5910 changeupper_info.upper_dev = upper_dev; 5911 changeupper_info.master = master; 5912 changeupper_info.linking = true; 5913 changeupper_info.upper_info = upper_info; 5914 5915 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev, 5916 &changeupper_info.info); 5917 ret = notifier_to_errno(ret); 5918 if (ret) 5919 return ret; 5920 5921 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv, 5922 master); 5923 if (ret) 5924 return ret; 5925 5926 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev, 5927 &changeupper_info.info); 5928 ret = notifier_to_errno(ret); 5929 if (ret) 5930 goto rollback; 5931 5932 return 0; 5933 5934 rollback: 5935 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 5936 5937 return ret; 5938 } 5939 5940 /** 5941 * netdev_upper_dev_link - Add a link to the upper device 5942 * @dev: device 5943 * @upper_dev: new upper device 5944 * 5945 * Adds a link to device which is upper to this one. The caller must hold 5946 * the RTNL lock. On a failure a negative errno code is returned. 5947 * On success the reference counts are adjusted and the function 5948 * returns zero. 5949 */ 5950 int netdev_upper_dev_link(struct net_device *dev, 5951 struct net_device *upper_dev) 5952 { 5953 return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL); 5954 } 5955 EXPORT_SYMBOL(netdev_upper_dev_link); 5956 5957 /** 5958 * netdev_master_upper_dev_link - Add a master link to the upper device 5959 * @dev: device 5960 * @upper_dev: new upper device 5961 * @upper_priv: upper device private 5962 * @upper_info: upper info to be passed down via notifier 5963 * 5964 * Adds a link to device which is upper to this one. In this case, only 5965 * one master upper device can be linked, although other non-master devices 5966 * might be linked as well. The caller must hold the RTNL lock. 5967 * On a failure a negative errno code is returned. On success the reference 5968 * counts are adjusted and the function returns zero. 5969 */ 5970 int netdev_master_upper_dev_link(struct net_device *dev, 5971 struct net_device *upper_dev, 5972 void *upper_priv, void *upper_info) 5973 { 5974 return __netdev_upper_dev_link(dev, upper_dev, true, 5975 upper_priv, upper_info); 5976 } 5977 EXPORT_SYMBOL(netdev_master_upper_dev_link); 5978 5979 /** 5980 * netdev_upper_dev_unlink - Removes a link to upper device 5981 * @dev: device 5982 * @upper_dev: new upper device 5983 * 5984 * Removes a link to device which is upper to this one. The caller must hold 5985 * the RTNL lock. 5986 */ 5987 void netdev_upper_dev_unlink(struct net_device *dev, 5988 struct net_device *upper_dev) 5989 { 5990 struct netdev_notifier_changeupper_info changeupper_info; 5991 ASSERT_RTNL(); 5992 5993 changeupper_info.upper_dev = upper_dev; 5994 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev; 5995 changeupper_info.linking = false; 5996 5997 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev, 5998 &changeupper_info.info); 5999 6000 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 6001 6002 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev, 6003 &changeupper_info.info); 6004 } 6005 EXPORT_SYMBOL(netdev_upper_dev_unlink); 6006 6007 /** 6008 * netdev_bonding_info_change - Dispatch event about slave change 6009 * @dev: device 6010 * @bonding_info: info to dispatch 6011 * 6012 * Send NETDEV_BONDING_INFO to netdev notifiers with info. 6013 * The caller must hold the RTNL lock. 6014 */ 6015 void netdev_bonding_info_change(struct net_device *dev, 6016 struct netdev_bonding_info *bonding_info) 6017 { 6018 struct netdev_notifier_bonding_info info; 6019 6020 memcpy(&info.bonding_info, bonding_info, 6021 sizeof(struct netdev_bonding_info)); 6022 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev, 6023 &info.info); 6024 } 6025 EXPORT_SYMBOL(netdev_bonding_info_change); 6026 6027 static void netdev_adjacent_add_links(struct net_device *dev) 6028 { 6029 struct netdev_adjacent *iter; 6030 6031 struct net *net = dev_net(dev); 6032 6033 list_for_each_entry(iter, &dev->adj_list.upper, list) { 6034 if (!net_eq(net, dev_net(iter->dev))) 6035 continue; 6036 netdev_adjacent_sysfs_add(iter->dev, dev, 6037 &iter->dev->adj_list.lower); 6038 netdev_adjacent_sysfs_add(dev, iter->dev, 6039 &dev->adj_list.upper); 6040 } 6041 6042 list_for_each_entry(iter, &dev->adj_list.lower, list) { 6043 if (!net_eq(net, dev_net(iter->dev))) 6044 continue; 6045 netdev_adjacent_sysfs_add(iter->dev, dev, 6046 &iter->dev->adj_list.upper); 6047 netdev_adjacent_sysfs_add(dev, iter->dev, 6048 &dev->adj_list.lower); 6049 } 6050 } 6051 6052 static void netdev_adjacent_del_links(struct net_device *dev) 6053 { 6054 struct netdev_adjacent *iter; 6055 6056 struct net *net = dev_net(dev); 6057 6058 list_for_each_entry(iter, &dev->adj_list.upper, list) { 6059 if (!net_eq(net, dev_net(iter->dev))) 6060 continue; 6061 netdev_adjacent_sysfs_del(iter->dev, dev->name, 6062 &iter->dev->adj_list.lower); 6063 netdev_adjacent_sysfs_del(dev, iter->dev->name, 6064 &dev->adj_list.upper); 6065 } 6066 6067 list_for_each_entry(iter, &dev->adj_list.lower, list) { 6068 if (!net_eq(net, dev_net(iter->dev))) 6069 continue; 6070 netdev_adjacent_sysfs_del(iter->dev, dev->name, 6071 &iter->dev->adj_list.upper); 6072 netdev_adjacent_sysfs_del(dev, iter->dev->name, 6073 &dev->adj_list.lower); 6074 } 6075 } 6076 6077 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname) 6078 { 6079 struct netdev_adjacent *iter; 6080 6081 struct net *net = dev_net(dev); 6082 6083 list_for_each_entry(iter, &dev->adj_list.upper, list) { 6084 if (!net_eq(net, dev_net(iter->dev))) 6085 continue; 6086 netdev_adjacent_sysfs_del(iter->dev, oldname, 6087 &iter->dev->adj_list.lower); 6088 netdev_adjacent_sysfs_add(iter->dev, dev, 6089 &iter->dev->adj_list.lower); 6090 } 6091 6092 list_for_each_entry(iter, &dev->adj_list.lower, list) { 6093 if (!net_eq(net, dev_net(iter->dev))) 6094 continue; 6095 netdev_adjacent_sysfs_del(iter->dev, oldname, 6096 &iter->dev->adj_list.upper); 6097 netdev_adjacent_sysfs_add(iter->dev, dev, 6098 &iter->dev->adj_list.upper); 6099 } 6100 } 6101 6102 void *netdev_lower_dev_get_private(struct net_device *dev, 6103 struct net_device *lower_dev) 6104 { 6105 struct netdev_adjacent *lower; 6106 6107 if (!lower_dev) 6108 return NULL; 6109 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower); 6110 if (!lower) 6111 return NULL; 6112 6113 return lower->private; 6114 } 6115 EXPORT_SYMBOL(netdev_lower_dev_get_private); 6116 6117 6118 int dev_get_nest_level(struct net_device *dev) 6119 { 6120 struct net_device *lower = NULL; 6121 struct list_head *iter; 6122 int max_nest = -1; 6123 int nest; 6124 6125 ASSERT_RTNL(); 6126 6127 netdev_for_each_lower_dev(dev, lower, iter) { 6128 nest = dev_get_nest_level(lower); 6129 if (max_nest < nest) 6130 max_nest = nest; 6131 } 6132 6133 return max_nest + 1; 6134 } 6135 EXPORT_SYMBOL(dev_get_nest_level); 6136 6137 /** 6138 * netdev_lower_change - Dispatch event about lower device state change 6139 * @lower_dev: device 6140 * @lower_state_info: state to dispatch 6141 * 6142 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info. 6143 * The caller must hold the RTNL lock. 6144 */ 6145 void netdev_lower_state_changed(struct net_device *lower_dev, 6146 void *lower_state_info) 6147 { 6148 struct netdev_notifier_changelowerstate_info changelowerstate_info; 6149 6150 ASSERT_RTNL(); 6151 changelowerstate_info.lower_state_info = lower_state_info; 6152 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev, 6153 &changelowerstate_info.info); 6154 } 6155 EXPORT_SYMBOL(netdev_lower_state_changed); 6156 6157 int netdev_default_l2upper_neigh_construct(struct net_device *dev, 6158 struct neighbour *n) 6159 { 6160 struct net_device *lower_dev, *stop_dev; 6161 struct list_head *iter; 6162 int err; 6163 6164 netdev_for_each_lower_dev(dev, lower_dev, iter) { 6165 if (!lower_dev->netdev_ops->ndo_neigh_construct) 6166 continue; 6167 err = lower_dev->netdev_ops->ndo_neigh_construct(lower_dev, n); 6168 if (err) { 6169 stop_dev = lower_dev; 6170 goto rollback; 6171 } 6172 } 6173 return 0; 6174 6175 rollback: 6176 netdev_for_each_lower_dev(dev, lower_dev, iter) { 6177 if (lower_dev == stop_dev) 6178 break; 6179 if (!lower_dev->netdev_ops->ndo_neigh_destroy) 6180 continue; 6181 lower_dev->netdev_ops->ndo_neigh_destroy(lower_dev, n); 6182 } 6183 return err; 6184 } 6185 EXPORT_SYMBOL_GPL(netdev_default_l2upper_neigh_construct); 6186 6187 void netdev_default_l2upper_neigh_destroy(struct net_device *dev, 6188 struct neighbour *n) 6189 { 6190 struct net_device *lower_dev; 6191 struct list_head *iter; 6192 6193 netdev_for_each_lower_dev(dev, lower_dev, iter) { 6194 if (!lower_dev->netdev_ops->ndo_neigh_destroy) 6195 continue; 6196 lower_dev->netdev_ops->ndo_neigh_destroy(lower_dev, n); 6197 } 6198 } 6199 EXPORT_SYMBOL_GPL(netdev_default_l2upper_neigh_destroy); 6200 6201 static void dev_change_rx_flags(struct net_device *dev, int flags) 6202 { 6203 const struct net_device_ops *ops = dev->netdev_ops; 6204 6205 if (ops->ndo_change_rx_flags) 6206 ops->ndo_change_rx_flags(dev, flags); 6207 } 6208 6209 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify) 6210 { 6211 unsigned int old_flags = dev->flags; 6212 kuid_t uid; 6213 kgid_t gid; 6214 6215 ASSERT_RTNL(); 6216 6217 dev->flags |= IFF_PROMISC; 6218 dev->promiscuity += inc; 6219 if (dev->promiscuity == 0) { 6220 /* 6221 * Avoid overflow. 6222 * If inc causes overflow, untouch promisc and return error. 6223 */ 6224 if (inc < 0) 6225 dev->flags &= ~IFF_PROMISC; 6226 else { 6227 dev->promiscuity -= inc; 6228 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n", 6229 dev->name); 6230 return -EOVERFLOW; 6231 } 6232 } 6233 if (dev->flags != old_flags) { 6234 pr_info("device %s %s promiscuous mode\n", 6235 dev->name, 6236 dev->flags & IFF_PROMISC ? "entered" : "left"); 6237 if (audit_enabled) { 6238 current_uid_gid(&uid, &gid); 6239 audit_log(current->audit_context, GFP_ATOMIC, 6240 AUDIT_ANOM_PROMISCUOUS, 6241 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u", 6242 dev->name, (dev->flags & IFF_PROMISC), 6243 (old_flags & IFF_PROMISC), 6244 from_kuid(&init_user_ns, audit_get_loginuid(current)), 6245 from_kuid(&init_user_ns, uid), 6246 from_kgid(&init_user_ns, gid), 6247 audit_get_sessionid(current)); 6248 } 6249 6250 dev_change_rx_flags(dev, IFF_PROMISC); 6251 } 6252 if (notify) 6253 __dev_notify_flags(dev, old_flags, IFF_PROMISC); 6254 return 0; 6255 } 6256 6257 /** 6258 * dev_set_promiscuity - update promiscuity count on a device 6259 * @dev: device 6260 * @inc: modifier 6261 * 6262 * Add or remove promiscuity from a device. While the count in the device 6263 * remains above zero the interface remains promiscuous. Once it hits zero 6264 * the device reverts back to normal filtering operation. A negative inc 6265 * value is used to drop promiscuity on the device. 6266 * Return 0 if successful or a negative errno code on error. 6267 */ 6268 int dev_set_promiscuity(struct net_device *dev, int inc) 6269 { 6270 unsigned int old_flags = dev->flags; 6271 int err; 6272 6273 err = __dev_set_promiscuity(dev, inc, true); 6274 if (err < 0) 6275 return err; 6276 if (dev->flags != old_flags) 6277 dev_set_rx_mode(dev); 6278 return err; 6279 } 6280 EXPORT_SYMBOL(dev_set_promiscuity); 6281 6282 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify) 6283 { 6284 unsigned int old_flags = dev->flags, old_gflags = dev->gflags; 6285 6286 ASSERT_RTNL(); 6287 6288 dev->flags |= IFF_ALLMULTI; 6289 dev->allmulti += inc; 6290 if (dev->allmulti == 0) { 6291 /* 6292 * Avoid overflow. 6293 * If inc causes overflow, untouch allmulti and return error. 6294 */ 6295 if (inc < 0) 6296 dev->flags &= ~IFF_ALLMULTI; 6297 else { 6298 dev->allmulti -= inc; 6299 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n", 6300 dev->name); 6301 return -EOVERFLOW; 6302 } 6303 } 6304 if (dev->flags ^ old_flags) { 6305 dev_change_rx_flags(dev, IFF_ALLMULTI); 6306 dev_set_rx_mode(dev); 6307 if (notify) 6308 __dev_notify_flags(dev, old_flags, 6309 dev->gflags ^ old_gflags); 6310 } 6311 return 0; 6312 } 6313 6314 /** 6315 * dev_set_allmulti - update allmulti count on a device 6316 * @dev: device 6317 * @inc: modifier 6318 * 6319 * Add or remove reception of all multicast frames to a device. While the 6320 * count in the device remains above zero the interface remains listening 6321 * to all interfaces. Once it hits zero the device reverts back to normal 6322 * filtering operation. A negative @inc value is used to drop the counter 6323 * when releasing a resource needing all multicasts. 6324 * Return 0 if successful or a negative errno code on error. 6325 */ 6326 6327 int dev_set_allmulti(struct net_device *dev, int inc) 6328 { 6329 return __dev_set_allmulti(dev, inc, true); 6330 } 6331 EXPORT_SYMBOL(dev_set_allmulti); 6332 6333 /* 6334 * Upload unicast and multicast address lists to device and 6335 * configure RX filtering. When the device doesn't support unicast 6336 * filtering it is put in promiscuous mode while unicast addresses 6337 * are present. 6338 */ 6339 void __dev_set_rx_mode(struct net_device *dev) 6340 { 6341 const struct net_device_ops *ops = dev->netdev_ops; 6342 6343 /* dev_open will call this function so the list will stay sane. */ 6344 if (!(dev->flags&IFF_UP)) 6345 return; 6346 6347 if (!netif_device_present(dev)) 6348 return; 6349 6350 if (!(dev->priv_flags & IFF_UNICAST_FLT)) { 6351 /* Unicast addresses changes may only happen under the rtnl, 6352 * therefore calling __dev_set_promiscuity here is safe. 6353 */ 6354 if (!netdev_uc_empty(dev) && !dev->uc_promisc) { 6355 __dev_set_promiscuity(dev, 1, false); 6356 dev->uc_promisc = true; 6357 } else if (netdev_uc_empty(dev) && dev->uc_promisc) { 6358 __dev_set_promiscuity(dev, -1, false); 6359 dev->uc_promisc = false; 6360 } 6361 } 6362 6363 if (ops->ndo_set_rx_mode) 6364 ops->ndo_set_rx_mode(dev); 6365 } 6366 6367 void dev_set_rx_mode(struct net_device *dev) 6368 { 6369 netif_addr_lock_bh(dev); 6370 __dev_set_rx_mode(dev); 6371 netif_addr_unlock_bh(dev); 6372 } 6373 6374 /** 6375 * dev_get_flags - get flags reported to userspace 6376 * @dev: device 6377 * 6378 * Get the combination of flag bits exported through APIs to userspace. 6379 */ 6380 unsigned int dev_get_flags(const struct net_device *dev) 6381 { 6382 unsigned int flags; 6383 6384 flags = (dev->flags & ~(IFF_PROMISC | 6385 IFF_ALLMULTI | 6386 IFF_RUNNING | 6387 IFF_LOWER_UP | 6388 IFF_DORMANT)) | 6389 (dev->gflags & (IFF_PROMISC | 6390 IFF_ALLMULTI)); 6391 6392 if (netif_running(dev)) { 6393 if (netif_oper_up(dev)) 6394 flags |= IFF_RUNNING; 6395 if (netif_carrier_ok(dev)) 6396 flags |= IFF_LOWER_UP; 6397 if (netif_dormant(dev)) 6398 flags |= IFF_DORMANT; 6399 } 6400 6401 return flags; 6402 } 6403 EXPORT_SYMBOL(dev_get_flags); 6404 6405 int __dev_change_flags(struct net_device *dev, unsigned int flags) 6406 { 6407 unsigned int old_flags = dev->flags; 6408 int ret; 6409 6410 ASSERT_RTNL(); 6411 6412 /* 6413 * Set the flags on our device. 6414 */ 6415 6416 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | 6417 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL | 6418 IFF_AUTOMEDIA)) | 6419 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | 6420 IFF_ALLMULTI)); 6421 6422 /* 6423 * Load in the correct multicast list now the flags have changed. 6424 */ 6425 6426 if ((old_flags ^ flags) & IFF_MULTICAST) 6427 dev_change_rx_flags(dev, IFF_MULTICAST); 6428 6429 dev_set_rx_mode(dev); 6430 6431 /* 6432 * Have we downed the interface. We handle IFF_UP ourselves 6433 * according to user attempts to set it, rather than blindly 6434 * setting it. 6435 */ 6436 6437 ret = 0; 6438 if ((old_flags ^ flags) & IFF_UP) 6439 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev); 6440 6441 if ((flags ^ dev->gflags) & IFF_PROMISC) { 6442 int inc = (flags & IFF_PROMISC) ? 1 : -1; 6443 unsigned int old_flags = dev->flags; 6444 6445 dev->gflags ^= IFF_PROMISC; 6446 6447 if (__dev_set_promiscuity(dev, inc, false) >= 0) 6448 if (dev->flags != old_flags) 6449 dev_set_rx_mode(dev); 6450 } 6451 6452 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI 6453 is important. Some (broken) drivers set IFF_PROMISC, when 6454 IFF_ALLMULTI is requested not asking us and not reporting. 6455 */ 6456 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { 6457 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; 6458 6459 dev->gflags ^= IFF_ALLMULTI; 6460 __dev_set_allmulti(dev, inc, false); 6461 } 6462 6463 return ret; 6464 } 6465 6466 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, 6467 unsigned int gchanges) 6468 { 6469 unsigned int changes = dev->flags ^ old_flags; 6470 6471 if (gchanges) 6472 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC); 6473 6474 if (changes & IFF_UP) { 6475 if (dev->flags & IFF_UP) 6476 call_netdevice_notifiers(NETDEV_UP, dev); 6477 else 6478 call_netdevice_notifiers(NETDEV_DOWN, dev); 6479 } 6480 6481 if (dev->flags & IFF_UP && 6482 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) { 6483 struct netdev_notifier_change_info change_info; 6484 6485 change_info.flags_changed = changes; 6486 call_netdevice_notifiers_info(NETDEV_CHANGE, dev, 6487 &change_info.info); 6488 } 6489 } 6490 6491 /** 6492 * dev_change_flags - change device settings 6493 * @dev: device 6494 * @flags: device state flags 6495 * 6496 * Change settings on device based state flags. The flags are 6497 * in the userspace exported format. 6498 */ 6499 int dev_change_flags(struct net_device *dev, unsigned int flags) 6500 { 6501 int ret; 6502 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; 6503 6504 ret = __dev_change_flags(dev, flags); 6505 if (ret < 0) 6506 return ret; 6507 6508 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); 6509 __dev_notify_flags(dev, old_flags, changes); 6510 return ret; 6511 } 6512 EXPORT_SYMBOL(dev_change_flags); 6513 6514 static int __dev_set_mtu(struct net_device *dev, int new_mtu) 6515 { 6516 const struct net_device_ops *ops = dev->netdev_ops; 6517 6518 if (ops->ndo_change_mtu) 6519 return ops->ndo_change_mtu(dev, new_mtu); 6520 6521 dev->mtu = new_mtu; 6522 return 0; 6523 } 6524 6525 /** 6526 * dev_set_mtu - Change maximum transfer unit 6527 * @dev: device 6528 * @new_mtu: new transfer unit 6529 * 6530 * Change the maximum transfer size of the network device. 6531 */ 6532 int dev_set_mtu(struct net_device *dev, int new_mtu) 6533 { 6534 int err, orig_mtu; 6535 6536 if (new_mtu == dev->mtu) 6537 return 0; 6538 6539 /* MTU must be positive, and in range */ 6540 if (new_mtu < 0 || new_mtu < dev->min_mtu) { 6541 net_err_ratelimited("%s: Invalid MTU %d requested, hw min %d\n", 6542 dev->name, new_mtu, dev->min_mtu); 6543 return -EINVAL; 6544 } 6545 6546 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) { 6547 net_err_ratelimited("%s: Invalid MTU %d requested, hw max %d\n", 6548 dev->name, new_mtu, dev->max_mtu); 6549 return -EINVAL; 6550 } 6551 6552 if (!netif_device_present(dev)) 6553 return -ENODEV; 6554 6555 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev); 6556 err = notifier_to_errno(err); 6557 if (err) 6558 return err; 6559 6560 orig_mtu = dev->mtu; 6561 err = __dev_set_mtu(dev, new_mtu); 6562 6563 if (!err) { 6564 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); 6565 err = notifier_to_errno(err); 6566 if (err) { 6567 /* setting mtu back and notifying everyone again, 6568 * so that they have a chance to revert changes. 6569 */ 6570 __dev_set_mtu(dev, orig_mtu); 6571 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); 6572 } 6573 } 6574 return err; 6575 } 6576 EXPORT_SYMBOL(dev_set_mtu); 6577 6578 /** 6579 * dev_set_group - Change group this device belongs to 6580 * @dev: device 6581 * @new_group: group this device should belong to 6582 */ 6583 void dev_set_group(struct net_device *dev, int new_group) 6584 { 6585 dev->group = new_group; 6586 } 6587 EXPORT_SYMBOL(dev_set_group); 6588 6589 /** 6590 * dev_set_mac_address - Change Media Access Control Address 6591 * @dev: device 6592 * @sa: new address 6593 * 6594 * Change the hardware (MAC) address of the device 6595 */ 6596 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa) 6597 { 6598 const struct net_device_ops *ops = dev->netdev_ops; 6599 int err; 6600 6601 if (!ops->ndo_set_mac_address) 6602 return -EOPNOTSUPP; 6603 if (sa->sa_family != dev->type) 6604 return -EINVAL; 6605 if (!netif_device_present(dev)) 6606 return -ENODEV; 6607 err = ops->ndo_set_mac_address(dev, sa); 6608 if (err) 6609 return err; 6610 dev->addr_assign_type = NET_ADDR_SET; 6611 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 6612 add_device_randomness(dev->dev_addr, dev->addr_len); 6613 return 0; 6614 } 6615 EXPORT_SYMBOL(dev_set_mac_address); 6616 6617 /** 6618 * dev_change_carrier - Change device carrier 6619 * @dev: device 6620 * @new_carrier: new value 6621 * 6622 * Change device carrier 6623 */ 6624 int dev_change_carrier(struct net_device *dev, bool new_carrier) 6625 { 6626 const struct net_device_ops *ops = dev->netdev_ops; 6627 6628 if (!ops->ndo_change_carrier) 6629 return -EOPNOTSUPP; 6630 if (!netif_device_present(dev)) 6631 return -ENODEV; 6632 return ops->ndo_change_carrier(dev, new_carrier); 6633 } 6634 EXPORT_SYMBOL(dev_change_carrier); 6635 6636 /** 6637 * dev_get_phys_port_id - Get device physical port ID 6638 * @dev: device 6639 * @ppid: port ID 6640 * 6641 * Get device physical port ID 6642 */ 6643 int dev_get_phys_port_id(struct net_device *dev, 6644 struct netdev_phys_item_id *ppid) 6645 { 6646 const struct net_device_ops *ops = dev->netdev_ops; 6647 6648 if (!ops->ndo_get_phys_port_id) 6649 return -EOPNOTSUPP; 6650 return ops->ndo_get_phys_port_id(dev, ppid); 6651 } 6652 EXPORT_SYMBOL(dev_get_phys_port_id); 6653 6654 /** 6655 * dev_get_phys_port_name - Get device physical port name 6656 * @dev: device 6657 * @name: port name 6658 * @len: limit of bytes to copy to name 6659 * 6660 * Get device physical port name 6661 */ 6662 int dev_get_phys_port_name(struct net_device *dev, 6663 char *name, size_t len) 6664 { 6665 const struct net_device_ops *ops = dev->netdev_ops; 6666 6667 if (!ops->ndo_get_phys_port_name) 6668 return -EOPNOTSUPP; 6669 return ops->ndo_get_phys_port_name(dev, name, len); 6670 } 6671 EXPORT_SYMBOL(dev_get_phys_port_name); 6672 6673 /** 6674 * dev_change_proto_down - update protocol port state information 6675 * @dev: device 6676 * @proto_down: new value 6677 * 6678 * This info can be used by switch drivers to set the phys state of the 6679 * port. 6680 */ 6681 int dev_change_proto_down(struct net_device *dev, bool proto_down) 6682 { 6683 const struct net_device_ops *ops = dev->netdev_ops; 6684 6685 if (!ops->ndo_change_proto_down) 6686 return -EOPNOTSUPP; 6687 if (!netif_device_present(dev)) 6688 return -ENODEV; 6689 return ops->ndo_change_proto_down(dev, proto_down); 6690 } 6691 EXPORT_SYMBOL(dev_change_proto_down); 6692 6693 /** 6694 * dev_change_xdp_fd - set or clear a bpf program for a device rx path 6695 * @dev: device 6696 * @fd: new program fd or negative value to clear 6697 * @flags: xdp-related flags 6698 * 6699 * Set or clear a bpf program for a device 6700 */ 6701 int dev_change_xdp_fd(struct net_device *dev, int fd, u32 flags) 6702 { 6703 const struct net_device_ops *ops = dev->netdev_ops; 6704 struct bpf_prog *prog = NULL; 6705 struct netdev_xdp xdp; 6706 int err; 6707 6708 ASSERT_RTNL(); 6709 6710 if (!ops->ndo_xdp) 6711 return -EOPNOTSUPP; 6712 if (fd >= 0) { 6713 if (flags & XDP_FLAGS_UPDATE_IF_NOEXIST) { 6714 memset(&xdp, 0, sizeof(xdp)); 6715 xdp.command = XDP_QUERY_PROG; 6716 6717 err = ops->ndo_xdp(dev, &xdp); 6718 if (err < 0) 6719 return err; 6720 if (xdp.prog_attached) 6721 return -EBUSY; 6722 } 6723 6724 prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_XDP); 6725 if (IS_ERR(prog)) 6726 return PTR_ERR(prog); 6727 } 6728 6729 memset(&xdp, 0, sizeof(xdp)); 6730 xdp.command = XDP_SETUP_PROG; 6731 xdp.prog = prog; 6732 6733 err = ops->ndo_xdp(dev, &xdp); 6734 if (err < 0 && prog) 6735 bpf_prog_put(prog); 6736 6737 return err; 6738 } 6739 EXPORT_SYMBOL(dev_change_xdp_fd); 6740 6741 /** 6742 * dev_new_index - allocate an ifindex 6743 * @net: the applicable net namespace 6744 * 6745 * Returns a suitable unique value for a new device interface 6746 * number. The caller must hold the rtnl semaphore or the 6747 * dev_base_lock to be sure it remains unique. 6748 */ 6749 static int dev_new_index(struct net *net) 6750 { 6751 int ifindex = net->ifindex; 6752 for (;;) { 6753 if (++ifindex <= 0) 6754 ifindex = 1; 6755 if (!__dev_get_by_index(net, ifindex)) 6756 return net->ifindex = ifindex; 6757 } 6758 } 6759 6760 /* Delayed registration/unregisteration */ 6761 static LIST_HEAD(net_todo_list); 6762 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq); 6763 6764 static void net_set_todo(struct net_device *dev) 6765 { 6766 list_add_tail(&dev->todo_list, &net_todo_list); 6767 dev_net(dev)->dev_unreg_count++; 6768 } 6769 6770 static void rollback_registered_many(struct list_head *head) 6771 { 6772 struct net_device *dev, *tmp; 6773 LIST_HEAD(close_head); 6774 6775 BUG_ON(dev_boot_phase); 6776 ASSERT_RTNL(); 6777 6778 list_for_each_entry_safe(dev, tmp, head, unreg_list) { 6779 /* Some devices call without registering 6780 * for initialization unwind. Remove those 6781 * devices and proceed with the remaining. 6782 */ 6783 if (dev->reg_state == NETREG_UNINITIALIZED) { 6784 pr_debug("unregister_netdevice: device %s/%p never was registered\n", 6785 dev->name, dev); 6786 6787 WARN_ON(1); 6788 list_del(&dev->unreg_list); 6789 continue; 6790 } 6791 dev->dismantle = true; 6792 BUG_ON(dev->reg_state != NETREG_REGISTERED); 6793 } 6794 6795 /* If device is running, close it first. */ 6796 list_for_each_entry(dev, head, unreg_list) 6797 list_add_tail(&dev->close_list, &close_head); 6798 dev_close_many(&close_head, true); 6799 6800 list_for_each_entry(dev, head, unreg_list) { 6801 /* And unlink it from device chain. */ 6802 unlist_netdevice(dev); 6803 6804 dev->reg_state = NETREG_UNREGISTERING; 6805 } 6806 flush_all_backlogs(); 6807 6808 synchronize_net(); 6809 6810 list_for_each_entry(dev, head, unreg_list) { 6811 struct sk_buff *skb = NULL; 6812 6813 /* Shutdown queueing discipline. */ 6814 dev_shutdown(dev); 6815 6816 6817 /* Notify protocols, that we are about to destroy 6818 this device. They should clean all the things. 6819 */ 6820 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 6821 6822 if (!dev->rtnl_link_ops || 6823 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 6824 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 6825 GFP_KERNEL); 6826 6827 /* 6828 * Flush the unicast and multicast chains 6829 */ 6830 dev_uc_flush(dev); 6831 dev_mc_flush(dev); 6832 6833 if (dev->netdev_ops->ndo_uninit) 6834 dev->netdev_ops->ndo_uninit(dev); 6835 6836 if (skb) 6837 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL); 6838 6839 /* Notifier chain MUST detach us all upper devices. */ 6840 WARN_ON(netdev_has_any_upper_dev(dev)); 6841 WARN_ON(netdev_has_any_lower_dev(dev)); 6842 6843 /* Remove entries from kobject tree */ 6844 netdev_unregister_kobject(dev); 6845 #ifdef CONFIG_XPS 6846 /* Remove XPS queueing entries */ 6847 netif_reset_xps_queues_gt(dev, 0); 6848 #endif 6849 } 6850 6851 synchronize_net(); 6852 6853 list_for_each_entry(dev, head, unreg_list) 6854 dev_put(dev); 6855 } 6856 6857 static void rollback_registered(struct net_device *dev) 6858 { 6859 LIST_HEAD(single); 6860 6861 list_add(&dev->unreg_list, &single); 6862 rollback_registered_many(&single); 6863 list_del(&single); 6864 } 6865 6866 static netdev_features_t netdev_sync_upper_features(struct net_device *lower, 6867 struct net_device *upper, netdev_features_t features) 6868 { 6869 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 6870 netdev_features_t feature; 6871 int feature_bit; 6872 6873 for_each_netdev_feature(&upper_disables, feature_bit) { 6874 feature = __NETIF_F_BIT(feature_bit); 6875 if (!(upper->wanted_features & feature) 6876 && (features & feature)) { 6877 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n", 6878 &feature, upper->name); 6879 features &= ~feature; 6880 } 6881 } 6882 6883 return features; 6884 } 6885 6886 static void netdev_sync_lower_features(struct net_device *upper, 6887 struct net_device *lower, netdev_features_t features) 6888 { 6889 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 6890 netdev_features_t feature; 6891 int feature_bit; 6892 6893 for_each_netdev_feature(&upper_disables, feature_bit) { 6894 feature = __NETIF_F_BIT(feature_bit); 6895 if (!(features & feature) && (lower->features & feature)) { 6896 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n", 6897 &feature, lower->name); 6898 lower->wanted_features &= ~feature; 6899 netdev_update_features(lower); 6900 6901 if (unlikely(lower->features & feature)) 6902 netdev_WARN(upper, "failed to disable %pNF on %s!\n", 6903 &feature, lower->name); 6904 } 6905 } 6906 } 6907 6908 static netdev_features_t netdev_fix_features(struct net_device *dev, 6909 netdev_features_t features) 6910 { 6911 /* Fix illegal checksum combinations */ 6912 if ((features & NETIF_F_HW_CSUM) && 6913 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { 6914 netdev_warn(dev, "mixed HW and IP checksum settings.\n"); 6915 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); 6916 } 6917 6918 /* TSO requires that SG is present as well. */ 6919 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) { 6920 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n"); 6921 features &= ~NETIF_F_ALL_TSO; 6922 } 6923 6924 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) && 6925 !(features & NETIF_F_IP_CSUM)) { 6926 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n"); 6927 features &= ~NETIF_F_TSO; 6928 features &= ~NETIF_F_TSO_ECN; 6929 } 6930 6931 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) && 6932 !(features & NETIF_F_IPV6_CSUM)) { 6933 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n"); 6934 features &= ~NETIF_F_TSO6; 6935 } 6936 6937 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */ 6938 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO)) 6939 features &= ~NETIF_F_TSO_MANGLEID; 6940 6941 /* TSO ECN requires that TSO is present as well. */ 6942 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN) 6943 features &= ~NETIF_F_TSO_ECN; 6944 6945 /* Software GSO depends on SG. */ 6946 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) { 6947 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n"); 6948 features &= ~NETIF_F_GSO; 6949 } 6950 6951 /* UFO needs SG and checksumming */ 6952 if (features & NETIF_F_UFO) { 6953 /* maybe split UFO into V4 and V6? */ 6954 if (!(features & NETIF_F_HW_CSUM) && 6955 ((features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) != 6956 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) { 6957 netdev_dbg(dev, 6958 "Dropping NETIF_F_UFO since no checksum offload features.\n"); 6959 features &= ~NETIF_F_UFO; 6960 } 6961 6962 if (!(features & NETIF_F_SG)) { 6963 netdev_dbg(dev, 6964 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n"); 6965 features &= ~NETIF_F_UFO; 6966 } 6967 } 6968 6969 /* GSO partial features require GSO partial be set */ 6970 if ((features & dev->gso_partial_features) && 6971 !(features & NETIF_F_GSO_PARTIAL)) { 6972 netdev_dbg(dev, 6973 "Dropping partially supported GSO features since no GSO partial.\n"); 6974 features &= ~dev->gso_partial_features; 6975 } 6976 6977 #ifdef CONFIG_NET_RX_BUSY_POLL 6978 if (dev->netdev_ops->ndo_busy_poll) 6979 features |= NETIF_F_BUSY_POLL; 6980 else 6981 #endif 6982 features &= ~NETIF_F_BUSY_POLL; 6983 6984 return features; 6985 } 6986 6987 int __netdev_update_features(struct net_device *dev) 6988 { 6989 struct net_device *upper, *lower; 6990 netdev_features_t features; 6991 struct list_head *iter; 6992 int err = -1; 6993 6994 ASSERT_RTNL(); 6995 6996 features = netdev_get_wanted_features(dev); 6997 6998 if (dev->netdev_ops->ndo_fix_features) 6999 features = dev->netdev_ops->ndo_fix_features(dev, features); 7000 7001 /* driver might be less strict about feature dependencies */ 7002 features = netdev_fix_features(dev, features); 7003 7004 /* some features can't be enabled if they're off an an upper device */ 7005 netdev_for_each_upper_dev_rcu(dev, upper, iter) 7006 features = netdev_sync_upper_features(dev, upper, features); 7007 7008 if (dev->features == features) 7009 goto sync_lower; 7010 7011 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", 7012 &dev->features, &features); 7013 7014 if (dev->netdev_ops->ndo_set_features) 7015 err = dev->netdev_ops->ndo_set_features(dev, features); 7016 else 7017 err = 0; 7018 7019 if (unlikely(err < 0)) { 7020 netdev_err(dev, 7021 "set_features() failed (%d); wanted %pNF, left %pNF\n", 7022 err, &features, &dev->features); 7023 /* return non-0 since some features might have changed and 7024 * it's better to fire a spurious notification than miss it 7025 */ 7026 return -1; 7027 } 7028 7029 sync_lower: 7030 /* some features must be disabled on lower devices when disabled 7031 * on an upper device (think: bonding master or bridge) 7032 */ 7033 netdev_for_each_lower_dev(dev, lower, iter) 7034 netdev_sync_lower_features(dev, lower, features); 7035 7036 if (!err) 7037 dev->features = features; 7038 7039 return err < 0 ? 0 : 1; 7040 } 7041 7042 /** 7043 * netdev_update_features - recalculate device features 7044 * @dev: the device to check 7045 * 7046 * Recalculate dev->features set and send notifications if it 7047 * has changed. Should be called after driver or hardware dependent 7048 * conditions might have changed that influence the features. 7049 */ 7050 void netdev_update_features(struct net_device *dev) 7051 { 7052 if (__netdev_update_features(dev)) 7053 netdev_features_change(dev); 7054 } 7055 EXPORT_SYMBOL(netdev_update_features); 7056 7057 /** 7058 * netdev_change_features - recalculate device features 7059 * @dev: the device to check 7060 * 7061 * Recalculate dev->features set and send notifications even 7062 * if they have not changed. Should be called instead of 7063 * netdev_update_features() if also dev->vlan_features might 7064 * have changed to allow the changes to be propagated to stacked 7065 * VLAN devices. 7066 */ 7067 void netdev_change_features(struct net_device *dev) 7068 { 7069 __netdev_update_features(dev); 7070 netdev_features_change(dev); 7071 } 7072 EXPORT_SYMBOL(netdev_change_features); 7073 7074 /** 7075 * netif_stacked_transfer_operstate - transfer operstate 7076 * @rootdev: the root or lower level device to transfer state from 7077 * @dev: the device to transfer operstate to 7078 * 7079 * Transfer operational state from root to device. This is normally 7080 * called when a stacking relationship exists between the root 7081 * device and the device(a leaf device). 7082 */ 7083 void netif_stacked_transfer_operstate(const struct net_device *rootdev, 7084 struct net_device *dev) 7085 { 7086 if (rootdev->operstate == IF_OPER_DORMANT) 7087 netif_dormant_on(dev); 7088 else 7089 netif_dormant_off(dev); 7090 7091 if (netif_carrier_ok(rootdev)) { 7092 if (!netif_carrier_ok(dev)) 7093 netif_carrier_on(dev); 7094 } else { 7095 if (netif_carrier_ok(dev)) 7096 netif_carrier_off(dev); 7097 } 7098 } 7099 EXPORT_SYMBOL(netif_stacked_transfer_operstate); 7100 7101 #ifdef CONFIG_SYSFS 7102 static int netif_alloc_rx_queues(struct net_device *dev) 7103 { 7104 unsigned int i, count = dev->num_rx_queues; 7105 struct netdev_rx_queue *rx; 7106 size_t sz = count * sizeof(*rx); 7107 7108 BUG_ON(count < 1); 7109 7110 rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 7111 if (!rx) { 7112 rx = vzalloc(sz); 7113 if (!rx) 7114 return -ENOMEM; 7115 } 7116 dev->_rx = rx; 7117 7118 for (i = 0; i < count; i++) 7119 rx[i].dev = dev; 7120 return 0; 7121 } 7122 #endif 7123 7124 static void netdev_init_one_queue(struct net_device *dev, 7125 struct netdev_queue *queue, void *_unused) 7126 { 7127 /* Initialize queue lock */ 7128 spin_lock_init(&queue->_xmit_lock); 7129 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); 7130 queue->xmit_lock_owner = -1; 7131 netdev_queue_numa_node_write(queue, NUMA_NO_NODE); 7132 queue->dev = dev; 7133 #ifdef CONFIG_BQL 7134 dql_init(&queue->dql, HZ); 7135 #endif 7136 } 7137 7138 static void netif_free_tx_queues(struct net_device *dev) 7139 { 7140 kvfree(dev->_tx); 7141 } 7142 7143 static int netif_alloc_netdev_queues(struct net_device *dev) 7144 { 7145 unsigned int count = dev->num_tx_queues; 7146 struct netdev_queue *tx; 7147 size_t sz = count * sizeof(*tx); 7148 7149 if (count < 1 || count > 0xffff) 7150 return -EINVAL; 7151 7152 tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 7153 if (!tx) { 7154 tx = vzalloc(sz); 7155 if (!tx) 7156 return -ENOMEM; 7157 } 7158 dev->_tx = tx; 7159 7160 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); 7161 spin_lock_init(&dev->tx_global_lock); 7162 7163 return 0; 7164 } 7165 7166 void netif_tx_stop_all_queues(struct net_device *dev) 7167 { 7168 unsigned int i; 7169 7170 for (i = 0; i < dev->num_tx_queues; i++) { 7171 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 7172 netif_tx_stop_queue(txq); 7173 } 7174 } 7175 EXPORT_SYMBOL(netif_tx_stop_all_queues); 7176 7177 /** 7178 * register_netdevice - register a network device 7179 * @dev: device to register 7180 * 7181 * Take a completed network device structure and add it to the kernel 7182 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 7183 * chain. 0 is returned on success. A negative errno code is returned 7184 * on a failure to set up the device, or if the name is a duplicate. 7185 * 7186 * Callers must hold the rtnl semaphore. You may want 7187 * register_netdev() instead of this. 7188 * 7189 * BUGS: 7190 * The locking appears insufficient to guarantee two parallel registers 7191 * will not get the same name. 7192 */ 7193 7194 int register_netdevice(struct net_device *dev) 7195 { 7196 int ret; 7197 struct net *net = dev_net(dev); 7198 7199 BUG_ON(dev_boot_phase); 7200 ASSERT_RTNL(); 7201 7202 might_sleep(); 7203 7204 /* When net_device's are persistent, this will be fatal. */ 7205 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); 7206 BUG_ON(!net); 7207 7208 spin_lock_init(&dev->addr_list_lock); 7209 netdev_set_addr_lockdep_class(dev); 7210 7211 ret = dev_get_valid_name(net, dev, dev->name); 7212 if (ret < 0) 7213 goto out; 7214 7215 /* Init, if this function is available */ 7216 if (dev->netdev_ops->ndo_init) { 7217 ret = dev->netdev_ops->ndo_init(dev); 7218 if (ret) { 7219 if (ret > 0) 7220 ret = -EIO; 7221 goto out; 7222 } 7223 } 7224 7225 if (((dev->hw_features | dev->features) & 7226 NETIF_F_HW_VLAN_CTAG_FILTER) && 7227 (!dev->netdev_ops->ndo_vlan_rx_add_vid || 7228 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { 7229 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n"); 7230 ret = -EINVAL; 7231 goto err_uninit; 7232 } 7233 7234 ret = -EBUSY; 7235 if (!dev->ifindex) 7236 dev->ifindex = dev_new_index(net); 7237 else if (__dev_get_by_index(net, dev->ifindex)) 7238 goto err_uninit; 7239 7240 /* Transfer changeable features to wanted_features and enable 7241 * software offloads (GSO and GRO). 7242 */ 7243 dev->hw_features |= NETIF_F_SOFT_FEATURES; 7244 dev->features |= NETIF_F_SOFT_FEATURES; 7245 dev->wanted_features = dev->features & dev->hw_features; 7246 7247 if (!(dev->flags & IFF_LOOPBACK)) 7248 dev->hw_features |= NETIF_F_NOCACHE_COPY; 7249 7250 /* If IPv4 TCP segmentation offload is supported we should also 7251 * allow the device to enable segmenting the frame with the option 7252 * of ignoring a static IP ID value. This doesn't enable the 7253 * feature itself but allows the user to enable it later. 7254 */ 7255 if (dev->hw_features & NETIF_F_TSO) 7256 dev->hw_features |= NETIF_F_TSO_MANGLEID; 7257 if (dev->vlan_features & NETIF_F_TSO) 7258 dev->vlan_features |= NETIF_F_TSO_MANGLEID; 7259 if (dev->mpls_features & NETIF_F_TSO) 7260 dev->mpls_features |= NETIF_F_TSO_MANGLEID; 7261 if (dev->hw_enc_features & NETIF_F_TSO) 7262 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 7263 7264 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. 7265 */ 7266 dev->vlan_features |= NETIF_F_HIGHDMA; 7267 7268 /* Make NETIF_F_SG inheritable to tunnel devices. 7269 */ 7270 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL; 7271 7272 /* Make NETIF_F_SG inheritable to MPLS. 7273 */ 7274 dev->mpls_features |= NETIF_F_SG; 7275 7276 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); 7277 ret = notifier_to_errno(ret); 7278 if (ret) 7279 goto err_uninit; 7280 7281 ret = netdev_register_kobject(dev); 7282 if (ret) 7283 goto err_uninit; 7284 dev->reg_state = NETREG_REGISTERED; 7285 7286 __netdev_update_features(dev); 7287 7288 /* 7289 * Default initial state at registry is that the 7290 * device is present. 7291 */ 7292 7293 set_bit(__LINK_STATE_PRESENT, &dev->state); 7294 7295 linkwatch_init_dev(dev); 7296 7297 dev_init_scheduler(dev); 7298 dev_hold(dev); 7299 list_netdevice(dev); 7300 add_device_randomness(dev->dev_addr, dev->addr_len); 7301 7302 /* If the device has permanent device address, driver should 7303 * set dev_addr and also addr_assign_type should be set to 7304 * NET_ADDR_PERM (default value). 7305 */ 7306 if (dev->addr_assign_type == NET_ADDR_PERM) 7307 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); 7308 7309 /* Notify protocols, that a new device appeared. */ 7310 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); 7311 ret = notifier_to_errno(ret); 7312 if (ret) { 7313 rollback_registered(dev); 7314 dev->reg_state = NETREG_UNREGISTERED; 7315 } 7316 /* 7317 * Prevent userspace races by waiting until the network 7318 * device is fully setup before sending notifications. 7319 */ 7320 if (!dev->rtnl_link_ops || 7321 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 7322 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); 7323 7324 out: 7325 return ret; 7326 7327 err_uninit: 7328 if (dev->netdev_ops->ndo_uninit) 7329 dev->netdev_ops->ndo_uninit(dev); 7330 goto out; 7331 } 7332 EXPORT_SYMBOL(register_netdevice); 7333 7334 /** 7335 * init_dummy_netdev - init a dummy network device for NAPI 7336 * @dev: device to init 7337 * 7338 * This takes a network device structure and initialize the minimum 7339 * amount of fields so it can be used to schedule NAPI polls without 7340 * registering a full blown interface. This is to be used by drivers 7341 * that need to tie several hardware interfaces to a single NAPI 7342 * poll scheduler due to HW limitations. 7343 */ 7344 int init_dummy_netdev(struct net_device *dev) 7345 { 7346 /* Clear everything. Note we don't initialize spinlocks 7347 * are they aren't supposed to be taken by any of the 7348 * NAPI code and this dummy netdev is supposed to be 7349 * only ever used for NAPI polls 7350 */ 7351 memset(dev, 0, sizeof(struct net_device)); 7352 7353 /* make sure we BUG if trying to hit standard 7354 * register/unregister code path 7355 */ 7356 dev->reg_state = NETREG_DUMMY; 7357 7358 /* NAPI wants this */ 7359 INIT_LIST_HEAD(&dev->napi_list); 7360 7361 /* a dummy interface is started by default */ 7362 set_bit(__LINK_STATE_PRESENT, &dev->state); 7363 set_bit(__LINK_STATE_START, &dev->state); 7364 7365 /* Note : We dont allocate pcpu_refcnt for dummy devices, 7366 * because users of this 'device' dont need to change 7367 * its refcount. 7368 */ 7369 7370 return 0; 7371 } 7372 EXPORT_SYMBOL_GPL(init_dummy_netdev); 7373 7374 7375 /** 7376 * register_netdev - register a network device 7377 * @dev: device to register 7378 * 7379 * Take a completed network device structure and add it to the kernel 7380 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 7381 * chain. 0 is returned on success. A negative errno code is returned 7382 * on a failure to set up the device, or if the name is a duplicate. 7383 * 7384 * This is a wrapper around register_netdevice that takes the rtnl semaphore 7385 * and expands the device name if you passed a format string to 7386 * alloc_netdev. 7387 */ 7388 int register_netdev(struct net_device *dev) 7389 { 7390 int err; 7391 7392 rtnl_lock(); 7393 err = register_netdevice(dev); 7394 rtnl_unlock(); 7395 return err; 7396 } 7397 EXPORT_SYMBOL(register_netdev); 7398 7399 int netdev_refcnt_read(const struct net_device *dev) 7400 { 7401 int i, refcnt = 0; 7402 7403 for_each_possible_cpu(i) 7404 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); 7405 return refcnt; 7406 } 7407 EXPORT_SYMBOL(netdev_refcnt_read); 7408 7409 /** 7410 * netdev_wait_allrefs - wait until all references are gone. 7411 * @dev: target net_device 7412 * 7413 * This is called when unregistering network devices. 7414 * 7415 * Any protocol or device that holds a reference should register 7416 * for netdevice notification, and cleanup and put back the 7417 * reference if they receive an UNREGISTER event. 7418 * We can get stuck here if buggy protocols don't correctly 7419 * call dev_put. 7420 */ 7421 static void netdev_wait_allrefs(struct net_device *dev) 7422 { 7423 unsigned long rebroadcast_time, warning_time; 7424 int refcnt; 7425 7426 linkwatch_forget_dev(dev); 7427 7428 rebroadcast_time = warning_time = jiffies; 7429 refcnt = netdev_refcnt_read(dev); 7430 7431 while (refcnt != 0) { 7432 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { 7433 rtnl_lock(); 7434 7435 /* Rebroadcast unregister notification */ 7436 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 7437 7438 __rtnl_unlock(); 7439 rcu_barrier(); 7440 rtnl_lock(); 7441 7442 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 7443 if (test_bit(__LINK_STATE_LINKWATCH_PENDING, 7444 &dev->state)) { 7445 /* We must not have linkwatch events 7446 * pending on unregister. If this 7447 * happens, we simply run the queue 7448 * unscheduled, resulting in a noop 7449 * for this device. 7450 */ 7451 linkwatch_run_queue(); 7452 } 7453 7454 __rtnl_unlock(); 7455 7456 rebroadcast_time = jiffies; 7457 } 7458 7459 msleep(250); 7460 7461 refcnt = netdev_refcnt_read(dev); 7462 7463 if (time_after(jiffies, warning_time + 10 * HZ)) { 7464 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", 7465 dev->name, refcnt); 7466 warning_time = jiffies; 7467 } 7468 } 7469 } 7470 7471 /* The sequence is: 7472 * 7473 * rtnl_lock(); 7474 * ... 7475 * register_netdevice(x1); 7476 * register_netdevice(x2); 7477 * ... 7478 * unregister_netdevice(y1); 7479 * unregister_netdevice(y2); 7480 * ... 7481 * rtnl_unlock(); 7482 * free_netdev(y1); 7483 * free_netdev(y2); 7484 * 7485 * We are invoked by rtnl_unlock(). 7486 * This allows us to deal with problems: 7487 * 1) We can delete sysfs objects which invoke hotplug 7488 * without deadlocking with linkwatch via keventd. 7489 * 2) Since we run with the RTNL semaphore not held, we can sleep 7490 * safely in order to wait for the netdev refcnt to drop to zero. 7491 * 7492 * We must not return until all unregister events added during 7493 * the interval the lock was held have been completed. 7494 */ 7495 void netdev_run_todo(void) 7496 { 7497 struct list_head list; 7498 7499 /* Snapshot list, allow later requests */ 7500 list_replace_init(&net_todo_list, &list); 7501 7502 __rtnl_unlock(); 7503 7504 7505 /* Wait for rcu callbacks to finish before next phase */ 7506 if (!list_empty(&list)) 7507 rcu_barrier(); 7508 7509 while (!list_empty(&list)) { 7510 struct net_device *dev 7511 = list_first_entry(&list, struct net_device, todo_list); 7512 list_del(&dev->todo_list); 7513 7514 rtnl_lock(); 7515 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 7516 __rtnl_unlock(); 7517 7518 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { 7519 pr_err("network todo '%s' but state %d\n", 7520 dev->name, dev->reg_state); 7521 dump_stack(); 7522 continue; 7523 } 7524 7525 dev->reg_state = NETREG_UNREGISTERED; 7526 7527 netdev_wait_allrefs(dev); 7528 7529 /* paranoia */ 7530 BUG_ON(netdev_refcnt_read(dev)); 7531 BUG_ON(!list_empty(&dev->ptype_all)); 7532 BUG_ON(!list_empty(&dev->ptype_specific)); 7533 WARN_ON(rcu_access_pointer(dev->ip_ptr)); 7534 WARN_ON(rcu_access_pointer(dev->ip6_ptr)); 7535 WARN_ON(dev->dn_ptr); 7536 7537 if (dev->destructor) 7538 dev->destructor(dev); 7539 7540 /* Report a network device has been unregistered */ 7541 rtnl_lock(); 7542 dev_net(dev)->dev_unreg_count--; 7543 __rtnl_unlock(); 7544 wake_up(&netdev_unregistering_wq); 7545 7546 /* Free network device */ 7547 kobject_put(&dev->dev.kobj); 7548 } 7549 } 7550 7551 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has 7552 * all the same fields in the same order as net_device_stats, with only 7553 * the type differing, but rtnl_link_stats64 may have additional fields 7554 * at the end for newer counters. 7555 */ 7556 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64, 7557 const struct net_device_stats *netdev_stats) 7558 { 7559 #if BITS_PER_LONG == 64 7560 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats)); 7561 memcpy(stats64, netdev_stats, sizeof(*stats64)); 7562 /* zero out counters that only exist in rtnl_link_stats64 */ 7563 memset((char *)stats64 + sizeof(*netdev_stats), 0, 7564 sizeof(*stats64) - sizeof(*netdev_stats)); 7565 #else 7566 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long); 7567 const unsigned long *src = (const unsigned long *)netdev_stats; 7568 u64 *dst = (u64 *)stats64; 7569 7570 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64)); 7571 for (i = 0; i < n; i++) 7572 dst[i] = src[i]; 7573 /* zero out counters that only exist in rtnl_link_stats64 */ 7574 memset((char *)stats64 + n * sizeof(u64), 0, 7575 sizeof(*stats64) - n * sizeof(u64)); 7576 #endif 7577 } 7578 EXPORT_SYMBOL(netdev_stats_to_stats64); 7579 7580 /** 7581 * dev_get_stats - get network device statistics 7582 * @dev: device to get statistics from 7583 * @storage: place to store stats 7584 * 7585 * Get network statistics from device. Return @storage. 7586 * The device driver may provide its own method by setting 7587 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats; 7588 * otherwise the internal statistics structure is used. 7589 */ 7590 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, 7591 struct rtnl_link_stats64 *storage) 7592 { 7593 const struct net_device_ops *ops = dev->netdev_ops; 7594 7595 if (ops->ndo_get_stats64) { 7596 memset(storage, 0, sizeof(*storage)); 7597 ops->ndo_get_stats64(dev, storage); 7598 } else if (ops->ndo_get_stats) { 7599 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); 7600 } else { 7601 netdev_stats_to_stats64(storage, &dev->stats); 7602 } 7603 storage->rx_dropped += atomic_long_read(&dev->rx_dropped); 7604 storage->tx_dropped += atomic_long_read(&dev->tx_dropped); 7605 storage->rx_nohandler += atomic_long_read(&dev->rx_nohandler); 7606 return storage; 7607 } 7608 EXPORT_SYMBOL(dev_get_stats); 7609 7610 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) 7611 { 7612 struct netdev_queue *queue = dev_ingress_queue(dev); 7613 7614 #ifdef CONFIG_NET_CLS_ACT 7615 if (queue) 7616 return queue; 7617 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 7618 if (!queue) 7619 return NULL; 7620 netdev_init_one_queue(dev, queue, NULL); 7621 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); 7622 queue->qdisc_sleeping = &noop_qdisc; 7623 rcu_assign_pointer(dev->ingress_queue, queue); 7624 #endif 7625 return queue; 7626 } 7627 7628 static const struct ethtool_ops default_ethtool_ops; 7629 7630 void netdev_set_default_ethtool_ops(struct net_device *dev, 7631 const struct ethtool_ops *ops) 7632 { 7633 if (dev->ethtool_ops == &default_ethtool_ops) 7634 dev->ethtool_ops = ops; 7635 } 7636 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops); 7637 7638 void netdev_freemem(struct net_device *dev) 7639 { 7640 char *addr = (char *)dev - dev->padded; 7641 7642 kvfree(addr); 7643 } 7644 7645 /** 7646 * alloc_netdev_mqs - allocate network device 7647 * @sizeof_priv: size of private data to allocate space for 7648 * @name: device name format string 7649 * @name_assign_type: origin of device name 7650 * @setup: callback to initialize device 7651 * @txqs: the number of TX subqueues to allocate 7652 * @rxqs: the number of RX subqueues to allocate 7653 * 7654 * Allocates a struct net_device with private data area for driver use 7655 * and performs basic initialization. Also allocates subqueue structs 7656 * for each queue on the device. 7657 */ 7658 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, 7659 unsigned char name_assign_type, 7660 void (*setup)(struct net_device *), 7661 unsigned int txqs, unsigned int rxqs) 7662 { 7663 struct net_device *dev; 7664 size_t alloc_size; 7665 struct net_device *p; 7666 7667 BUG_ON(strlen(name) >= sizeof(dev->name)); 7668 7669 if (txqs < 1) { 7670 pr_err("alloc_netdev: Unable to allocate device with zero queues\n"); 7671 return NULL; 7672 } 7673 7674 #ifdef CONFIG_SYSFS 7675 if (rxqs < 1) { 7676 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n"); 7677 return NULL; 7678 } 7679 #endif 7680 7681 alloc_size = sizeof(struct net_device); 7682 if (sizeof_priv) { 7683 /* ensure 32-byte alignment of private area */ 7684 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN); 7685 alloc_size += sizeof_priv; 7686 } 7687 /* ensure 32-byte alignment of whole construct */ 7688 alloc_size += NETDEV_ALIGN - 1; 7689 7690 p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 7691 if (!p) 7692 p = vzalloc(alloc_size); 7693 if (!p) 7694 return NULL; 7695 7696 dev = PTR_ALIGN(p, NETDEV_ALIGN); 7697 dev->padded = (char *)dev - (char *)p; 7698 7699 dev->pcpu_refcnt = alloc_percpu(int); 7700 if (!dev->pcpu_refcnt) 7701 goto free_dev; 7702 7703 if (dev_addr_init(dev)) 7704 goto free_pcpu; 7705 7706 dev_mc_init(dev); 7707 dev_uc_init(dev); 7708 7709 dev_net_set(dev, &init_net); 7710 7711 dev->gso_max_size = GSO_MAX_SIZE; 7712 dev->gso_max_segs = GSO_MAX_SEGS; 7713 7714 INIT_LIST_HEAD(&dev->napi_list); 7715 INIT_LIST_HEAD(&dev->unreg_list); 7716 INIT_LIST_HEAD(&dev->close_list); 7717 INIT_LIST_HEAD(&dev->link_watch_list); 7718 INIT_LIST_HEAD(&dev->adj_list.upper); 7719 INIT_LIST_HEAD(&dev->adj_list.lower); 7720 INIT_LIST_HEAD(&dev->ptype_all); 7721 INIT_LIST_HEAD(&dev->ptype_specific); 7722 #ifdef CONFIG_NET_SCHED 7723 hash_init(dev->qdisc_hash); 7724 #endif 7725 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; 7726 setup(dev); 7727 7728 if (!dev->tx_queue_len) { 7729 dev->priv_flags |= IFF_NO_QUEUE; 7730 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; 7731 } 7732 7733 dev->num_tx_queues = txqs; 7734 dev->real_num_tx_queues = txqs; 7735 if (netif_alloc_netdev_queues(dev)) 7736 goto free_all; 7737 7738 #ifdef CONFIG_SYSFS 7739 dev->num_rx_queues = rxqs; 7740 dev->real_num_rx_queues = rxqs; 7741 if (netif_alloc_rx_queues(dev)) 7742 goto free_all; 7743 #endif 7744 7745 strcpy(dev->name, name); 7746 dev->name_assign_type = name_assign_type; 7747 dev->group = INIT_NETDEV_GROUP; 7748 if (!dev->ethtool_ops) 7749 dev->ethtool_ops = &default_ethtool_ops; 7750 7751 nf_hook_ingress_init(dev); 7752 7753 return dev; 7754 7755 free_all: 7756 free_netdev(dev); 7757 return NULL; 7758 7759 free_pcpu: 7760 free_percpu(dev->pcpu_refcnt); 7761 free_dev: 7762 netdev_freemem(dev); 7763 return NULL; 7764 } 7765 EXPORT_SYMBOL(alloc_netdev_mqs); 7766 7767 /** 7768 * free_netdev - free network device 7769 * @dev: device 7770 * 7771 * This function does the last stage of destroying an allocated device 7772 * interface. The reference to the device object is released. 7773 * If this is the last reference then it will be freed. 7774 * Must be called in process context. 7775 */ 7776 void free_netdev(struct net_device *dev) 7777 { 7778 struct napi_struct *p, *n; 7779 7780 might_sleep(); 7781 netif_free_tx_queues(dev); 7782 #ifdef CONFIG_SYSFS 7783 kvfree(dev->_rx); 7784 #endif 7785 7786 kfree(rcu_dereference_protected(dev->ingress_queue, 1)); 7787 7788 /* Flush device addresses */ 7789 dev_addr_flush(dev); 7790 7791 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) 7792 netif_napi_del(p); 7793 7794 free_percpu(dev->pcpu_refcnt); 7795 dev->pcpu_refcnt = NULL; 7796 7797 /* Compatibility with error handling in drivers */ 7798 if (dev->reg_state == NETREG_UNINITIALIZED) { 7799 netdev_freemem(dev); 7800 return; 7801 } 7802 7803 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); 7804 dev->reg_state = NETREG_RELEASED; 7805 7806 /* will free via device release */ 7807 put_device(&dev->dev); 7808 } 7809 EXPORT_SYMBOL(free_netdev); 7810 7811 /** 7812 * synchronize_net - Synchronize with packet receive processing 7813 * 7814 * Wait for packets currently being received to be done. 7815 * Does not block later packets from starting. 7816 */ 7817 void synchronize_net(void) 7818 { 7819 might_sleep(); 7820 if (rtnl_is_locked()) 7821 synchronize_rcu_expedited(); 7822 else 7823 synchronize_rcu(); 7824 } 7825 EXPORT_SYMBOL(synchronize_net); 7826 7827 /** 7828 * unregister_netdevice_queue - remove device from the kernel 7829 * @dev: device 7830 * @head: list 7831 * 7832 * This function shuts down a device interface and removes it 7833 * from the kernel tables. 7834 * If head not NULL, device is queued to be unregistered later. 7835 * 7836 * Callers must hold the rtnl semaphore. You may want 7837 * unregister_netdev() instead of this. 7838 */ 7839 7840 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) 7841 { 7842 ASSERT_RTNL(); 7843 7844 if (head) { 7845 list_move_tail(&dev->unreg_list, head); 7846 } else { 7847 rollback_registered(dev); 7848 /* Finish processing unregister after unlock */ 7849 net_set_todo(dev); 7850 } 7851 } 7852 EXPORT_SYMBOL(unregister_netdevice_queue); 7853 7854 /** 7855 * unregister_netdevice_many - unregister many devices 7856 * @head: list of devices 7857 * 7858 * Note: As most callers use a stack allocated list_head, 7859 * we force a list_del() to make sure stack wont be corrupted later. 7860 */ 7861 void unregister_netdevice_many(struct list_head *head) 7862 { 7863 struct net_device *dev; 7864 7865 if (!list_empty(head)) { 7866 rollback_registered_many(head); 7867 list_for_each_entry(dev, head, unreg_list) 7868 net_set_todo(dev); 7869 list_del(head); 7870 } 7871 } 7872 EXPORT_SYMBOL(unregister_netdevice_many); 7873 7874 /** 7875 * unregister_netdev - remove device from the kernel 7876 * @dev: device 7877 * 7878 * This function shuts down a device interface and removes it 7879 * from the kernel tables. 7880 * 7881 * This is just a wrapper for unregister_netdevice that takes 7882 * the rtnl semaphore. In general you want to use this and not 7883 * unregister_netdevice. 7884 */ 7885 void unregister_netdev(struct net_device *dev) 7886 { 7887 rtnl_lock(); 7888 unregister_netdevice(dev); 7889 rtnl_unlock(); 7890 } 7891 EXPORT_SYMBOL(unregister_netdev); 7892 7893 /** 7894 * dev_change_net_namespace - move device to different nethost namespace 7895 * @dev: device 7896 * @net: network namespace 7897 * @pat: If not NULL name pattern to try if the current device name 7898 * is already taken in the destination network namespace. 7899 * 7900 * This function shuts down a device interface and moves it 7901 * to a new network namespace. On success 0 is returned, on 7902 * a failure a netagive errno code is returned. 7903 * 7904 * Callers must hold the rtnl semaphore. 7905 */ 7906 7907 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat) 7908 { 7909 int err; 7910 7911 ASSERT_RTNL(); 7912 7913 /* Don't allow namespace local devices to be moved. */ 7914 err = -EINVAL; 7915 if (dev->features & NETIF_F_NETNS_LOCAL) 7916 goto out; 7917 7918 /* Ensure the device has been registrered */ 7919 if (dev->reg_state != NETREG_REGISTERED) 7920 goto out; 7921 7922 /* Get out if there is nothing todo */ 7923 err = 0; 7924 if (net_eq(dev_net(dev), net)) 7925 goto out; 7926 7927 /* Pick the destination device name, and ensure 7928 * we can use it in the destination network namespace. 7929 */ 7930 err = -EEXIST; 7931 if (__dev_get_by_name(net, dev->name)) { 7932 /* We get here if we can't use the current device name */ 7933 if (!pat) 7934 goto out; 7935 if (dev_get_valid_name(net, dev, pat) < 0) 7936 goto out; 7937 } 7938 7939 /* 7940 * And now a mini version of register_netdevice unregister_netdevice. 7941 */ 7942 7943 /* If device is running close it first. */ 7944 dev_close(dev); 7945 7946 /* And unlink it from device chain */ 7947 err = -ENODEV; 7948 unlist_netdevice(dev); 7949 7950 synchronize_net(); 7951 7952 /* Shutdown queueing discipline. */ 7953 dev_shutdown(dev); 7954 7955 /* Notify protocols, that we are about to destroy 7956 this device. They should clean all the things. 7957 7958 Note that dev->reg_state stays at NETREG_REGISTERED. 7959 This is wanted because this way 8021q and macvlan know 7960 the device is just moving and can keep their slaves up. 7961 */ 7962 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 7963 rcu_barrier(); 7964 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 7965 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL); 7966 7967 /* 7968 * Flush the unicast and multicast chains 7969 */ 7970 dev_uc_flush(dev); 7971 dev_mc_flush(dev); 7972 7973 /* Send a netdev-removed uevent to the old namespace */ 7974 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); 7975 netdev_adjacent_del_links(dev); 7976 7977 /* Actually switch the network namespace */ 7978 dev_net_set(dev, net); 7979 7980 /* If there is an ifindex conflict assign a new one */ 7981 if (__dev_get_by_index(net, dev->ifindex)) 7982 dev->ifindex = dev_new_index(net); 7983 7984 /* Send a netdev-add uevent to the new namespace */ 7985 kobject_uevent(&dev->dev.kobj, KOBJ_ADD); 7986 netdev_adjacent_add_links(dev); 7987 7988 /* Fixup kobjects */ 7989 err = device_rename(&dev->dev, dev->name); 7990 WARN_ON(err); 7991 7992 /* Add the device back in the hashes */ 7993 list_netdevice(dev); 7994 7995 /* Notify protocols, that a new device appeared. */ 7996 call_netdevice_notifiers(NETDEV_REGISTER, dev); 7997 7998 /* 7999 * Prevent userspace races by waiting until the network 8000 * device is fully setup before sending notifications. 8001 */ 8002 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); 8003 8004 synchronize_net(); 8005 err = 0; 8006 out: 8007 return err; 8008 } 8009 EXPORT_SYMBOL_GPL(dev_change_net_namespace); 8010 8011 static int dev_cpu_dead(unsigned int oldcpu) 8012 { 8013 struct sk_buff **list_skb; 8014 struct sk_buff *skb; 8015 unsigned int cpu; 8016 struct softnet_data *sd, *oldsd; 8017 8018 local_irq_disable(); 8019 cpu = smp_processor_id(); 8020 sd = &per_cpu(softnet_data, cpu); 8021 oldsd = &per_cpu(softnet_data, oldcpu); 8022 8023 /* Find end of our completion_queue. */ 8024 list_skb = &sd->completion_queue; 8025 while (*list_skb) 8026 list_skb = &(*list_skb)->next; 8027 /* Append completion queue from offline CPU. */ 8028 *list_skb = oldsd->completion_queue; 8029 oldsd->completion_queue = NULL; 8030 8031 /* Append output queue from offline CPU. */ 8032 if (oldsd->output_queue) { 8033 *sd->output_queue_tailp = oldsd->output_queue; 8034 sd->output_queue_tailp = oldsd->output_queue_tailp; 8035 oldsd->output_queue = NULL; 8036 oldsd->output_queue_tailp = &oldsd->output_queue; 8037 } 8038 /* Append NAPI poll list from offline CPU, with one exception : 8039 * process_backlog() must be called by cpu owning percpu backlog. 8040 * We properly handle process_queue & input_pkt_queue later. 8041 */ 8042 while (!list_empty(&oldsd->poll_list)) { 8043 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, 8044 struct napi_struct, 8045 poll_list); 8046 8047 list_del_init(&napi->poll_list); 8048 if (napi->poll == process_backlog) 8049 napi->state = 0; 8050 else 8051 ____napi_schedule(sd, napi); 8052 } 8053 8054 raise_softirq_irqoff(NET_TX_SOFTIRQ); 8055 local_irq_enable(); 8056 8057 /* Process offline CPU's input_pkt_queue */ 8058 while ((skb = __skb_dequeue(&oldsd->process_queue))) { 8059 netif_rx_ni(skb); 8060 input_queue_head_incr(oldsd); 8061 } 8062 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { 8063 netif_rx_ni(skb); 8064 input_queue_head_incr(oldsd); 8065 } 8066 8067 return 0; 8068 } 8069 8070 /** 8071 * netdev_increment_features - increment feature set by one 8072 * @all: current feature set 8073 * @one: new feature set 8074 * @mask: mask feature set 8075 * 8076 * Computes a new feature set after adding a device with feature set 8077 * @one to the master device with current feature set @all. Will not 8078 * enable anything that is off in @mask. Returns the new feature set. 8079 */ 8080 netdev_features_t netdev_increment_features(netdev_features_t all, 8081 netdev_features_t one, netdev_features_t mask) 8082 { 8083 if (mask & NETIF_F_HW_CSUM) 8084 mask |= NETIF_F_CSUM_MASK; 8085 mask |= NETIF_F_VLAN_CHALLENGED; 8086 8087 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask; 8088 all &= one | ~NETIF_F_ALL_FOR_ALL; 8089 8090 /* If one device supports hw checksumming, set for all. */ 8091 if (all & NETIF_F_HW_CSUM) 8092 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM); 8093 8094 return all; 8095 } 8096 EXPORT_SYMBOL(netdev_increment_features); 8097 8098 static struct hlist_head * __net_init netdev_create_hash(void) 8099 { 8100 int i; 8101 struct hlist_head *hash; 8102 8103 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL); 8104 if (hash != NULL) 8105 for (i = 0; i < NETDEV_HASHENTRIES; i++) 8106 INIT_HLIST_HEAD(&hash[i]); 8107 8108 return hash; 8109 } 8110 8111 /* Initialize per network namespace state */ 8112 static int __net_init netdev_init(struct net *net) 8113 { 8114 if (net != &init_net) 8115 INIT_LIST_HEAD(&net->dev_base_head); 8116 8117 net->dev_name_head = netdev_create_hash(); 8118 if (net->dev_name_head == NULL) 8119 goto err_name; 8120 8121 net->dev_index_head = netdev_create_hash(); 8122 if (net->dev_index_head == NULL) 8123 goto err_idx; 8124 8125 return 0; 8126 8127 err_idx: 8128 kfree(net->dev_name_head); 8129 err_name: 8130 return -ENOMEM; 8131 } 8132 8133 /** 8134 * netdev_drivername - network driver for the device 8135 * @dev: network device 8136 * 8137 * Determine network driver for device. 8138 */ 8139 const char *netdev_drivername(const struct net_device *dev) 8140 { 8141 const struct device_driver *driver; 8142 const struct device *parent; 8143 const char *empty = ""; 8144 8145 parent = dev->dev.parent; 8146 if (!parent) 8147 return empty; 8148 8149 driver = parent->driver; 8150 if (driver && driver->name) 8151 return driver->name; 8152 return empty; 8153 } 8154 8155 static void __netdev_printk(const char *level, const struct net_device *dev, 8156 struct va_format *vaf) 8157 { 8158 if (dev && dev->dev.parent) { 8159 dev_printk_emit(level[1] - '0', 8160 dev->dev.parent, 8161 "%s %s %s%s: %pV", 8162 dev_driver_string(dev->dev.parent), 8163 dev_name(dev->dev.parent), 8164 netdev_name(dev), netdev_reg_state(dev), 8165 vaf); 8166 } else if (dev) { 8167 printk("%s%s%s: %pV", 8168 level, netdev_name(dev), netdev_reg_state(dev), vaf); 8169 } else { 8170 printk("%s(NULL net_device): %pV", level, vaf); 8171 } 8172 } 8173 8174 void netdev_printk(const char *level, const struct net_device *dev, 8175 const char *format, ...) 8176 { 8177 struct va_format vaf; 8178 va_list args; 8179 8180 va_start(args, format); 8181 8182 vaf.fmt = format; 8183 vaf.va = &args; 8184 8185 __netdev_printk(level, dev, &vaf); 8186 8187 va_end(args); 8188 } 8189 EXPORT_SYMBOL(netdev_printk); 8190 8191 #define define_netdev_printk_level(func, level) \ 8192 void func(const struct net_device *dev, const char *fmt, ...) \ 8193 { \ 8194 struct va_format vaf; \ 8195 va_list args; \ 8196 \ 8197 va_start(args, fmt); \ 8198 \ 8199 vaf.fmt = fmt; \ 8200 vaf.va = &args; \ 8201 \ 8202 __netdev_printk(level, dev, &vaf); \ 8203 \ 8204 va_end(args); \ 8205 } \ 8206 EXPORT_SYMBOL(func); 8207 8208 define_netdev_printk_level(netdev_emerg, KERN_EMERG); 8209 define_netdev_printk_level(netdev_alert, KERN_ALERT); 8210 define_netdev_printk_level(netdev_crit, KERN_CRIT); 8211 define_netdev_printk_level(netdev_err, KERN_ERR); 8212 define_netdev_printk_level(netdev_warn, KERN_WARNING); 8213 define_netdev_printk_level(netdev_notice, KERN_NOTICE); 8214 define_netdev_printk_level(netdev_info, KERN_INFO); 8215 8216 static void __net_exit netdev_exit(struct net *net) 8217 { 8218 kfree(net->dev_name_head); 8219 kfree(net->dev_index_head); 8220 } 8221 8222 static struct pernet_operations __net_initdata netdev_net_ops = { 8223 .init = netdev_init, 8224 .exit = netdev_exit, 8225 }; 8226 8227 static void __net_exit default_device_exit(struct net *net) 8228 { 8229 struct net_device *dev, *aux; 8230 /* 8231 * Push all migratable network devices back to the 8232 * initial network namespace 8233 */ 8234 rtnl_lock(); 8235 for_each_netdev_safe(net, dev, aux) { 8236 int err; 8237 char fb_name[IFNAMSIZ]; 8238 8239 /* Ignore unmoveable devices (i.e. loopback) */ 8240 if (dev->features & NETIF_F_NETNS_LOCAL) 8241 continue; 8242 8243 /* Leave virtual devices for the generic cleanup */ 8244 if (dev->rtnl_link_ops) 8245 continue; 8246 8247 /* Push remaining network devices to init_net */ 8248 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); 8249 err = dev_change_net_namespace(dev, &init_net, fb_name); 8250 if (err) { 8251 pr_emerg("%s: failed to move %s to init_net: %d\n", 8252 __func__, dev->name, err); 8253 BUG(); 8254 } 8255 } 8256 rtnl_unlock(); 8257 } 8258 8259 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list) 8260 { 8261 /* Return with the rtnl_lock held when there are no network 8262 * devices unregistering in any network namespace in net_list. 8263 */ 8264 struct net *net; 8265 bool unregistering; 8266 DEFINE_WAIT_FUNC(wait, woken_wake_function); 8267 8268 add_wait_queue(&netdev_unregistering_wq, &wait); 8269 for (;;) { 8270 unregistering = false; 8271 rtnl_lock(); 8272 list_for_each_entry(net, net_list, exit_list) { 8273 if (net->dev_unreg_count > 0) { 8274 unregistering = true; 8275 break; 8276 } 8277 } 8278 if (!unregistering) 8279 break; 8280 __rtnl_unlock(); 8281 8282 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); 8283 } 8284 remove_wait_queue(&netdev_unregistering_wq, &wait); 8285 } 8286 8287 static void __net_exit default_device_exit_batch(struct list_head *net_list) 8288 { 8289 /* At exit all network devices most be removed from a network 8290 * namespace. Do this in the reverse order of registration. 8291 * Do this across as many network namespaces as possible to 8292 * improve batching efficiency. 8293 */ 8294 struct net_device *dev; 8295 struct net *net; 8296 LIST_HEAD(dev_kill_list); 8297 8298 /* To prevent network device cleanup code from dereferencing 8299 * loopback devices or network devices that have been freed 8300 * wait here for all pending unregistrations to complete, 8301 * before unregistring the loopback device and allowing the 8302 * network namespace be freed. 8303 * 8304 * The netdev todo list containing all network devices 8305 * unregistrations that happen in default_device_exit_batch 8306 * will run in the rtnl_unlock() at the end of 8307 * default_device_exit_batch. 8308 */ 8309 rtnl_lock_unregistering(net_list); 8310 list_for_each_entry(net, net_list, exit_list) { 8311 for_each_netdev_reverse(net, dev) { 8312 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) 8313 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); 8314 else 8315 unregister_netdevice_queue(dev, &dev_kill_list); 8316 } 8317 } 8318 unregister_netdevice_many(&dev_kill_list); 8319 rtnl_unlock(); 8320 } 8321 8322 static struct pernet_operations __net_initdata default_device_ops = { 8323 .exit = default_device_exit, 8324 .exit_batch = default_device_exit_batch, 8325 }; 8326 8327 /* 8328 * Initialize the DEV module. At boot time this walks the device list and 8329 * unhooks any devices that fail to initialise (normally hardware not 8330 * present) and leaves us with a valid list of present and active devices. 8331 * 8332 */ 8333 8334 /* 8335 * This is called single threaded during boot, so no need 8336 * to take the rtnl semaphore. 8337 */ 8338 static int __init net_dev_init(void) 8339 { 8340 int i, rc = -ENOMEM; 8341 8342 BUG_ON(!dev_boot_phase); 8343 8344 if (dev_proc_init()) 8345 goto out; 8346 8347 if (netdev_kobject_init()) 8348 goto out; 8349 8350 INIT_LIST_HEAD(&ptype_all); 8351 for (i = 0; i < PTYPE_HASH_SIZE; i++) 8352 INIT_LIST_HEAD(&ptype_base[i]); 8353 8354 INIT_LIST_HEAD(&offload_base); 8355 8356 if (register_pernet_subsys(&netdev_net_ops)) 8357 goto out; 8358 8359 /* 8360 * Initialise the packet receive queues. 8361 */ 8362 8363 for_each_possible_cpu(i) { 8364 struct work_struct *flush = per_cpu_ptr(&flush_works, i); 8365 struct softnet_data *sd = &per_cpu(softnet_data, i); 8366 8367 INIT_WORK(flush, flush_backlog); 8368 8369 skb_queue_head_init(&sd->input_pkt_queue); 8370 skb_queue_head_init(&sd->process_queue); 8371 INIT_LIST_HEAD(&sd->poll_list); 8372 sd->output_queue_tailp = &sd->output_queue; 8373 #ifdef CONFIG_RPS 8374 sd->csd.func = rps_trigger_softirq; 8375 sd->csd.info = sd; 8376 sd->cpu = i; 8377 #endif 8378 8379 sd->backlog.poll = process_backlog; 8380 sd->backlog.weight = weight_p; 8381 } 8382 8383 dev_boot_phase = 0; 8384 8385 /* The loopback device is special if any other network devices 8386 * is present in a network namespace the loopback device must 8387 * be present. Since we now dynamically allocate and free the 8388 * loopback device ensure this invariant is maintained by 8389 * keeping the loopback device as the first device on the 8390 * list of network devices. Ensuring the loopback devices 8391 * is the first device that appears and the last network device 8392 * that disappears. 8393 */ 8394 if (register_pernet_device(&loopback_net_ops)) 8395 goto out; 8396 8397 if (register_pernet_device(&default_device_ops)) 8398 goto out; 8399 8400 open_softirq(NET_TX_SOFTIRQ, net_tx_action); 8401 open_softirq(NET_RX_SOFTIRQ, net_rx_action); 8402 8403 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead", 8404 NULL, dev_cpu_dead); 8405 WARN_ON(rc < 0); 8406 dst_subsys_init(); 8407 rc = 0; 8408 out: 8409 return rc; 8410 } 8411 8412 subsys_initcall(net_dev_init); 8413