1 /* 2 * NET3 Protocol independent device support routines. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 * 9 * Derived from the non IP parts of dev.c 1.0.19 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Mark Evans, <evansmp@uhura.aston.ac.uk> 13 * 14 * Additional Authors: 15 * Florian la Roche <rzsfl@rz.uni-sb.de> 16 * Alan Cox <gw4pts@gw4pts.ampr.org> 17 * David Hinds <dahinds@users.sourceforge.net> 18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 19 * Adam Sulmicki <adam@cfar.umd.edu> 20 * Pekka Riikonen <priikone@poesidon.pspt.fi> 21 * 22 * Changes: 23 * D.J. Barrow : Fixed bug where dev->refcnt gets set 24 * to 2 if register_netdev gets called 25 * before net_dev_init & also removed a 26 * few lines of code in the process. 27 * Alan Cox : device private ioctl copies fields back. 28 * Alan Cox : Transmit queue code does relevant 29 * stunts to keep the queue safe. 30 * Alan Cox : Fixed double lock. 31 * Alan Cox : Fixed promisc NULL pointer trap 32 * ???????? : Support the full private ioctl range 33 * Alan Cox : Moved ioctl permission check into 34 * drivers 35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI 36 * Alan Cox : 100 backlog just doesn't cut it when 37 * you start doing multicast video 8) 38 * Alan Cox : Rewrote net_bh and list manager. 39 * Alan Cox : Fix ETH_P_ALL echoback lengths. 40 * Alan Cox : Took out transmit every packet pass 41 * Saved a few bytes in the ioctl handler 42 * Alan Cox : Network driver sets packet type before 43 * calling netif_rx. Saves a function 44 * call a packet. 45 * Alan Cox : Hashed net_bh() 46 * Richard Kooijman: Timestamp fixes. 47 * Alan Cox : Wrong field in SIOCGIFDSTADDR 48 * Alan Cox : Device lock protection. 49 * Alan Cox : Fixed nasty side effect of device close 50 * changes. 51 * Rudi Cilibrasi : Pass the right thing to 52 * set_mac_address() 53 * Dave Miller : 32bit quantity for the device lock to 54 * make it work out on a Sparc. 55 * Bjorn Ekwall : Added KERNELD hack. 56 * Alan Cox : Cleaned up the backlog initialise. 57 * Craig Metz : SIOCGIFCONF fix if space for under 58 * 1 device. 59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there 60 * is no device open function. 61 * Andi Kleen : Fix error reporting for SIOCGIFCONF 62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF 63 * Cyrus Durgin : Cleaned for KMOD 64 * Adam Sulmicki : Bug Fix : Network Device Unload 65 * A network device unload needs to purge 66 * the backlog queue. 67 * Paul Rusty Russell : SIOCSIFNAME 68 * Pekka Riikonen : Netdev boot-time settings code 69 * Andrew Morton : Make unregister_netdevice wait 70 * indefinitely on dev->refcnt 71 * J Hadi Salim : - Backlog queue sampling 72 * - netif_rx() feedback 73 */ 74 75 #include <linux/uaccess.h> 76 #include <linux/bitops.h> 77 #include <linux/capability.h> 78 #include <linux/cpu.h> 79 #include <linux/types.h> 80 #include <linux/kernel.h> 81 #include <linux/hash.h> 82 #include <linux/slab.h> 83 #include <linux/sched.h> 84 #include <linux/mutex.h> 85 #include <linux/string.h> 86 #include <linux/mm.h> 87 #include <linux/socket.h> 88 #include <linux/sockios.h> 89 #include <linux/errno.h> 90 #include <linux/interrupt.h> 91 #include <linux/if_ether.h> 92 #include <linux/netdevice.h> 93 #include <linux/etherdevice.h> 94 #include <linux/ethtool.h> 95 #include <linux/notifier.h> 96 #include <linux/skbuff.h> 97 #include <linux/bpf.h> 98 #include <net/net_namespace.h> 99 #include <net/sock.h> 100 #include <net/busy_poll.h> 101 #include <linux/rtnetlink.h> 102 #include <linux/stat.h> 103 #include <net/dst.h> 104 #include <net/dst_metadata.h> 105 #include <net/pkt_sched.h> 106 #include <net/checksum.h> 107 #include <net/xfrm.h> 108 #include <linux/highmem.h> 109 #include <linux/init.h> 110 #include <linux/module.h> 111 #include <linux/netpoll.h> 112 #include <linux/rcupdate.h> 113 #include <linux/delay.h> 114 #include <net/iw_handler.h> 115 #include <asm/current.h> 116 #include <linux/audit.h> 117 #include <linux/dmaengine.h> 118 #include <linux/err.h> 119 #include <linux/ctype.h> 120 #include <linux/if_arp.h> 121 #include <linux/if_vlan.h> 122 #include <linux/ip.h> 123 #include <net/ip.h> 124 #include <net/mpls.h> 125 #include <linux/ipv6.h> 126 #include <linux/in.h> 127 #include <linux/jhash.h> 128 #include <linux/random.h> 129 #include <trace/events/napi.h> 130 #include <trace/events/net.h> 131 #include <trace/events/skb.h> 132 #include <linux/pci.h> 133 #include <linux/inetdevice.h> 134 #include <linux/cpu_rmap.h> 135 #include <linux/static_key.h> 136 #include <linux/hashtable.h> 137 #include <linux/vmalloc.h> 138 #include <linux/if_macvlan.h> 139 #include <linux/errqueue.h> 140 #include <linux/hrtimer.h> 141 #include <linux/netfilter_ingress.h> 142 #include <linux/crash_dump.h> 143 144 #include "net-sysfs.h" 145 146 /* Instead of increasing this, you should create a hash table. */ 147 #define MAX_GRO_SKBS 8 148 149 /* This should be increased if a protocol with a bigger head is added. */ 150 #define GRO_MAX_HEAD (MAX_HEADER + 128) 151 152 static DEFINE_SPINLOCK(ptype_lock); 153 static DEFINE_SPINLOCK(offload_lock); 154 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; 155 struct list_head ptype_all __read_mostly; /* Taps */ 156 static struct list_head offload_base __read_mostly; 157 158 static int netif_rx_internal(struct sk_buff *skb); 159 static int call_netdevice_notifiers_info(unsigned long val, 160 struct net_device *dev, 161 struct netdev_notifier_info *info); 162 163 /* 164 * The @dev_base_head list is protected by @dev_base_lock and the rtnl 165 * semaphore. 166 * 167 * Pure readers hold dev_base_lock for reading, or rcu_read_lock() 168 * 169 * Writers must hold the rtnl semaphore while they loop through the 170 * dev_base_head list, and hold dev_base_lock for writing when they do the 171 * actual updates. This allows pure readers to access the list even 172 * while a writer is preparing to update it. 173 * 174 * To put it another way, dev_base_lock is held for writing only to 175 * protect against pure readers; the rtnl semaphore provides the 176 * protection against other writers. 177 * 178 * See, for example usages, register_netdevice() and 179 * unregister_netdevice(), which must be called with the rtnl 180 * semaphore held. 181 */ 182 DEFINE_RWLOCK(dev_base_lock); 183 EXPORT_SYMBOL(dev_base_lock); 184 185 /* protects napi_hash addition/deletion and napi_gen_id */ 186 static DEFINE_SPINLOCK(napi_hash_lock); 187 188 static unsigned int napi_gen_id = NR_CPUS; 189 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8); 190 191 static seqcount_t devnet_rename_seq; 192 193 static inline void dev_base_seq_inc(struct net *net) 194 { 195 while (++net->dev_base_seq == 0) 196 ; 197 } 198 199 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name) 200 { 201 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ)); 202 203 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; 204 } 205 206 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) 207 { 208 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; 209 } 210 211 static inline void rps_lock(struct softnet_data *sd) 212 { 213 #ifdef CONFIG_RPS 214 spin_lock(&sd->input_pkt_queue.lock); 215 #endif 216 } 217 218 static inline void rps_unlock(struct softnet_data *sd) 219 { 220 #ifdef CONFIG_RPS 221 spin_unlock(&sd->input_pkt_queue.lock); 222 #endif 223 } 224 225 /* Device list insertion */ 226 static void list_netdevice(struct net_device *dev) 227 { 228 struct net *net = dev_net(dev); 229 230 ASSERT_RTNL(); 231 232 write_lock_bh(&dev_base_lock); 233 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); 234 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); 235 hlist_add_head_rcu(&dev->index_hlist, 236 dev_index_hash(net, dev->ifindex)); 237 write_unlock_bh(&dev_base_lock); 238 239 dev_base_seq_inc(net); 240 } 241 242 /* Device list removal 243 * caller must respect a RCU grace period before freeing/reusing dev 244 */ 245 static void unlist_netdevice(struct net_device *dev) 246 { 247 ASSERT_RTNL(); 248 249 /* Unlink dev from the device chain */ 250 write_lock_bh(&dev_base_lock); 251 list_del_rcu(&dev->dev_list); 252 hlist_del_rcu(&dev->name_hlist); 253 hlist_del_rcu(&dev->index_hlist); 254 write_unlock_bh(&dev_base_lock); 255 256 dev_base_seq_inc(dev_net(dev)); 257 } 258 259 /* 260 * Our notifier list 261 */ 262 263 static RAW_NOTIFIER_HEAD(netdev_chain); 264 265 /* 266 * Device drivers call our routines to queue packets here. We empty the 267 * queue in the local softnet handler. 268 */ 269 270 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data); 271 EXPORT_PER_CPU_SYMBOL(softnet_data); 272 273 #ifdef CONFIG_LOCKDEP 274 /* 275 * register_netdevice() inits txq->_xmit_lock and sets lockdep class 276 * according to dev->type 277 */ 278 static const unsigned short netdev_lock_type[] = { 279 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25, 280 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET, 281 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM, 282 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, 283 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, 284 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, 285 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, 286 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, 287 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, 288 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, 289 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, 290 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, 291 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM, 292 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE, 293 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE}; 294 295 static const char *const netdev_lock_name[] = { 296 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", 297 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET", 298 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM", 299 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", 300 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", 301 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", 302 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", 303 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", 304 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", 305 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", 306 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", 307 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", 308 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM", 309 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE", 310 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"}; 311 312 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; 313 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; 314 315 static inline unsigned short netdev_lock_pos(unsigned short dev_type) 316 { 317 int i; 318 319 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++) 320 if (netdev_lock_type[i] == dev_type) 321 return i; 322 /* the last key is used by default */ 323 return ARRAY_SIZE(netdev_lock_type) - 1; 324 } 325 326 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 327 unsigned short dev_type) 328 { 329 int i; 330 331 i = netdev_lock_pos(dev_type); 332 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i], 333 netdev_lock_name[i]); 334 } 335 336 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 337 { 338 int i; 339 340 i = netdev_lock_pos(dev->type); 341 lockdep_set_class_and_name(&dev->addr_list_lock, 342 &netdev_addr_lock_key[i], 343 netdev_lock_name[i]); 344 } 345 #else 346 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 347 unsigned short dev_type) 348 { 349 } 350 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 351 { 352 } 353 #endif 354 355 /******************************************************************************* 356 * 357 * Protocol management and registration routines 358 * 359 *******************************************************************************/ 360 361 362 /* 363 * Add a protocol ID to the list. Now that the input handler is 364 * smarter we can dispense with all the messy stuff that used to be 365 * here. 366 * 367 * BEWARE!!! Protocol handlers, mangling input packets, 368 * MUST BE last in hash buckets and checking protocol handlers 369 * MUST start from promiscuous ptype_all chain in net_bh. 370 * It is true now, do not change it. 371 * Explanation follows: if protocol handler, mangling packet, will 372 * be the first on list, it is not able to sense, that packet 373 * is cloned and should be copied-on-write, so that it will 374 * change it and subsequent readers will get broken packet. 375 * --ANK (980803) 376 */ 377 378 static inline struct list_head *ptype_head(const struct packet_type *pt) 379 { 380 if (pt->type == htons(ETH_P_ALL)) 381 return pt->dev ? &pt->dev->ptype_all : &ptype_all; 382 else 383 return pt->dev ? &pt->dev->ptype_specific : 384 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; 385 } 386 387 /** 388 * dev_add_pack - add packet handler 389 * @pt: packet type declaration 390 * 391 * Add a protocol handler to the networking stack. The passed &packet_type 392 * is linked into kernel lists and may not be freed until it has been 393 * removed from the kernel lists. 394 * 395 * This call does not sleep therefore it can not 396 * guarantee all CPU's that are in middle of receiving packets 397 * will see the new packet type (until the next received packet). 398 */ 399 400 void dev_add_pack(struct packet_type *pt) 401 { 402 struct list_head *head = ptype_head(pt); 403 404 spin_lock(&ptype_lock); 405 list_add_rcu(&pt->list, head); 406 spin_unlock(&ptype_lock); 407 } 408 EXPORT_SYMBOL(dev_add_pack); 409 410 /** 411 * __dev_remove_pack - remove packet handler 412 * @pt: packet type declaration 413 * 414 * Remove a protocol handler that was previously added to the kernel 415 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 416 * from the kernel lists and can be freed or reused once this function 417 * returns. 418 * 419 * The packet type might still be in use by receivers 420 * and must not be freed until after all the CPU's have gone 421 * through a quiescent state. 422 */ 423 void __dev_remove_pack(struct packet_type *pt) 424 { 425 struct list_head *head = ptype_head(pt); 426 struct packet_type *pt1; 427 428 spin_lock(&ptype_lock); 429 430 list_for_each_entry(pt1, head, list) { 431 if (pt == pt1) { 432 list_del_rcu(&pt->list); 433 goto out; 434 } 435 } 436 437 pr_warn("dev_remove_pack: %p not found\n", pt); 438 out: 439 spin_unlock(&ptype_lock); 440 } 441 EXPORT_SYMBOL(__dev_remove_pack); 442 443 /** 444 * dev_remove_pack - remove packet handler 445 * @pt: packet type declaration 446 * 447 * Remove a protocol handler that was previously added to the kernel 448 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 449 * from the kernel lists and can be freed or reused once this function 450 * returns. 451 * 452 * This call sleeps to guarantee that no CPU is looking at the packet 453 * type after return. 454 */ 455 void dev_remove_pack(struct packet_type *pt) 456 { 457 __dev_remove_pack(pt); 458 459 synchronize_net(); 460 } 461 EXPORT_SYMBOL(dev_remove_pack); 462 463 464 /** 465 * dev_add_offload - register offload handlers 466 * @po: protocol offload declaration 467 * 468 * Add protocol offload handlers to the networking stack. The passed 469 * &proto_offload is linked into kernel lists and may not be freed until 470 * it has been removed from the kernel lists. 471 * 472 * This call does not sleep therefore it can not 473 * guarantee all CPU's that are in middle of receiving packets 474 * will see the new offload handlers (until the next received packet). 475 */ 476 void dev_add_offload(struct packet_offload *po) 477 { 478 struct packet_offload *elem; 479 480 spin_lock(&offload_lock); 481 list_for_each_entry(elem, &offload_base, list) { 482 if (po->priority < elem->priority) 483 break; 484 } 485 list_add_rcu(&po->list, elem->list.prev); 486 spin_unlock(&offload_lock); 487 } 488 EXPORT_SYMBOL(dev_add_offload); 489 490 /** 491 * __dev_remove_offload - remove offload handler 492 * @po: packet offload declaration 493 * 494 * Remove a protocol offload handler that was previously added to the 495 * kernel offload handlers by dev_add_offload(). The passed &offload_type 496 * is removed from the kernel lists and can be freed or reused once this 497 * function returns. 498 * 499 * The packet type might still be in use by receivers 500 * and must not be freed until after all the CPU's have gone 501 * through a quiescent state. 502 */ 503 static void __dev_remove_offload(struct packet_offload *po) 504 { 505 struct list_head *head = &offload_base; 506 struct packet_offload *po1; 507 508 spin_lock(&offload_lock); 509 510 list_for_each_entry(po1, head, list) { 511 if (po == po1) { 512 list_del_rcu(&po->list); 513 goto out; 514 } 515 } 516 517 pr_warn("dev_remove_offload: %p not found\n", po); 518 out: 519 spin_unlock(&offload_lock); 520 } 521 522 /** 523 * dev_remove_offload - remove packet offload handler 524 * @po: packet offload declaration 525 * 526 * Remove a packet offload handler that was previously added to the kernel 527 * offload handlers by dev_add_offload(). The passed &offload_type is 528 * removed from the kernel lists and can be freed or reused once this 529 * function returns. 530 * 531 * This call sleeps to guarantee that no CPU is looking at the packet 532 * type after return. 533 */ 534 void dev_remove_offload(struct packet_offload *po) 535 { 536 __dev_remove_offload(po); 537 538 synchronize_net(); 539 } 540 EXPORT_SYMBOL(dev_remove_offload); 541 542 /****************************************************************************** 543 * 544 * Device Boot-time Settings Routines 545 * 546 ******************************************************************************/ 547 548 /* Boot time configuration table */ 549 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX]; 550 551 /** 552 * netdev_boot_setup_add - add new setup entry 553 * @name: name of the device 554 * @map: configured settings for the device 555 * 556 * Adds new setup entry to the dev_boot_setup list. The function 557 * returns 0 on error and 1 on success. This is a generic routine to 558 * all netdevices. 559 */ 560 static int netdev_boot_setup_add(char *name, struct ifmap *map) 561 { 562 struct netdev_boot_setup *s; 563 int i; 564 565 s = dev_boot_setup; 566 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 567 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') { 568 memset(s[i].name, 0, sizeof(s[i].name)); 569 strlcpy(s[i].name, name, IFNAMSIZ); 570 memcpy(&s[i].map, map, sizeof(s[i].map)); 571 break; 572 } 573 } 574 575 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1; 576 } 577 578 /** 579 * netdev_boot_setup_check - check boot time settings 580 * @dev: the netdevice 581 * 582 * Check boot time settings for the device. 583 * The found settings are set for the device to be used 584 * later in the device probing. 585 * Returns 0 if no settings found, 1 if they are. 586 */ 587 int netdev_boot_setup_check(struct net_device *dev) 588 { 589 struct netdev_boot_setup *s = dev_boot_setup; 590 int i; 591 592 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 593 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' && 594 !strcmp(dev->name, s[i].name)) { 595 dev->irq = s[i].map.irq; 596 dev->base_addr = s[i].map.base_addr; 597 dev->mem_start = s[i].map.mem_start; 598 dev->mem_end = s[i].map.mem_end; 599 return 1; 600 } 601 } 602 return 0; 603 } 604 EXPORT_SYMBOL(netdev_boot_setup_check); 605 606 607 /** 608 * netdev_boot_base - get address from boot time settings 609 * @prefix: prefix for network device 610 * @unit: id for network device 611 * 612 * Check boot time settings for the base address of device. 613 * The found settings are set for the device to be used 614 * later in the device probing. 615 * Returns 0 if no settings found. 616 */ 617 unsigned long netdev_boot_base(const char *prefix, int unit) 618 { 619 const struct netdev_boot_setup *s = dev_boot_setup; 620 char name[IFNAMSIZ]; 621 int i; 622 623 sprintf(name, "%s%d", prefix, unit); 624 625 /* 626 * If device already registered then return base of 1 627 * to indicate not to probe for this interface 628 */ 629 if (__dev_get_by_name(&init_net, name)) 630 return 1; 631 632 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) 633 if (!strcmp(name, s[i].name)) 634 return s[i].map.base_addr; 635 return 0; 636 } 637 638 /* 639 * Saves at boot time configured settings for any netdevice. 640 */ 641 int __init netdev_boot_setup(char *str) 642 { 643 int ints[5]; 644 struct ifmap map; 645 646 str = get_options(str, ARRAY_SIZE(ints), ints); 647 if (!str || !*str) 648 return 0; 649 650 /* Save settings */ 651 memset(&map, 0, sizeof(map)); 652 if (ints[0] > 0) 653 map.irq = ints[1]; 654 if (ints[0] > 1) 655 map.base_addr = ints[2]; 656 if (ints[0] > 2) 657 map.mem_start = ints[3]; 658 if (ints[0] > 3) 659 map.mem_end = ints[4]; 660 661 /* Add new entry to the list */ 662 return netdev_boot_setup_add(str, &map); 663 } 664 665 __setup("netdev=", netdev_boot_setup); 666 667 /******************************************************************************* 668 * 669 * Device Interface Subroutines 670 * 671 *******************************************************************************/ 672 673 /** 674 * dev_get_iflink - get 'iflink' value of a interface 675 * @dev: targeted interface 676 * 677 * Indicates the ifindex the interface is linked to. 678 * Physical interfaces have the same 'ifindex' and 'iflink' values. 679 */ 680 681 int dev_get_iflink(const struct net_device *dev) 682 { 683 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) 684 return dev->netdev_ops->ndo_get_iflink(dev); 685 686 return dev->ifindex; 687 } 688 EXPORT_SYMBOL(dev_get_iflink); 689 690 /** 691 * dev_fill_metadata_dst - Retrieve tunnel egress information. 692 * @dev: targeted interface 693 * @skb: The packet. 694 * 695 * For better visibility of tunnel traffic OVS needs to retrieve 696 * egress tunnel information for a packet. Following API allows 697 * user to get this info. 698 */ 699 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb) 700 { 701 struct ip_tunnel_info *info; 702 703 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst) 704 return -EINVAL; 705 706 info = skb_tunnel_info_unclone(skb); 707 if (!info) 708 return -ENOMEM; 709 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX))) 710 return -EINVAL; 711 712 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb); 713 } 714 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst); 715 716 /** 717 * __dev_get_by_name - find a device by its name 718 * @net: the applicable net namespace 719 * @name: name to find 720 * 721 * Find an interface by name. Must be called under RTNL semaphore 722 * or @dev_base_lock. If the name is found a pointer to the device 723 * is returned. If the name is not found then %NULL is returned. The 724 * reference counters are not incremented so the caller must be 725 * careful with locks. 726 */ 727 728 struct net_device *__dev_get_by_name(struct net *net, const char *name) 729 { 730 struct net_device *dev; 731 struct hlist_head *head = dev_name_hash(net, name); 732 733 hlist_for_each_entry(dev, head, name_hlist) 734 if (!strncmp(dev->name, name, IFNAMSIZ)) 735 return dev; 736 737 return NULL; 738 } 739 EXPORT_SYMBOL(__dev_get_by_name); 740 741 /** 742 * dev_get_by_name_rcu - find a device by its name 743 * @net: the applicable net namespace 744 * @name: name to find 745 * 746 * Find an interface by name. 747 * If the name is found a pointer to the device is returned. 748 * If the name is not found then %NULL is returned. 749 * The reference counters are not incremented so the caller must be 750 * careful with locks. The caller must hold RCU lock. 751 */ 752 753 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name) 754 { 755 struct net_device *dev; 756 struct hlist_head *head = dev_name_hash(net, name); 757 758 hlist_for_each_entry_rcu(dev, head, name_hlist) 759 if (!strncmp(dev->name, name, IFNAMSIZ)) 760 return dev; 761 762 return NULL; 763 } 764 EXPORT_SYMBOL(dev_get_by_name_rcu); 765 766 /** 767 * dev_get_by_name - find a device by its name 768 * @net: the applicable net namespace 769 * @name: name to find 770 * 771 * Find an interface by name. This can be called from any 772 * context and does its own locking. The returned handle has 773 * the usage count incremented and the caller must use dev_put() to 774 * release it when it is no longer needed. %NULL is returned if no 775 * matching device is found. 776 */ 777 778 struct net_device *dev_get_by_name(struct net *net, const char *name) 779 { 780 struct net_device *dev; 781 782 rcu_read_lock(); 783 dev = dev_get_by_name_rcu(net, name); 784 if (dev) 785 dev_hold(dev); 786 rcu_read_unlock(); 787 return dev; 788 } 789 EXPORT_SYMBOL(dev_get_by_name); 790 791 /** 792 * __dev_get_by_index - find a device by its ifindex 793 * @net: the applicable net namespace 794 * @ifindex: index of device 795 * 796 * Search for an interface by index. Returns %NULL if the device 797 * is not found or a pointer to the device. The device has not 798 * had its reference counter increased so the caller must be careful 799 * about locking. The caller must hold either the RTNL semaphore 800 * or @dev_base_lock. 801 */ 802 803 struct net_device *__dev_get_by_index(struct net *net, int ifindex) 804 { 805 struct net_device *dev; 806 struct hlist_head *head = dev_index_hash(net, ifindex); 807 808 hlist_for_each_entry(dev, head, index_hlist) 809 if (dev->ifindex == ifindex) 810 return dev; 811 812 return NULL; 813 } 814 EXPORT_SYMBOL(__dev_get_by_index); 815 816 /** 817 * dev_get_by_index_rcu - find a device by its ifindex 818 * @net: the applicable net namespace 819 * @ifindex: index of device 820 * 821 * Search for an interface by index. Returns %NULL if the device 822 * is not found or a pointer to the device. The device has not 823 * had its reference counter increased so the caller must be careful 824 * about locking. The caller must hold RCU lock. 825 */ 826 827 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex) 828 { 829 struct net_device *dev; 830 struct hlist_head *head = dev_index_hash(net, ifindex); 831 832 hlist_for_each_entry_rcu(dev, head, index_hlist) 833 if (dev->ifindex == ifindex) 834 return dev; 835 836 return NULL; 837 } 838 EXPORT_SYMBOL(dev_get_by_index_rcu); 839 840 841 /** 842 * dev_get_by_index - find a device by its ifindex 843 * @net: the applicable net namespace 844 * @ifindex: index of device 845 * 846 * Search for an interface by index. Returns NULL if the device 847 * is not found or a pointer to the device. The device returned has 848 * had a reference added and the pointer is safe until the user calls 849 * dev_put to indicate they have finished with it. 850 */ 851 852 struct net_device *dev_get_by_index(struct net *net, int ifindex) 853 { 854 struct net_device *dev; 855 856 rcu_read_lock(); 857 dev = dev_get_by_index_rcu(net, ifindex); 858 if (dev) 859 dev_hold(dev); 860 rcu_read_unlock(); 861 return dev; 862 } 863 EXPORT_SYMBOL(dev_get_by_index); 864 865 /** 866 * netdev_get_name - get a netdevice name, knowing its ifindex. 867 * @net: network namespace 868 * @name: a pointer to the buffer where the name will be stored. 869 * @ifindex: the ifindex of the interface to get the name from. 870 * 871 * The use of raw_seqcount_begin() and cond_resched() before 872 * retrying is required as we want to give the writers a chance 873 * to complete when CONFIG_PREEMPT is not set. 874 */ 875 int netdev_get_name(struct net *net, char *name, int ifindex) 876 { 877 struct net_device *dev; 878 unsigned int seq; 879 880 retry: 881 seq = raw_seqcount_begin(&devnet_rename_seq); 882 rcu_read_lock(); 883 dev = dev_get_by_index_rcu(net, ifindex); 884 if (!dev) { 885 rcu_read_unlock(); 886 return -ENODEV; 887 } 888 889 strcpy(name, dev->name); 890 rcu_read_unlock(); 891 if (read_seqcount_retry(&devnet_rename_seq, seq)) { 892 cond_resched(); 893 goto retry; 894 } 895 896 return 0; 897 } 898 899 /** 900 * dev_getbyhwaddr_rcu - find a device by its hardware address 901 * @net: the applicable net namespace 902 * @type: media type of device 903 * @ha: hardware address 904 * 905 * Search for an interface by MAC address. Returns NULL if the device 906 * is not found or a pointer to the device. 907 * The caller must hold RCU or RTNL. 908 * The returned device has not had its ref count increased 909 * and the caller must therefore be careful about locking 910 * 911 */ 912 913 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type, 914 const char *ha) 915 { 916 struct net_device *dev; 917 918 for_each_netdev_rcu(net, dev) 919 if (dev->type == type && 920 !memcmp(dev->dev_addr, ha, dev->addr_len)) 921 return dev; 922 923 return NULL; 924 } 925 EXPORT_SYMBOL(dev_getbyhwaddr_rcu); 926 927 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type) 928 { 929 struct net_device *dev; 930 931 ASSERT_RTNL(); 932 for_each_netdev(net, dev) 933 if (dev->type == type) 934 return dev; 935 936 return NULL; 937 } 938 EXPORT_SYMBOL(__dev_getfirstbyhwtype); 939 940 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) 941 { 942 struct net_device *dev, *ret = NULL; 943 944 rcu_read_lock(); 945 for_each_netdev_rcu(net, dev) 946 if (dev->type == type) { 947 dev_hold(dev); 948 ret = dev; 949 break; 950 } 951 rcu_read_unlock(); 952 return ret; 953 } 954 EXPORT_SYMBOL(dev_getfirstbyhwtype); 955 956 /** 957 * __dev_get_by_flags - find any device with given flags 958 * @net: the applicable net namespace 959 * @if_flags: IFF_* values 960 * @mask: bitmask of bits in if_flags to check 961 * 962 * Search for any interface with the given flags. Returns NULL if a device 963 * is not found or a pointer to the device. Must be called inside 964 * rtnl_lock(), and result refcount is unchanged. 965 */ 966 967 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags, 968 unsigned short mask) 969 { 970 struct net_device *dev, *ret; 971 972 ASSERT_RTNL(); 973 974 ret = NULL; 975 for_each_netdev(net, dev) { 976 if (((dev->flags ^ if_flags) & mask) == 0) { 977 ret = dev; 978 break; 979 } 980 } 981 return ret; 982 } 983 EXPORT_SYMBOL(__dev_get_by_flags); 984 985 /** 986 * dev_valid_name - check if name is okay for network device 987 * @name: name string 988 * 989 * Network device names need to be valid file names to 990 * to allow sysfs to work. We also disallow any kind of 991 * whitespace. 992 */ 993 bool dev_valid_name(const char *name) 994 { 995 if (*name == '\0') 996 return false; 997 if (strlen(name) >= IFNAMSIZ) 998 return false; 999 if (!strcmp(name, ".") || !strcmp(name, "..")) 1000 return false; 1001 1002 while (*name) { 1003 if (*name == '/' || *name == ':' || isspace(*name)) 1004 return false; 1005 name++; 1006 } 1007 return true; 1008 } 1009 EXPORT_SYMBOL(dev_valid_name); 1010 1011 /** 1012 * __dev_alloc_name - allocate a name for a device 1013 * @net: network namespace to allocate the device name in 1014 * @name: name format string 1015 * @buf: scratch buffer and result name string 1016 * 1017 * Passed a format string - eg "lt%d" it will try and find a suitable 1018 * id. It scans list of devices to build up a free map, then chooses 1019 * the first empty slot. The caller must hold the dev_base or rtnl lock 1020 * while allocating the name and adding the device in order to avoid 1021 * duplicates. 1022 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1023 * Returns the number of the unit assigned or a negative errno code. 1024 */ 1025 1026 static int __dev_alloc_name(struct net *net, const char *name, char *buf) 1027 { 1028 int i = 0; 1029 const char *p; 1030 const int max_netdevices = 8*PAGE_SIZE; 1031 unsigned long *inuse; 1032 struct net_device *d; 1033 1034 p = strnchr(name, IFNAMSIZ-1, '%'); 1035 if (p) { 1036 /* 1037 * Verify the string as this thing may have come from 1038 * the user. There must be either one "%d" and no other "%" 1039 * characters. 1040 */ 1041 if (p[1] != 'd' || strchr(p + 2, '%')) 1042 return -EINVAL; 1043 1044 /* Use one page as a bit array of possible slots */ 1045 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC); 1046 if (!inuse) 1047 return -ENOMEM; 1048 1049 for_each_netdev(net, d) { 1050 if (!sscanf(d->name, name, &i)) 1051 continue; 1052 if (i < 0 || i >= max_netdevices) 1053 continue; 1054 1055 /* avoid cases where sscanf is not exact inverse of printf */ 1056 snprintf(buf, IFNAMSIZ, name, i); 1057 if (!strncmp(buf, d->name, IFNAMSIZ)) 1058 set_bit(i, inuse); 1059 } 1060 1061 i = find_first_zero_bit(inuse, max_netdevices); 1062 free_page((unsigned long) inuse); 1063 } 1064 1065 if (buf != name) 1066 snprintf(buf, IFNAMSIZ, name, i); 1067 if (!__dev_get_by_name(net, buf)) 1068 return i; 1069 1070 /* It is possible to run out of possible slots 1071 * when the name is long and there isn't enough space left 1072 * for the digits, or if all bits are used. 1073 */ 1074 return -ENFILE; 1075 } 1076 1077 /** 1078 * dev_alloc_name - allocate a name for a device 1079 * @dev: device 1080 * @name: name format string 1081 * 1082 * Passed a format string - eg "lt%d" it will try and find a suitable 1083 * id. It scans list of devices to build up a free map, then chooses 1084 * the first empty slot. The caller must hold the dev_base or rtnl lock 1085 * while allocating the name and adding the device in order to avoid 1086 * duplicates. 1087 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1088 * Returns the number of the unit assigned or a negative errno code. 1089 */ 1090 1091 int dev_alloc_name(struct net_device *dev, const char *name) 1092 { 1093 char buf[IFNAMSIZ]; 1094 struct net *net; 1095 int ret; 1096 1097 BUG_ON(!dev_net(dev)); 1098 net = dev_net(dev); 1099 ret = __dev_alloc_name(net, name, buf); 1100 if (ret >= 0) 1101 strlcpy(dev->name, buf, IFNAMSIZ); 1102 return ret; 1103 } 1104 EXPORT_SYMBOL(dev_alloc_name); 1105 1106 static int dev_alloc_name_ns(struct net *net, 1107 struct net_device *dev, 1108 const char *name) 1109 { 1110 char buf[IFNAMSIZ]; 1111 int ret; 1112 1113 ret = __dev_alloc_name(net, name, buf); 1114 if (ret >= 0) 1115 strlcpy(dev->name, buf, IFNAMSIZ); 1116 return ret; 1117 } 1118 1119 static int dev_get_valid_name(struct net *net, 1120 struct net_device *dev, 1121 const char *name) 1122 { 1123 BUG_ON(!net); 1124 1125 if (!dev_valid_name(name)) 1126 return -EINVAL; 1127 1128 if (strchr(name, '%')) 1129 return dev_alloc_name_ns(net, dev, name); 1130 else if (__dev_get_by_name(net, name)) 1131 return -EEXIST; 1132 else if (dev->name != name) 1133 strlcpy(dev->name, name, IFNAMSIZ); 1134 1135 return 0; 1136 } 1137 1138 /** 1139 * dev_change_name - change name of a device 1140 * @dev: device 1141 * @newname: name (or format string) must be at least IFNAMSIZ 1142 * 1143 * Change name of a device, can pass format strings "eth%d". 1144 * for wildcarding. 1145 */ 1146 int dev_change_name(struct net_device *dev, const char *newname) 1147 { 1148 unsigned char old_assign_type; 1149 char oldname[IFNAMSIZ]; 1150 int err = 0; 1151 int ret; 1152 struct net *net; 1153 1154 ASSERT_RTNL(); 1155 BUG_ON(!dev_net(dev)); 1156 1157 net = dev_net(dev); 1158 if (dev->flags & IFF_UP) 1159 return -EBUSY; 1160 1161 write_seqcount_begin(&devnet_rename_seq); 1162 1163 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) { 1164 write_seqcount_end(&devnet_rename_seq); 1165 return 0; 1166 } 1167 1168 memcpy(oldname, dev->name, IFNAMSIZ); 1169 1170 err = dev_get_valid_name(net, dev, newname); 1171 if (err < 0) { 1172 write_seqcount_end(&devnet_rename_seq); 1173 return err; 1174 } 1175 1176 if (oldname[0] && !strchr(oldname, '%')) 1177 netdev_info(dev, "renamed from %s\n", oldname); 1178 1179 old_assign_type = dev->name_assign_type; 1180 dev->name_assign_type = NET_NAME_RENAMED; 1181 1182 rollback: 1183 ret = device_rename(&dev->dev, dev->name); 1184 if (ret) { 1185 memcpy(dev->name, oldname, IFNAMSIZ); 1186 dev->name_assign_type = old_assign_type; 1187 write_seqcount_end(&devnet_rename_seq); 1188 return ret; 1189 } 1190 1191 write_seqcount_end(&devnet_rename_seq); 1192 1193 netdev_adjacent_rename_links(dev, oldname); 1194 1195 write_lock_bh(&dev_base_lock); 1196 hlist_del_rcu(&dev->name_hlist); 1197 write_unlock_bh(&dev_base_lock); 1198 1199 synchronize_rcu(); 1200 1201 write_lock_bh(&dev_base_lock); 1202 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); 1203 write_unlock_bh(&dev_base_lock); 1204 1205 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); 1206 ret = notifier_to_errno(ret); 1207 1208 if (ret) { 1209 /* err >= 0 after dev_alloc_name() or stores the first errno */ 1210 if (err >= 0) { 1211 err = ret; 1212 write_seqcount_begin(&devnet_rename_seq); 1213 memcpy(dev->name, oldname, IFNAMSIZ); 1214 memcpy(oldname, newname, IFNAMSIZ); 1215 dev->name_assign_type = old_assign_type; 1216 old_assign_type = NET_NAME_RENAMED; 1217 goto rollback; 1218 } else { 1219 pr_err("%s: name change rollback failed: %d\n", 1220 dev->name, ret); 1221 } 1222 } 1223 1224 return err; 1225 } 1226 1227 /** 1228 * dev_set_alias - change ifalias of a device 1229 * @dev: device 1230 * @alias: name up to IFALIASZ 1231 * @len: limit of bytes to copy from info 1232 * 1233 * Set ifalias for a device, 1234 */ 1235 int dev_set_alias(struct net_device *dev, const char *alias, size_t len) 1236 { 1237 char *new_ifalias; 1238 1239 ASSERT_RTNL(); 1240 1241 if (len >= IFALIASZ) 1242 return -EINVAL; 1243 1244 if (!len) { 1245 kfree(dev->ifalias); 1246 dev->ifalias = NULL; 1247 return 0; 1248 } 1249 1250 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL); 1251 if (!new_ifalias) 1252 return -ENOMEM; 1253 dev->ifalias = new_ifalias; 1254 1255 strlcpy(dev->ifalias, alias, len+1); 1256 return len; 1257 } 1258 1259 1260 /** 1261 * netdev_features_change - device changes features 1262 * @dev: device to cause notification 1263 * 1264 * Called to indicate a device has changed features. 1265 */ 1266 void netdev_features_change(struct net_device *dev) 1267 { 1268 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); 1269 } 1270 EXPORT_SYMBOL(netdev_features_change); 1271 1272 /** 1273 * netdev_state_change - device changes state 1274 * @dev: device to cause notification 1275 * 1276 * Called to indicate a device has changed state. This function calls 1277 * the notifier chains for netdev_chain and sends a NEWLINK message 1278 * to the routing socket. 1279 */ 1280 void netdev_state_change(struct net_device *dev) 1281 { 1282 if (dev->flags & IFF_UP) { 1283 struct netdev_notifier_change_info change_info; 1284 1285 change_info.flags_changed = 0; 1286 call_netdevice_notifiers_info(NETDEV_CHANGE, dev, 1287 &change_info.info); 1288 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL); 1289 } 1290 } 1291 EXPORT_SYMBOL(netdev_state_change); 1292 1293 /** 1294 * netdev_notify_peers - notify network peers about existence of @dev 1295 * @dev: network device 1296 * 1297 * Generate traffic such that interested network peers are aware of 1298 * @dev, such as by generating a gratuitous ARP. This may be used when 1299 * a device wants to inform the rest of the network about some sort of 1300 * reconfiguration such as a failover event or virtual machine 1301 * migration. 1302 */ 1303 void netdev_notify_peers(struct net_device *dev) 1304 { 1305 rtnl_lock(); 1306 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev); 1307 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev); 1308 rtnl_unlock(); 1309 } 1310 EXPORT_SYMBOL(netdev_notify_peers); 1311 1312 static int __dev_open(struct net_device *dev) 1313 { 1314 const struct net_device_ops *ops = dev->netdev_ops; 1315 int ret; 1316 1317 ASSERT_RTNL(); 1318 1319 if (!netif_device_present(dev)) 1320 return -ENODEV; 1321 1322 /* Block netpoll from trying to do any rx path servicing. 1323 * If we don't do this there is a chance ndo_poll_controller 1324 * or ndo_poll may be running while we open the device 1325 */ 1326 netpoll_poll_disable(dev); 1327 1328 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev); 1329 ret = notifier_to_errno(ret); 1330 if (ret) 1331 return ret; 1332 1333 set_bit(__LINK_STATE_START, &dev->state); 1334 1335 if (ops->ndo_validate_addr) 1336 ret = ops->ndo_validate_addr(dev); 1337 1338 if (!ret && ops->ndo_open) 1339 ret = ops->ndo_open(dev); 1340 1341 netpoll_poll_enable(dev); 1342 1343 if (ret) 1344 clear_bit(__LINK_STATE_START, &dev->state); 1345 else { 1346 dev->flags |= IFF_UP; 1347 dev_set_rx_mode(dev); 1348 dev_activate(dev); 1349 add_device_randomness(dev->dev_addr, dev->addr_len); 1350 } 1351 1352 return ret; 1353 } 1354 1355 /** 1356 * dev_open - prepare an interface for use. 1357 * @dev: device to open 1358 * 1359 * Takes a device from down to up state. The device's private open 1360 * function is invoked and then the multicast lists are loaded. Finally 1361 * the device is moved into the up state and a %NETDEV_UP message is 1362 * sent to the netdev notifier chain. 1363 * 1364 * Calling this function on an active interface is a nop. On a failure 1365 * a negative errno code is returned. 1366 */ 1367 int dev_open(struct net_device *dev) 1368 { 1369 int ret; 1370 1371 if (dev->flags & IFF_UP) 1372 return 0; 1373 1374 ret = __dev_open(dev); 1375 if (ret < 0) 1376 return ret; 1377 1378 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); 1379 call_netdevice_notifiers(NETDEV_UP, dev); 1380 1381 return ret; 1382 } 1383 EXPORT_SYMBOL(dev_open); 1384 1385 static int __dev_close_many(struct list_head *head) 1386 { 1387 struct net_device *dev; 1388 1389 ASSERT_RTNL(); 1390 might_sleep(); 1391 1392 list_for_each_entry(dev, head, close_list) { 1393 /* Temporarily disable netpoll until the interface is down */ 1394 netpoll_poll_disable(dev); 1395 1396 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); 1397 1398 clear_bit(__LINK_STATE_START, &dev->state); 1399 1400 /* Synchronize to scheduled poll. We cannot touch poll list, it 1401 * can be even on different cpu. So just clear netif_running(). 1402 * 1403 * dev->stop() will invoke napi_disable() on all of it's 1404 * napi_struct instances on this device. 1405 */ 1406 smp_mb__after_atomic(); /* Commit netif_running(). */ 1407 } 1408 1409 dev_deactivate_many(head); 1410 1411 list_for_each_entry(dev, head, close_list) { 1412 const struct net_device_ops *ops = dev->netdev_ops; 1413 1414 /* 1415 * Call the device specific close. This cannot fail. 1416 * Only if device is UP 1417 * 1418 * We allow it to be called even after a DETACH hot-plug 1419 * event. 1420 */ 1421 if (ops->ndo_stop) 1422 ops->ndo_stop(dev); 1423 1424 dev->flags &= ~IFF_UP; 1425 netpoll_poll_enable(dev); 1426 } 1427 1428 return 0; 1429 } 1430 1431 static int __dev_close(struct net_device *dev) 1432 { 1433 int retval; 1434 LIST_HEAD(single); 1435 1436 list_add(&dev->close_list, &single); 1437 retval = __dev_close_many(&single); 1438 list_del(&single); 1439 1440 return retval; 1441 } 1442 1443 int dev_close_many(struct list_head *head, bool unlink) 1444 { 1445 struct net_device *dev, *tmp; 1446 1447 /* Remove the devices that don't need to be closed */ 1448 list_for_each_entry_safe(dev, tmp, head, close_list) 1449 if (!(dev->flags & IFF_UP)) 1450 list_del_init(&dev->close_list); 1451 1452 __dev_close_many(head); 1453 1454 list_for_each_entry_safe(dev, tmp, head, close_list) { 1455 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); 1456 call_netdevice_notifiers(NETDEV_DOWN, dev); 1457 if (unlink) 1458 list_del_init(&dev->close_list); 1459 } 1460 1461 return 0; 1462 } 1463 EXPORT_SYMBOL(dev_close_many); 1464 1465 /** 1466 * dev_close - shutdown an interface. 1467 * @dev: device to shutdown 1468 * 1469 * This function moves an active device into down state. A 1470 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device 1471 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier 1472 * chain. 1473 */ 1474 int dev_close(struct net_device *dev) 1475 { 1476 if (dev->flags & IFF_UP) { 1477 LIST_HEAD(single); 1478 1479 list_add(&dev->close_list, &single); 1480 dev_close_many(&single, true); 1481 list_del(&single); 1482 } 1483 return 0; 1484 } 1485 EXPORT_SYMBOL(dev_close); 1486 1487 1488 /** 1489 * dev_disable_lro - disable Large Receive Offload on a device 1490 * @dev: device 1491 * 1492 * Disable Large Receive Offload (LRO) on a net device. Must be 1493 * called under RTNL. This is needed if received packets may be 1494 * forwarded to another interface. 1495 */ 1496 void dev_disable_lro(struct net_device *dev) 1497 { 1498 struct net_device *lower_dev; 1499 struct list_head *iter; 1500 1501 dev->wanted_features &= ~NETIF_F_LRO; 1502 netdev_update_features(dev); 1503 1504 if (unlikely(dev->features & NETIF_F_LRO)) 1505 netdev_WARN(dev, "failed to disable LRO!\n"); 1506 1507 netdev_for_each_lower_dev(dev, lower_dev, iter) 1508 dev_disable_lro(lower_dev); 1509 } 1510 EXPORT_SYMBOL(dev_disable_lro); 1511 1512 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val, 1513 struct net_device *dev) 1514 { 1515 struct netdev_notifier_info info; 1516 1517 netdev_notifier_info_init(&info, dev); 1518 return nb->notifier_call(nb, val, &info); 1519 } 1520 1521 static int dev_boot_phase = 1; 1522 1523 /** 1524 * register_netdevice_notifier - register a network notifier block 1525 * @nb: notifier 1526 * 1527 * Register a notifier to be called when network device events occur. 1528 * The notifier passed is linked into the kernel structures and must 1529 * not be reused until it has been unregistered. A negative errno code 1530 * is returned on a failure. 1531 * 1532 * When registered all registration and up events are replayed 1533 * to the new notifier to allow device to have a race free 1534 * view of the network device list. 1535 */ 1536 1537 int register_netdevice_notifier(struct notifier_block *nb) 1538 { 1539 struct net_device *dev; 1540 struct net_device *last; 1541 struct net *net; 1542 int err; 1543 1544 rtnl_lock(); 1545 err = raw_notifier_chain_register(&netdev_chain, nb); 1546 if (err) 1547 goto unlock; 1548 if (dev_boot_phase) 1549 goto unlock; 1550 for_each_net(net) { 1551 for_each_netdev(net, dev) { 1552 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev); 1553 err = notifier_to_errno(err); 1554 if (err) 1555 goto rollback; 1556 1557 if (!(dev->flags & IFF_UP)) 1558 continue; 1559 1560 call_netdevice_notifier(nb, NETDEV_UP, dev); 1561 } 1562 } 1563 1564 unlock: 1565 rtnl_unlock(); 1566 return err; 1567 1568 rollback: 1569 last = dev; 1570 for_each_net(net) { 1571 for_each_netdev(net, dev) { 1572 if (dev == last) 1573 goto outroll; 1574 1575 if (dev->flags & IFF_UP) { 1576 call_netdevice_notifier(nb, NETDEV_GOING_DOWN, 1577 dev); 1578 call_netdevice_notifier(nb, NETDEV_DOWN, dev); 1579 } 1580 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); 1581 } 1582 } 1583 1584 outroll: 1585 raw_notifier_chain_unregister(&netdev_chain, nb); 1586 goto unlock; 1587 } 1588 EXPORT_SYMBOL(register_netdevice_notifier); 1589 1590 /** 1591 * unregister_netdevice_notifier - unregister a network notifier block 1592 * @nb: notifier 1593 * 1594 * Unregister a notifier previously registered by 1595 * register_netdevice_notifier(). The notifier is unlinked into the 1596 * kernel structures and may then be reused. A negative errno code 1597 * is returned on a failure. 1598 * 1599 * After unregistering unregister and down device events are synthesized 1600 * for all devices on the device list to the removed notifier to remove 1601 * the need for special case cleanup code. 1602 */ 1603 1604 int unregister_netdevice_notifier(struct notifier_block *nb) 1605 { 1606 struct net_device *dev; 1607 struct net *net; 1608 int err; 1609 1610 rtnl_lock(); 1611 err = raw_notifier_chain_unregister(&netdev_chain, nb); 1612 if (err) 1613 goto unlock; 1614 1615 for_each_net(net) { 1616 for_each_netdev(net, dev) { 1617 if (dev->flags & IFF_UP) { 1618 call_netdevice_notifier(nb, NETDEV_GOING_DOWN, 1619 dev); 1620 call_netdevice_notifier(nb, NETDEV_DOWN, dev); 1621 } 1622 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); 1623 } 1624 } 1625 unlock: 1626 rtnl_unlock(); 1627 return err; 1628 } 1629 EXPORT_SYMBOL(unregister_netdevice_notifier); 1630 1631 /** 1632 * call_netdevice_notifiers_info - call all network notifier blocks 1633 * @val: value passed unmodified to notifier function 1634 * @dev: net_device pointer passed unmodified to notifier function 1635 * @info: notifier information data 1636 * 1637 * Call all network notifier blocks. Parameters and return value 1638 * are as for raw_notifier_call_chain(). 1639 */ 1640 1641 static int call_netdevice_notifiers_info(unsigned long val, 1642 struct net_device *dev, 1643 struct netdev_notifier_info *info) 1644 { 1645 ASSERT_RTNL(); 1646 netdev_notifier_info_init(info, dev); 1647 return raw_notifier_call_chain(&netdev_chain, val, info); 1648 } 1649 1650 /** 1651 * call_netdevice_notifiers - call all network notifier blocks 1652 * @val: value passed unmodified to notifier function 1653 * @dev: net_device pointer passed unmodified to notifier function 1654 * 1655 * Call all network notifier blocks. Parameters and return value 1656 * are as for raw_notifier_call_chain(). 1657 */ 1658 1659 int call_netdevice_notifiers(unsigned long val, struct net_device *dev) 1660 { 1661 struct netdev_notifier_info info; 1662 1663 return call_netdevice_notifiers_info(val, dev, &info); 1664 } 1665 EXPORT_SYMBOL(call_netdevice_notifiers); 1666 1667 #ifdef CONFIG_NET_INGRESS 1668 static struct static_key ingress_needed __read_mostly; 1669 1670 void net_inc_ingress_queue(void) 1671 { 1672 static_key_slow_inc(&ingress_needed); 1673 } 1674 EXPORT_SYMBOL_GPL(net_inc_ingress_queue); 1675 1676 void net_dec_ingress_queue(void) 1677 { 1678 static_key_slow_dec(&ingress_needed); 1679 } 1680 EXPORT_SYMBOL_GPL(net_dec_ingress_queue); 1681 #endif 1682 1683 #ifdef CONFIG_NET_EGRESS 1684 static struct static_key egress_needed __read_mostly; 1685 1686 void net_inc_egress_queue(void) 1687 { 1688 static_key_slow_inc(&egress_needed); 1689 } 1690 EXPORT_SYMBOL_GPL(net_inc_egress_queue); 1691 1692 void net_dec_egress_queue(void) 1693 { 1694 static_key_slow_dec(&egress_needed); 1695 } 1696 EXPORT_SYMBOL_GPL(net_dec_egress_queue); 1697 #endif 1698 1699 static struct static_key netstamp_needed __read_mostly; 1700 #ifdef HAVE_JUMP_LABEL 1701 static atomic_t netstamp_needed_deferred; 1702 static atomic_t netstamp_wanted; 1703 static void netstamp_clear(struct work_struct *work) 1704 { 1705 int deferred = atomic_xchg(&netstamp_needed_deferred, 0); 1706 int wanted; 1707 1708 wanted = atomic_add_return(deferred, &netstamp_wanted); 1709 if (wanted > 0) 1710 static_key_enable(&netstamp_needed); 1711 else 1712 static_key_disable(&netstamp_needed); 1713 } 1714 static DECLARE_WORK(netstamp_work, netstamp_clear); 1715 #endif 1716 1717 void net_enable_timestamp(void) 1718 { 1719 #ifdef HAVE_JUMP_LABEL 1720 int wanted; 1721 1722 while (1) { 1723 wanted = atomic_read(&netstamp_wanted); 1724 if (wanted <= 0) 1725 break; 1726 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted) 1727 return; 1728 } 1729 atomic_inc(&netstamp_needed_deferred); 1730 schedule_work(&netstamp_work); 1731 #else 1732 static_key_slow_inc(&netstamp_needed); 1733 #endif 1734 } 1735 EXPORT_SYMBOL(net_enable_timestamp); 1736 1737 void net_disable_timestamp(void) 1738 { 1739 #ifdef HAVE_JUMP_LABEL 1740 int wanted; 1741 1742 while (1) { 1743 wanted = atomic_read(&netstamp_wanted); 1744 if (wanted <= 1) 1745 break; 1746 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted) 1747 return; 1748 } 1749 atomic_dec(&netstamp_needed_deferred); 1750 schedule_work(&netstamp_work); 1751 #else 1752 static_key_slow_dec(&netstamp_needed); 1753 #endif 1754 } 1755 EXPORT_SYMBOL(net_disable_timestamp); 1756 1757 static inline void net_timestamp_set(struct sk_buff *skb) 1758 { 1759 skb->tstamp = 0; 1760 if (static_key_false(&netstamp_needed)) 1761 __net_timestamp(skb); 1762 } 1763 1764 #define net_timestamp_check(COND, SKB) \ 1765 if (static_key_false(&netstamp_needed)) { \ 1766 if ((COND) && !(SKB)->tstamp) \ 1767 __net_timestamp(SKB); \ 1768 } \ 1769 1770 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb) 1771 { 1772 unsigned int len; 1773 1774 if (!(dev->flags & IFF_UP)) 1775 return false; 1776 1777 len = dev->mtu + dev->hard_header_len + VLAN_HLEN; 1778 if (skb->len <= len) 1779 return true; 1780 1781 /* if TSO is enabled, we don't care about the length as the packet 1782 * could be forwarded without being segmented before 1783 */ 1784 if (skb_is_gso(skb)) 1785 return true; 1786 1787 return false; 1788 } 1789 EXPORT_SYMBOL_GPL(is_skb_forwardable); 1790 1791 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 1792 { 1793 int ret = ____dev_forward_skb(dev, skb); 1794 1795 if (likely(!ret)) { 1796 skb->protocol = eth_type_trans(skb, dev); 1797 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); 1798 } 1799 1800 return ret; 1801 } 1802 EXPORT_SYMBOL_GPL(__dev_forward_skb); 1803 1804 /** 1805 * dev_forward_skb - loopback an skb to another netif 1806 * 1807 * @dev: destination network device 1808 * @skb: buffer to forward 1809 * 1810 * return values: 1811 * NET_RX_SUCCESS (no congestion) 1812 * NET_RX_DROP (packet was dropped, but freed) 1813 * 1814 * dev_forward_skb can be used for injecting an skb from the 1815 * start_xmit function of one device into the receive queue 1816 * of another device. 1817 * 1818 * The receiving device may be in another namespace, so 1819 * we have to clear all information in the skb that could 1820 * impact namespace isolation. 1821 */ 1822 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 1823 { 1824 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb); 1825 } 1826 EXPORT_SYMBOL_GPL(dev_forward_skb); 1827 1828 static inline int deliver_skb(struct sk_buff *skb, 1829 struct packet_type *pt_prev, 1830 struct net_device *orig_dev) 1831 { 1832 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) 1833 return -ENOMEM; 1834 atomic_inc(&skb->users); 1835 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 1836 } 1837 1838 static inline void deliver_ptype_list_skb(struct sk_buff *skb, 1839 struct packet_type **pt, 1840 struct net_device *orig_dev, 1841 __be16 type, 1842 struct list_head *ptype_list) 1843 { 1844 struct packet_type *ptype, *pt_prev = *pt; 1845 1846 list_for_each_entry_rcu(ptype, ptype_list, list) { 1847 if (ptype->type != type) 1848 continue; 1849 if (pt_prev) 1850 deliver_skb(skb, pt_prev, orig_dev); 1851 pt_prev = ptype; 1852 } 1853 *pt = pt_prev; 1854 } 1855 1856 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) 1857 { 1858 if (!ptype->af_packet_priv || !skb->sk) 1859 return false; 1860 1861 if (ptype->id_match) 1862 return ptype->id_match(ptype, skb->sk); 1863 else if ((struct sock *)ptype->af_packet_priv == skb->sk) 1864 return true; 1865 1866 return false; 1867 } 1868 1869 /* 1870 * Support routine. Sends outgoing frames to any network 1871 * taps currently in use. 1872 */ 1873 1874 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) 1875 { 1876 struct packet_type *ptype; 1877 struct sk_buff *skb2 = NULL; 1878 struct packet_type *pt_prev = NULL; 1879 struct list_head *ptype_list = &ptype_all; 1880 1881 rcu_read_lock(); 1882 again: 1883 list_for_each_entry_rcu(ptype, ptype_list, list) { 1884 /* Never send packets back to the socket 1885 * they originated from - MvS (miquels@drinkel.ow.org) 1886 */ 1887 if (skb_loop_sk(ptype, skb)) 1888 continue; 1889 1890 if (pt_prev) { 1891 deliver_skb(skb2, pt_prev, skb->dev); 1892 pt_prev = ptype; 1893 continue; 1894 } 1895 1896 /* need to clone skb, done only once */ 1897 skb2 = skb_clone(skb, GFP_ATOMIC); 1898 if (!skb2) 1899 goto out_unlock; 1900 1901 net_timestamp_set(skb2); 1902 1903 /* skb->nh should be correctly 1904 * set by sender, so that the second statement is 1905 * just protection against buggy protocols. 1906 */ 1907 skb_reset_mac_header(skb2); 1908 1909 if (skb_network_header(skb2) < skb2->data || 1910 skb_network_header(skb2) > skb_tail_pointer(skb2)) { 1911 net_crit_ratelimited("protocol %04x is buggy, dev %s\n", 1912 ntohs(skb2->protocol), 1913 dev->name); 1914 skb_reset_network_header(skb2); 1915 } 1916 1917 skb2->transport_header = skb2->network_header; 1918 skb2->pkt_type = PACKET_OUTGOING; 1919 pt_prev = ptype; 1920 } 1921 1922 if (ptype_list == &ptype_all) { 1923 ptype_list = &dev->ptype_all; 1924 goto again; 1925 } 1926 out_unlock: 1927 if (pt_prev) 1928 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); 1929 rcu_read_unlock(); 1930 } 1931 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit); 1932 1933 /** 1934 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change 1935 * @dev: Network device 1936 * @txq: number of queues available 1937 * 1938 * If real_num_tx_queues is changed the tc mappings may no longer be 1939 * valid. To resolve this verify the tc mapping remains valid and if 1940 * not NULL the mapping. With no priorities mapping to this 1941 * offset/count pair it will no longer be used. In the worst case TC0 1942 * is invalid nothing can be done so disable priority mappings. If is 1943 * expected that drivers will fix this mapping if they can before 1944 * calling netif_set_real_num_tx_queues. 1945 */ 1946 static void netif_setup_tc(struct net_device *dev, unsigned int txq) 1947 { 1948 int i; 1949 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 1950 1951 /* If TC0 is invalidated disable TC mapping */ 1952 if (tc->offset + tc->count > txq) { 1953 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n"); 1954 dev->num_tc = 0; 1955 return; 1956 } 1957 1958 /* Invalidated prio to tc mappings set to TC0 */ 1959 for (i = 1; i < TC_BITMASK + 1; i++) { 1960 int q = netdev_get_prio_tc_map(dev, i); 1961 1962 tc = &dev->tc_to_txq[q]; 1963 if (tc->offset + tc->count > txq) { 1964 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n", 1965 i, q); 1966 netdev_set_prio_tc_map(dev, i, 0); 1967 } 1968 } 1969 } 1970 1971 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq) 1972 { 1973 if (dev->num_tc) { 1974 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 1975 int i; 1976 1977 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) { 1978 if ((txq - tc->offset) < tc->count) 1979 return i; 1980 } 1981 1982 return -1; 1983 } 1984 1985 return 0; 1986 } 1987 1988 #ifdef CONFIG_XPS 1989 static DEFINE_MUTEX(xps_map_mutex); 1990 #define xmap_dereference(P) \ 1991 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex)) 1992 1993 static bool remove_xps_queue(struct xps_dev_maps *dev_maps, 1994 int tci, u16 index) 1995 { 1996 struct xps_map *map = NULL; 1997 int pos; 1998 1999 if (dev_maps) 2000 map = xmap_dereference(dev_maps->cpu_map[tci]); 2001 if (!map) 2002 return false; 2003 2004 for (pos = map->len; pos--;) { 2005 if (map->queues[pos] != index) 2006 continue; 2007 2008 if (map->len > 1) { 2009 map->queues[pos] = map->queues[--map->len]; 2010 break; 2011 } 2012 2013 RCU_INIT_POINTER(dev_maps->cpu_map[tci], NULL); 2014 kfree_rcu(map, rcu); 2015 return false; 2016 } 2017 2018 return true; 2019 } 2020 2021 static bool remove_xps_queue_cpu(struct net_device *dev, 2022 struct xps_dev_maps *dev_maps, 2023 int cpu, u16 offset, u16 count) 2024 { 2025 int num_tc = dev->num_tc ? : 1; 2026 bool active = false; 2027 int tci; 2028 2029 for (tci = cpu * num_tc; num_tc--; tci++) { 2030 int i, j; 2031 2032 for (i = count, j = offset; i--; j++) { 2033 if (!remove_xps_queue(dev_maps, cpu, j)) 2034 break; 2035 } 2036 2037 active |= i < 0; 2038 } 2039 2040 return active; 2041 } 2042 2043 static void netif_reset_xps_queues(struct net_device *dev, u16 offset, 2044 u16 count) 2045 { 2046 struct xps_dev_maps *dev_maps; 2047 int cpu, i; 2048 bool active = false; 2049 2050 mutex_lock(&xps_map_mutex); 2051 dev_maps = xmap_dereference(dev->xps_maps); 2052 2053 if (!dev_maps) 2054 goto out_no_maps; 2055 2056 for_each_possible_cpu(cpu) 2057 active |= remove_xps_queue_cpu(dev, dev_maps, cpu, 2058 offset, count); 2059 2060 if (!active) { 2061 RCU_INIT_POINTER(dev->xps_maps, NULL); 2062 kfree_rcu(dev_maps, rcu); 2063 } 2064 2065 for (i = offset + (count - 1); count--; i--) 2066 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i), 2067 NUMA_NO_NODE); 2068 2069 out_no_maps: 2070 mutex_unlock(&xps_map_mutex); 2071 } 2072 2073 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index) 2074 { 2075 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index); 2076 } 2077 2078 static struct xps_map *expand_xps_map(struct xps_map *map, 2079 int cpu, u16 index) 2080 { 2081 struct xps_map *new_map; 2082 int alloc_len = XPS_MIN_MAP_ALLOC; 2083 int i, pos; 2084 2085 for (pos = 0; map && pos < map->len; pos++) { 2086 if (map->queues[pos] != index) 2087 continue; 2088 return map; 2089 } 2090 2091 /* Need to add queue to this CPU's existing map */ 2092 if (map) { 2093 if (pos < map->alloc_len) 2094 return map; 2095 2096 alloc_len = map->alloc_len * 2; 2097 } 2098 2099 /* Need to allocate new map to store queue on this CPU's map */ 2100 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL, 2101 cpu_to_node(cpu)); 2102 if (!new_map) 2103 return NULL; 2104 2105 for (i = 0; i < pos; i++) 2106 new_map->queues[i] = map->queues[i]; 2107 new_map->alloc_len = alloc_len; 2108 new_map->len = pos; 2109 2110 return new_map; 2111 } 2112 2113 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask, 2114 u16 index) 2115 { 2116 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL; 2117 int i, cpu, tci, numa_node_id = -2; 2118 int maps_sz, num_tc = 1, tc = 0; 2119 struct xps_map *map, *new_map; 2120 bool active = false; 2121 2122 if (dev->num_tc) { 2123 num_tc = dev->num_tc; 2124 tc = netdev_txq_to_tc(dev, index); 2125 if (tc < 0) 2126 return -EINVAL; 2127 } 2128 2129 maps_sz = XPS_DEV_MAPS_SIZE(num_tc); 2130 if (maps_sz < L1_CACHE_BYTES) 2131 maps_sz = L1_CACHE_BYTES; 2132 2133 mutex_lock(&xps_map_mutex); 2134 2135 dev_maps = xmap_dereference(dev->xps_maps); 2136 2137 /* allocate memory for queue storage */ 2138 for_each_cpu_and(cpu, cpu_online_mask, mask) { 2139 if (!new_dev_maps) 2140 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL); 2141 if (!new_dev_maps) { 2142 mutex_unlock(&xps_map_mutex); 2143 return -ENOMEM; 2144 } 2145 2146 tci = cpu * num_tc + tc; 2147 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[tci]) : 2148 NULL; 2149 2150 map = expand_xps_map(map, cpu, index); 2151 if (!map) 2152 goto error; 2153 2154 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map); 2155 } 2156 2157 if (!new_dev_maps) 2158 goto out_no_new_maps; 2159 2160 for_each_possible_cpu(cpu) { 2161 /* copy maps belonging to foreign traffic classes */ 2162 for (i = tc, tci = cpu * num_tc; dev_maps && i--; tci++) { 2163 /* fill in the new device map from the old device map */ 2164 map = xmap_dereference(dev_maps->cpu_map[tci]); 2165 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map); 2166 } 2167 2168 /* We need to explicitly update tci as prevous loop 2169 * could break out early if dev_maps is NULL. 2170 */ 2171 tci = cpu * num_tc + tc; 2172 2173 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) { 2174 /* add queue to CPU maps */ 2175 int pos = 0; 2176 2177 map = xmap_dereference(new_dev_maps->cpu_map[tci]); 2178 while ((pos < map->len) && (map->queues[pos] != index)) 2179 pos++; 2180 2181 if (pos == map->len) 2182 map->queues[map->len++] = index; 2183 #ifdef CONFIG_NUMA 2184 if (numa_node_id == -2) 2185 numa_node_id = cpu_to_node(cpu); 2186 else if (numa_node_id != cpu_to_node(cpu)) 2187 numa_node_id = -1; 2188 #endif 2189 } else if (dev_maps) { 2190 /* fill in the new device map from the old device map */ 2191 map = xmap_dereference(dev_maps->cpu_map[tci]); 2192 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map); 2193 } 2194 2195 /* copy maps belonging to foreign traffic classes */ 2196 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) { 2197 /* fill in the new device map from the old device map */ 2198 map = xmap_dereference(dev_maps->cpu_map[tci]); 2199 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map); 2200 } 2201 } 2202 2203 rcu_assign_pointer(dev->xps_maps, new_dev_maps); 2204 2205 /* Cleanup old maps */ 2206 if (!dev_maps) 2207 goto out_no_old_maps; 2208 2209 for_each_possible_cpu(cpu) { 2210 for (i = num_tc, tci = cpu * num_tc; i--; tci++) { 2211 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]); 2212 map = xmap_dereference(dev_maps->cpu_map[tci]); 2213 if (map && map != new_map) 2214 kfree_rcu(map, rcu); 2215 } 2216 } 2217 2218 kfree_rcu(dev_maps, rcu); 2219 2220 out_no_old_maps: 2221 dev_maps = new_dev_maps; 2222 active = true; 2223 2224 out_no_new_maps: 2225 /* update Tx queue numa node */ 2226 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index), 2227 (numa_node_id >= 0) ? numa_node_id : 2228 NUMA_NO_NODE); 2229 2230 if (!dev_maps) 2231 goto out_no_maps; 2232 2233 /* removes queue from unused CPUs */ 2234 for_each_possible_cpu(cpu) { 2235 for (i = tc, tci = cpu * num_tc; i--; tci++) 2236 active |= remove_xps_queue(dev_maps, tci, index); 2237 if (!cpumask_test_cpu(cpu, mask) || !cpu_online(cpu)) 2238 active |= remove_xps_queue(dev_maps, tci, index); 2239 for (i = num_tc - tc, tci++; --i; tci++) 2240 active |= remove_xps_queue(dev_maps, tci, index); 2241 } 2242 2243 /* free map if not active */ 2244 if (!active) { 2245 RCU_INIT_POINTER(dev->xps_maps, NULL); 2246 kfree_rcu(dev_maps, rcu); 2247 } 2248 2249 out_no_maps: 2250 mutex_unlock(&xps_map_mutex); 2251 2252 return 0; 2253 error: 2254 /* remove any maps that we added */ 2255 for_each_possible_cpu(cpu) { 2256 for (i = num_tc, tci = cpu * num_tc; i--; tci++) { 2257 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]); 2258 map = dev_maps ? 2259 xmap_dereference(dev_maps->cpu_map[tci]) : 2260 NULL; 2261 if (new_map && new_map != map) 2262 kfree(new_map); 2263 } 2264 } 2265 2266 mutex_unlock(&xps_map_mutex); 2267 2268 kfree(new_dev_maps); 2269 return -ENOMEM; 2270 } 2271 EXPORT_SYMBOL(netif_set_xps_queue); 2272 2273 #endif 2274 void netdev_reset_tc(struct net_device *dev) 2275 { 2276 #ifdef CONFIG_XPS 2277 netif_reset_xps_queues_gt(dev, 0); 2278 #endif 2279 dev->num_tc = 0; 2280 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq)); 2281 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map)); 2282 } 2283 EXPORT_SYMBOL(netdev_reset_tc); 2284 2285 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset) 2286 { 2287 if (tc >= dev->num_tc) 2288 return -EINVAL; 2289 2290 #ifdef CONFIG_XPS 2291 netif_reset_xps_queues(dev, offset, count); 2292 #endif 2293 dev->tc_to_txq[tc].count = count; 2294 dev->tc_to_txq[tc].offset = offset; 2295 return 0; 2296 } 2297 EXPORT_SYMBOL(netdev_set_tc_queue); 2298 2299 int netdev_set_num_tc(struct net_device *dev, u8 num_tc) 2300 { 2301 if (num_tc > TC_MAX_QUEUE) 2302 return -EINVAL; 2303 2304 #ifdef CONFIG_XPS 2305 netif_reset_xps_queues_gt(dev, 0); 2306 #endif 2307 dev->num_tc = num_tc; 2308 return 0; 2309 } 2310 EXPORT_SYMBOL(netdev_set_num_tc); 2311 2312 /* 2313 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues 2314 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed. 2315 */ 2316 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) 2317 { 2318 int rc; 2319 2320 if (txq < 1 || txq > dev->num_tx_queues) 2321 return -EINVAL; 2322 2323 if (dev->reg_state == NETREG_REGISTERED || 2324 dev->reg_state == NETREG_UNREGISTERING) { 2325 ASSERT_RTNL(); 2326 2327 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, 2328 txq); 2329 if (rc) 2330 return rc; 2331 2332 if (dev->num_tc) 2333 netif_setup_tc(dev, txq); 2334 2335 if (txq < dev->real_num_tx_queues) { 2336 qdisc_reset_all_tx_gt(dev, txq); 2337 #ifdef CONFIG_XPS 2338 netif_reset_xps_queues_gt(dev, txq); 2339 #endif 2340 } 2341 } 2342 2343 dev->real_num_tx_queues = txq; 2344 return 0; 2345 } 2346 EXPORT_SYMBOL(netif_set_real_num_tx_queues); 2347 2348 #ifdef CONFIG_SYSFS 2349 /** 2350 * netif_set_real_num_rx_queues - set actual number of RX queues used 2351 * @dev: Network device 2352 * @rxq: Actual number of RX queues 2353 * 2354 * This must be called either with the rtnl_lock held or before 2355 * registration of the net device. Returns 0 on success, or a 2356 * negative error code. If called before registration, it always 2357 * succeeds. 2358 */ 2359 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) 2360 { 2361 int rc; 2362 2363 if (rxq < 1 || rxq > dev->num_rx_queues) 2364 return -EINVAL; 2365 2366 if (dev->reg_state == NETREG_REGISTERED) { 2367 ASSERT_RTNL(); 2368 2369 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, 2370 rxq); 2371 if (rc) 2372 return rc; 2373 } 2374 2375 dev->real_num_rx_queues = rxq; 2376 return 0; 2377 } 2378 EXPORT_SYMBOL(netif_set_real_num_rx_queues); 2379 #endif 2380 2381 /** 2382 * netif_get_num_default_rss_queues - default number of RSS queues 2383 * 2384 * This routine should set an upper limit on the number of RSS queues 2385 * used by default by multiqueue devices. 2386 */ 2387 int netif_get_num_default_rss_queues(void) 2388 { 2389 return is_kdump_kernel() ? 2390 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus()); 2391 } 2392 EXPORT_SYMBOL(netif_get_num_default_rss_queues); 2393 2394 static void __netif_reschedule(struct Qdisc *q) 2395 { 2396 struct softnet_data *sd; 2397 unsigned long flags; 2398 2399 local_irq_save(flags); 2400 sd = this_cpu_ptr(&softnet_data); 2401 q->next_sched = NULL; 2402 *sd->output_queue_tailp = q; 2403 sd->output_queue_tailp = &q->next_sched; 2404 raise_softirq_irqoff(NET_TX_SOFTIRQ); 2405 local_irq_restore(flags); 2406 } 2407 2408 void __netif_schedule(struct Qdisc *q) 2409 { 2410 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) 2411 __netif_reschedule(q); 2412 } 2413 EXPORT_SYMBOL(__netif_schedule); 2414 2415 struct dev_kfree_skb_cb { 2416 enum skb_free_reason reason; 2417 }; 2418 2419 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb) 2420 { 2421 return (struct dev_kfree_skb_cb *)skb->cb; 2422 } 2423 2424 void netif_schedule_queue(struct netdev_queue *txq) 2425 { 2426 rcu_read_lock(); 2427 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) { 2428 struct Qdisc *q = rcu_dereference(txq->qdisc); 2429 2430 __netif_schedule(q); 2431 } 2432 rcu_read_unlock(); 2433 } 2434 EXPORT_SYMBOL(netif_schedule_queue); 2435 2436 void netif_tx_wake_queue(struct netdev_queue *dev_queue) 2437 { 2438 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { 2439 struct Qdisc *q; 2440 2441 rcu_read_lock(); 2442 q = rcu_dereference(dev_queue->qdisc); 2443 __netif_schedule(q); 2444 rcu_read_unlock(); 2445 } 2446 } 2447 EXPORT_SYMBOL(netif_tx_wake_queue); 2448 2449 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason) 2450 { 2451 unsigned long flags; 2452 2453 if (likely(atomic_read(&skb->users) == 1)) { 2454 smp_rmb(); 2455 atomic_set(&skb->users, 0); 2456 } else if (likely(!atomic_dec_and_test(&skb->users))) { 2457 return; 2458 } 2459 get_kfree_skb_cb(skb)->reason = reason; 2460 local_irq_save(flags); 2461 skb->next = __this_cpu_read(softnet_data.completion_queue); 2462 __this_cpu_write(softnet_data.completion_queue, skb); 2463 raise_softirq_irqoff(NET_TX_SOFTIRQ); 2464 local_irq_restore(flags); 2465 } 2466 EXPORT_SYMBOL(__dev_kfree_skb_irq); 2467 2468 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason) 2469 { 2470 if (in_irq() || irqs_disabled()) 2471 __dev_kfree_skb_irq(skb, reason); 2472 else 2473 dev_kfree_skb(skb); 2474 } 2475 EXPORT_SYMBOL(__dev_kfree_skb_any); 2476 2477 2478 /** 2479 * netif_device_detach - mark device as removed 2480 * @dev: network device 2481 * 2482 * Mark device as removed from system and therefore no longer available. 2483 */ 2484 void netif_device_detach(struct net_device *dev) 2485 { 2486 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && 2487 netif_running(dev)) { 2488 netif_tx_stop_all_queues(dev); 2489 } 2490 } 2491 EXPORT_SYMBOL(netif_device_detach); 2492 2493 /** 2494 * netif_device_attach - mark device as attached 2495 * @dev: network device 2496 * 2497 * Mark device as attached from system and restart if needed. 2498 */ 2499 void netif_device_attach(struct net_device *dev) 2500 { 2501 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && 2502 netif_running(dev)) { 2503 netif_tx_wake_all_queues(dev); 2504 __netdev_watchdog_up(dev); 2505 } 2506 } 2507 EXPORT_SYMBOL(netif_device_attach); 2508 2509 /* 2510 * Returns a Tx hash based on the given packet descriptor a Tx queues' number 2511 * to be used as a distribution range. 2512 */ 2513 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb, 2514 unsigned int num_tx_queues) 2515 { 2516 u32 hash; 2517 u16 qoffset = 0; 2518 u16 qcount = num_tx_queues; 2519 2520 if (skb_rx_queue_recorded(skb)) { 2521 hash = skb_get_rx_queue(skb); 2522 while (unlikely(hash >= num_tx_queues)) 2523 hash -= num_tx_queues; 2524 return hash; 2525 } 2526 2527 if (dev->num_tc) { 2528 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); 2529 2530 qoffset = dev->tc_to_txq[tc].offset; 2531 qcount = dev->tc_to_txq[tc].count; 2532 } 2533 2534 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset; 2535 } 2536 EXPORT_SYMBOL(__skb_tx_hash); 2537 2538 static void skb_warn_bad_offload(const struct sk_buff *skb) 2539 { 2540 static const netdev_features_t null_features; 2541 struct net_device *dev = skb->dev; 2542 const char *name = ""; 2543 2544 if (!net_ratelimit()) 2545 return; 2546 2547 if (dev) { 2548 if (dev->dev.parent) 2549 name = dev_driver_string(dev->dev.parent); 2550 else 2551 name = netdev_name(dev); 2552 } 2553 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d " 2554 "gso_type=%d ip_summed=%d\n", 2555 name, dev ? &dev->features : &null_features, 2556 skb->sk ? &skb->sk->sk_route_caps : &null_features, 2557 skb->len, skb->data_len, skb_shinfo(skb)->gso_size, 2558 skb_shinfo(skb)->gso_type, skb->ip_summed); 2559 } 2560 2561 /* 2562 * Invalidate hardware checksum when packet is to be mangled, and 2563 * complete checksum manually on outgoing path. 2564 */ 2565 int skb_checksum_help(struct sk_buff *skb) 2566 { 2567 __wsum csum; 2568 int ret = 0, offset; 2569 2570 if (skb->ip_summed == CHECKSUM_COMPLETE) 2571 goto out_set_summed; 2572 2573 if (unlikely(skb_shinfo(skb)->gso_size)) { 2574 skb_warn_bad_offload(skb); 2575 return -EINVAL; 2576 } 2577 2578 /* Before computing a checksum, we should make sure no frag could 2579 * be modified by an external entity : checksum could be wrong. 2580 */ 2581 if (skb_has_shared_frag(skb)) { 2582 ret = __skb_linearize(skb); 2583 if (ret) 2584 goto out; 2585 } 2586 2587 offset = skb_checksum_start_offset(skb); 2588 BUG_ON(offset >= skb_headlen(skb)); 2589 csum = skb_checksum(skb, offset, skb->len - offset, 0); 2590 2591 offset += skb->csum_offset; 2592 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb)); 2593 2594 if (skb_cloned(skb) && 2595 !skb_clone_writable(skb, offset + sizeof(__sum16))) { 2596 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2597 if (ret) 2598 goto out; 2599 } 2600 2601 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0; 2602 out_set_summed: 2603 skb->ip_summed = CHECKSUM_NONE; 2604 out: 2605 return ret; 2606 } 2607 EXPORT_SYMBOL(skb_checksum_help); 2608 2609 __be16 skb_network_protocol(struct sk_buff *skb, int *depth) 2610 { 2611 __be16 type = skb->protocol; 2612 2613 /* Tunnel gso handlers can set protocol to ethernet. */ 2614 if (type == htons(ETH_P_TEB)) { 2615 struct ethhdr *eth; 2616 2617 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr)))) 2618 return 0; 2619 2620 eth = (struct ethhdr *)skb_mac_header(skb); 2621 type = eth->h_proto; 2622 } 2623 2624 return __vlan_get_protocol(skb, type, depth); 2625 } 2626 2627 /** 2628 * skb_mac_gso_segment - mac layer segmentation handler. 2629 * @skb: buffer to segment 2630 * @features: features for the output path (see dev->features) 2631 */ 2632 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, 2633 netdev_features_t features) 2634 { 2635 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); 2636 struct packet_offload *ptype; 2637 int vlan_depth = skb->mac_len; 2638 __be16 type = skb_network_protocol(skb, &vlan_depth); 2639 2640 if (unlikely(!type)) 2641 return ERR_PTR(-EINVAL); 2642 2643 __skb_pull(skb, vlan_depth); 2644 2645 rcu_read_lock(); 2646 list_for_each_entry_rcu(ptype, &offload_base, list) { 2647 if (ptype->type == type && ptype->callbacks.gso_segment) { 2648 segs = ptype->callbacks.gso_segment(skb, features); 2649 break; 2650 } 2651 } 2652 rcu_read_unlock(); 2653 2654 __skb_push(skb, skb->data - skb_mac_header(skb)); 2655 2656 return segs; 2657 } 2658 EXPORT_SYMBOL(skb_mac_gso_segment); 2659 2660 2661 /* openvswitch calls this on rx path, so we need a different check. 2662 */ 2663 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) 2664 { 2665 if (tx_path) 2666 return skb->ip_summed != CHECKSUM_PARTIAL && 2667 skb->ip_summed != CHECKSUM_NONE; 2668 2669 return skb->ip_summed == CHECKSUM_NONE; 2670 } 2671 2672 /** 2673 * __skb_gso_segment - Perform segmentation on skb. 2674 * @skb: buffer to segment 2675 * @features: features for the output path (see dev->features) 2676 * @tx_path: whether it is called in TX path 2677 * 2678 * This function segments the given skb and returns a list of segments. 2679 * 2680 * It may return NULL if the skb requires no segmentation. This is 2681 * only possible when GSO is used for verifying header integrity. 2682 * 2683 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb. 2684 */ 2685 struct sk_buff *__skb_gso_segment(struct sk_buff *skb, 2686 netdev_features_t features, bool tx_path) 2687 { 2688 struct sk_buff *segs; 2689 2690 if (unlikely(skb_needs_check(skb, tx_path))) { 2691 int err; 2692 2693 /* We're going to init ->check field in TCP or UDP header */ 2694 err = skb_cow_head(skb, 0); 2695 if (err < 0) 2696 return ERR_PTR(err); 2697 } 2698 2699 /* Only report GSO partial support if it will enable us to 2700 * support segmentation on this frame without needing additional 2701 * work. 2702 */ 2703 if (features & NETIF_F_GSO_PARTIAL) { 2704 netdev_features_t partial_features = NETIF_F_GSO_ROBUST; 2705 struct net_device *dev = skb->dev; 2706 2707 partial_features |= dev->features & dev->gso_partial_features; 2708 if (!skb_gso_ok(skb, features | partial_features)) 2709 features &= ~NETIF_F_GSO_PARTIAL; 2710 } 2711 2712 BUILD_BUG_ON(SKB_SGO_CB_OFFSET + 2713 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb)); 2714 2715 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb); 2716 SKB_GSO_CB(skb)->encap_level = 0; 2717 2718 skb_reset_mac_header(skb); 2719 skb_reset_mac_len(skb); 2720 2721 segs = skb_mac_gso_segment(skb, features); 2722 2723 if (unlikely(skb_needs_check(skb, tx_path))) 2724 skb_warn_bad_offload(skb); 2725 2726 return segs; 2727 } 2728 EXPORT_SYMBOL(__skb_gso_segment); 2729 2730 /* Take action when hardware reception checksum errors are detected. */ 2731 #ifdef CONFIG_BUG 2732 void netdev_rx_csum_fault(struct net_device *dev) 2733 { 2734 if (net_ratelimit()) { 2735 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>"); 2736 dump_stack(); 2737 } 2738 } 2739 EXPORT_SYMBOL(netdev_rx_csum_fault); 2740 #endif 2741 2742 /* Actually, we should eliminate this check as soon as we know, that: 2743 * 1. IOMMU is present and allows to map all the memory. 2744 * 2. No high memory really exists on this machine. 2745 */ 2746 2747 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) 2748 { 2749 #ifdef CONFIG_HIGHMEM 2750 int i; 2751 2752 if (!(dev->features & NETIF_F_HIGHDMA)) { 2753 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2754 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2755 2756 if (PageHighMem(skb_frag_page(frag))) 2757 return 1; 2758 } 2759 } 2760 2761 if (PCI_DMA_BUS_IS_PHYS) { 2762 struct device *pdev = dev->dev.parent; 2763 2764 if (!pdev) 2765 return 0; 2766 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2767 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2768 dma_addr_t addr = page_to_phys(skb_frag_page(frag)); 2769 2770 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask) 2771 return 1; 2772 } 2773 } 2774 #endif 2775 return 0; 2776 } 2777 2778 /* If MPLS offload request, verify we are testing hardware MPLS features 2779 * instead of standard features for the netdev. 2780 */ 2781 #if IS_ENABLED(CONFIG_NET_MPLS_GSO) 2782 static netdev_features_t net_mpls_features(struct sk_buff *skb, 2783 netdev_features_t features, 2784 __be16 type) 2785 { 2786 if (eth_p_mpls(type)) 2787 features &= skb->dev->mpls_features; 2788 2789 return features; 2790 } 2791 #else 2792 static netdev_features_t net_mpls_features(struct sk_buff *skb, 2793 netdev_features_t features, 2794 __be16 type) 2795 { 2796 return features; 2797 } 2798 #endif 2799 2800 static netdev_features_t harmonize_features(struct sk_buff *skb, 2801 netdev_features_t features) 2802 { 2803 int tmp; 2804 __be16 type; 2805 2806 type = skb_network_protocol(skb, &tmp); 2807 features = net_mpls_features(skb, features, type); 2808 2809 if (skb->ip_summed != CHECKSUM_NONE && 2810 !can_checksum_protocol(features, type)) { 2811 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 2812 } 2813 if (illegal_highdma(skb->dev, skb)) 2814 features &= ~NETIF_F_SG; 2815 2816 return features; 2817 } 2818 2819 netdev_features_t passthru_features_check(struct sk_buff *skb, 2820 struct net_device *dev, 2821 netdev_features_t features) 2822 { 2823 return features; 2824 } 2825 EXPORT_SYMBOL(passthru_features_check); 2826 2827 static netdev_features_t dflt_features_check(const struct sk_buff *skb, 2828 struct net_device *dev, 2829 netdev_features_t features) 2830 { 2831 return vlan_features_check(skb, features); 2832 } 2833 2834 static netdev_features_t gso_features_check(const struct sk_buff *skb, 2835 struct net_device *dev, 2836 netdev_features_t features) 2837 { 2838 u16 gso_segs = skb_shinfo(skb)->gso_segs; 2839 2840 if (gso_segs > dev->gso_max_segs) 2841 return features & ~NETIF_F_GSO_MASK; 2842 2843 /* Support for GSO partial features requires software 2844 * intervention before we can actually process the packets 2845 * so we need to strip support for any partial features now 2846 * and we can pull them back in after we have partially 2847 * segmented the frame. 2848 */ 2849 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)) 2850 features &= ~dev->gso_partial_features; 2851 2852 /* Make sure to clear the IPv4 ID mangling feature if the 2853 * IPv4 header has the potential to be fragmented. 2854 */ 2855 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { 2856 struct iphdr *iph = skb->encapsulation ? 2857 inner_ip_hdr(skb) : ip_hdr(skb); 2858 2859 if (!(iph->frag_off & htons(IP_DF))) 2860 features &= ~NETIF_F_TSO_MANGLEID; 2861 } 2862 2863 return features; 2864 } 2865 2866 netdev_features_t netif_skb_features(struct sk_buff *skb) 2867 { 2868 struct net_device *dev = skb->dev; 2869 netdev_features_t features = dev->features; 2870 2871 if (skb_is_gso(skb)) 2872 features = gso_features_check(skb, dev, features); 2873 2874 /* If encapsulation offload request, verify we are testing 2875 * hardware encapsulation features instead of standard 2876 * features for the netdev 2877 */ 2878 if (skb->encapsulation) 2879 features &= dev->hw_enc_features; 2880 2881 if (skb_vlan_tagged(skb)) 2882 features = netdev_intersect_features(features, 2883 dev->vlan_features | 2884 NETIF_F_HW_VLAN_CTAG_TX | 2885 NETIF_F_HW_VLAN_STAG_TX); 2886 2887 if (dev->netdev_ops->ndo_features_check) 2888 features &= dev->netdev_ops->ndo_features_check(skb, dev, 2889 features); 2890 else 2891 features &= dflt_features_check(skb, dev, features); 2892 2893 return harmonize_features(skb, features); 2894 } 2895 EXPORT_SYMBOL(netif_skb_features); 2896 2897 static int xmit_one(struct sk_buff *skb, struct net_device *dev, 2898 struct netdev_queue *txq, bool more) 2899 { 2900 unsigned int len; 2901 int rc; 2902 2903 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all)) 2904 dev_queue_xmit_nit(skb, dev); 2905 2906 len = skb->len; 2907 trace_net_dev_start_xmit(skb, dev); 2908 rc = netdev_start_xmit(skb, dev, txq, more); 2909 trace_net_dev_xmit(skb, rc, dev, len); 2910 2911 return rc; 2912 } 2913 2914 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev, 2915 struct netdev_queue *txq, int *ret) 2916 { 2917 struct sk_buff *skb = first; 2918 int rc = NETDEV_TX_OK; 2919 2920 while (skb) { 2921 struct sk_buff *next = skb->next; 2922 2923 skb->next = NULL; 2924 rc = xmit_one(skb, dev, txq, next != NULL); 2925 if (unlikely(!dev_xmit_complete(rc))) { 2926 skb->next = next; 2927 goto out; 2928 } 2929 2930 skb = next; 2931 if (netif_xmit_stopped(txq) && skb) { 2932 rc = NETDEV_TX_BUSY; 2933 break; 2934 } 2935 } 2936 2937 out: 2938 *ret = rc; 2939 return skb; 2940 } 2941 2942 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, 2943 netdev_features_t features) 2944 { 2945 if (skb_vlan_tag_present(skb) && 2946 !vlan_hw_offload_capable(features, skb->vlan_proto)) 2947 skb = __vlan_hwaccel_push_inside(skb); 2948 return skb; 2949 } 2950 2951 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev) 2952 { 2953 netdev_features_t features; 2954 2955 features = netif_skb_features(skb); 2956 skb = validate_xmit_vlan(skb, features); 2957 if (unlikely(!skb)) 2958 goto out_null; 2959 2960 if (netif_needs_gso(skb, features)) { 2961 struct sk_buff *segs; 2962 2963 segs = skb_gso_segment(skb, features); 2964 if (IS_ERR(segs)) { 2965 goto out_kfree_skb; 2966 } else if (segs) { 2967 consume_skb(skb); 2968 skb = segs; 2969 } 2970 } else { 2971 if (skb_needs_linearize(skb, features) && 2972 __skb_linearize(skb)) 2973 goto out_kfree_skb; 2974 2975 /* If packet is not checksummed and device does not 2976 * support checksumming for this protocol, complete 2977 * checksumming here. 2978 */ 2979 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2980 if (skb->encapsulation) 2981 skb_set_inner_transport_header(skb, 2982 skb_checksum_start_offset(skb)); 2983 else 2984 skb_set_transport_header(skb, 2985 skb_checksum_start_offset(skb)); 2986 if (!(features & NETIF_F_CSUM_MASK) && 2987 skb_checksum_help(skb)) 2988 goto out_kfree_skb; 2989 } 2990 } 2991 2992 return skb; 2993 2994 out_kfree_skb: 2995 kfree_skb(skb); 2996 out_null: 2997 atomic_long_inc(&dev->tx_dropped); 2998 return NULL; 2999 } 3000 3001 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev) 3002 { 3003 struct sk_buff *next, *head = NULL, *tail; 3004 3005 for (; skb != NULL; skb = next) { 3006 next = skb->next; 3007 skb->next = NULL; 3008 3009 /* in case skb wont be segmented, point to itself */ 3010 skb->prev = skb; 3011 3012 skb = validate_xmit_skb(skb, dev); 3013 if (!skb) 3014 continue; 3015 3016 if (!head) 3017 head = skb; 3018 else 3019 tail->next = skb; 3020 /* If skb was segmented, skb->prev points to 3021 * the last segment. If not, it still contains skb. 3022 */ 3023 tail = skb->prev; 3024 } 3025 return head; 3026 } 3027 EXPORT_SYMBOL_GPL(validate_xmit_skb_list); 3028 3029 static void qdisc_pkt_len_init(struct sk_buff *skb) 3030 { 3031 const struct skb_shared_info *shinfo = skb_shinfo(skb); 3032 3033 qdisc_skb_cb(skb)->pkt_len = skb->len; 3034 3035 /* To get more precise estimation of bytes sent on wire, 3036 * we add to pkt_len the headers size of all segments 3037 */ 3038 if (shinfo->gso_size) { 3039 unsigned int hdr_len; 3040 u16 gso_segs = shinfo->gso_segs; 3041 3042 /* mac layer + network layer */ 3043 hdr_len = skb_transport_header(skb) - skb_mac_header(skb); 3044 3045 /* + transport layer */ 3046 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) 3047 hdr_len += tcp_hdrlen(skb); 3048 else 3049 hdr_len += sizeof(struct udphdr); 3050 3051 if (shinfo->gso_type & SKB_GSO_DODGY) 3052 gso_segs = DIV_ROUND_UP(skb->len - hdr_len, 3053 shinfo->gso_size); 3054 3055 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; 3056 } 3057 } 3058 3059 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 3060 struct net_device *dev, 3061 struct netdev_queue *txq) 3062 { 3063 spinlock_t *root_lock = qdisc_lock(q); 3064 struct sk_buff *to_free = NULL; 3065 bool contended; 3066 int rc; 3067 3068 qdisc_calculate_pkt_len(skb, q); 3069 /* 3070 * Heuristic to force contended enqueues to serialize on a 3071 * separate lock before trying to get qdisc main lock. 3072 * This permits qdisc->running owner to get the lock more 3073 * often and dequeue packets faster. 3074 */ 3075 contended = qdisc_is_running(q); 3076 if (unlikely(contended)) 3077 spin_lock(&q->busylock); 3078 3079 spin_lock(root_lock); 3080 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { 3081 __qdisc_drop(skb, &to_free); 3082 rc = NET_XMIT_DROP; 3083 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && 3084 qdisc_run_begin(q)) { 3085 /* 3086 * This is a work-conserving queue; there are no old skbs 3087 * waiting to be sent out; and the qdisc is not running - 3088 * xmit the skb directly. 3089 */ 3090 3091 qdisc_bstats_update(q, skb); 3092 3093 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) { 3094 if (unlikely(contended)) { 3095 spin_unlock(&q->busylock); 3096 contended = false; 3097 } 3098 __qdisc_run(q); 3099 } else 3100 qdisc_run_end(q); 3101 3102 rc = NET_XMIT_SUCCESS; 3103 } else { 3104 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK; 3105 if (qdisc_run_begin(q)) { 3106 if (unlikely(contended)) { 3107 spin_unlock(&q->busylock); 3108 contended = false; 3109 } 3110 __qdisc_run(q); 3111 } 3112 } 3113 spin_unlock(root_lock); 3114 if (unlikely(to_free)) 3115 kfree_skb_list(to_free); 3116 if (unlikely(contended)) 3117 spin_unlock(&q->busylock); 3118 return rc; 3119 } 3120 3121 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO) 3122 static void skb_update_prio(struct sk_buff *skb) 3123 { 3124 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap); 3125 3126 if (!skb->priority && skb->sk && map) { 3127 unsigned int prioidx = 3128 sock_cgroup_prioidx(&skb->sk->sk_cgrp_data); 3129 3130 if (prioidx < map->priomap_len) 3131 skb->priority = map->priomap[prioidx]; 3132 } 3133 } 3134 #else 3135 #define skb_update_prio(skb) 3136 #endif 3137 3138 DEFINE_PER_CPU(int, xmit_recursion); 3139 EXPORT_SYMBOL(xmit_recursion); 3140 3141 /** 3142 * dev_loopback_xmit - loop back @skb 3143 * @net: network namespace this loopback is happening in 3144 * @sk: sk needed to be a netfilter okfn 3145 * @skb: buffer to transmit 3146 */ 3147 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb) 3148 { 3149 skb_reset_mac_header(skb); 3150 __skb_pull(skb, skb_network_offset(skb)); 3151 skb->pkt_type = PACKET_LOOPBACK; 3152 skb->ip_summed = CHECKSUM_UNNECESSARY; 3153 WARN_ON(!skb_dst(skb)); 3154 skb_dst_force(skb); 3155 netif_rx_ni(skb); 3156 return 0; 3157 } 3158 EXPORT_SYMBOL(dev_loopback_xmit); 3159 3160 #ifdef CONFIG_NET_EGRESS 3161 static struct sk_buff * 3162 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) 3163 { 3164 struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list); 3165 struct tcf_result cl_res; 3166 3167 if (!cl) 3168 return skb; 3169 3170 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */ 3171 qdisc_bstats_cpu_update(cl->q, skb); 3172 3173 switch (tc_classify(skb, cl, &cl_res, false)) { 3174 case TC_ACT_OK: 3175 case TC_ACT_RECLASSIFY: 3176 skb->tc_index = TC_H_MIN(cl_res.classid); 3177 break; 3178 case TC_ACT_SHOT: 3179 qdisc_qstats_cpu_drop(cl->q); 3180 *ret = NET_XMIT_DROP; 3181 kfree_skb(skb); 3182 return NULL; 3183 case TC_ACT_STOLEN: 3184 case TC_ACT_QUEUED: 3185 *ret = NET_XMIT_SUCCESS; 3186 consume_skb(skb); 3187 return NULL; 3188 case TC_ACT_REDIRECT: 3189 /* No need to push/pop skb's mac_header here on egress! */ 3190 skb_do_redirect(skb); 3191 *ret = NET_XMIT_SUCCESS; 3192 return NULL; 3193 default: 3194 break; 3195 } 3196 3197 return skb; 3198 } 3199 #endif /* CONFIG_NET_EGRESS */ 3200 3201 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb) 3202 { 3203 #ifdef CONFIG_XPS 3204 struct xps_dev_maps *dev_maps; 3205 struct xps_map *map; 3206 int queue_index = -1; 3207 3208 rcu_read_lock(); 3209 dev_maps = rcu_dereference(dev->xps_maps); 3210 if (dev_maps) { 3211 unsigned int tci = skb->sender_cpu - 1; 3212 3213 if (dev->num_tc) { 3214 tci *= dev->num_tc; 3215 tci += netdev_get_prio_tc_map(dev, skb->priority); 3216 } 3217 3218 map = rcu_dereference(dev_maps->cpu_map[tci]); 3219 if (map) { 3220 if (map->len == 1) 3221 queue_index = map->queues[0]; 3222 else 3223 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb), 3224 map->len)]; 3225 if (unlikely(queue_index >= dev->real_num_tx_queues)) 3226 queue_index = -1; 3227 } 3228 } 3229 rcu_read_unlock(); 3230 3231 return queue_index; 3232 #else 3233 return -1; 3234 #endif 3235 } 3236 3237 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb) 3238 { 3239 struct sock *sk = skb->sk; 3240 int queue_index = sk_tx_queue_get(sk); 3241 3242 if (queue_index < 0 || skb->ooo_okay || 3243 queue_index >= dev->real_num_tx_queues) { 3244 int new_index = get_xps_queue(dev, skb); 3245 3246 if (new_index < 0) 3247 new_index = skb_tx_hash(dev, skb); 3248 3249 if (queue_index != new_index && sk && 3250 sk_fullsock(sk) && 3251 rcu_access_pointer(sk->sk_dst_cache)) 3252 sk_tx_queue_set(sk, new_index); 3253 3254 queue_index = new_index; 3255 } 3256 3257 return queue_index; 3258 } 3259 3260 struct netdev_queue *netdev_pick_tx(struct net_device *dev, 3261 struct sk_buff *skb, 3262 void *accel_priv) 3263 { 3264 int queue_index = 0; 3265 3266 #ifdef CONFIG_XPS 3267 u32 sender_cpu = skb->sender_cpu - 1; 3268 3269 if (sender_cpu >= (u32)NR_CPUS) 3270 skb->sender_cpu = raw_smp_processor_id() + 1; 3271 #endif 3272 3273 if (dev->real_num_tx_queues != 1) { 3274 const struct net_device_ops *ops = dev->netdev_ops; 3275 3276 if (ops->ndo_select_queue) 3277 queue_index = ops->ndo_select_queue(dev, skb, accel_priv, 3278 __netdev_pick_tx); 3279 else 3280 queue_index = __netdev_pick_tx(dev, skb); 3281 3282 if (!accel_priv) 3283 queue_index = netdev_cap_txqueue(dev, queue_index); 3284 } 3285 3286 skb_set_queue_mapping(skb, queue_index); 3287 return netdev_get_tx_queue(dev, queue_index); 3288 } 3289 3290 /** 3291 * __dev_queue_xmit - transmit a buffer 3292 * @skb: buffer to transmit 3293 * @accel_priv: private data used for L2 forwarding offload 3294 * 3295 * Queue a buffer for transmission to a network device. The caller must 3296 * have set the device and priority and built the buffer before calling 3297 * this function. The function can be called from an interrupt. 3298 * 3299 * A negative errno code is returned on a failure. A success does not 3300 * guarantee the frame will be transmitted as it may be dropped due 3301 * to congestion or traffic shaping. 3302 * 3303 * ----------------------------------------------------------------------------------- 3304 * I notice this method can also return errors from the queue disciplines, 3305 * including NET_XMIT_DROP, which is a positive value. So, errors can also 3306 * be positive. 3307 * 3308 * Regardless of the return value, the skb is consumed, so it is currently 3309 * difficult to retry a send to this method. (You can bump the ref count 3310 * before sending to hold a reference for retry if you are careful.) 3311 * 3312 * When calling this method, interrupts MUST be enabled. This is because 3313 * the BH enable code must have IRQs enabled so that it will not deadlock. 3314 * --BLG 3315 */ 3316 static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv) 3317 { 3318 struct net_device *dev = skb->dev; 3319 struct netdev_queue *txq; 3320 struct Qdisc *q; 3321 int rc = -ENOMEM; 3322 3323 skb_reset_mac_header(skb); 3324 3325 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP)) 3326 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED); 3327 3328 /* Disable soft irqs for various locks below. Also 3329 * stops preemption for RCU. 3330 */ 3331 rcu_read_lock_bh(); 3332 3333 skb_update_prio(skb); 3334 3335 qdisc_pkt_len_init(skb); 3336 #ifdef CONFIG_NET_CLS_ACT 3337 skb->tc_at_ingress = 0; 3338 # ifdef CONFIG_NET_EGRESS 3339 if (static_key_false(&egress_needed)) { 3340 skb = sch_handle_egress(skb, &rc, dev); 3341 if (!skb) 3342 goto out; 3343 } 3344 # endif 3345 #endif 3346 /* If device/qdisc don't need skb->dst, release it right now while 3347 * its hot in this cpu cache. 3348 */ 3349 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) 3350 skb_dst_drop(skb); 3351 else 3352 skb_dst_force(skb); 3353 3354 txq = netdev_pick_tx(dev, skb, accel_priv); 3355 q = rcu_dereference_bh(txq->qdisc); 3356 3357 trace_net_dev_queue(skb); 3358 if (q->enqueue) { 3359 rc = __dev_xmit_skb(skb, q, dev, txq); 3360 goto out; 3361 } 3362 3363 /* The device has no queue. Common case for software devices: 3364 * loopback, all the sorts of tunnels... 3365 3366 * Really, it is unlikely that netif_tx_lock protection is necessary 3367 * here. (f.e. loopback and IP tunnels are clean ignoring statistics 3368 * counters.) 3369 * However, it is possible, that they rely on protection 3370 * made by us here. 3371 3372 * Check this and shot the lock. It is not prone from deadlocks. 3373 *Either shot noqueue qdisc, it is even simpler 8) 3374 */ 3375 if (dev->flags & IFF_UP) { 3376 int cpu = smp_processor_id(); /* ok because BHs are off */ 3377 3378 if (txq->xmit_lock_owner != cpu) { 3379 if (unlikely(__this_cpu_read(xmit_recursion) > 3380 XMIT_RECURSION_LIMIT)) 3381 goto recursion_alert; 3382 3383 skb = validate_xmit_skb(skb, dev); 3384 if (!skb) 3385 goto out; 3386 3387 HARD_TX_LOCK(dev, txq, cpu); 3388 3389 if (!netif_xmit_stopped(txq)) { 3390 __this_cpu_inc(xmit_recursion); 3391 skb = dev_hard_start_xmit(skb, dev, txq, &rc); 3392 __this_cpu_dec(xmit_recursion); 3393 if (dev_xmit_complete(rc)) { 3394 HARD_TX_UNLOCK(dev, txq); 3395 goto out; 3396 } 3397 } 3398 HARD_TX_UNLOCK(dev, txq); 3399 net_crit_ratelimited("Virtual device %s asks to queue packet!\n", 3400 dev->name); 3401 } else { 3402 /* Recursion is detected! It is possible, 3403 * unfortunately 3404 */ 3405 recursion_alert: 3406 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", 3407 dev->name); 3408 } 3409 } 3410 3411 rc = -ENETDOWN; 3412 rcu_read_unlock_bh(); 3413 3414 atomic_long_inc(&dev->tx_dropped); 3415 kfree_skb_list(skb); 3416 return rc; 3417 out: 3418 rcu_read_unlock_bh(); 3419 return rc; 3420 } 3421 3422 int dev_queue_xmit(struct sk_buff *skb) 3423 { 3424 return __dev_queue_xmit(skb, NULL); 3425 } 3426 EXPORT_SYMBOL(dev_queue_xmit); 3427 3428 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv) 3429 { 3430 return __dev_queue_xmit(skb, accel_priv); 3431 } 3432 EXPORT_SYMBOL(dev_queue_xmit_accel); 3433 3434 3435 /************************************************************************* 3436 * Receiver routines 3437 *************************************************************************/ 3438 3439 int netdev_max_backlog __read_mostly = 1000; 3440 EXPORT_SYMBOL(netdev_max_backlog); 3441 3442 int netdev_tstamp_prequeue __read_mostly = 1; 3443 int netdev_budget __read_mostly = 300; 3444 int weight_p __read_mostly = 64; /* old backlog weight */ 3445 int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */ 3446 int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */ 3447 int dev_rx_weight __read_mostly = 64; 3448 int dev_tx_weight __read_mostly = 64; 3449 3450 /* Called with irq disabled */ 3451 static inline void ____napi_schedule(struct softnet_data *sd, 3452 struct napi_struct *napi) 3453 { 3454 list_add_tail(&napi->poll_list, &sd->poll_list); 3455 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 3456 } 3457 3458 #ifdef CONFIG_RPS 3459 3460 /* One global table that all flow-based protocols share. */ 3461 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; 3462 EXPORT_SYMBOL(rps_sock_flow_table); 3463 u32 rps_cpu_mask __read_mostly; 3464 EXPORT_SYMBOL(rps_cpu_mask); 3465 3466 struct static_key rps_needed __read_mostly; 3467 EXPORT_SYMBOL(rps_needed); 3468 struct static_key rfs_needed __read_mostly; 3469 EXPORT_SYMBOL(rfs_needed); 3470 3471 static struct rps_dev_flow * 3472 set_rps_cpu(struct net_device *dev, struct sk_buff *skb, 3473 struct rps_dev_flow *rflow, u16 next_cpu) 3474 { 3475 if (next_cpu < nr_cpu_ids) { 3476 #ifdef CONFIG_RFS_ACCEL 3477 struct netdev_rx_queue *rxqueue; 3478 struct rps_dev_flow_table *flow_table; 3479 struct rps_dev_flow *old_rflow; 3480 u32 flow_id; 3481 u16 rxq_index; 3482 int rc; 3483 3484 /* Should we steer this flow to a different hardware queue? */ 3485 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || 3486 !(dev->features & NETIF_F_NTUPLE)) 3487 goto out; 3488 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); 3489 if (rxq_index == skb_get_rx_queue(skb)) 3490 goto out; 3491 3492 rxqueue = dev->_rx + rxq_index; 3493 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3494 if (!flow_table) 3495 goto out; 3496 flow_id = skb_get_hash(skb) & flow_table->mask; 3497 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, 3498 rxq_index, flow_id); 3499 if (rc < 0) 3500 goto out; 3501 old_rflow = rflow; 3502 rflow = &flow_table->flows[flow_id]; 3503 rflow->filter = rc; 3504 if (old_rflow->filter == rflow->filter) 3505 old_rflow->filter = RPS_NO_FILTER; 3506 out: 3507 #endif 3508 rflow->last_qtail = 3509 per_cpu(softnet_data, next_cpu).input_queue_head; 3510 } 3511 3512 rflow->cpu = next_cpu; 3513 return rflow; 3514 } 3515 3516 /* 3517 * get_rps_cpu is called from netif_receive_skb and returns the target 3518 * CPU from the RPS map of the receiving queue for a given skb. 3519 * rcu_read_lock must be held on entry. 3520 */ 3521 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, 3522 struct rps_dev_flow **rflowp) 3523 { 3524 const struct rps_sock_flow_table *sock_flow_table; 3525 struct netdev_rx_queue *rxqueue = dev->_rx; 3526 struct rps_dev_flow_table *flow_table; 3527 struct rps_map *map; 3528 int cpu = -1; 3529 u32 tcpu; 3530 u32 hash; 3531 3532 if (skb_rx_queue_recorded(skb)) { 3533 u16 index = skb_get_rx_queue(skb); 3534 3535 if (unlikely(index >= dev->real_num_rx_queues)) { 3536 WARN_ONCE(dev->real_num_rx_queues > 1, 3537 "%s received packet on queue %u, but number " 3538 "of RX queues is %u\n", 3539 dev->name, index, dev->real_num_rx_queues); 3540 goto done; 3541 } 3542 rxqueue += index; 3543 } 3544 3545 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */ 3546 3547 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3548 map = rcu_dereference(rxqueue->rps_map); 3549 if (!flow_table && !map) 3550 goto done; 3551 3552 skb_reset_network_header(skb); 3553 hash = skb_get_hash(skb); 3554 if (!hash) 3555 goto done; 3556 3557 sock_flow_table = rcu_dereference(rps_sock_flow_table); 3558 if (flow_table && sock_flow_table) { 3559 struct rps_dev_flow *rflow; 3560 u32 next_cpu; 3561 u32 ident; 3562 3563 /* First check into global flow table if there is a match */ 3564 ident = sock_flow_table->ents[hash & sock_flow_table->mask]; 3565 if ((ident ^ hash) & ~rps_cpu_mask) 3566 goto try_rps; 3567 3568 next_cpu = ident & rps_cpu_mask; 3569 3570 /* OK, now we know there is a match, 3571 * we can look at the local (per receive queue) flow table 3572 */ 3573 rflow = &flow_table->flows[hash & flow_table->mask]; 3574 tcpu = rflow->cpu; 3575 3576 /* 3577 * If the desired CPU (where last recvmsg was done) is 3578 * different from current CPU (one in the rx-queue flow 3579 * table entry), switch if one of the following holds: 3580 * - Current CPU is unset (>= nr_cpu_ids). 3581 * - Current CPU is offline. 3582 * - The current CPU's queue tail has advanced beyond the 3583 * last packet that was enqueued using this table entry. 3584 * This guarantees that all previous packets for the flow 3585 * have been dequeued, thus preserving in order delivery. 3586 */ 3587 if (unlikely(tcpu != next_cpu) && 3588 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) || 3589 ((int)(per_cpu(softnet_data, tcpu).input_queue_head - 3590 rflow->last_qtail)) >= 0)) { 3591 tcpu = next_cpu; 3592 rflow = set_rps_cpu(dev, skb, rflow, next_cpu); 3593 } 3594 3595 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { 3596 *rflowp = rflow; 3597 cpu = tcpu; 3598 goto done; 3599 } 3600 } 3601 3602 try_rps: 3603 3604 if (map) { 3605 tcpu = map->cpus[reciprocal_scale(hash, map->len)]; 3606 if (cpu_online(tcpu)) { 3607 cpu = tcpu; 3608 goto done; 3609 } 3610 } 3611 3612 done: 3613 return cpu; 3614 } 3615 3616 #ifdef CONFIG_RFS_ACCEL 3617 3618 /** 3619 * rps_may_expire_flow - check whether an RFS hardware filter may be removed 3620 * @dev: Device on which the filter was set 3621 * @rxq_index: RX queue index 3622 * @flow_id: Flow ID passed to ndo_rx_flow_steer() 3623 * @filter_id: Filter ID returned by ndo_rx_flow_steer() 3624 * 3625 * Drivers that implement ndo_rx_flow_steer() should periodically call 3626 * this function for each installed filter and remove the filters for 3627 * which it returns %true. 3628 */ 3629 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, 3630 u32 flow_id, u16 filter_id) 3631 { 3632 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; 3633 struct rps_dev_flow_table *flow_table; 3634 struct rps_dev_flow *rflow; 3635 bool expire = true; 3636 unsigned int cpu; 3637 3638 rcu_read_lock(); 3639 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3640 if (flow_table && flow_id <= flow_table->mask) { 3641 rflow = &flow_table->flows[flow_id]; 3642 cpu = ACCESS_ONCE(rflow->cpu); 3643 if (rflow->filter == filter_id && cpu < nr_cpu_ids && 3644 ((int)(per_cpu(softnet_data, cpu).input_queue_head - 3645 rflow->last_qtail) < 3646 (int)(10 * flow_table->mask))) 3647 expire = false; 3648 } 3649 rcu_read_unlock(); 3650 return expire; 3651 } 3652 EXPORT_SYMBOL(rps_may_expire_flow); 3653 3654 #endif /* CONFIG_RFS_ACCEL */ 3655 3656 /* Called from hardirq (IPI) context */ 3657 static void rps_trigger_softirq(void *data) 3658 { 3659 struct softnet_data *sd = data; 3660 3661 ____napi_schedule(sd, &sd->backlog); 3662 sd->received_rps++; 3663 } 3664 3665 #endif /* CONFIG_RPS */ 3666 3667 /* 3668 * Check if this softnet_data structure is another cpu one 3669 * If yes, queue it to our IPI list and return 1 3670 * If no, return 0 3671 */ 3672 static int rps_ipi_queued(struct softnet_data *sd) 3673 { 3674 #ifdef CONFIG_RPS 3675 struct softnet_data *mysd = this_cpu_ptr(&softnet_data); 3676 3677 if (sd != mysd) { 3678 sd->rps_ipi_next = mysd->rps_ipi_list; 3679 mysd->rps_ipi_list = sd; 3680 3681 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 3682 return 1; 3683 } 3684 #endif /* CONFIG_RPS */ 3685 return 0; 3686 } 3687 3688 #ifdef CONFIG_NET_FLOW_LIMIT 3689 int netdev_flow_limit_table_len __read_mostly = (1 << 12); 3690 #endif 3691 3692 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) 3693 { 3694 #ifdef CONFIG_NET_FLOW_LIMIT 3695 struct sd_flow_limit *fl; 3696 struct softnet_data *sd; 3697 unsigned int old_flow, new_flow; 3698 3699 if (qlen < (netdev_max_backlog >> 1)) 3700 return false; 3701 3702 sd = this_cpu_ptr(&softnet_data); 3703 3704 rcu_read_lock(); 3705 fl = rcu_dereference(sd->flow_limit); 3706 if (fl) { 3707 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1); 3708 old_flow = fl->history[fl->history_head]; 3709 fl->history[fl->history_head] = new_flow; 3710 3711 fl->history_head++; 3712 fl->history_head &= FLOW_LIMIT_HISTORY - 1; 3713 3714 if (likely(fl->buckets[old_flow])) 3715 fl->buckets[old_flow]--; 3716 3717 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { 3718 fl->count++; 3719 rcu_read_unlock(); 3720 return true; 3721 } 3722 } 3723 rcu_read_unlock(); 3724 #endif 3725 return false; 3726 } 3727 3728 /* 3729 * enqueue_to_backlog is called to queue an skb to a per CPU backlog 3730 * queue (may be a remote CPU queue). 3731 */ 3732 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, 3733 unsigned int *qtail) 3734 { 3735 struct softnet_data *sd; 3736 unsigned long flags; 3737 unsigned int qlen; 3738 3739 sd = &per_cpu(softnet_data, cpu); 3740 3741 local_irq_save(flags); 3742 3743 rps_lock(sd); 3744 if (!netif_running(skb->dev)) 3745 goto drop; 3746 qlen = skb_queue_len(&sd->input_pkt_queue); 3747 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) { 3748 if (qlen) { 3749 enqueue: 3750 __skb_queue_tail(&sd->input_pkt_queue, skb); 3751 input_queue_tail_incr_save(sd, qtail); 3752 rps_unlock(sd); 3753 local_irq_restore(flags); 3754 return NET_RX_SUCCESS; 3755 } 3756 3757 /* Schedule NAPI for backlog device 3758 * We can use non atomic operation since we own the queue lock 3759 */ 3760 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) { 3761 if (!rps_ipi_queued(sd)) 3762 ____napi_schedule(sd, &sd->backlog); 3763 } 3764 goto enqueue; 3765 } 3766 3767 drop: 3768 sd->dropped++; 3769 rps_unlock(sd); 3770 3771 local_irq_restore(flags); 3772 3773 atomic_long_inc(&skb->dev->rx_dropped); 3774 kfree_skb(skb); 3775 return NET_RX_DROP; 3776 } 3777 3778 static int netif_rx_internal(struct sk_buff *skb) 3779 { 3780 int ret; 3781 3782 net_timestamp_check(netdev_tstamp_prequeue, skb); 3783 3784 trace_netif_rx(skb); 3785 #ifdef CONFIG_RPS 3786 if (static_key_false(&rps_needed)) { 3787 struct rps_dev_flow voidflow, *rflow = &voidflow; 3788 int cpu; 3789 3790 preempt_disable(); 3791 rcu_read_lock(); 3792 3793 cpu = get_rps_cpu(skb->dev, skb, &rflow); 3794 if (cpu < 0) 3795 cpu = smp_processor_id(); 3796 3797 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 3798 3799 rcu_read_unlock(); 3800 preempt_enable(); 3801 } else 3802 #endif 3803 { 3804 unsigned int qtail; 3805 3806 ret = enqueue_to_backlog(skb, get_cpu(), &qtail); 3807 put_cpu(); 3808 } 3809 return ret; 3810 } 3811 3812 /** 3813 * netif_rx - post buffer to the network code 3814 * @skb: buffer to post 3815 * 3816 * This function receives a packet from a device driver and queues it for 3817 * the upper (protocol) levels to process. It always succeeds. The buffer 3818 * may be dropped during processing for congestion control or by the 3819 * protocol layers. 3820 * 3821 * return values: 3822 * NET_RX_SUCCESS (no congestion) 3823 * NET_RX_DROP (packet was dropped) 3824 * 3825 */ 3826 3827 int netif_rx(struct sk_buff *skb) 3828 { 3829 trace_netif_rx_entry(skb); 3830 3831 return netif_rx_internal(skb); 3832 } 3833 EXPORT_SYMBOL(netif_rx); 3834 3835 int netif_rx_ni(struct sk_buff *skb) 3836 { 3837 int err; 3838 3839 trace_netif_rx_ni_entry(skb); 3840 3841 preempt_disable(); 3842 err = netif_rx_internal(skb); 3843 if (local_softirq_pending()) 3844 do_softirq(); 3845 preempt_enable(); 3846 3847 return err; 3848 } 3849 EXPORT_SYMBOL(netif_rx_ni); 3850 3851 static __latent_entropy void net_tx_action(struct softirq_action *h) 3852 { 3853 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 3854 3855 if (sd->completion_queue) { 3856 struct sk_buff *clist; 3857 3858 local_irq_disable(); 3859 clist = sd->completion_queue; 3860 sd->completion_queue = NULL; 3861 local_irq_enable(); 3862 3863 while (clist) { 3864 struct sk_buff *skb = clist; 3865 3866 clist = clist->next; 3867 3868 WARN_ON(atomic_read(&skb->users)); 3869 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED)) 3870 trace_consume_skb(skb); 3871 else 3872 trace_kfree_skb(skb, net_tx_action); 3873 3874 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) 3875 __kfree_skb(skb); 3876 else 3877 __kfree_skb_defer(skb); 3878 } 3879 3880 __kfree_skb_flush(); 3881 } 3882 3883 if (sd->output_queue) { 3884 struct Qdisc *head; 3885 3886 local_irq_disable(); 3887 head = sd->output_queue; 3888 sd->output_queue = NULL; 3889 sd->output_queue_tailp = &sd->output_queue; 3890 local_irq_enable(); 3891 3892 while (head) { 3893 struct Qdisc *q = head; 3894 spinlock_t *root_lock; 3895 3896 head = head->next_sched; 3897 3898 root_lock = qdisc_lock(q); 3899 spin_lock(root_lock); 3900 /* We need to make sure head->next_sched is read 3901 * before clearing __QDISC_STATE_SCHED 3902 */ 3903 smp_mb__before_atomic(); 3904 clear_bit(__QDISC_STATE_SCHED, &q->state); 3905 qdisc_run(q); 3906 spin_unlock(root_lock); 3907 } 3908 } 3909 } 3910 3911 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE) 3912 /* This hook is defined here for ATM LANE */ 3913 int (*br_fdb_test_addr_hook)(struct net_device *dev, 3914 unsigned char *addr) __read_mostly; 3915 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); 3916 #endif 3917 3918 static inline struct sk_buff * 3919 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, 3920 struct net_device *orig_dev) 3921 { 3922 #ifdef CONFIG_NET_CLS_ACT 3923 struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list); 3924 struct tcf_result cl_res; 3925 3926 /* If there's at least one ingress present somewhere (so 3927 * we get here via enabled static key), remaining devices 3928 * that are not configured with an ingress qdisc will bail 3929 * out here. 3930 */ 3931 if (!cl) 3932 return skb; 3933 if (*pt_prev) { 3934 *ret = deliver_skb(skb, *pt_prev, orig_dev); 3935 *pt_prev = NULL; 3936 } 3937 3938 qdisc_skb_cb(skb)->pkt_len = skb->len; 3939 skb->tc_at_ingress = 1; 3940 qdisc_bstats_cpu_update(cl->q, skb); 3941 3942 switch (tc_classify(skb, cl, &cl_res, false)) { 3943 case TC_ACT_OK: 3944 case TC_ACT_RECLASSIFY: 3945 skb->tc_index = TC_H_MIN(cl_res.classid); 3946 break; 3947 case TC_ACT_SHOT: 3948 qdisc_qstats_cpu_drop(cl->q); 3949 kfree_skb(skb); 3950 return NULL; 3951 case TC_ACT_STOLEN: 3952 case TC_ACT_QUEUED: 3953 consume_skb(skb); 3954 return NULL; 3955 case TC_ACT_REDIRECT: 3956 /* skb_mac_header check was done by cls/act_bpf, so 3957 * we can safely push the L2 header back before 3958 * redirecting to another netdev 3959 */ 3960 __skb_push(skb, skb->mac_len); 3961 skb_do_redirect(skb); 3962 return NULL; 3963 default: 3964 break; 3965 } 3966 #endif /* CONFIG_NET_CLS_ACT */ 3967 return skb; 3968 } 3969 3970 /** 3971 * netdev_is_rx_handler_busy - check if receive handler is registered 3972 * @dev: device to check 3973 * 3974 * Check if a receive handler is already registered for a given device. 3975 * Return true if there one. 3976 * 3977 * The caller must hold the rtnl_mutex. 3978 */ 3979 bool netdev_is_rx_handler_busy(struct net_device *dev) 3980 { 3981 ASSERT_RTNL(); 3982 return dev && rtnl_dereference(dev->rx_handler); 3983 } 3984 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy); 3985 3986 /** 3987 * netdev_rx_handler_register - register receive handler 3988 * @dev: device to register a handler for 3989 * @rx_handler: receive handler to register 3990 * @rx_handler_data: data pointer that is used by rx handler 3991 * 3992 * Register a receive handler for a device. This handler will then be 3993 * called from __netif_receive_skb. A negative errno code is returned 3994 * on a failure. 3995 * 3996 * The caller must hold the rtnl_mutex. 3997 * 3998 * For a general description of rx_handler, see enum rx_handler_result. 3999 */ 4000 int netdev_rx_handler_register(struct net_device *dev, 4001 rx_handler_func_t *rx_handler, 4002 void *rx_handler_data) 4003 { 4004 if (netdev_is_rx_handler_busy(dev)) 4005 return -EBUSY; 4006 4007 /* Note: rx_handler_data must be set before rx_handler */ 4008 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); 4009 rcu_assign_pointer(dev->rx_handler, rx_handler); 4010 4011 return 0; 4012 } 4013 EXPORT_SYMBOL_GPL(netdev_rx_handler_register); 4014 4015 /** 4016 * netdev_rx_handler_unregister - unregister receive handler 4017 * @dev: device to unregister a handler from 4018 * 4019 * Unregister a receive handler from a device. 4020 * 4021 * The caller must hold the rtnl_mutex. 4022 */ 4023 void netdev_rx_handler_unregister(struct net_device *dev) 4024 { 4025 4026 ASSERT_RTNL(); 4027 RCU_INIT_POINTER(dev->rx_handler, NULL); 4028 /* a reader seeing a non NULL rx_handler in a rcu_read_lock() 4029 * section has a guarantee to see a non NULL rx_handler_data 4030 * as well. 4031 */ 4032 synchronize_net(); 4033 RCU_INIT_POINTER(dev->rx_handler_data, NULL); 4034 } 4035 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); 4036 4037 /* 4038 * Limit the use of PFMEMALLOC reserves to those protocols that implement 4039 * the special handling of PFMEMALLOC skbs. 4040 */ 4041 static bool skb_pfmemalloc_protocol(struct sk_buff *skb) 4042 { 4043 switch (skb->protocol) { 4044 case htons(ETH_P_ARP): 4045 case htons(ETH_P_IP): 4046 case htons(ETH_P_IPV6): 4047 case htons(ETH_P_8021Q): 4048 case htons(ETH_P_8021AD): 4049 return true; 4050 default: 4051 return false; 4052 } 4053 } 4054 4055 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev, 4056 int *ret, struct net_device *orig_dev) 4057 { 4058 #ifdef CONFIG_NETFILTER_INGRESS 4059 if (nf_hook_ingress_active(skb)) { 4060 int ingress_retval; 4061 4062 if (*pt_prev) { 4063 *ret = deliver_skb(skb, *pt_prev, orig_dev); 4064 *pt_prev = NULL; 4065 } 4066 4067 rcu_read_lock(); 4068 ingress_retval = nf_hook_ingress(skb); 4069 rcu_read_unlock(); 4070 return ingress_retval; 4071 } 4072 #endif /* CONFIG_NETFILTER_INGRESS */ 4073 return 0; 4074 } 4075 4076 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc) 4077 { 4078 struct packet_type *ptype, *pt_prev; 4079 rx_handler_func_t *rx_handler; 4080 struct net_device *orig_dev; 4081 bool deliver_exact = false; 4082 int ret = NET_RX_DROP; 4083 __be16 type; 4084 4085 net_timestamp_check(!netdev_tstamp_prequeue, skb); 4086 4087 trace_netif_receive_skb(skb); 4088 4089 orig_dev = skb->dev; 4090 4091 skb_reset_network_header(skb); 4092 if (!skb_transport_header_was_set(skb)) 4093 skb_reset_transport_header(skb); 4094 skb_reset_mac_len(skb); 4095 4096 pt_prev = NULL; 4097 4098 another_round: 4099 skb->skb_iif = skb->dev->ifindex; 4100 4101 __this_cpu_inc(softnet_data.processed); 4102 4103 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 4104 skb->protocol == cpu_to_be16(ETH_P_8021AD)) { 4105 skb = skb_vlan_untag(skb); 4106 if (unlikely(!skb)) 4107 goto out; 4108 } 4109 4110 if (skb_skip_tc_classify(skb)) 4111 goto skip_classify; 4112 4113 if (pfmemalloc) 4114 goto skip_taps; 4115 4116 list_for_each_entry_rcu(ptype, &ptype_all, list) { 4117 if (pt_prev) 4118 ret = deliver_skb(skb, pt_prev, orig_dev); 4119 pt_prev = ptype; 4120 } 4121 4122 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { 4123 if (pt_prev) 4124 ret = deliver_skb(skb, pt_prev, orig_dev); 4125 pt_prev = ptype; 4126 } 4127 4128 skip_taps: 4129 #ifdef CONFIG_NET_INGRESS 4130 if (static_key_false(&ingress_needed)) { 4131 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev); 4132 if (!skb) 4133 goto out; 4134 4135 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0) 4136 goto out; 4137 } 4138 #endif 4139 skb_reset_tc(skb); 4140 skip_classify: 4141 if (pfmemalloc && !skb_pfmemalloc_protocol(skb)) 4142 goto drop; 4143 4144 if (skb_vlan_tag_present(skb)) { 4145 if (pt_prev) { 4146 ret = deliver_skb(skb, pt_prev, orig_dev); 4147 pt_prev = NULL; 4148 } 4149 if (vlan_do_receive(&skb)) 4150 goto another_round; 4151 else if (unlikely(!skb)) 4152 goto out; 4153 } 4154 4155 rx_handler = rcu_dereference(skb->dev->rx_handler); 4156 if (rx_handler) { 4157 if (pt_prev) { 4158 ret = deliver_skb(skb, pt_prev, orig_dev); 4159 pt_prev = NULL; 4160 } 4161 switch (rx_handler(&skb)) { 4162 case RX_HANDLER_CONSUMED: 4163 ret = NET_RX_SUCCESS; 4164 goto out; 4165 case RX_HANDLER_ANOTHER: 4166 goto another_round; 4167 case RX_HANDLER_EXACT: 4168 deliver_exact = true; 4169 case RX_HANDLER_PASS: 4170 break; 4171 default: 4172 BUG(); 4173 } 4174 } 4175 4176 if (unlikely(skb_vlan_tag_present(skb))) { 4177 if (skb_vlan_tag_get_id(skb)) 4178 skb->pkt_type = PACKET_OTHERHOST; 4179 /* Note: we might in the future use prio bits 4180 * and set skb->priority like in vlan_do_receive() 4181 * For the time being, just ignore Priority Code Point 4182 */ 4183 skb->vlan_tci = 0; 4184 } 4185 4186 type = skb->protocol; 4187 4188 /* deliver only exact match when indicated */ 4189 if (likely(!deliver_exact)) { 4190 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4191 &ptype_base[ntohs(type) & 4192 PTYPE_HASH_MASK]); 4193 } 4194 4195 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4196 &orig_dev->ptype_specific); 4197 4198 if (unlikely(skb->dev != orig_dev)) { 4199 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4200 &skb->dev->ptype_specific); 4201 } 4202 4203 if (pt_prev) { 4204 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) 4205 goto drop; 4206 else 4207 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 4208 } else { 4209 drop: 4210 if (!deliver_exact) 4211 atomic_long_inc(&skb->dev->rx_dropped); 4212 else 4213 atomic_long_inc(&skb->dev->rx_nohandler); 4214 kfree_skb(skb); 4215 /* Jamal, now you will not able to escape explaining 4216 * me how you were going to use this. :-) 4217 */ 4218 ret = NET_RX_DROP; 4219 } 4220 4221 out: 4222 return ret; 4223 } 4224 4225 static int __netif_receive_skb(struct sk_buff *skb) 4226 { 4227 int ret; 4228 4229 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) { 4230 unsigned long pflags = current->flags; 4231 4232 /* 4233 * PFMEMALLOC skbs are special, they should 4234 * - be delivered to SOCK_MEMALLOC sockets only 4235 * - stay away from userspace 4236 * - have bounded memory usage 4237 * 4238 * Use PF_MEMALLOC as this saves us from propagating the allocation 4239 * context down to all allocation sites. 4240 */ 4241 current->flags |= PF_MEMALLOC; 4242 ret = __netif_receive_skb_core(skb, true); 4243 tsk_restore_flags(current, pflags, PF_MEMALLOC); 4244 } else 4245 ret = __netif_receive_skb_core(skb, false); 4246 4247 return ret; 4248 } 4249 4250 static int netif_receive_skb_internal(struct sk_buff *skb) 4251 { 4252 int ret; 4253 4254 net_timestamp_check(netdev_tstamp_prequeue, skb); 4255 4256 if (skb_defer_rx_timestamp(skb)) 4257 return NET_RX_SUCCESS; 4258 4259 rcu_read_lock(); 4260 4261 #ifdef CONFIG_RPS 4262 if (static_key_false(&rps_needed)) { 4263 struct rps_dev_flow voidflow, *rflow = &voidflow; 4264 int cpu = get_rps_cpu(skb->dev, skb, &rflow); 4265 4266 if (cpu >= 0) { 4267 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 4268 rcu_read_unlock(); 4269 return ret; 4270 } 4271 } 4272 #endif 4273 ret = __netif_receive_skb(skb); 4274 rcu_read_unlock(); 4275 return ret; 4276 } 4277 4278 /** 4279 * netif_receive_skb - process receive buffer from network 4280 * @skb: buffer to process 4281 * 4282 * netif_receive_skb() is the main receive data processing function. 4283 * It always succeeds. The buffer may be dropped during processing 4284 * for congestion control or by the protocol layers. 4285 * 4286 * This function may only be called from softirq context and interrupts 4287 * should be enabled. 4288 * 4289 * Return values (usually ignored): 4290 * NET_RX_SUCCESS: no congestion 4291 * NET_RX_DROP: packet was dropped 4292 */ 4293 int netif_receive_skb(struct sk_buff *skb) 4294 { 4295 trace_netif_receive_skb_entry(skb); 4296 4297 return netif_receive_skb_internal(skb); 4298 } 4299 EXPORT_SYMBOL(netif_receive_skb); 4300 4301 DEFINE_PER_CPU(struct work_struct, flush_works); 4302 4303 /* Network device is going away, flush any packets still pending */ 4304 static void flush_backlog(struct work_struct *work) 4305 { 4306 struct sk_buff *skb, *tmp; 4307 struct softnet_data *sd; 4308 4309 local_bh_disable(); 4310 sd = this_cpu_ptr(&softnet_data); 4311 4312 local_irq_disable(); 4313 rps_lock(sd); 4314 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { 4315 if (skb->dev->reg_state == NETREG_UNREGISTERING) { 4316 __skb_unlink(skb, &sd->input_pkt_queue); 4317 kfree_skb(skb); 4318 input_queue_head_incr(sd); 4319 } 4320 } 4321 rps_unlock(sd); 4322 local_irq_enable(); 4323 4324 skb_queue_walk_safe(&sd->process_queue, skb, tmp) { 4325 if (skb->dev->reg_state == NETREG_UNREGISTERING) { 4326 __skb_unlink(skb, &sd->process_queue); 4327 kfree_skb(skb); 4328 input_queue_head_incr(sd); 4329 } 4330 } 4331 local_bh_enable(); 4332 } 4333 4334 static void flush_all_backlogs(void) 4335 { 4336 unsigned int cpu; 4337 4338 get_online_cpus(); 4339 4340 for_each_online_cpu(cpu) 4341 queue_work_on(cpu, system_highpri_wq, 4342 per_cpu_ptr(&flush_works, cpu)); 4343 4344 for_each_online_cpu(cpu) 4345 flush_work(per_cpu_ptr(&flush_works, cpu)); 4346 4347 put_online_cpus(); 4348 } 4349 4350 static int napi_gro_complete(struct sk_buff *skb) 4351 { 4352 struct packet_offload *ptype; 4353 __be16 type = skb->protocol; 4354 struct list_head *head = &offload_base; 4355 int err = -ENOENT; 4356 4357 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); 4358 4359 if (NAPI_GRO_CB(skb)->count == 1) { 4360 skb_shinfo(skb)->gso_size = 0; 4361 goto out; 4362 } 4363 4364 rcu_read_lock(); 4365 list_for_each_entry_rcu(ptype, head, list) { 4366 if (ptype->type != type || !ptype->callbacks.gro_complete) 4367 continue; 4368 4369 err = ptype->callbacks.gro_complete(skb, 0); 4370 break; 4371 } 4372 rcu_read_unlock(); 4373 4374 if (err) { 4375 WARN_ON(&ptype->list == head); 4376 kfree_skb(skb); 4377 return NET_RX_SUCCESS; 4378 } 4379 4380 out: 4381 return netif_receive_skb_internal(skb); 4382 } 4383 4384 /* napi->gro_list contains packets ordered by age. 4385 * youngest packets at the head of it. 4386 * Complete skbs in reverse order to reduce latencies. 4387 */ 4388 void napi_gro_flush(struct napi_struct *napi, bool flush_old) 4389 { 4390 struct sk_buff *skb, *prev = NULL; 4391 4392 /* scan list and build reverse chain */ 4393 for (skb = napi->gro_list; skb != NULL; skb = skb->next) { 4394 skb->prev = prev; 4395 prev = skb; 4396 } 4397 4398 for (skb = prev; skb; skb = prev) { 4399 skb->next = NULL; 4400 4401 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) 4402 return; 4403 4404 prev = skb->prev; 4405 napi_gro_complete(skb); 4406 napi->gro_count--; 4407 } 4408 4409 napi->gro_list = NULL; 4410 } 4411 EXPORT_SYMBOL(napi_gro_flush); 4412 4413 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb) 4414 { 4415 struct sk_buff *p; 4416 unsigned int maclen = skb->dev->hard_header_len; 4417 u32 hash = skb_get_hash_raw(skb); 4418 4419 for (p = napi->gro_list; p; p = p->next) { 4420 unsigned long diffs; 4421 4422 NAPI_GRO_CB(p)->flush = 0; 4423 4424 if (hash != skb_get_hash_raw(p)) { 4425 NAPI_GRO_CB(p)->same_flow = 0; 4426 continue; 4427 } 4428 4429 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; 4430 diffs |= p->vlan_tci ^ skb->vlan_tci; 4431 diffs |= skb_metadata_dst_cmp(p, skb); 4432 if (maclen == ETH_HLEN) 4433 diffs |= compare_ether_header(skb_mac_header(p), 4434 skb_mac_header(skb)); 4435 else if (!diffs) 4436 diffs = memcmp(skb_mac_header(p), 4437 skb_mac_header(skb), 4438 maclen); 4439 NAPI_GRO_CB(p)->same_flow = !diffs; 4440 } 4441 } 4442 4443 static void skb_gro_reset_offset(struct sk_buff *skb) 4444 { 4445 const struct skb_shared_info *pinfo = skb_shinfo(skb); 4446 const skb_frag_t *frag0 = &pinfo->frags[0]; 4447 4448 NAPI_GRO_CB(skb)->data_offset = 0; 4449 NAPI_GRO_CB(skb)->frag0 = NULL; 4450 NAPI_GRO_CB(skb)->frag0_len = 0; 4451 4452 if (skb_mac_header(skb) == skb_tail_pointer(skb) && 4453 pinfo->nr_frags && 4454 !PageHighMem(skb_frag_page(frag0))) { 4455 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); 4456 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int, 4457 skb_frag_size(frag0), 4458 skb->end - skb->tail); 4459 } 4460 } 4461 4462 static void gro_pull_from_frag0(struct sk_buff *skb, int grow) 4463 { 4464 struct skb_shared_info *pinfo = skb_shinfo(skb); 4465 4466 BUG_ON(skb->end - skb->tail < grow); 4467 4468 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); 4469 4470 skb->data_len -= grow; 4471 skb->tail += grow; 4472 4473 pinfo->frags[0].page_offset += grow; 4474 skb_frag_size_sub(&pinfo->frags[0], grow); 4475 4476 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { 4477 skb_frag_unref(skb, 0); 4478 memmove(pinfo->frags, pinfo->frags + 1, 4479 --pinfo->nr_frags * sizeof(pinfo->frags[0])); 4480 } 4481 } 4482 4483 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 4484 { 4485 struct sk_buff **pp = NULL; 4486 struct packet_offload *ptype; 4487 __be16 type = skb->protocol; 4488 struct list_head *head = &offload_base; 4489 int same_flow; 4490 enum gro_result ret; 4491 int grow; 4492 4493 if (!(skb->dev->features & NETIF_F_GRO)) 4494 goto normal; 4495 4496 if (skb->csum_bad) 4497 goto normal; 4498 4499 gro_list_prepare(napi, skb); 4500 4501 rcu_read_lock(); 4502 list_for_each_entry_rcu(ptype, head, list) { 4503 if (ptype->type != type || !ptype->callbacks.gro_receive) 4504 continue; 4505 4506 skb_set_network_header(skb, skb_gro_offset(skb)); 4507 skb_reset_mac_len(skb); 4508 NAPI_GRO_CB(skb)->same_flow = 0; 4509 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb); 4510 NAPI_GRO_CB(skb)->free = 0; 4511 NAPI_GRO_CB(skb)->encap_mark = 0; 4512 NAPI_GRO_CB(skb)->recursion_counter = 0; 4513 NAPI_GRO_CB(skb)->is_fou = 0; 4514 NAPI_GRO_CB(skb)->is_atomic = 1; 4515 NAPI_GRO_CB(skb)->gro_remcsum_start = 0; 4516 4517 /* Setup for GRO checksum validation */ 4518 switch (skb->ip_summed) { 4519 case CHECKSUM_COMPLETE: 4520 NAPI_GRO_CB(skb)->csum = skb->csum; 4521 NAPI_GRO_CB(skb)->csum_valid = 1; 4522 NAPI_GRO_CB(skb)->csum_cnt = 0; 4523 break; 4524 case CHECKSUM_UNNECESSARY: 4525 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; 4526 NAPI_GRO_CB(skb)->csum_valid = 0; 4527 break; 4528 default: 4529 NAPI_GRO_CB(skb)->csum_cnt = 0; 4530 NAPI_GRO_CB(skb)->csum_valid = 0; 4531 } 4532 4533 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb); 4534 break; 4535 } 4536 rcu_read_unlock(); 4537 4538 if (&ptype->list == head) 4539 goto normal; 4540 4541 if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) { 4542 ret = GRO_CONSUMED; 4543 goto ok; 4544 } 4545 4546 same_flow = NAPI_GRO_CB(skb)->same_flow; 4547 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; 4548 4549 if (pp) { 4550 struct sk_buff *nskb = *pp; 4551 4552 *pp = nskb->next; 4553 nskb->next = NULL; 4554 napi_gro_complete(nskb); 4555 napi->gro_count--; 4556 } 4557 4558 if (same_flow) 4559 goto ok; 4560 4561 if (NAPI_GRO_CB(skb)->flush) 4562 goto normal; 4563 4564 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) { 4565 struct sk_buff *nskb = napi->gro_list; 4566 4567 /* locate the end of the list to select the 'oldest' flow */ 4568 while (nskb->next) { 4569 pp = &nskb->next; 4570 nskb = *pp; 4571 } 4572 *pp = NULL; 4573 nskb->next = NULL; 4574 napi_gro_complete(nskb); 4575 } else { 4576 napi->gro_count++; 4577 } 4578 NAPI_GRO_CB(skb)->count = 1; 4579 NAPI_GRO_CB(skb)->age = jiffies; 4580 NAPI_GRO_CB(skb)->last = skb; 4581 skb_shinfo(skb)->gso_size = skb_gro_len(skb); 4582 skb->next = napi->gro_list; 4583 napi->gro_list = skb; 4584 ret = GRO_HELD; 4585 4586 pull: 4587 grow = skb_gro_offset(skb) - skb_headlen(skb); 4588 if (grow > 0) 4589 gro_pull_from_frag0(skb, grow); 4590 ok: 4591 return ret; 4592 4593 normal: 4594 ret = GRO_NORMAL; 4595 goto pull; 4596 } 4597 4598 struct packet_offload *gro_find_receive_by_type(__be16 type) 4599 { 4600 struct list_head *offload_head = &offload_base; 4601 struct packet_offload *ptype; 4602 4603 list_for_each_entry_rcu(ptype, offload_head, list) { 4604 if (ptype->type != type || !ptype->callbacks.gro_receive) 4605 continue; 4606 return ptype; 4607 } 4608 return NULL; 4609 } 4610 EXPORT_SYMBOL(gro_find_receive_by_type); 4611 4612 struct packet_offload *gro_find_complete_by_type(__be16 type) 4613 { 4614 struct list_head *offload_head = &offload_base; 4615 struct packet_offload *ptype; 4616 4617 list_for_each_entry_rcu(ptype, offload_head, list) { 4618 if (ptype->type != type || !ptype->callbacks.gro_complete) 4619 continue; 4620 return ptype; 4621 } 4622 return NULL; 4623 } 4624 EXPORT_SYMBOL(gro_find_complete_by_type); 4625 4626 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) 4627 { 4628 switch (ret) { 4629 case GRO_NORMAL: 4630 if (netif_receive_skb_internal(skb)) 4631 ret = GRO_DROP; 4632 break; 4633 4634 case GRO_DROP: 4635 kfree_skb(skb); 4636 break; 4637 4638 case GRO_MERGED_FREE: 4639 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) { 4640 skb_dst_drop(skb); 4641 secpath_reset(skb); 4642 kmem_cache_free(skbuff_head_cache, skb); 4643 } else { 4644 __kfree_skb(skb); 4645 } 4646 break; 4647 4648 case GRO_HELD: 4649 case GRO_MERGED: 4650 case GRO_CONSUMED: 4651 break; 4652 } 4653 4654 return ret; 4655 } 4656 4657 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 4658 { 4659 skb_mark_napi_id(skb, napi); 4660 trace_napi_gro_receive_entry(skb); 4661 4662 skb_gro_reset_offset(skb); 4663 4664 return napi_skb_finish(dev_gro_receive(napi, skb), skb); 4665 } 4666 EXPORT_SYMBOL(napi_gro_receive); 4667 4668 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) 4669 { 4670 if (unlikely(skb->pfmemalloc)) { 4671 consume_skb(skb); 4672 return; 4673 } 4674 __skb_pull(skb, skb_headlen(skb)); 4675 /* restore the reserve we had after netdev_alloc_skb_ip_align() */ 4676 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); 4677 skb->vlan_tci = 0; 4678 skb->dev = napi->dev; 4679 skb->skb_iif = 0; 4680 skb->encapsulation = 0; 4681 skb_shinfo(skb)->gso_type = 0; 4682 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 4683 secpath_reset(skb); 4684 4685 napi->skb = skb; 4686 } 4687 4688 struct sk_buff *napi_get_frags(struct napi_struct *napi) 4689 { 4690 struct sk_buff *skb = napi->skb; 4691 4692 if (!skb) { 4693 skb = napi_alloc_skb(napi, GRO_MAX_HEAD); 4694 if (skb) { 4695 napi->skb = skb; 4696 skb_mark_napi_id(skb, napi); 4697 } 4698 } 4699 return skb; 4700 } 4701 EXPORT_SYMBOL(napi_get_frags); 4702 4703 static gro_result_t napi_frags_finish(struct napi_struct *napi, 4704 struct sk_buff *skb, 4705 gro_result_t ret) 4706 { 4707 switch (ret) { 4708 case GRO_NORMAL: 4709 case GRO_HELD: 4710 __skb_push(skb, ETH_HLEN); 4711 skb->protocol = eth_type_trans(skb, skb->dev); 4712 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb)) 4713 ret = GRO_DROP; 4714 break; 4715 4716 case GRO_DROP: 4717 case GRO_MERGED_FREE: 4718 napi_reuse_skb(napi, skb); 4719 break; 4720 4721 case GRO_MERGED: 4722 case GRO_CONSUMED: 4723 break; 4724 } 4725 4726 return ret; 4727 } 4728 4729 /* Upper GRO stack assumes network header starts at gro_offset=0 4730 * Drivers could call both napi_gro_frags() and napi_gro_receive() 4731 * We copy ethernet header into skb->data to have a common layout. 4732 */ 4733 static struct sk_buff *napi_frags_skb(struct napi_struct *napi) 4734 { 4735 struct sk_buff *skb = napi->skb; 4736 const struct ethhdr *eth; 4737 unsigned int hlen = sizeof(*eth); 4738 4739 napi->skb = NULL; 4740 4741 skb_reset_mac_header(skb); 4742 skb_gro_reset_offset(skb); 4743 4744 eth = skb_gro_header_fast(skb, 0); 4745 if (unlikely(skb_gro_header_hard(skb, hlen))) { 4746 eth = skb_gro_header_slow(skb, hlen, 0); 4747 if (unlikely(!eth)) { 4748 net_warn_ratelimited("%s: dropping impossible skb from %s\n", 4749 __func__, napi->dev->name); 4750 napi_reuse_skb(napi, skb); 4751 return NULL; 4752 } 4753 } else { 4754 gro_pull_from_frag0(skb, hlen); 4755 NAPI_GRO_CB(skb)->frag0 += hlen; 4756 NAPI_GRO_CB(skb)->frag0_len -= hlen; 4757 } 4758 __skb_pull(skb, hlen); 4759 4760 /* 4761 * This works because the only protocols we care about don't require 4762 * special handling. 4763 * We'll fix it up properly in napi_frags_finish() 4764 */ 4765 skb->protocol = eth->h_proto; 4766 4767 return skb; 4768 } 4769 4770 gro_result_t napi_gro_frags(struct napi_struct *napi) 4771 { 4772 struct sk_buff *skb = napi_frags_skb(napi); 4773 4774 if (!skb) 4775 return GRO_DROP; 4776 4777 trace_napi_gro_frags_entry(skb); 4778 4779 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); 4780 } 4781 EXPORT_SYMBOL(napi_gro_frags); 4782 4783 /* Compute the checksum from gro_offset and return the folded value 4784 * after adding in any pseudo checksum. 4785 */ 4786 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb) 4787 { 4788 __wsum wsum; 4789 __sum16 sum; 4790 4791 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); 4792 4793 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ 4794 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); 4795 if (likely(!sum)) { 4796 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 4797 !skb->csum_complete_sw) 4798 netdev_rx_csum_fault(skb->dev); 4799 } 4800 4801 NAPI_GRO_CB(skb)->csum = wsum; 4802 NAPI_GRO_CB(skb)->csum_valid = 1; 4803 4804 return sum; 4805 } 4806 EXPORT_SYMBOL(__skb_gro_checksum_complete); 4807 4808 /* 4809 * net_rps_action_and_irq_enable sends any pending IPI's for rps. 4810 * Note: called with local irq disabled, but exits with local irq enabled. 4811 */ 4812 static void net_rps_action_and_irq_enable(struct softnet_data *sd) 4813 { 4814 #ifdef CONFIG_RPS 4815 struct softnet_data *remsd = sd->rps_ipi_list; 4816 4817 if (remsd) { 4818 sd->rps_ipi_list = NULL; 4819 4820 local_irq_enable(); 4821 4822 /* Send pending IPI's to kick RPS processing on remote cpus. */ 4823 while (remsd) { 4824 struct softnet_data *next = remsd->rps_ipi_next; 4825 4826 if (cpu_online(remsd->cpu)) 4827 smp_call_function_single_async(remsd->cpu, 4828 &remsd->csd); 4829 remsd = next; 4830 } 4831 } else 4832 #endif 4833 local_irq_enable(); 4834 } 4835 4836 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) 4837 { 4838 #ifdef CONFIG_RPS 4839 return sd->rps_ipi_list != NULL; 4840 #else 4841 return false; 4842 #endif 4843 } 4844 4845 static int process_backlog(struct napi_struct *napi, int quota) 4846 { 4847 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); 4848 bool again = true; 4849 int work = 0; 4850 4851 /* Check if we have pending ipi, its better to send them now, 4852 * not waiting net_rx_action() end. 4853 */ 4854 if (sd_has_rps_ipi_waiting(sd)) { 4855 local_irq_disable(); 4856 net_rps_action_and_irq_enable(sd); 4857 } 4858 4859 napi->weight = dev_rx_weight; 4860 while (again) { 4861 struct sk_buff *skb; 4862 4863 while ((skb = __skb_dequeue(&sd->process_queue))) { 4864 rcu_read_lock(); 4865 __netif_receive_skb(skb); 4866 rcu_read_unlock(); 4867 input_queue_head_incr(sd); 4868 if (++work >= quota) 4869 return work; 4870 4871 } 4872 4873 local_irq_disable(); 4874 rps_lock(sd); 4875 if (skb_queue_empty(&sd->input_pkt_queue)) { 4876 /* 4877 * Inline a custom version of __napi_complete(). 4878 * only current cpu owns and manipulates this napi, 4879 * and NAPI_STATE_SCHED is the only possible flag set 4880 * on backlog. 4881 * We can use a plain write instead of clear_bit(), 4882 * and we dont need an smp_mb() memory barrier. 4883 */ 4884 napi->state = 0; 4885 again = false; 4886 } else { 4887 skb_queue_splice_tail_init(&sd->input_pkt_queue, 4888 &sd->process_queue); 4889 } 4890 rps_unlock(sd); 4891 local_irq_enable(); 4892 } 4893 4894 return work; 4895 } 4896 4897 /** 4898 * __napi_schedule - schedule for receive 4899 * @n: entry to schedule 4900 * 4901 * The entry's receive function will be scheduled to run. 4902 * Consider using __napi_schedule_irqoff() if hard irqs are masked. 4903 */ 4904 void __napi_schedule(struct napi_struct *n) 4905 { 4906 unsigned long flags; 4907 4908 local_irq_save(flags); 4909 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 4910 local_irq_restore(flags); 4911 } 4912 EXPORT_SYMBOL(__napi_schedule); 4913 4914 /** 4915 * napi_schedule_prep - check if napi can be scheduled 4916 * @n: napi context 4917 * 4918 * Test if NAPI routine is already running, and if not mark 4919 * it as running. This is used as a condition variable 4920 * insure only one NAPI poll instance runs. We also make 4921 * sure there is no pending NAPI disable. 4922 */ 4923 bool napi_schedule_prep(struct napi_struct *n) 4924 { 4925 unsigned long val, new; 4926 4927 do { 4928 val = READ_ONCE(n->state); 4929 if (unlikely(val & NAPIF_STATE_DISABLE)) 4930 return false; 4931 new = val | NAPIF_STATE_SCHED; 4932 4933 /* Sets STATE_MISSED bit if STATE_SCHED was already set 4934 * This was suggested by Alexander Duyck, as compiler 4935 * emits better code than : 4936 * if (val & NAPIF_STATE_SCHED) 4937 * new |= NAPIF_STATE_MISSED; 4938 */ 4939 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED * 4940 NAPIF_STATE_MISSED; 4941 } while (cmpxchg(&n->state, val, new) != val); 4942 4943 return !(val & NAPIF_STATE_SCHED); 4944 } 4945 EXPORT_SYMBOL(napi_schedule_prep); 4946 4947 /** 4948 * __napi_schedule_irqoff - schedule for receive 4949 * @n: entry to schedule 4950 * 4951 * Variant of __napi_schedule() assuming hard irqs are masked 4952 */ 4953 void __napi_schedule_irqoff(struct napi_struct *n) 4954 { 4955 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 4956 } 4957 EXPORT_SYMBOL(__napi_schedule_irqoff); 4958 4959 bool napi_complete_done(struct napi_struct *n, int work_done) 4960 { 4961 unsigned long flags, val, new; 4962 4963 /* 4964 * 1) Don't let napi dequeue from the cpu poll list 4965 * just in case its running on a different cpu. 4966 * 2) If we are busy polling, do nothing here, we have 4967 * the guarantee we will be called later. 4968 */ 4969 if (unlikely(n->state & (NAPIF_STATE_NPSVC | 4970 NAPIF_STATE_IN_BUSY_POLL))) 4971 return false; 4972 4973 if (n->gro_list) { 4974 unsigned long timeout = 0; 4975 4976 if (work_done) 4977 timeout = n->dev->gro_flush_timeout; 4978 4979 if (timeout) 4980 hrtimer_start(&n->timer, ns_to_ktime(timeout), 4981 HRTIMER_MODE_REL_PINNED); 4982 else 4983 napi_gro_flush(n, false); 4984 } 4985 if (unlikely(!list_empty(&n->poll_list))) { 4986 /* If n->poll_list is not empty, we need to mask irqs */ 4987 local_irq_save(flags); 4988 list_del_init(&n->poll_list); 4989 local_irq_restore(flags); 4990 } 4991 4992 do { 4993 val = READ_ONCE(n->state); 4994 4995 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED)); 4996 4997 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED); 4998 4999 /* If STATE_MISSED was set, leave STATE_SCHED set, 5000 * because we will call napi->poll() one more time. 5001 * This C code was suggested by Alexander Duyck to help gcc. 5002 */ 5003 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED * 5004 NAPIF_STATE_SCHED; 5005 } while (cmpxchg(&n->state, val, new) != val); 5006 5007 if (unlikely(val & NAPIF_STATE_MISSED)) { 5008 __napi_schedule(n); 5009 return false; 5010 } 5011 5012 return true; 5013 } 5014 EXPORT_SYMBOL(napi_complete_done); 5015 5016 /* must be called under rcu_read_lock(), as we dont take a reference */ 5017 static struct napi_struct *napi_by_id(unsigned int napi_id) 5018 { 5019 unsigned int hash = napi_id % HASH_SIZE(napi_hash); 5020 struct napi_struct *napi; 5021 5022 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node) 5023 if (napi->napi_id == napi_id) 5024 return napi; 5025 5026 return NULL; 5027 } 5028 5029 #if defined(CONFIG_NET_RX_BUSY_POLL) 5030 5031 #define BUSY_POLL_BUDGET 8 5032 5033 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock) 5034 { 5035 int rc; 5036 5037 /* Busy polling means there is a high chance device driver hard irq 5038 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was 5039 * set in napi_schedule_prep(). 5040 * Since we are about to call napi->poll() once more, we can safely 5041 * clear NAPI_STATE_MISSED. 5042 * 5043 * Note: x86 could use a single "lock and ..." instruction 5044 * to perform these two clear_bit() 5045 */ 5046 clear_bit(NAPI_STATE_MISSED, &napi->state); 5047 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state); 5048 5049 local_bh_disable(); 5050 5051 /* All we really want here is to re-enable device interrupts. 5052 * Ideally, a new ndo_busy_poll_stop() could avoid another round. 5053 */ 5054 rc = napi->poll(napi, BUSY_POLL_BUDGET); 5055 netpoll_poll_unlock(have_poll_lock); 5056 if (rc == BUSY_POLL_BUDGET) 5057 __napi_schedule(napi); 5058 local_bh_enable(); 5059 if (local_softirq_pending()) 5060 do_softirq(); 5061 } 5062 5063 bool sk_busy_loop(struct sock *sk, int nonblock) 5064 { 5065 unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0; 5066 int (*napi_poll)(struct napi_struct *napi, int budget); 5067 void *have_poll_lock = NULL; 5068 struct napi_struct *napi; 5069 int rc; 5070 5071 restart: 5072 rc = false; 5073 napi_poll = NULL; 5074 5075 rcu_read_lock(); 5076 5077 napi = napi_by_id(sk->sk_napi_id); 5078 if (!napi) 5079 goto out; 5080 5081 preempt_disable(); 5082 for (;;) { 5083 rc = 0; 5084 local_bh_disable(); 5085 if (!napi_poll) { 5086 unsigned long val = READ_ONCE(napi->state); 5087 5088 /* If multiple threads are competing for this napi, 5089 * we avoid dirtying napi->state as much as we can. 5090 */ 5091 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED | 5092 NAPIF_STATE_IN_BUSY_POLL)) 5093 goto count; 5094 if (cmpxchg(&napi->state, val, 5095 val | NAPIF_STATE_IN_BUSY_POLL | 5096 NAPIF_STATE_SCHED) != val) 5097 goto count; 5098 have_poll_lock = netpoll_poll_lock(napi); 5099 napi_poll = napi->poll; 5100 } 5101 rc = napi_poll(napi, BUSY_POLL_BUDGET); 5102 trace_napi_poll(napi, rc, BUSY_POLL_BUDGET); 5103 count: 5104 if (rc > 0) 5105 __NET_ADD_STATS(sock_net(sk), 5106 LINUX_MIB_BUSYPOLLRXPACKETS, rc); 5107 local_bh_enable(); 5108 5109 if (nonblock || !skb_queue_empty(&sk->sk_receive_queue) || 5110 busy_loop_timeout(end_time)) 5111 break; 5112 5113 if (unlikely(need_resched())) { 5114 if (napi_poll) 5115 busy_poll_stop(napi, have_poll_lock); 5116 preempt_enable(); 5117 rcu_read_unlock(); 5118 cond_resched(); 5119 rc = !skb_queue_empty(&sk->sk_receive_queue); 5120 if (rc || busy_loop_timeout(end_time)) 5121 return rc; 5122 goto restart; 5123 } 5124 cpu_relax(); 5125 } 5126 if (napi_poll) 5127 busy_poll_stop(napi, have_poll_lock); 5128 preempt_enable(); 5129 rc = !skb_queue_empty(&sk->sk_receive_queue); 5130 out: 5131 rcu_read_unlock(); 5132 return rc; 5133 } 5134 EXPORT_SYMBOL(sk_busy_loop); 5135 5136 #endif /* CONFIG_NET_RX_BUSY_POLL */ 5137 5138 static void napi_hash_add(struct napi_struct *napi) 5139 { 5140 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) || 5141 test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) 5142 return; 5143 5144 spin_lock(&napi_hash_lock); 5145 5146 /* 0..NR_CPUS+1 range is reserved for sender_cpu use */ 5147 do { 5148 if (unlikely(++napi_gen_id < NR_CPUS + 1)) 5149 napi_gen_id = NR_CPUS + 1; 5150 } while (napi_by_id(napi_gen_id)); 5151 napi->napi_id = napi_gen_id; 5152 5153 hlist_add_head_rcu(&napi->napi_hash_node, 5154 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); 5155 5156 spin_unlock(&napi_hash_lock); 5157 } 5158 5159 /* Warning : caller is responsible to make sure rcu grace period 5160 * is respected before freeing memory containing @napi 5161 */ 5162 bool napi_hash_del(struct napi_struct *napi) 5163 { 5164 bool rcu_sync_needed = false; 5165 5166 spin_lock(&napi_hash_lock); 5167 5168 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) { 5169 rcu_sync_needed = true; 5170 hlist_del_rcu(&napi->napi_hash_node); 5171 } 5172 spin_unlock(&napi_hash_lock); 5173 return rcu_sync_needed; 5174 } 5175 EXPORT_SYMBOL_GPL(napi_hash_del); 5176 5177 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer) 5178 { 5179 struct napi_struct *napi; 5180 5181 napi = container_of(timer, struct napi_struct, timer); 5182 5183 /* Note : we use a relaxed variant of napi_schedule_prep() not setting 5184 * NAPI_STATE_MISSED, since we do not react to a device IRQ. 5185 */ 5186 if (napi->gro_list && !napi_disable_pending(napi) && 5187 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) 5188 __napi_schedule_irqoff(napi); 5189 5190 return HRTIMER_NORESTART; 5191 } 5192 5193 void netif_napi_add(struct net_device *dev, struct napi_struct *napi, 5194 int (*poll)(struct napi_struct *, int), int weight) 5195 { 5196 INIT_LIST_HEAD(&napi->poll_list); 5197 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); 5198 napi->timer.function = napi_watchdog; 5199 napi->gro_count = 0; 5200 napi->gro_list = NULL; 5201 napi->skb = NULL; 5202 napi->poll = poll; 5203 if (weight > NAPI_POLL_WEIGHT) 5204 pr_err_once("netif_napi_add() called with weight %d on device %s\n", 5205 weight, dev->name); 5206 napi->weight = weight; 5207 list_add(&napi->dev_list, &dev->napi_list); 5208 napi->dev = dev; 5209 #ifdef CONFIG_NETPOLL 5210 napi->poll_owner = -1; 5211 #endif 5212 set_bit(NAPI_STATE_SCHED, &napi->state); 5213 napi_hash_add(napi); 5214 } 5215 EXPORT_SYMBOL(netif_napi_add); 5216 5217 void napi_disable(struct napi_struct *n) 5218 { 5219 might_sleep(); 5220 set_bit(NAPI_STATE_DISABLE, &n->state); 5221 5222 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state)) 5223 msleep(1); 5224 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state)) 5225 msleep(1); 5226 5227 hrtimer_cancel(&n->timer); 5228 5229 clear_bit(NAPI_STATE_DISABLE, &n->state); 5230 } 5231 EXPORT_SYMBOL(napi_disable); 5232 5233 /* Must be called in process context */ 5234 void netif_napi_del(struct napi_struct *napi) 5235 { 5236 might_sleep(); 5237 if (napi_hash_del(napi)) 5238 synchronize_net(); 5239 list_del_init(&napi->dev_list); 5240 napi_free_frags(napi); 5241 5242 kfree_skb_list(napi->gro_list); 5243 napi->gro_list = NULL; 5244 napi->gro_count = 0; 5245 } 5246 EXPORT_SYMBOL(netif_napi_del); 5247 5248 static int napi_poll(struct napi_struct *n, struct list_head *repoll) 5249 { 5250 void *have; 5251 int work, weight; 5252 5253 list_del_init(&n->poll_list); 5254 5255 have = netpoll_poll_lock(n); 5256 5257 weight = n->weight; 5258 5259 /* This NAPI_STATE_SCHED test is for avoiding a race 5260 * with netpoll's poll_napi(). Only the entity which 5261 * obtains the lock and sees NAPI_STATE_SCHED set will 5262 * actually make the ->poll() call. Therefore we avoid 5263 * accidentally calling ->poll() when NAPI is not scheduled. 5264 */ 5265 work = 0; 5266 if (test_bit(NAPI_STATE_SCHED, &n->state)) { 5267 work = n->poll(n, weight); 5268 trace_napi_poll(n, work, weight); 5269 } 5270 5271 WARN_ON_ONCE(work > weight); 5272 5273 if (likely(work < weight)) 5274 goto out_unlock; 5275 5276 /* Drivers must not modify the NAPI state if they 5277 * consume the entire weight. In such cases this code 5278 * still "owns" the NAPI instance and therefore can 5279 * move the instance around on the list at-will. 5280 */ 5281 if (unlikely(napi_disable_pending(n))) { 5282 napi_complete(n); 5283 goto out_unlock; 5284 } 5285 5286 if (n->gro_list) { 5287 /* flush too old packets 5288 * If HZ < 1000, flush all packets. 5289 */ 5290 napi_gro_flush(n, HZ >= 1000); 5291 } 5292 5293 /* Some drivers may have called napi_schedule 5294 * prior to exhausting their budget. 5295 */ 5296 if (unlikely(!list_empty(&n->poll_list))) { 5297 pr_warn_once("%s: Budget exhausted after napi rescheduled\n", 5298 n->dev ? n->dev->name : "backlog"); 5299 goto out_unlock; 5300 } 5301 5302 list_add_tail(&n->poll_list, repoll); 5303 5304 out_unlock: 5305 netpoll_poll_unlock(have); 5306 5307 return work; 5308 } 5309 5310 static __latent_entropy void net_rx_action(struct softirq_action *h) 5311 { 5312 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 5313 unsigned long time_limit = jiffies + 2; 5314 int budget = netdev_budget; 5315 LIST_HEAD(list); 5316 LIST_HEAD(repoll); 5317 5318 local_irq_disable(); 5319 list_splice_init(&sd->poll_list, &list); 5320 local_irq_enable(); 5321 5322 for (;;) { 5323 struct napi_struct *n; 5324 5325 if (list_empty(&list)) { 5326 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll)) 5327 goto out; 5328 break; 5329 } 5330 5331 n = list_first_entry(&list, struct napi_struct, poll_list); 5332 budget -= napi_poll(n, &repoll); 5333 5334 /* If softirq window is exhausted then punt. 5335 * Allow this to run for 2 jiffies since which will allow 5336 * an average latency of 1.5/HZ. 5337 */ 5338 if (unlikely(budget <= 0 || 5339 time_after_eq(jiffies, time_limit))) { 5340 sd->time_squeeze++; 5341 break; 5342 } 5343 } 5344 5345 local_irq_disable(); 5346 5347 list_splice_tail_init(&sd->poll_list, &list); 5348 list_splice_tail(&repoll, &list); 5349 list_splice(&list, &sd->poll_list); 5350 if (!list_empty(&sd->poll_list)) 5351 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 5352 5353 net_rps_action_and_irq_enable(sd); 5354 out: 5355 __kfree_skb_flush(); 5356 } 5357 5358 struct netdev_adjacent { 5359 struct net_device *dev; 5360 5361 /* upper master flag, there can only be one master device per list */ 5362 bool master; 5363 5364 /* counter for the number of times this device was added to us */ 5365 u16 ref_nr; 5366 5367 /* private field for the users */ 5368 void *private; 5369 5370 struct list_head list; 5371 struct rcu_head rcu; 5372 }; 5373 5374 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev, 5375 struct list_head *adj_list) 5376 { 5377 struct netdev_adjacent *adj; 5378 5379 list_for_each_entry(adj, adj_list, list) { 5380 if (adj->dev == adj_dev) 5381 return adj; 5382 } 5383 return NULL; 5384 } 5385 5386 static int __netdev_has_upper_dev(struct net_device *upper_dev, void *data) 5387 { 5388 struct net_device *dev = data; 5389 5390 return upper_dev == dev; 5391 } 5392 5393 /** 5394 * netdev_has_upper_dev - Check if device is linked to an upper device 5395 * @dev: device 5396 * @upper_dev: upper device to check 5397 * 5398 * Find out if a device is linked to specified upper device and return true 5399 * in case it is. Note that this checks only immediate upper device, 5400 * not through a complete stack of devices. The caller must hold the RTNL lock. 5401 */ 5402 bool netdev_has_upper_dev(struct net_device *dev, 5403 struct net_device *upper_dev) 5404 { 5405 ASSERT_RTNL(); 5406 5407 return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev, 5408 upper_dev); 5409 } 5410 EXPORT_SYMBOL(netdev_has_upper_dev); 5411 5412 /** 5413 * netdev_has_upper_dev_all - Check if device is linked to an upper device 5414 * @dev: device 5415 * @upper_dev: upper device to check 5416 * 5417 * Find out if a device is linked to specified upper device and return true 5418 * in case it is. Note that this checks the entire upper device chain. 5419 * The caller must hold rcu lock. 5420 */ 5421 5422 bool netdev_has_upper_dev_all_rcu(struct net_device *dev, 5423 struct net_device *upper_dev) 5424 { 5425 return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev, 5426 upper_dev); 5427 } 5428 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu); 5429 5430 /** 5431 * netdev_has_any_upper_dev - Check if device is linked to some device 5432 * @dev: device 5433 * 5434 * Find out if a device is linked to an upper device and return true in case 5435 * it is. The caller must hold the RTNL lock. 5436 */ 5437 static bool netdev_has_any_upper_dev(struct net_device *dev) 5438 { 5439 ASSERT_RTNL(); 5440 5441 return !list_empty(&dev->adj_list.upper); 5442 } 5443 5444 /** 5445 * netdev_master_upper_dev_get - Get master upper device 5446 * @dev: device 5447 * 5448 * Find a master upper device and return pointer to it or NULL in case 5449 * it's not there. The caller must hold the RTNL lock. 5450 */ 5451 struct net_device *netdev_master_upper_dev_get(struct net_device *dev) 5452 { 5453 struct netdev_adjacent *upper; 5454 5455 ASSERT_RTNL(); 5456 5457 if (list_empty(&dev->adj_list.upper)) 5458 return NULL; 5459 5460 upper = list_first_entry(&dev->adj_list.upper, 5461 struct netdev_adjacent, list); 5462 if (likely(upper->master)) 5463 return upper->dev; 5464 return NULL; 5465 } 5466 EXPORT_SYMBOL(netdev_master_upper_dev_get); 5467 5468 /** 5469 * netdev_has_any_lower_dev - Check if device is linked to some device 5470 * @dev: device 5471 * 5472 * Find out if a device is linked to a lower device and return true in case 5473 * it is. The caller must hold the RTNL lock. 5474 */ 5475 static bool netdev_has_any_lower_dev(struct net_device *dev) 5476 { 5477 ASSERT_RTNL(); 5478 5479 return !list_empty(&dev->adj_list.lower); 5480 } 5481 5482 void *netdev_adjacent_get_private(struct list_head *adj_list) 5483 { 5484 struct netdev_adjacent *adj; 5485 5486 adj = list_entry(adj_list, struct netdev_adjacent, list); 5487 5488 return adj->private; 5489 } 5490 EXPORT_SYMBOL(netdev_adjacent_get_private); 5491 5492 /** 5493 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list 5494 * @dev: device 5495 * @iter: list_head ** of the current position 5496 * 5497 * Gets the next device from the dev's upper list, starting from iter 5498 * position. The caller must hold RCU read lock. 5499 */ 5500 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev, 5501 struct list_head **iter) 5502 { 5503 struct netdev_adjacent *upper; 5504 5505 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 5506 5507 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5508 5509 if (&upper->list == &dev->adj_list.upper) 5510 return NULL; 5511 5512 *iter = &upper->list; 5513 5514 return upper->dev; 5515 } 5516 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu); 5517 5518 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev, 5519 struct list_head **iter) 5520 { 5521 struct netdev_adjacent *upper; 5522 5523 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 5524 5525 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5526 5527 if (&upper->list == &dev->adj_list.upper) 5528 return NULL; 5529 5530 *iter = &upper->list; 5531 5532 return upper->dev; 5533 } 5534 5535 int netdev_walk_all_upper_dev_rcu(struct net_device *dev, 5536 int (*fn)(struct net_device *dev, 5537 void *data), 5538 void *data) 5539 { 5540 struct net_device *udev; 5541 struct list_head *iter; 5542 int ret; 5543 5544 for (iter = &dev->adj_list.upper, 5545 udev = netdev_next_upper_dev_rcu(dev, &iter); 5546 udev; 5547 udev = netdev_next_upper_dev_rcu(dev, &iter)) { 5548 /* first is the upper device itself */ 5549 ret = fn(udev, data); 5550 if (ret) 5551 return ret; 5552 5553 /* then look at all of its upper devices */ 5554 ret = netdev_walk_all_upper_dev_rcu(udev, fn, data); 5555 if (ret) 5556 return ret; 5557 } 5558 5559 return 0; 5560 } 5561 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu); 5562 5563 /** 5564 * netdev_lower_get_next_private - Get the next ->private from the 5565 * lower neighbour list 5566 * @dev: device 5567 * @iter: list_head ** of the current position 5568 * 5569 * Gets the next netdev_adjacent->private from the dev's lower neighbour 5570 * list, starting from iter position. The caller must hold either hold the 5571 * RTNL lock or its own locking that guarantees that the neighbour lower 5572 * list will remain unchanged. 5573 */ 5574 void *netdev_lower_get_next_private(struct net_device *dev, 5575 struct list_head **iter) 5576 { 5577 struct netdev_adjacent *lower; 5578 5579 lower = list_entry(*iter, struct netdev_adjacent, list); 5580 5581 if (&lower->list == &dev->adj_list.lower) 5582 return NULL; 5583 5584 *iter = lower->list.next; 5585 5586 return lower->private; 5587 } 5588 EXPORT_SYMBOL(netdev_lower_get_next_private); 5589 5590 /** 5591 * netdev_lower_get_next_private_rcu - Get the next ->private from the 5592 * lower neighbour list, RCU 5593 * variant 5594 * @dev: device 5595 * @iter: list_head ** of the current position 5596 * 5597 * Gets the next netdev_adjacent->private from the dev's lower neighbour 5598 * list, starting from iter position. The caller must hold RCU read lock. 5599 */ 5600 void *netdev_lower_get_next_private_rcu(struct net_device *dev, 5601 struct list_head **iter) 5602 { 5603 struct netdev_adjacent *lower; 5604 5605 WARN_ON_ONCE(!rcu_read_lock_held()); 5606 5607 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5608 5609 if (&lower->list == &dev->adj_list.lower) 5610 return NULL; 5611 5612 *iter = &lower->list; 5613 5614 return lower->private; 5615 } 5616 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu); 5617 5618 /** 5619 * netdev_lower_get_next - Get the next device from the lower neighbour 5620 * list 5621 * @dev: device 5622 * @iter: list_head ** of the current position 5623 * 5624 * Gets the next netdev_adjacent from the dev's lower neighbour 5625 * list, starting from iter position. The caller must hold RTNL lock or 5626 * its own locking that guarantees that the neighbour lower 5627 * list will remain unchanged. 5628 */ 5629 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter) 5630 { 5631 struct netdev_adjacent *lower; 5632 5633 lower = list_entry(*iter, struct netdev_adjacent, list); 5634 5635 if (&lower->list == &dev->adj_list.lower) 5636 return NULL; 5637 5638 *iter = lower->list.next; 5639 5640 return lower->dev; 5641 } 5642 EXPORT_SYMBOL(netdev_lower_get_next); 5643 5644 static struct net_device *netdev_next_lower_dev(struct net_device *dev, 5645 struct list_head **iter) 5646 { 5647 struct netdev_adjacent *lower; 5648 5649 lower = list_entry((*iter)->next, struct netdev_adjacent, list); 5650 5651 if (&lower->list == &dev->adj_list.lower) 5652 return NULL; 5653 5654 *iter = &lower->list; 5655 5656 return lower->dev; 5657 } 5658 5659 int netdev_walk_all_lower_dev(struct net_device *dev, 5660 int (*fn)(struct net_device *dev, 5661 void *data), 5662 void *data) 5663 { 5664 struct net_device *ldev; 5665 struct list_head *iter; 5666 int ret; 5667 5668 for (iter = &dev->adj_list.lower, 5669 ldev = netdev_next_lower_dev(dev, &iter); 5670 ldev; 5671 ldev = netdev_next_lower_dev(dev, &iter)) { 5672 /* first is the lower device itself */ 5673 ret = fn(ldev, data); 5674 if (ret) 5675 return ret; 5676 5677 /* then look at all of its lower devices */ 5678 ret = netdev_walk_all_lower_dev(ldev, fn, data); 5679 if (ret) 5680 return ret; 5681 } 5682 5683 return 0; 5684 } 5685 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev); 5686 5687 static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev, 5688 struct list_head **iter) 5689 { 5690 struct netdev_adjacent *lower; 5691 5692 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5693 if (&lower->list == &dev->adj_list.lower) 5694 return NULL; 5695 5696 *iter = &lower->list; 5697 5698 return lower->dev; 5699 } 5700 5701 int netdev_walk_all_lower_dev_rcu(struct net_device *dev, 5702 int (*fn)(struct net_device *dev, 5703 void *data), 5704 void *data) 5705 { 5706 struct net_device *ldev; 5707 struct list_head *iter; 5708 int ret; 5709 5710 for (iter = &dev->adj_list.lower, 5711 ldev = netdev_next_lower_dev_rcu(dev, &iter); 5712 ldev; 5713 ldev = netdev_next_lower_dev_rcu(dev, &iter)) { 5714 /* first is the lower device itself */ 5715 ret = fn(ldev, data); 5716 if (ret) 5717 return ret; 5718 5719 /* then look at all of its lower devices */ 5720 ret = netdev_walk_all_lower_dev_rcu(ldev, fn, data); 5721 if (ret) 5722 return ret; 5723 } 5724 5725 return 0; 5726 } 5727 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu); 5728 5729 /** 5730 * netdev_lower_get_first_private_rcu - Get the first ->private from the 5731 * lower neighbour list, RCU 5732 * variant 5733 * @dev: device 5734 * 5735 * Gets the first netdev_adjacent->private from the dev's lower neighbour 5736 * list. The caller must hold RCU read lock. 5737 */ 5738 void *netdev_lower_get_first_private_rcu(struct net_device *dev) 5739 { 5740 struct netdev_adjacent *lower; 5741 5742 lower = list_first_or_null_rcu(&dev->adj_list.lower, 5743 struct netdev_adjacent, list); 5744 if (lower) 5745 return lower->private; 5746 return NULL; 5747 } 5748 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu); 5749 5750 /** 5751 * netdev_master_upper_dev_get_rcu - Get master upper device 5752 * @dev: device 5753 * 5754 * Find a master upper device and return pointer to it or NULL in case 5755 * it's not there. The caller must hold the RCU read lock. 5756 */ 5757 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev) 5758 { 5759 struct netdev_adjacent *upper; 5760 5761 upper = list_first_or_null_rcu(&dev->adj_list.upper, 5762 struct netdev_adjacent, list); 5763 if (upper && likely(upper->master)) 5764 return upper->dev; 5765 return NULL; 5766 } 5767 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu); 5768 5769 static int netdev_adjacent_sysfs_add(struct net_device *dev, 5770 struct net_device *adj_dev, 5771 struct list_head *dev_list) 5772 { 5773 char linkname[IFNAMSIZ+7]; 5774 5775 sprintf(linkname, dev_list == &dev->adj_list.upper ? 5776 "upper_%s" : "lower_%s", adj_dev->name); 5777 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), 5778 linkname); 5779 } 5780 static void netdev_adjacent_sysfs_del(struct net_device *dev, 5781 char *name, 5782 struct list_head *dev_list) 5783 { 5784 char linkname[IFNAMSIZ+7]; 5785 5786 sprintf(linkname, dev_list == &dev->adj_list.upper ? 5787 "upper_%s" : "lower_%s", name); 5788 sysfs_remove_link(&(dev->dev.kobj), linkname); 5789 } 5790 5791 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev, 5792 struct net_device *adj_dev, 5793 struct list_head *dev_list) 5794 { 5795 return (dev_list == &dev->adj_list.upper || 5796 dev_list == &dev->adj_list.lower) && 5797 net_eq(dev_net(dev), dev_net(adj_dev)); 5798 } 5799 5800 static int __netdev_adjacent_dev_insert(struct net_device *dev, 5801 struct net_device *adj_dev, 5802 struct list_head *dev_list, 5803 void *private, bool master) 5804 { 5805 struct netdev_adjacent *adj; 5806 int ret; 5807 5808 adj = __netdev_find_adj(adj_dev, dev_list); 5809 5810 if (adj) { 5811 adj->ref_nr += 1; 5812 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n", 5813 dev->name, adj_dev->name, adj->ref_nr); 5814 5815 return 0; 5816 } 5817 5818 adj = kmalloc(sizeof(*adj), GFP_KERNEL); 5819 if (!adj) 5820 return -ENOMEM; 5821 5822 adj->dev = adj_dev; 5823 adj->master = master; 5824 adj->ref_nr = 1; 5825 adj->private = private; 5826 dev_hold(adj_dev); 5827 5828 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n", 5829 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name); 5830 5831 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) { 5832 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list); 5833 if (ret) 5834 goto free_adj; 5835 } 5836 5837 /* Ensure that master link is always the first item in list. */ 5838 if (master) { 5839 ret = sysfs_create_link(&(dev->dev.kobj), 5840 &(adj_dev->dev.kobj), "master"); 5841 if (ret) 5842 goto remove_symlinks; 5843 5844 list_add_rcu(&adj->list, dev_list); 5845 } else { 5846 list_add_tail_rcu(&adj->list, dev_list); 5847 } 5848 5849 return 0; 5850 5851 remove_symlinks: 5852 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 5853 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 5854 free_adj: 5855 kfree(adj); 5856 dev_put(adj_dev); 5857 5858 return ret; 5859 } 5860 5861 static void __netdev_adjacent_dev_remove(struct net_device *dev, 5862 struct net_device *adj_dev, 5863 u16 ref_nr, 5864 struct list_head *dev_list) 5865 { 5866 struct netdev_adjacent *adj; 5867 5868 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n", 5869 dev->name, adj_dev->name, ref_nr); 5870 5871 adj = __netdev_find_adj(adj_dev, dev_list); 5872 5873 if (!adj) { 5874 pr_err("Adjacency does not exist for device %s from %s\n", 5875 dev->name, adj_dev->name); 5876 WARN_ON(1); 5877 return; 5878 } 5879 5880 if (adj->ref_nr > ref_nr) { 5881 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n", 5882 dev->name, adj_dev->name, ref_nr, 5883 adj->ref_nr - ref_nr); 5884 adj->ref_nr -= ref_nr; 5885 return; 5886 } 5887 5888 if (adj->master) 5889 sysfs_remove_link(&(dev->dev.kobj), "master"); 5890 5891 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 5892 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 5893 5894 list_del_rcu(&adj->list); 5895 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n", 5896 adj_dev->name, dev->name, adj_dev->name); 5897 dev_put(adj_dev); 5898 kfree_rcu(adj, rcu); 5899 } 5900 5901 static int __netdev_adjacent_dev_link_lists(struct net_device *dev, 5902 struct net_device *upper_dev, 5903 struct list_head *up_list, 5904 struct list_head *down_list, 5905 void *private, bool master) 5906 { 5907 int ret; 5908 5909 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, 5910 private, master); 5911 if (ret) 5912 return ret; 5913 5914 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, 5915 private, false); 5916 if (ret) { 5917 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list); 5918 return ret; 5919 } 5920 5921 return 0; 5922 } 5923 5924 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev, 5925 struct net_device *upper_dev, 5926 u16 ref_nr, 5927 struct list_head *up_list, 5928 struct list_head *down_list) 5929 { 5930 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list); 5931 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list); 5932 } 5933 5934 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev, 5935 struct net_device *upper_dev, 5936 void *private, bool master) 5937 { 5938 return __netdev_adjacent_dev_link_lists(dev, upper_dev, 5939 &dev->adj_list.upper, 5940 &upper_dev->adj_list.lower, 5941 private, master); 5942 } 5943 5944 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev, 5945 struct net_device *upper_dev) 5946 { 5947 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1, 5948 &dev->adj_list.upper, 5949 &upper_dev->adj_list.lower); 5950 } 5951 5952 static int __netdev_upper_dev_link(struct net_device *dev, 5953 struct net_device *upper_dev, bool master, 5954 void *upper_priv, void *upper_info) 5955 { 5956 struct netdev_notifier_changeupper_info changeupper_info; 5957 int ret = 0; 5958 5959 ASSERT_RTNL(); 5960 5961 if (dev == upper_dev) 5962 return -EBUSY; 5963 5964 /* To prevent loops, check if dev is not upper device to upper_dev. */ 5965 if (netdev_has_upper_dev(upper_dev, dev)) 5966 return -EBUSY; 5967 5968 if (netdev_has_upper_dev(dev, upper_dev)) 5969 return -EEXIST; 5970 5971 if (master && netdev_master_upper_dev_get(dev)) 5972 return -EBUSY; 5973 5974 changeupper_info.upper_dev = upper_dev; 5975 changeupper_info.master = master; 5976 changeupper_info.linking = true; 5977 changeupper_info.upper_info = upper_info; 5978 5979 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev, 5980 &changeupper_info.info); 5981 ret = notifier_to_errno(ret); 5982 if (ret) 5983 return ret; 5984 5985 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv, 5986 master); 5987 if (ret) 5988 return ret; 5989 5990 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev, 5991 &changeupper_info.info); 5992 ret = notifier_to_errno(ret); 5993 if (ret) 5994 goto rollback; 5995 5996 return 0; 5997 5998 rollback: 5999 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 6000 6001 return ret; 6002 } 6003 6004 /** 6005 * netdev_upper_dev_link - Add a link to the upper device 6006 * @dev: device 6007 * @upper_dev: new upper device 6008 * 6009 * Adds a link to device which is upper to this one. The caller must hold 6010 * the RTNL lock. On a failure a negative errno code is returned. 6011 * On success the reference counts are adjusted and the function 6012 * returns zero. 6013 */ 6014 int netdev_upper_dev_link(struct net_device *dev, 6015 struct net_device *upper_dev) 6016 { 6017 return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL); 6018 } 6019 EXPORT_SYMBOL(netdev_upper_dev_link); 6020 6021 /** 6022 * netdev_master_upper_dev_link - Add a master link to the upper device 6023 * @dev: device 6024 * @upper_dev: new upper device 6025 * @upper_priv: upper device private 6026 * @upper_info: upper info to be passed down via notifier 6027 * 6028 * Adds a link to device which is upper to this one. In this case, only 6029 * one master upper device can be linked, although other non-master devices 6030 * might be linked as well. The caller must hold the RTNL lock. 6031 * On a failure a negative errno code is returned. On success the reference 6032 * counts are adjusted and the function returns zero. 6033 */ 6034 int netdev_master_upper_dev_link(struct net_device *dev, 6035 struct net_device *upper_dev, 6036 void *upper_priv, void *upper_info) 6037 { 6038 return __netdev_upper_dev_link(dev, upper_dev, true, 6039 upper_priv, upper_info); 6040 } 6041 EXPORT_SYMBOL(netdev_master_upper_dev_link); 6042 6043 /** 6044 * netdev_upper_dev_unlink - Removes a link to upper device 6045 * @dev: device 6046 * @upper_dev: new upper device 6047 * 6048 * Removes a link to device which is upper to this one. The caller must hold 6049 * the RTNL lock. 6050 */ 6051 void netdev_upper_dev_unlink(struct net_device *dev, 6052 struct net_device *upper_dev) 6053 { 6054 struct netdev_notifier_changeupper_info changeupper_info; 6055 6056 ASSERT_RTNL(); 6057 6058 changeupper_info.upper_dev = upper_dev; 6059 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev; 6060 changeupper_info.linking = false; 6061 6062 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev, 6063 &changeupper_info.info); 6064 6065 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 6066 6067 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev, 6068 &changeupper_info.info); 6069 } 6070 EXPORT_SYMBOL(netdev_upper_dev_unlink); 6071 6072 /** 6073 * netdev_bonding_info_change - Dispatch event about slave change 6074 * @dev: device 6075 * @bonding_info: info to dispatch 6076 * 6077 * Send NETDEV_BONDING_INFO to netdev notifiers with info. 6078 * The caller must hold the RTNL lock. 6079 */ 6080 void netdev_bonding_info_change(struct net_device *dev, 6081 struct netdev_bonding_info *bonding_info) 6082 { 6083 struct netdev_notifier_bonding_info info; 6084 6085 memcpy(&info.bonding_info, bonding_info, 6086 sizeof(struct netdev_bonding_info)); 6087 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev, 6088 &info.info); 6089 } 6090 EXPORT_SYMBOL(netdev_bonding_info_change); 6091 6092 static void netdev_adjacent_add_links(struct net_device *dev) 6093 { 6094 struct netdev_adjacent *iter; 6095 6096 struct net *net = dev_net(dev); 6097 6098 list_for_each_entry(iter, &dev->adj_list.upper, list) { 6099 if (!net_eq(net, dev_net(iter->dev))) 6100 continue; 6101 netdev_adjacent_sysfs_add(iter->dev, dev, 6102 &iter->dev->adj_list.lower); 6103 netdev_adjacent_sysfs_add(dev, iter->dev, 6104 &dev->adj_list.upper); 6105 } 6106 6107 list_for_each_entry(iter, &dev->adj_list.lower, list) { 6108 if (!net_eq(net, dev_net(iter->dev))) 6109 continue; 6110 netdev_adjacent_sysfs_add(iter->dev, dev, 6111 &iter->dev->adj_list.upper); 6112 netdev_adjacent_sysfs_add(dev, iter->dev, 6113 &dev->adj_list.lower); 6114 } 6115 } 6116 6117 static void netdev_adjacent_del_links(struct net_device *dev) 6118 { 6119 struct netdev_adjacent *iter; 6120 6121 struct net *net = dev_net(dev); 6122 6123 list_for_each_entry(iter, &dev->adj_list.upper, list) { 6124 if (!net_eq(net, dev_net(iter->dev))) 6125 continue; 6126 netdev_adjacent_sysfs_del(iter->dev, dev->name, 6127 &iter->dev->adj_list.lower); 6128 netdev_adjacent_sysfs_del(dev, iter->dev->name, 6129 &dev->adj_list.upper); 6130 } 6131 6132 list_for_each_entry(iter, &dev->adj_list.lower, list) { 6133 if (!net_eq(net, dev_net(iter->dev))) 6134 continue; 6135 netdev_adjacent_sysfs_del(iter->dev, dev->name, 6136 &iter->dev->adj_list.upper); 6137 netdev_adjacent_sysfs_del(dev, iter->dev->name, 6138 &dev->adj_list.lower); 6139 } 6140 } 6141 6142 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname) 6143 { 6144 struct netdev_adjacent *iter; 6145 6146 struct net *net = dev_net(dev); 6147 6148 list_for_each_entry(iter, &dev->adj_list.upper, list) { 6149 if (!net_eq(net, dev_net(iter->dev))) 6150 continue; 6151 netdev_adjacent_sysfs_del(iter->dev, oldname, 6152 &iter->dev->adj_list.lower); 6153 netdev_adjacent_sysfs_add(iter->dev, dev, 6154 &iter->dev->adj_list.lower); 6155 } 6156 6157 list_for_each_entry(iter, &dev->adj_list.lower, list) { 6158 if (!net_eq(net, dev_net(iter->dev))) 6159 continue; 6160 netdev_adjacent_sysfs_del(iter->dev, oldname, 6161 &iter->dev->adj_list.upper); 6162 netdev_adjacent_sysfs_add(iter->dev, dev, 6163 &iter->dev->adj_list.upper); 6164 } 6165 } 6166 6167 void *netdev_lower_dev_get_private(struct net_device *dev, 6168 struct net_device *lower_dev) 6169 { 6170 struct netdev_adjacent *lower; 6171 6172 if (!lower_dev) 6173 return NULL; 6174 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower); 6175 if (!lower) 6176 return NULL; 6177 6178 return lower->private; 6179 } 6180 EXPORT_SYMBOL(netdev_lower_dev_get_private); 6181 6182 6183 int dev_get_nest_level(struct net_device *dev) 6184 { 6185 struct net_device *lower = NULL; 6186 struct list_head *iter; 6187 int max_nest = -1; 6188 int nest; 6189 6190 ASSERT_RTNL(); 6191 6192 netdev_for_each_lower_dev(dev, lower, iter) { 6193 nest = dev_get_nest_level(lower); 6194 if (max_nest < nest) 6195 max_nest = nest; 6196 } 6197 6198 return max_nest + 1; 6199 } 6200 EXPORT_SYMBOL(dev_get_nest_level); 6201 6202 /** 6203 * netdev_lower_change - Dispatch event about lower device state change 6204 * @lower_dev: device 6205 * @lower_state_info: state to dispatch 6206 * 6207 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info. 6208 * The caller must hold the RTNL lock. 6209 */ 6210 void netdev_lower_state_changed(struct net_device *lower_dev, 6211 void *lower_state_info) 6212 { 6213 struct netdev_notifier_changelowerstate_info changelowerstate_info; 6214 6215 ASSERT_RTNL(); 6216 changelowerstate_info.lower_state_info = lower_state_info; 6217 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev, 6218 &changelowerstate_info.info); 6219 } 6220 EXPORT_SYMBOL(netdev_lower_state_changed); 6221 6222 static void dev_change_rx_flags(struct net_device *dev, int flags) 6223 { 6224 const struct net_device_ops *ops = dev->netdev_ops; 6225 6226 if (ops->ndo_change_rx_flags) 6227 ops->ndo_change_rx_flags(dev, flags); 6228 } 6229 6230 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify) 6231 { 6232 unsigned int old_flags = dev->flags; 6233 kuid_t uid; 6234 kgid_t gid; 6235 6236 ASSERT_RTNL(); 6237 6238 dev->flags |= IFF_PROMISC; 6239 dev->promiscuity += inc; 6240 if (dev->promiscuity == 0) { 6241 /* 6242 * Avoid overflow. 6243 * If inc causes overflow, untouch promisc and return error. 6244 */ 6245 if (inc < 0) 6246 dev->flags &= ~IFF_PROMISC; 6247 else { 6248 dev->promiscuity -= inc; 6249 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n", 6250 dev->name); 6251 return -EOVERFLOW; 6252 } 6253 } 6254 if (dev->flags != old_flags) { 6255 pr_info("device %s %s promiscuous mode\n", 6256 dev->name, 6257 dev->flags & IFF_PROMISC ? "entered" : "left"); 6258 if (audit_enabled) { 6259 current_uid_gid(&uid, &gid); 6260 audit_log(current->audit_context, GFP_ATOMIC, 6261 AUDIT_ANOM_PROMISCUOUS, 6262 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u", 6263 dev->name, (dev->flags & IFF_PROMISC), 6264 (old_flags & IFF_PROMISC), 6265 from_kuid(&init_user_ns, audit_get_loginuid(current)), 6266 from_kuid(&init_user_ns, uid), 6267 from_kgid(&init_user_ns, gid), 6268 audit_get_sessionid(current)); 6269 } 6270 6271 dev_change_rx_flags(dev, IFF_PROMISC); 6272 } 6273 if (notify) 6274 __dev_notify_flags(dev, old_flags, IFF_PROMISC); 6275 return 0; 6276 } 6277 6278 /** 6279 * dev_set_promiscuity - update promiscuity count on a device 6280 * @dev: device 6281 * @inc: modifier 6282 * 6283 * Add or remove promiscuity from a device. While the count in the device 6284 * remains above zero the interface remains promiscuous. Once it hits zero 6285 * the device reverts back to normal filtering operation. A negative inc 6286 * value is used to drop promiscuity on the device. 6287 * Return 0 if successful or a negative errno code on error. 6288 */ 6289 int dev_set_promiscuity(struct net_device *dev, int inc) 6290 { 6291 unsigned int old_flags = dev->flags; 6292 int err; 6293 6294 err = __dev_set_promiscuity(dev, inc, true); 6295 if (err < 0) 6296 return err; 6297 if (dev->flags != old_flags) 6298 dev_set_rx_mode(dev); 6299 return err; 6300 } 6301 EXPORT_SYMBOL(dev_set_promiscuity); 6302 6303 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify) 6304 { 6305 unsigned int old_flags = dev->flags, old_gflags = dev->gflags; 6306 6307 ASSERT_RTNL(); 6308 6309 dev->flags |= IFF_ALLMULTI; 6310 dev->allmulti += inc; 6311 if (dev->allmulti == 0) { 6312 /* 6313 * Avoid overflow. 6314 * If inc causes overflow, untouch allmulti and return error. 6315 */ 6316 if (inc < 0) 6317 dev->flags &= ~IFF_ALLMULTI; 6318 else { 6319 dev->allmulti -= inc; 6320 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n", 6321 dev->name); 6322 return -EOVERFLOW; 6323 } 6324 } 6325 if (dev->flags ^ old_flags) { 6326 dev_change_rx_flags(dev, IFF_ALLMULTI); 6327 dev_set_rx_mode(dev); 6328 if (notify) 6329 __dev_notify_flags(dev, old_flags, 6330 dev->gflags ^ old_gflags); 6331 } 6332 return 0; 6333 } 6334 6335 /** 6336 * dev_set_allmulti - update allmulti count on a device 6337 * @dev: device 6338 * @inc: modifier 6339 * 6340 * Add or remove reception of all multicast frames to a device. While the 6341 * count in the device remains above zero the interface remains listening 6342 * to all interfaces. Once it hits zero the device reverts back to normal 6343 * filtering operation. A negative @inc value is used to drop the counter 6344 * when releasing a resource needing all multicasts. 6345 * Return 0 if successful or a negative errno code on error. 6346 */ 6347 6348 int dev_set_allmulti(struct net_device *dev, int inc) 6349 { 6350 return __dev_set_allmulti(dev, inc, true); 6351 } 6352 EXPORT_SYMBOL(dev_set_allmulti); 6353 6354 /* 6355 * Upload unicast and multicast address lists to device and 6356 * configure RX filtering. When the device doesn't support unicast 6357 * filtering it is put in promiscuous mode while unicast addresses 6358 * are present. 6359 */ 6360 void __dev_set_rx_mode(struct net_device *dev) 6361 { 6362 const struct net_device_ops *ops = dev->netdev_ops; 6363 6364 /* dev_open will call this function so the list will stay sane. */ 6365 if (!(dev->flags&IFF_UP)) 6366 return; 6367 6368 if (!netif_device_present(dev)) 6369 return; 6370 6371 if (!(dev->priv_flags & IFF_UNICAST_FLT)) { 6372 /* Unicast addresses changes may only happen under the rtnl, 6373 * therefore calling __dev_set_promiscuity here is safe. 6374 */ 6375 if (!netdev_uc_empty(dev) && !dev->uc_promisc) { 6376 __dev_set_promiscuity(dev, 1, false); 6377 dev->uc_promisc = true; 6378 } else if (netdev_uc_empty(dev) && dev->uc_promisc) { 6379 __dev_set_promiscuity(dev, -1, false); 6380 dev->uc_promisc = false; 6381 } 6382 } 6383 6384 if (ops->ndo_set_rx_mode) 6385 ops->ndo_set_rx_mode(dev); 6386 } 6387 6388 void dev_set_rx_mode(struct net_device *dev) 6389 { 6390 netif_addr_lock_bh(dev); 6391 __dev_set_rx_mode(dev); 6392 netif_addr_unlock_bh(dev); 6393 } 6394 6395 /** 6396 * dev_get_flags - get flags reported to userspace 6397 * @dev: device 6398 * 6399 * Get the combination of flag bits exported through APIs to userspace. 6400 */ 6401 unsigned int dev_get_flags(const struct net_device *dev) 6402 { 6403 unsigned int flags; 6404 6405 flags = (dev->flags & ~(IFF_PROMISC | 6406 IFF_ALLMULTI | 6407 IFF_RUNNING | 6408 IFF_LOWER_UP | 6409 IFF_DORMANT)) | 6410 (dev->gflags & (IFF_PROMISC | 6411 IFF_ALLMULTI)); 6412 6413 if (netif_running(dev)) { 6414 if (netif_oper_up(dev)) 6415 flags |= IFF_RUNNING; 6416 if (netif_carrier_ok(dev)) 6417 flags |= IFF_LOWER_UP; 6418 if (netif_dormant(dev)) 6419 flags |= IFF_DORMANT; 6420 } 6421 6422 return flags; 6423 } 6424 EXPORT_SYMBOL(dev_get_flags); 6425 6426 int __dev_change_flags(struct net_device *dev, unsigned int flags) 6427 { 6428 unsigned int old_flags = dev->flags; 6429 int ret; 6430 6431 ASSERT_RTNL(); 6432 6433 /* 6434 * Set the flags on our device. 6435 */ 6436 6437 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | 6438 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL | 6439 IFF_AUTOMEDIA)) | 6440 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | 6441 IFF_ALLMULTI)); 6442 6443 /* 6444 * Load in the correct multicast list now the flags have changed. 6445 */ 6446 6447 if ((old_flags ^ flags) & IFF_MULTICAST) 6448 dev_change_rx_flags(dev, IFF_MULTICAST); 6449 6450 dev_set_rx_mode(dev); 6451 6452 /* 6453 * Have we downed the interface. We handle IFF_UP ourselves 6454 * according to user attempts to set it, rather than blindly 6455 * setting it. 6456 */ 6457 6458 ret = 0; 6459 if ((old_flags ^ flags) & IFF_UP) 6460 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev); 6461 6462 if ((flags ^ dev->gflags) & IFF_PROMISC) { 6463 int inc = (flags & IFF_PROMISC) ? 1 : -1; 6464 unsigned int old_flags = dev->flags; 6465 6466 dev->gflags ^= IFF_PROMISC; 6467 6468 if (__dev_set_promiscuity(dev, inc, false) >= 0) 6469 if (dev->flags != old_flags) 6470 dev_set_rx_mode(dev); 6471 } 6472 6473 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI 6474 * is important. Some (broken) drivers set IFF_PROMISC, when 6475 * IFF_ALLMULTI is requested not asking us and not reporting. 6476 */ 6477 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { 6478 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; 6479 6480 dev->gflags ^= IFF_ALLMULTI; 6481 __dev_set_allmulti(dev, inc, false); 6482 } 6483 6484 return ret; 6485 } 6486 6487 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, 6488 unsigned int gchanges) 6489 { 6490 unsigned int changes = dev->flags ^ old_flags; 6491 6492 if (gchanges) 6493 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC); 6494 6495 if (changes & IFF_UP) { 6496 if (dev->flags & IFF_UP) 6497 call_netdevice_notifiers(NETDEV_UP, dev); 6498 else 6499 call_netdevice_notifiers(NETDEV_DOWN, dev); 6500 } 6501 6502 if (dev->flags & IFF_UP && 6503 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) { 6504 struct netdev_notifier_change_info change_info; 6505 6506 change_info.flags_changed = changes; 6507 call_netdevice_notifiers_info(NETDEV_CHANGE, dev, 6508 &change_info.info); 6509 } 6510 } 6511 6512 /** 6513 * dev_change_flags - change device settings 6514 * @dev: device 6515 * @flags: device state flags 6516 * 6517 * Change settings on device based state flags. The flags are 6518 * in the userspace exported format. 6519 */ 6520 int dev_change_flags(struct net_device *dev, unsigned int flags) 6521 { 6522 int ret; 6523 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; 6524 6525 ret = __dev_change_flags(dev, flags); 6526 if (ret < 0) 6527 return ret; 6528 6529 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); 6530 __dev_notify_flags(dev, old_flags, changes); 6531 return ret; 6532 } 6533 EXPORT_SYMBOL(dev_change_flags); 6534 6535 static int __dev_set_mtu(struct net_device *dev, int new_mtu) 6536 { 6537 const struct net_device_ops *ops = dev->netdev_ops; 6538 6539 if (ops->ndo_change_mtu) 6540 return ops->ndo_change_mtu(dev, new_mtu); 6541 6542 dev->mtu = new_mtu; 6543 return 0; 6544 } 6545 6546 /** 6547 * dev_set_mtu - Change maximum transfer unit 6548 * @dev: device 6549 * @new_mtu: new transfer unit 6550 * 6551 * Change the maximum transfer size of the network device. 6552 */ 6553 int dev_set_mtu(struct net_device *dev, int new_mtu) 6554 { 6555 int err, orig_mtu; 6556 6557 if (new_mtu == dev->mtu) 6558 return 0; 6559 6560 /* MTU must be positive, and in range */ 6561 if (new_mtu < 0 || new_mtu < dev->min_mtu) { 6562 net_err_ratelimited("%s: Invalid MTU %d requested, hw min %d\n", 6563 dev->name, new_mtu, dev->min_mtu); 6564 return -EINVAL; 6565 } 6566 6567 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) { 6568 net_err_ratelimited("%s: Invalid MTU %d requested, hw max %d\n", 6569 dev->name, new_mtu, dev->max_mtu); 6570 return -EINVAL; 6571 } 6572 6573 if (!netif_device_present(dev)) 6574 return -ENODEV; 6575 6576 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev); 6577 err = notifier_to_errno(err); 6578 if (err) 6579 return err; 6580 6581 orig_mtu = dev->mtu; 6582 err = __dev_set_mtu(dev, new_mtu); 6583 6584 if (!err) { 6585 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); 6586 err = notifier_to_errno(err); 6587 if (err) { 6588 /* setting mtu back and notifying everyone again, 6589 * so that they have a chance to revert changes. 6590 */ 6591 __dev_set_mtu(dev, orig_mtu); 6592 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); 6593 } 6594 } 6595 return err; 6596 } 6597 EXPORT_SYMBOL(dev_set_mtu); 6598 6599 /** 6600 * dev_set_group - Change group this device belongs to 6601 * @dev: device 6602 * @new_group: group this device should belong to 6603 */ 6604 void dev_set_group(struct net_device *dev, int new_group) 6605 { 6606 dev->group = new_group; 6607 } 6608 EXPORT_SYMBOL(dev_set_group); 6609 6610 /** 6611 * dev_set_mac_address - Change Media Access Control Address 6612 * @dev: device 6613 * @sa: new address 6614 * 6615 * Change the hardware (MAC) address of the device 6616 */ 6617 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa) 6618 { 6619 const struct net_device_ops *ops = dev->netdev_ops; 6620 int err; 6621 6622 if (!ops->ndo_set_mac_address) 6623 return -EOPNOTSUPP; 6624 if (sa->sa_family != dev->type) 6625 return -EINVAL; 6626 if (!netif_device_present(dev)) 6627 return -ENODEV; 6628 err = ops->ndo_set_mac_address(dev, sa); 6629 if (err) 6630 return err; 6631 dev->addr_assign_type = NET_ADDR_SET; 6632 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 6633 add_device_randomness(dev->dev_addr, dev->addr_len); 6634 return 0; 6635 } 6636 EXPORT_SYMBOL(dev_set_mac_address); 6637 6638 /** 6639 * dev_change_carrier - Change device carrier 6640 * @dev: device 6641 * @new_carrier: new value 6642 * 6643 * Change device carrier 6644 */ 6645 int dev_change_carrier(struct net_device *dev, bool new_carrier) 6646 { 6647 const struct net_device_ops *ops = dev->netdev_ops; 6648 6649 if (!ops->ndo_change_carrier) 6650 return -EOPNOTSUPP; 6651 if (!netif_device_present(dev)) 6652 return -ENODEV; 6653 return ops->ndo_change_carrier(dev, new_carrier); 6654 } 6655 EXPORT_SYMBOL(dev_change_carrier); 6656 6657 /** 6658 * dev_get_phys_port_id - Get device physical port ID 6659 * @dev: device 6660 * @ppid: port ID 6661 * 6662 * Get device physical port ID 6663 */ 6664 int dev_get_phys_port_id(struct net_device *dev, 6665 struct netdev_phys_item_id *ppid) 6666 { 6667 const struct net_device_ops *ops = dev->netdev_ops; 6668 6669 if (!ops->ndo_get_phys_port_id) 6670 return -EOPNOTSUPP; 6671 return ops->ndo_get_phys_port_id(dev, ppid); 6672 } 6673 EXPORT_SYMBOL(dev_get_phys_port_id); 6674 6675 /** 6676 * dev_get_phys_port_name - Get device physical port name 6677 * @dev: device 6678 * @name: port name 6679 * @len: limit of bytes to copy to name 6680 * 6681 * Get device physical port name 6682 */ 6683 int dev_get_phys_port_name(struct net_device *dev, 6684 char *name, size_t len) 6685 { 6686 const struct net_device_ops *ops = dev->netdev_ops; 6687 6688 if (!ops->ndo_get_phys_port_name) 6689 return -EOPNOTSUPP; 6690 return ops->ndo_get_phys_port_name(dev, name, len); 6691 } 6692 EXPORT_SYMBOL(dev_get_phys_port_name); 6693 6694 /** 6695 * dev_change_proto_down - update protocol port state information 6696 * @dev: device 6697 * @proto_down: new value 6698 * 6699 * This info can be used by switch drivers to set the phys state of the 6700 * port. 6701 */ 6702 int dev_change_proto_down(struct net_device *dev, bool proto_down) 6703 { 6704 const struct net_device_ops *ops = dev->netdev_ops; 6705 6706 if (!ops->ndo_change_proto_down) 6707 return -EOPNOTSUPP; 6708 if (!netif_device_present(dev)) 6709 return -ENODEV; 6710 return ops->ndo_change_proto_down(dev, proto_down); 6711 } 6712 EXPORT_SYMBOL(dev_change_proto_down); 6713 6714 /** 6715 * dev_change_xdp_fd - set or clear a bpf program for a device rx path 6716 * @dev: device 6717 * @fd: new program fd or negative value to clear 6718 * @flags: xdp-related flags 6719 * 6720 * Set or clear a bpf program for a device 6721 */ 6722 int dev_change_xdp_fd(struct net_device *dev, int fd, u32 flags) 6723 { 6724 const struct net_device_ops *ops = dev->netdev_ops; 6725 struct bpf_prog *prog = NULL; 6726 struct netdev_xdp xdp; 6727 int err; 6728 6729 ASSERT_RTNL(); 6730 6731 if (!ops->ndo_xdp) 6732 return -EOPNOTSUPP; 6733 if (fd >= 0) { 6734 if (flags & XDP_FLAGS_UPDATE_IF_NOEXIST) { 6735 memset(&xdp, 0, sizeof(xdp)); 6736 xdp.command = XDP_QUERY_PROG; 6737 6738 err = ops->ndo_xdp(dev, &xdp); 6739 if (err < 0) 6740 return err; 6741 if (xdp.prog_attached) 6742 return -EBUSY; 6743 } 6744 6745 prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_XDP); 6746 if (IS_ERR(prog)) 6747 return PTR_ERR(prog); 6748 } 6749 6750 memset(&xdp, 0, sizeof(xdp)); 6751 xdp.command = XDP_SETUP_PROG; 6752 xdp.prog = prog; 6753 6754 err = ops->ndo_xdp(dev, &xdp); 6755 if (err < 0 && prog) 6756 bpf_prog_put(prog); 6757 6758 return err; 6759 } 6760 EXPORT_SYMBOL(dev_change_xdp_fd); 6761 6762 /** 6763 * dev_new_index - allocate an ifindex 6764 * @net: the applicable net namespace 6765 * 6766 * Returns a suitable unique value for a new device interface 6767 * number. The caller must hold the rtnl semaphore or the 6768 * dev_base_lock to be sure it remains unique. 6769 */ 6770 static int dev_new_index(struct net *net) 6771 { 6772 int ifindex = net->ifindex; 6773 6774 for (;;) { 6775 if (++ifindex <= 0) 6776 ifindex = 1; 6777 if (!__dev_get_by_index(net, ifindex)) 6778 return net->ifindex = ifindex; 6779 } 6780 } 6781 6782 /* Delayed registration/unregisteration */ 6783 static LIST_HEAD(net_todo_list); 6784 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq); 6785 6786 static void net_set_todo(struct net_device *dev) 6787 { 6788 list_add_tail(&dev->todo_list, &net_todo_list); 6789 dev_net(dev)->dev_unreg_count++; 6790 } 6791 6792 static void rollback_registered_many(struct list_head *head) 6793 { 6794 struct net_device *dev, *tmp; 6795 LIST_HEAD(close_head); 6796 6797 BUG_ON(dev_boot_phase); 6798 ASSERT_RTNL(); 6799 6800 list_for_each_entry_safe(dev, tmp, head, unreg_list) { 6801 /* Some devices call without registering 6802 * for initialization unwind. Remove those 6803 * devices and proceed with the remaining. 6804 */ 6805 if (dev->reg_state == NETREG_UNINITIALIZED) { 6806 pr_debug("unregister_netdevice: device %s/%p never was registered\n", 6807 dev->name, dev); 6808 6809 WARN_ON(1); 6810 list_del(&dev->unreg_list); 6811 continue; 6812 } 6813 dev->dismantle = true; 6814 BUG_ON(dev->reg_state != NETREG_REGISTERED); 6815 } 6816 6817 /* If device is running, close it first. */ 6818 list_for_each_entry(dev, head, unreg_list) 6819 list_add_tail(&dev->close_list, &close_head); 6820 dev_close_many(&close_head, true); 6821 6822 list_for_each_entry(dev, head, unreg_list) { 6823 /* And unlink it from device chain. */ 6824 unlist_netdevice(dev); 6825 6826 dev->reg_state = NETREG_UNREGISTERING; 6827 } 6828 flush_all_backlogs(); 6829 6830 synchronize_net(); 6831 6832 list_for_each_entry(dev, head, unreg_list) { 6833 struct sk_buff *skb = NULL; 6834 6835 /* Shutdown queueing discipline. */ 6836 dev_shutdown(dev); 6837 6838 6839 /* Notify protocols, that we are about to destroy 6840 * this device. They should clean all the things. 6841 */ 6842 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 6843 6844 if (!dev->rtnl_link_ops || 6845 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 6846 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 6847 GFP_KERNEL); 6848 6849 /* 6850 * Flush the unicast and multicast chains 6851 */ 6852 dev_uc_flush(dev); 6853 dev_mc_flush(dev); 6854 6855 if (dev->netdev_ops->ndo_uninit) 6856 dev->netdev_ops->ndo_uninit(dev); 6857 6858 if (skb) 6859 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL); 6860 6861 /* Notifier chain MUST detach us all upper devices. */ 6862 WARN_ON(netdev_has_any_upper_dev(dev)); 6863 WARN_ON(netdev_has_any_lower_dev(dev)); 6864 6865 /* Remove entries from kobject tree */ 6866 netdev_unregister_kobject(dev); 6867 #ifdef CONFIG_XPS 6868 /* Remove XPS queueing entries */ 6869 netif_reset_xps_queues_gt(dev, 0); 6870 #endif 6871 } 6872 6873 synchronize_net(); 6874 6875 list_for_each_entry(dev, head, unreg_list) 6876 dev_put(dev); 6877 } 6878 6879 static void rollback_registered(struct net_device *dev) 6880 { 6881 LIST_HEAD(single); 6882 6883 list_add(&dev->unreg_list, &single); 6884 rollback_registered_many(&single); 6885 list_del(&single); 6886 } 6887 6888 static netdev_features_t netdev_sync_upper_features(struct net_device *lower, 6889 struct net_device *upper, netdev_features_t features) 6890 { 6891 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 6892 netdev_features_t feature; 6893 int feature_bit; 6894 6895 for_each_netdev_feature(&upper_disables, feature_bit) { 6896 feature = __NETIF_F_BIT(feature_bit); 6897 if (!(upper->wanted_features & feature) 6898 && (features & feature)) { 6899 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n", 6900 &feature, upper->name); 6901 features &= ~feature; 6902 } 6903 } 6904 6905 return features; 6906 } 6907 6908 static void netdev_sync_lower_features(struct net_device *upper, 6909 struct net_device *lower, netdev_features_t features) 6910 { 6911 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 6912 netdev_features_t feature; 6913 int feature_bit; 6914 6915 for_each_netdev_feature(&upper_disables, feature_bit) { 6916 feature = __NETIF_F_BIT(feature_bit); 6917 if (!(features & feature) && (lower->features & feature)) { 6918 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n", 6919 &feature, lower->name); 6920 lower->wanted_features &= ~feature; 6921 netdev_update_features(lower); 6922 6923 if (unlikely(lower->features & feature)) 6924 netdev_WARN(upper, "failed to disable %pNF on %s!\n", 6925 &feature, lower->name); 6926 } 6927 } 6928 } 6929 6930 static netdev_features_t netdev_fix_features(struct net_device *dev, 6931 netdev_features_t features) 6932 { 6933 /* Fix illegal checksum combinations */ 6934 if ((features & NETIF_F_HW_CSUM) && 6935 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { 6936 netdev_warn(dev, "mixed HW and IP checksum settings.\n"); 6937 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); 6938 } 6939 6940 /* TSO requires that SG is present as well. */ 6941 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) { 6942 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n"); 6943 features &= ~NETIF_F_ALL_TSO; 6944 } 6945 6946 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) && 6947 !(features & NETIF_F_IP_CSUM)) { 6948 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n"); 6949 features &= ~NETIF_F_TSO; 6950 features &= ~NETIF_F_TSO_ECN; 6951 } 6952 6953 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) && 6954 !(features & NETIF_F_IPV6_CSUM)) { 6955 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n"); 6956 features &= ~NETIF_F_TSO6; 6957 } 6958 6959 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */ 6960 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO)) 6961 features &= ~NETIF_F_TSO_MANGLEID; 6962 6963 /* TSO ECN requires that TSO is present as well. */ 6964 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN) 6965 features &= ~NETIF_F_TSO_ECN; 6966 6967 /* Software GSO depends on SG. */ 6968 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) { 6969 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n"); 6970 features &= ~NETIF_F_GSO; 6971 } 6972 6973 /* UFO needs SG and checksumming */ 6974 if (features & NETIF_F_UFO) { 6975 /* maybe split UFO into V4 and V6? */ 6976 if (!(features & NETIF_F_HW_CSUM) && 6977 ((features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) != 6978 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) { 6979 netdev_dbg(dev, 6980 "Dropping NETIF_F_UFO since no checksum offload features.\n"); 6981 features &= ~NETIF_F_UFO; 6982 } 6983 6984 if (!(features & NETIF_F_SG)) { 6985 netdev_dbg(dev, 6986 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n"); 6987 features &= ~NETIF_F_UFO; 6988 } 6989 } 6990 6991 /* GSO partial features require GSO partial be set */ 6992 if ((features & dev->gso_partial_features) && 6993 !(features & NETIF_F_GSO_PARTIAL)) { 6994 netdev_dbg(dev, 6995 "Dropping partially supported GSO features since no GSO partial.\n"); 6996 features &= ~dev->gso_partial_features; 6997 } 6998 6999 return features; 7000 } 7001 7002 int __netdev_update_features(struct net_device *dev) 7003 { 7004 struct net_device *upper, *lower; 7005 netdev_features_t features; 7006 struct list_head *iter; 7007 int err = -1; 7008 7009 ASSERT_RTNL(); 7010 7011 features = netdev_get_wanted_features(dev); 7012 7013 if (dev->netdev_ops->ndo_fix_features) 7014 features = dev->netdev_ops->ndo_fix_features(dev, features); 7015 7016 /* driver might be less strict about feature dependencies */ 7017 features = netdev_fix_features(dev, features); 7018 7019 /* some features can't be enabled if they're off an an upper device */ 7020 netdev_for_each_upper_dev_rcu(dev, upper, iter) 7021 features = netdev_sync_upper_features(dev, upper, features); 7022 7023 if (dev->features == features) 7024 goto sync_lower; 7025 7026 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", 7027 &dev->features, &features); 7028 7029 if (dev->netdev_ops->ndo_set_features) 7030 err = dev->netdev_ops->ndo_set_features(dev, features); 7031 else 7032 err = 0; 7033 7034 if (unlikely(err < 0)) { 7035 netdev_err(dev, 7036 "set_features() failed (%d); wanted %pNF, left %pNF\n", 7037 err, &features, &dev->features); 7038 /* return non-0 since some features might have changed and 7039 * it's better to fire a spurious notification than miss it 7040 */ 7041 return -1; 7042 } 7043 7044 sync_lower: 7045 /* some features must be disabled on lower devices when disabled 7046 * on an upper device (think: bonding master or bridge) 7047 */ 7048 netdev_for_each_lower_dev(dev, lower, iter) 7049 netdev_sync_lower_features(dev, lower, features); 7050 7051 if (!err) 7052 dev->features = features; 7053 7054 return err < 0 ? 0 : 1; 7055 } 7056 7057 /** 7058 * netdev_update_features - recalculate device features 7059 * @dev: the device to check 7060 * 7061 * Recalculate dev->features set and send notifications if it 7062 * has changed. Should be called after driver or hardware dependent 7063 * conditions might have changed that influence the features. 7064 */ 7065 void netdev_update_features(struct net_device *dev) 7066 { 7067 if (__netdev_update_features(dev)) 7068 netdev_features_change(dev); 7069 } 7070 EXPORT_SYMBOL(netdev_update_features); 7071 7072 /** 7073 * netdev_change_features - recalculate device features 7074 * @dev: the device to check 7075 * 7076 * Recalculate dev->features set and send notifications even 7077 * if they have not changed. Should be called instead of 7078 * netdev_update_features() if also dev->vlan_features might 7079 * have changed to allow the changes to be propagated to stacked 7080 * VLAN devices. 7081 */ 7082 void netdev_change_features(struct net_device *dev) 7083 { 7084 __netdev_update_features(dev); 7085 netdev_features_change(dev); 7086 } 7087 EXPORT_SYMBOL(netdev_change_features); 7088 7089 /** 7090 * netif_stacked_transfer_operstate - transfer operstate 7091 * @rootdev: the root or lower level device to transfer state from 7092 * @dev: the device to transfer operstate to 7093 * 7094 * Transfer operational state from root to device. This is normally 7095 * called when a stacking relationship exists between the root 7096 * device and the device(a leaf device). 7097 */ 7098 void netif_stacked_transfer_operstate(const struct net_device *rootdev, 7099 struct net_device *dev) 7100 { 7101 if (rootdev->operstate == IF_OPER_DORMANT) 7102 netif_dormant_on(dev); 7103 else 7104 netif_dormant_off(dev); 7105 7106 if (netif_carrier_ok(rootdev)) { 7107 if (!netif_carrier_ok(dev)) 7108 netif_carrier_on(dev); 7109 } else { 7110 if (netif_carrier_ok(dev)) 7111 netif_carrier_off(dev); 7112 } 7113 } 7114 EXPORT_SYMBOL(netif_stacked_transfer_operstate); 7115 7116 #ifdef CONFIG_SYSFS 7117 static int netif_alloc_rx_queues(struct net_device *dev) 7118 { 7119 unsigned int i, count = dev->num_rx_queues; 7120 struct netdev_rx_queue *rx; 7121 size_t sz = count * sizeof(*rx); 7122 7123 BUG_ON(count < 1); 7124 7125 rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 7126 if (!rx) { 7127 rx = vzalloc(sz); 7128 if (!rx) 7129 return -ENOMEM; 7130 } 7131 dev->_rx = rx; 7132 7133 for (i = 0; i < count; i++) 7134 rx[i].dev = dev; 7135 return 0; 7136 } 7137 #endif 7138 7139 static void netdev_init_one_queue(struct net_device *dev, 7140 struct netdev_queue *queue, void *_unused) 7141 { 7142 /* Initialize queue lock */ 7143 spin_lock_init(&queue->_xmit_lock); 7144 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); 7145 queue->xmit_lock_owner = -1; 7146 netdev_queue_numa_node_write(queue, NUMA_NO_NODE); 7147 queue->dev = dev; 7148 #ifdef CONFIG_BQL 7149 dql_init(&queue->dql, HZ); 7150 #endif 7151 } 7152 7153 static void netif_free_tx_queues(struct net_device *dev) 7154 { 7155 kvfree(dev->_tx); 7156 } 7157 7158 static int netif_alloc_netdev_queues(struct net_device *dev) 7159 { 7160 unsigned int count = dev->num_tx_queues; 7161 struct netdev_queue *tx; 7162 size_t sz = count * sizeof(*tx); 7163 7164 if (count < 1 || count > 0xffff) 7165 return -EINVAL; 7166 7167 tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 7168 if (!tx) { 7169 tx = vzalloc(sz); 7170 if (!tx) 7171 return -ENOMEM; 7172 } 7173 dev->_tx = tx; 7174 7175 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); 7176 spin_lock_init(&dev->tx_global_lock); 7177 7178 return 0; 7179 } 7180 7181 void netif_tx_stop_all_queues(struct net_device *dev) 7182 { 7183 unsigned int i; 7184 7185 for (i = 0; i < dev->num_tx_queues; i++) { 7186 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 7187 7188 netif_tx_stop_queue(txq); 7189 } 7190 } 7191 EXPORT_SYMBOL(netif_tx_stop_all_queues); 7192 7193 /** 7194 * register_netdevice - register a network device 7195 * @dev: device to register 7196 * 7197 * Take a completed network device structure and add it to the kernel 7198 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 7199 * chain. 0 is returned on success. A negative errno code is returned 7200 * on a failure to set up the device, or if the name is a duplicate. 7201 * 7202 * Callers must hold the rtnl semaphore. You may want 7203 * register_netdev() instead of this. 7204 * 7205 * BUGS: 7206 * The locking appears insufficient to guarantee two parallel registers 7207 * will not get the same name. 7208 */ 7209 7210 int register_netdevice(struct net_device *dev) 7211 { 7212 int ret; 7213 struct net *net = dev_net(dev); 7214 7215 BUG_ON(dev_boot_phase); 7216 ASSERT_RTNL(); 7217 7218 might_sleep(); 7219 7220 /* When net_device's are persistent, this will be fatal. */ 7221 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); 7222 BUG_ON(!net); 7223 7224 spin_lock_init(&dev->addr_list_lock); 7225 netdev_set_addr_lockdep_class(dev); 7226 7227 ret = dev_get_valid_name(net, dev, dev->name); 7228 if (ret < 0) 7229 goto out; 7230 7231 /* Init, if this function is available */ 7232 if (dev->netdev_ops->ndo_init) { 7233 ret = dev->netdev_ops->ndo_init(dev); 7234 if (ret) { 7235 if (ret > 0) 7236 ret = -EIO; 7237 goto out; 7238 } 7239 } 7240 7241 if (((dev->hw_features | dev->features) & 7242 NETIF_F_HW_VLAN_CTAG_FILTER) && 7243 (!dev->netdev_ops->ndo_vlan_rx_add_vid || 7244 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { 7245 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n"); 7246 ret = -EINVAL; 7247 goto err_uninit; 7248 } 7249 7250 ret = -EBUSY; 7251 if (!dev->ifindex) 7252 dev->ifindex = dev_new_index(net); 7253 else if (__dev_get_by_index(net, dev->ifindex)) 7254 goto err_uninit; 7255 7256 /* Transfer changeable features to wanted_features and enable 7257 * software offloads (GSO and GRO). 7258 */ 7259 dev->hw_features |= NETIF_F_SOFT_FEATURES; 7260 dev->features |= NETIF_F_SOFT_FEATURES; 7261 dev->wanted_features = dev->features & dev->hw_features; 7262 7263 if (!(dev->flags & IFF_LOOPBACK)) 7264 dev->hw_features |= NETIF_F_NOCACHE_COPY; 7265 7266 /* If IPv4 TCP segmentation offload is supported we should also 7267 * allow the device to enable segmenting the frame with the option 7268 * of ignoring a static IP ID value. This doesn't enable the 7269 * feature itself but allows the user to enable it later. 7270 */ 7271 if (dev->hw_features & NETIF_F_TSO) 7272 dev->hw_features |= NETIF_F_TSO_MANGLEID; 7273 if (dev->vlan_features & NETIF_F_TSO) 7274 dev->vlan_features |= NETIF_F_TSO_MANGLEID; 7275 if (dev->mpls_features & NETIF_F_TSO) 7276 dev->mpls_features |= NETIF_F_TSO_MANGLEID; 7277 if (dev->hw_enc_features & NETIF_F_TSO) 7278 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 7279 7280 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. 7281 */ 7282 dev->vlan_features |= NETIF_F_HIGHDMA; 7283 7284 /* Make NETIF_F_SG inheritable to tunnel devices. 7285 */ 7286 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL; 7287 7288 /* Make NETIF_F_SG inheritable to MPLS. 7289 */ 7290 dev->mpls_features |= NETIF_F_SG; 7291 7292 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); 7293 ret = notifier_to_errno(ret); 7294 if (ret) 7295 goto err_uninit; 7296 7297 ret = netdev_register_kobject(dev); 7298 if (ret) 7299 goto err_uninit; 7300 dev->reg_state = NETREG_REGISTERED; 7301 7302 __netdev_update_features(dev); 7303 7304 /* 7305 * Default initial state at registry is that the 7306 * device is present. 7307 */ 7308 7309 set_bit(__LINK_STATE_PRESENT, &dev->state); 7310 7311 linkwatch_init_dev(dev); 7312 7313 dev_init_scheduler(dev); 7314 dev_hold(dev); 7315 list_netdevice(dev); 7316 add_device_randomness(dev->dev_addr, dev->addr_len); 7317 7318 /* If the device has permanent device address, driver should 7319 * set dev_addr and also addr_assign_type should be set to 7320 * NET_ADDR_PERM (default value). 7321 */ 7322 if (dev->addr_assign_type == NET_ADDR_PERM) 7323 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); 7324 7325 /* Notify protocols, that a new device appeared. */ 7326 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); 7327 ret = notifier_to_errno(ret); 7328 if (ret) { 7329 rollback_registered(dev); 7330 dev->reg_state = NETREG_UNREGISTERED; 7331 } 7332 /* 7333 * Prevent userspace races by waiting until the network 7334 * device is fully setup before sending notifications. 7335 */ 7336 if (!dev->rtnl_link_ops || 7337 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 7338 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); 7339 7340 out: 7341 return ret; 7342 7343 err_uninit: 7344 if (dev->netdev_ops->ndo_uninit) 7345 dev->netdev_ops->ndo_uninit(dev); 7346 goto out; 7347 } 7348 EXPORT_SYMBOL(register_netdevice); 7349 7350 /** 7351 * init_dummy_netdev - init a dummy network device for NAPI 7352 * @dev: device to init 7353 * 7354 * This takes a network device structure and initialize the minimum 7355 * amount of fields so it can be used to schedule NAPI polls without 7356 * registering a full blown interface. This is to be used by drivers 7357 * that need to tie several hardware interfaces to a single NAPI 7358 * poll scheduler due to HW limitations. 7359 */ 7360 int init_dummy_netdev(struct net_device *dev) 7361 { 7362 /* Clear everything. Note we don't initialize spinlocks 7363 * are they aren't supposed to be taken by any of the 7364 * NAPI code and this dummy netdev is supposed to be 7365 * only ever used for NAPI polls 7366 */ 7367 memset(dev, 0, sizeof(struct net_device)); 7368 7369 /* make sure we BUG if trying to hit standard 7370 * register/unregister code path 7371 */ 7372 dev->reg_state = NETREG_DUMMY; 7373 7374 /* NAPI wants this */ 7375 INIT_LIST_HEAD(&dev->napi_list); 7376 7377 /* a dummy interface is started by default */ 7378 set_bit(__LINK_STATE_PRESENT, &dev->state); 7379 set_bit(__LINK_STATE_START, &dev->state); 7380 7381 /* Note : We dont allocate pcpu_refcnt for dummy devices, 7382 * because users of this 'device' dont need to change 7383 * its refcount. 7384 */ 7385 7386 return 0; 7387 } 7388 EXPORT_SYMBOL_GPL(init_dummy_netdev); 7389 7390 7391 /** 7392 * register_netdev - register a network device 7393 * @dev: device to register 7394 * 7395 * Take a completed network device structure and add it to the kernel 7396 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 7397 * chain. 0 is returned on success. A negative errno code is returned 7398 * on a failure to set up the device, or if the name is a duplicate. 7399 * 7400 * This is a wrapper around register_netdevice that takes the rtnl semaphore 7401 * and expands the device name if you passed a format string to 7402 * alloc_netdev. 7403 */ 7404 int register_netdev(struct net_device *dev) 7405 { 7406 int err; 7407 7408 rtnl_lock(); 7409 err = register_netdevice(dev); 7410 rtnl_unlock(); 7411 return err; 7412 } 7413 EXPORT_SYMBOL(register_netdev); 7414 7415 int netdev_refcnt_read(const struct net_device *dev) 7416 { 7417 int i, refcnt = 0; 7418 7419 for_each_possible_cpu(i) 7420 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); 7421 return refcnt; 7422 } 7423 EXPORT_SYMBOL(netdev_refcnt_read); 7424 7425 /** 7426 * netdev_wait_allrefs - wait until all references are gone. 7427 * @dev: target net_device 7428 * 7429 * This is called when unregistering network devices. 7430 * 7431 * Any protocol or device that holds a reference should register 7432 * for netdevice notification, and cleanup and put back the 7433 * reference if they receive an UNREGISTER event. 7434 * We can get stuck here if buggy protocols don't correctly 7435 * call dev_put. 7436 */ 7437 static void netdev_wait_allrefs(struct net_device *dev) 7438 { 7439 unsigned long rebroadcast_time, warning_time; 7440 int refcnt; 7441 7442 linkwatch_forget_dev(dev); 7443 7444 rebroadcast_time = warning_time = jiffies; 7445 refcnt = netdev_refcnt_read(dev); 7446 7447 while (refcnt != 0) { 7448 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { 7449 rtnl_lock(); 7450 7451 /* Rebroadcast unregister notification */ 7452 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 7453 7454 __rtnl_unlock(); 7455 rcu_barrier(); 7456 rtnl_lock(); 7457 7458 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 7459 if (test_bit(__LINK_STATE_LINKWATCH_PENDING, 7460 &dev->state)) { 7461 /* We must not have linkwatch events 7462 * pending on unregister. If this 7463 * happens, we simply run the queue 7464 * unscheduled, resulting in a noop 7465 * for this device. 7466 */ 7467 linkwatch_run_queue(); 7468 } 7469 7470 __rtnl_unlock(); 7471 7472 rebroadcast_time = jiffies; 7473 } 7474 7475 msleep(250); 7476 7477 refcnt = netdev_refcnt_read(dev); 7478 7479 if (time_after(jiffies, warning_time + 10 * HZ)) { 7480 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", 7481 dev->name, refcnt); 7482 warning_time = jiffies; 7483 } 7484 } 7485 } 7486 7487 /* The sequence is: 7488 * 7489 * rtnl_lock(); 7490 * ... 7491 * register_netdevice(x1); 7492 * register_netdevice(x2); 7493 * ... 7494 * unregister_netdevice(y1); 7495 * unregister_netdevice(y2); 7496 * ... 7497 * rtnl_unlock(); 7498 * free_netdev(y1); 7499 * free_netdev(y2); 7500 * 7501 * We are invoked by rtnl_unlock(). 7502 * This allows us to deal with problems: 7503 * 1) We can delete sysfs objects which invoke hotplug 7504 * without deadlocking with linkwatch via keventd. 7505 * 2) Since we run with the RTNL semaphore not held, we can sleep 7506 * safely in order to wait for the netdev refcnt to drop to zero. 7507 * 7508 * We must not return until all unregister events added during 7509 * the interval the lock was held have been completed. 7510 */ 7511 void netdev_run_todo(void) 7512 { 7513 struct list_head list; 7514 7515 /* Snapshot list, allow later requests */ 7516 list_replace_init(&net_todo_list, &list); 7517 7518 __rtnl_unlock(); 7519 7520 7521 /* Wait for rcu callbacks to finish before next phase */ 7522 if (!list_empty(&list)) 7523 rcu_barrier(); 7524 7525 while (!list_empty(&list)) { 7526 struct net_device *dev 7527 = list_first_entry(&list, struct net_device, todo_list); 7528 list_del(&dev->todo_list); 7529 7530 rtnl_lock(); 7531 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 7532 __rtnl_unlock(); 7533 7534 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { 7535 pr_err("network todo '%s' but state %d\n", 7536 dev->name, dev->reg_state); 7537 dump_stack(); 7538 continue; 7539 } 7540 7541 dev->reg_state = NETREG_UNREGISTERED; 7542 7543 netdev_wait_allrefs(dev); 7544 7545 /* paranoia */ 7546 BUG_ON(netdev_refcnt_read(dev)); 7547 BUG_ON(!list_empty(&dev->ptype_all)); 7548 BUG_ON(!list_empty(&dev->ptype_specific)); 7549 WARN_ON(rcu_access_pointer(dev->ip_ptr)); 7550 WARN_ON(rcu_access_pointer(dev->ip6_ptr)); 7551 WARN_ON(dev->dn_ptr); 7552 7553 if (dev->destructor) 7554 dev->destructor(dev); 7555 7556 /* Report a network device has been unregistered */ 7557 rtnl_lock(); 7558 dev_net(dev)->dev_unreg_count--; 7559 __rtnl_unlock(); 7560 wake_up(&netdev_unregistering_wq); 7561 7562 /* Free network device */ 7563 kobject_put(&dev->dev.kobj); 7564 } 7565 } 7566 7567 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has 7568 * all the same fields in the same order as net_device_stats, with only 7569 * the type differing, but rtnl_link_stats64 may have additional fields 7570 * at the end for newer counters. 7571 */ 7572 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64, 7573 const struct net_device_stats *netdev_stats) 7574 { 7575 #if BITS_PER_LONG == 64 7576 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats)); 7577 memcpy(stats64, netdev_stats, sizeof(*stats64)); 7578 /* zero out counters that only exist in rtnl_link_stats64 */ 7579 memset((char *)stats64 + sizeof(*netdev_stats), 0, 7580 sizeof(*stats64) - sizeof(*netdev_stats)); 7581 #else 7582 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long); 7583 const unsigned long *src = (const unsigned long *)netdev_stats; 7584 u64 *dst = (u64 *)stats64; 7585 7586 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64)); 7587 for (i = 0; i < n; i++) 7588 dst[i] = src[i]; 7589 /* zero out counters that only exist in rtnl_link_stats64 */ 7590 memset((char *)stats64 + n * sizeof(u64), 0, 7591 sizeof(*stats64) - n * sizeof(u64)); 7592 #endif 7593 } 7594 EXPORT_SYMBOL(netdev_stats_to_stats64); 7595 7596 /** 7597 * dev_get_stats - get network device statistics 7598 * @dev: device to get statistics from 7599 * @storage: place to store stats 7600 * 7601 * Get network statistics from device. Return @storage. 7602 * The device driver may provide its own method by setting 7603 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats; 7604 * otherwise the internal statistics structure is used. 7605 */ 7606 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, 7607 struct rtnl_link_stats64 *storage) 7608 { 7609 const struct net_device_ops *ops = dev->netdev_ops; 7610 7611 if (ops->ndo_get_stats64) { 7612 memset(storage, 0, sizeof(*storage)); 7613 ops->ndo_get_stats64(dev, storage); 7614 } else if (ops->ndo_get_stats) { 7615 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); 7616 } else { 7617 netdev_stats_to_stats64(storage, &dev->stats); 7618 } 7619 storage->rx_dropped += atomic_long_read(&dev->rx_dropped); 7620 storage->tx_dropped += atomic_long_read(&dev->tx_dropped); 7621 storage->rx_nohandler += atomic_long_read(&dev->rx_nohandler); 7622 return storage; 7623 } 7624 EXPORT_SYMBOL(dev_get_stats); 7625 7626 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) 7627 { 7628 struct netdev_queue *queue = dev_ingress_queue(dev); 7629 7630 #ifdef CONFIG_NET_CLS_ACT 7631 if (queue) 7632 return queue; 7633 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 7634 if (!queue) 7635 return NULL; 7636 netdev_init_one_queue(dev, queue, NULL); 7637 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); 7638 queue->qdisc_sleeping = &noop_qdisc; 7639 rcu_assign_pointer(dev->ingress_queue, queue); 7640 #endif 7641 return queue; 7642 } 7643 7644 static const struct ethtool_ops default_ethtool_ops; 7645 7646 void netdev_set_default_ethtool_ops(struct net_device *dev, 7647 const struct ethtool_ops *ops) 7648 { 7649 if (dev->ethtool_ops == &default_ethtool_ops) 7650 dev->ethtool_ops = ops; 7651 } 7652 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops); 7653 7654 void netdev_freemem(struct net_device *dev) 7655 { 7656 char *addr = (char *)dev - dev->padded; 7657 7658 kvfree(addr); 7659 } 7660 7661 /** 7662 * alloc_netdev_mqs - allocate network device 7663 * @sizeof_priv: size of private data to allocate space for 7664 * @name: device name format string 7665 * @name_assign_type: origin of device name 7666 * @setup: callback to initialize device 7667 * @txqs: the number of TX subqueues to allocate 7668 * @rxqs: the number of RX subqueues to allocate 7669 * 7670 * Allocates a struct net_device with private data area for driver use 7671 * and performs basic initialization. Also allocates subqueue structs 7672 * for each queue on the device. 7673 */ 7674 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, 7675 unsigned char name_assign_type, 7676 void (*setup)(struct net_device *), 7677 unsigned int txqs, unsigned int rxqs) 7678 { 7679 struct net_device *dev; 7680 size_t alloc_size; 7681 struct net_device *p; 7682 7683 BUG_ON(strlen(name) >= sizeof(dev->name)); 7684 7685 if (txqs < 1) { 7686 pr_err("alloc_netdev: Unable to allocate device with zero queues\n"); 7687 return NULL; 7688 } 7689 7690 #ifdef CONFIG_SYSFS 7691 if (rxqs < 1) { 7692 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n"); 7693 return NULL; 7694 } 7695 #endif 7696 7697 alloc_size = sizeof(struct net_device); 7698 if (sizeof_priv) { 7699 /* ensure 32-byte alignment of private area */ 7700 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN); 7701 alloc_size += sizeof_priv; 7702 } 7703 /* ensure 32-byte alignment of whole construct */ 7704 alloc_size += NETDEV_ALIGN - 1; 7705 7706 p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 7707 if (!p) 7708 p = vzalloc(alloc_size); 7709 if (!p) 7710 return NULL; 7711 7712 dev = PTR_ALIGN(p, NETDEV_ALIGN); 7713 dev->padded = (char *)dev - (char *)p; 7714 7715 dev->pcpu_refcnt = alloc_percpu(int); 7716 if (!dev->pcpu_refcnt) 7717 goto free_dev; 7718 7719 if (dev_addr_init(dev)) 7720 goto free_pcpu; 7721 7722 dev_mc_init(dev); 7723 dev_uc_init(dev); 7724 7725 dev_net_set(dev, &init_net); 7726 7727 dev->gso_max_size = GSO_MAX_SIZE; 7728 dev->gso_max_segs = GSO_MAX_SEGS; 7729 7730 INIT_LIST_HEAD(&dev->napi_list); 7731 INIT_LIST_HEAD(&dev->unreg_list); 7732 INIT_LIST_HEAD(&dev->close_list); 7733 INIT_LIST_HEAD(&dev->link_watch_list); 7734 INIT_LIST_HEAD(&dev->adj_list.upper); 7735 INIT_LIST_HEAD(&dev->adj_list.lower); 7736 INIT_LIST_HEAD(&dev->ptype_all); 7737 INIT_LIST_HEAD(&dev->ptype_specific); 7738 #ifdef CONFIG_NET_SCHED 7739 hash_init(dev->qdisc_hash); 7740 #endif 7741 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; 7742 setup(dev); 7743 7744 if (!dev->tx_queue_len) { 7745 dev->priv_flags |= IFF_NO_QUEUE; 7746 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; 7747 } 7748 7749 dev->num_tx_queues = txqs; 7750 dev->real_num_tx_queues = txqs; 7751 if (netif_alloc_netdev_queues(dev)) 7752 goto free_all; 7753 7754 #ifdef CONFIG_SYSFS 7755 dev->num_rx_queues = rxqs; 7756 dev->real_num_rx_queues = rxqs; 7757 if (netif_alloc_rx_queues(dev)) 7758 goto free_all; 7759 #endif 7760 7761 strcpy(dev->name, name); 7762 dev->name_assign_type = name_assign_type; 7763 dev->group = INIT_NETDEV_GROUP; 7764 if (!dev->ethtool_ops) 7765 dev->ethtool_ops = &default_ethtool_ops; 7766 7767 nf_hook_ingress_init(dev); 7768 7769 return dev; 7770 7771 free_all: 7772 free_netdev(dev); 7773 return NULL; 7774 7775 free_pcpu: 7776 free_percpu(dev->pcpu_refcnt); 7777 free_dev: 7778 netdev_freemem(dev); 7779 return NULL; 7780 } 7781 EXPORT_SYMBOL(alloc_netdev_mqs); 7782 7783 /** 7784 * free_netdev - free network device 7785 * @dev: device 7786 * 7787 * This function does the last stage of destroying an allocated device 7788 * interface. The reference to the device object is released. If this 7789 * is the last reference then it will be freed.Must be called in process 7790 * context. 7791 */ 7792 void free_netdev(struct net_device *dev) 7793 { 7794 struct napi_struct *p, *n; 7795 7796 might_sleep(); 7797 netif_free_tx_queues(dev); 7798 #ifdef CONFIG_SYSFS 7799 kvfree(dev->_rx); 7800 #endif 7801 7802 kfree(rcu_dereference_protected(dev->ingress_queue, 1)); 7803 7804 /* Flush device addresses */ 7805 dev_addr_flush(dev); 7806 7807 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) 7808 netif_napi_del(p); 7809 7810 free_percpu(dev->pcpu_refcnt); 7811 dev->pcpu_refcnt = NULL; 7812 7813 /* Compatibility with error handling in drivers */ 7814 if (dev->reg_state == NETREG_UNINITIALIZED) { 7815 netdev_freemem(dev); 7816 return; 7817 } 7818 7819 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); 7820 dev->reg_state = NETREG_RELEASED; 7821 7822 /* will free via device release */ 7823 put_device(&dev->dev); 7824 } 7825 EXPORT_SYMBOL(free_netdev); 7826 7827 /** 7828 * synchronize_net - Synchronize with packet receive processing 7829 * 7830 * Wait for packets currently being received to be done. 7831 * Does not block later packets from starting. 7832 */ 7833 void synchronize_net(void) 7834 { 7835 might_sleep(); 7836 if (rtnl_is_locked()) 7837 synchronize_rcu_expedited(); 7838 else 7839 synchronize_rcu(); 7840 } 7841 EXPORT_SYMBOL(synchronize_net); 7842 7843 /** 7844 * unregister_netdevice_queue - remove device from the kernel 7845 * @dev: device 7846 * @head: list 7847 * 7848 * This function shuts down a device interface and removes it 7849 * from the kernel tables. 7850 * If head not NULL, device is queued to be unregistered later. 7851 * 7852 * Callers must hold the rtnl semaphore. You may want 7853 * unregister_netdev() instead of this. 7854 */ 7855 7856 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) 7857 { 7858 ASSERT_RTNL(); 7859 7860 if (head) { 7861 list_move_tail(&dev->unreg_list, head); 7862 } else { 7863 rollback_registered(dev); 7864 /* Finish processing unregister after unlock */ 7865 net_set_todo(dev); 7866 } 7867 } 7868 EXPORT_SYMBOL(unregister_netdevice_queue); 7869 7870 /** 7871 * unregister_netdevice_many - unregister many devices 7872 * @head: list of devices 7873 * 7874 * Note: As most callers use a stack allocated list_head, 7875 * we force a list_del() to make sure stack wont be corrupted later. 7876 */ 7877 void unregister_netdevice_many(struct list_head *head) 7878 { 7879 struct net_device *dev; 7880 7881 if (!list_empty(head)) { 7882 rollback_registered_many(head); 7883 list_for_each_entry(dev, head, unreg_list) 7884 net_set_todo(dev); 7885 list_del(head); 7886 } 7887 } 7888 EXPORT_SYMBOL(unregister_netdevice_many); 7889 7890 /** 7891 * unregister_netdev - remove device from the kernel 7892 * @dev: device 7893 * 7894 * This function shuts down a device interface and removes it 7895 * from the kernel tables. 7896 * 7897 * This is just a wrapper for unregister_netdevice that takes 7898 * the rtnl semaphore. In general you want to use this and not 7899 * unregister_netdevice. 7900 */ 7901 void unregister_netdev(struct net_device *dev) 7902 { 7903 rtnl_lock(); 7904 unregister_netdevice(dev); 7905 rtnl_unlock(); 7906 } 7907 EXPORT_SYMBOL(unregister_netdev); 7908 7909 /** 7910 * dev_change_net_namespace - move device to different nethost namespace 7911 * @dev: device 7912 * @net: network namespace 7913 * @pat: If not NULL name pattern to try if the current device name 7914 * is already taken in the destination network namespace. 7915 * 7916 * This function shuts down a device interface and moves it 7917 * to a new network namespace. On success 0 is returned, on 7918 * a failure a netagive errno code is returned. 7919 * 7920 * Callers must hold the rtnl semaphore. 7921 */ 7922 7923 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat) 7924 { 7925 int err; 7926 7927 ASSERT_RTNL(); 7928 7929 /* Don't allow namespace local devices to be moved. */ 7930 err = -EINVAL; 7931 if (dev->features & NETIF_F_NETNS_LOCAL) 7932 goto out; 7933 7934 /* Ensure the device has been registrered */ 7935 if (dev->reg_state != NETREG_REGISTERED) 7936 goto out; 7937 7938 /* Get out if there is nothing todo */ 7939 err = 0; 7940 if (net_eq(dev_net(dev), net)) 7941 goto out; 7942 7943 /* Pick the destination device name, and ensure 7944 * we can use it in the destination network namespace. 7945 */ 7946 err = -EEXIST; 7947 if (__dev_get_by_name(net, dev->name)) { 7948 /* We get here if we can't use the current device name */ 7949 if (!pat) 7950 goto out; 7951 if (dev_get_valid_name(net, dev, pat) < 0) 7952 goto out; 7953 } 7954 7955 /* 7956 * And now a mini version of register_netdevice unregister_netdevice. 7957 */ 7958 7959 /* If device is running close it first. */ 7960 dev_close(dev); 7961 7962 /* And unlink it from device chain */ 7963 err = -ENODEV; 7964 unlist_netdevice(dev); 7965 7966 synchronize_net(); 7967 7968 /* Shutdown queueing discipline. */ 7969 dev_shutdown(dev); 7970 7971 /* Notify protocols, that we are about to destroy 7972 * this device. They should clean all the things. 7973 * 7974 * Note that dev->reg_state stays at NETREG_REGISTERED. 7975 * This is wanted because this way 8021q and macvlan know 7976 * the device is just moving and can keep their slaves up. 7977 */ 7978 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 7979 rcu_barrier(); 7980 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 7981 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL); 7982 7983 /* 7984 * Flush the unicast and multicast chains 7985 */ 7986 dev_uc_flush(dev); 7987 dev_mc_flush(dev); 7988 7989 /* Send a netdev-removed uevent to the old namespace */ 7990 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); 7991 netdev_adjacent_del_links(dev); 7992 7993 /* Actually switch the network namespace */ 7994 dev_net_set(dev, net); 7995 7996 /* If there is an ifindex conflict assign a new one */ 7997 if (__dev_get_by_index(net, dev->ifindex)) 7998 dev->ifindex = dev_new_index(net); 7999 8000 /* Send a netdev-add uevent to the new namespace */ 8001 kobject_uevent(&dev->dev.kobj, KOBJ_ADD); 8002 netdev_adjacent_add_links(dev); 8003 8004 /* Fixup kobjects */ 8005 err = device_rename(&dev->dev, dev->name); 8006 WARN_ON(err); 8007 8008 /* Add the device back in the hashes */ 8009 list_netdevice(dev); 8010 8011 /* Notify protocols, that a new device appeared. */ 8012 call_netdevice_notifiers(NETDEV_REGISTER, dev); 8013 8014 /* 8015 * Prevent userspace races by waiting until the network 8016 * device is fully setup before sending notifications. 8017 */ 8018 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); 8019 8020 synchronize_net(); 8021 err = 0; 8022 out: 8023 return err; 8024 } 8025 EXPORT_SYMBOL_GPL(dev_change_net_namespace); 8026 8027 static int dev_cpu_dead(unsigned int oldcpu) 8028 { 8029 struct sk_buff **list_skb; 8030 struct sk_buff *skb; 8031 unsigned int cpu; 8032 struct softnet_data *sd, *oldsd; 8033 8034 local_irq_disable(); 8035 cpu = smp_processor_id(); 8036 sd = &per_cpu(softnet_data, cpu); 8037 oldsd = &per_cpu(softnet_data, oldcpu); 8038 8039 /* Find end of our completion_queue. */ 8040 list_skb = &sd->completion_queue; 8041 while (*list_skb) 8042 list_skb = &(*list_skb)->next; 8043 /* Append completion queue from offline CPU. */ 8044 *list_skb = oldsd->completion_queue; 8045 oldsd->completion_queue = NULL; 8046 8047 /* Append output queue from offline CPU. */ 8048 if (oldsd->output_queue) { 8049 *sd->output_queue_tailp = oldsd->output_queue; 8050 sd->output_queue_tailp = oldsd->output_queue_tailp; 8051 oldsd->output_queue = NULL; 8052 oldsd->output_queue_tailp = &oldsd->output_queue; 8053 } 8054 /* Append NAPI poll list from offline CPU, with one exception : 8055 * process_backlog() must be called by cpu owning percpu backlog. 8056 * We properly handle process_queue & input_pkt_queue later. 8057 */ 8058 while (!list_empty(&oldsd->poll_list)) { 8059 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, 8060 struct napi_struct, 8061 poll_list); 8062 8063 list_del_init(&napi->poll_list); 8064 if (napi->poll == process_backlog) 8065 napi->state = 0; 8066 else 8067 ____napi_schedule(sd, napi); 8068 } 8069 8070 raise_softirq_irqoff(NET_TX_SOFTIRQ); 8071 local_irq_enable(); 8072 8073 /* Process offline CPU's input_pkt_queue */ 8074 while ((skb = __skb_dequeue(&oldsd->process_queue))) { 8075 netif_rx_ni(skb); 8076 input_queue_head_incr(oldsd); 8077 } 8078 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { 8079 netif_rx_ni(skb); 8080 input_queue_head_incr(oldsd); 8081 } 8082 8083 return 0; 8084 } 8085 8086 /** 8087 * netdev_increment_features - increment feature set by one 8088 * @all: current feature set 8089 * @one: new feature set 8090 * @mask: mask feature set 8091 * 8092 * Computes a new feature set after adding a device with feature set 8093 * @one to the master device with current feature set @all. Will not 8094 * enable anything that is off in @mask. Returns the new feature set. 8095 */ 8096 netdev_features_t netdev_increment_features(netdev_features_t all, 8097 netdev_features_t one, netdev_features_t mask) 8098 { 8099 if (mask & NETIF_F_HW_CSUM) 8100 mask |= NETIF_F_CSUM_MASK; 8101 mask |= NETIF_F_VLAN_CHALLENGED; 8102 8103 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask; 8104 all &= one | ~NETIF_F_ALL_FOR_ALL; 8105 8106 /* If one device supports hw checksumming, set for all. */ 8107 if (all & NETIF_F_HW_CSUM) 8108 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM); 8109 8110 return all; 8111 } 8112 EXPORT_SYMBOL(netdev_increment_features); 8113 8114 static struct hlist_head * __net_init netdev_create_hash(void) 8115 { 8116 int i; 8117 struct hlist_head *hash; 8118 8119 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL); 8120 if (hash != NULL) 8121 for (i = 0; i < NETDEV_HASHENTRIES; i++) 8122 INIT_HLIST_HEAD(&hash[i]); 8123 8124 return hash; 8125 } 8126 8127 /* Initialize per network namespace state */ 8128 static int __net_init netdev_init(struct net *net) 8129 { 8130 if (net != &init_net) 8131 INIT_LIST_HEAD(&net->dev_base_head); 8132 8133 net->dev_name_head = netdev_create_hash(); 8134 if (net->dev_name_head == NULL) 8135 goto err_name; 8136 8137 net->dev_index_head = netdev_create_hash(); 8138 if (net->dev_index_head == NULL) 8139 goto err_idx; 8140 8141 return 0; 8142 8143 err_idx: 8144 kfree(net->dev_name_head); 8145 err_name: 8146 return -ENOMEM; 8147 } 8148 8149 /** 8150 * netdev_drivername - network driver for the device 8151 * @dev: network device 8152 * 8153 * Determine network driver for device. 8154 */ 8155 const char *netdev_drivername(const struct net_device *dev) 8156 { 8157 const struct device_driver *driver; 8158 const struct device *parent; 8159 const char *empty = ""; 8160 8161 parent = dev->dev.parent; 8162 if (!parent) 8163 return empty; 8164 8165 driver = parent->driver; 8166 if (driver && driver->name) 8167 return driver->name; 8168 return empty; 8169 } 8170 8171 static void __netdev_printk(const char *level, const struct net_device *dev, 8172 struct va_format *vaf) 8173 { 8174 if (dev && dev->dev.parent) { 8175 dev_printk_emit(level[1] - '0', 8176 dev->dev.parent, 8177 "%s %s %s%s: %pV", 8178 dev_driver_string(dev->dev.parent), 8179 dev_name(dev->dev.parent), 8180 netdev_name(dev), netdev_reg_state(dev), 8181 vaf); 8182 } else if (dev) { 8183 printk("%s%s%s: %pV", 8184 level, netdev_name(dev), netdev_reg_state(dev), vaf); 8185 } else { 8186 printk("%s(NULL net_device): %pV", level, vaf); 8187 } 8188 } 8189 8190 void netdev_printk(const char *level, const struct net_device *dev, 8191 const char *format, ...) 8192 { 8193 struct va_format vaf; 8194 va_list args; 8195 8196 va_start(args, format); 8197 8198 vaf.fmt = format; 8199 vaf.va = &args; 8200 8201 __netdev_printk(level, dev, &vaf); 8202 8203 va_end(args); 8204 } 8205 EXPORT_SYMBOL(netdev_printk); 8206 8207 #define define_netdev_printk_level(func, level) \ 8208 void func(const struct net_device *dev, const char *fmt, ...) \ 8209 { \ 8210 struct va_format vaf; \ 8211 va_list args; \ 8212 \ 8213 va_start(args, fmt); \ 8214 \ 8215 vaf.fmt = fmt; \ 8216 vaf.va = &args; \ 8217 \ 8218 __netdev_printk(level, dev, &vaf); \ 8219 \ 8220 va_end(args); \ 8221 } \ 8222 EXPORT_SYMBOL(func); 8223 8224 define_netdev_printk_level(netdev_emerg, KERN_EMERG); 8225 define_netdev_printk_level(netdev_alert, KERN_ALERT); 8226 define_netdev_printk_level(netdev_crit, KERN_CRIT); 8227 define_netdev_printk_level(netdev_err, KERN_ERR); 8228 define_netdev_printk_level(netdev_warn, KERN_WARNING); 8229 define_netdev_printk_level(netdev_notice, KERN_NOTICE); 8230 define_netdev_printk_level(netdev_info, KERN_INFO); 8231 8232 static void __net_exit netdev_exit(struct net *net) 8233 { 8234 kfree(net->dev_name_head); 8235 kfree(net->dev_index_head); 8236 } 8237 8238 static struct pernet_operations __net_initdata netdev_net_ops = { 8239 .init = netdev_init, 8240 .exit = netdev_exit, 8241 }; 8242 8243 static void __net_exit default_device_exit(struct net *net) 8244 { 8245 struct net_device *dev, *aux; 8246 /* 8247 * Push all migratable network devices back to the 8248 * initial network namespace 8249 */ 8250 rtnl_lock(); 8251 for_each_netdev_safe(net, dev, aux) { 8252 int err; 8253 char fb_name[IFNAMSIZ]; 8254 8255 /* Ignore unmoveable devices (i.e. loopback) */ 8256 if (dev->features & NETIF_F_NETNS_LOCAL) 8257 continue; 8258 8259 /* Leave virtual devices for the generic cleanup */ 8260 if (dev->rtnl_link_ops) 8261 continue; 8262 8263 /* Push remaining network devices to init_net */ 8264 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); 8265 err = dev_change_net_namespace(dev, &init_net, fb_name); 8266 if (err) { 8267 pr_emerg("%s: failed to move %s to init_net: %d\n", 8268 __func__, dev->name, err); 8269 BUG(); 8270 } 8271 } 8272 rtnl_unlock(); 8273 } 8274 8275 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list) 8276 { 8277 /* Return with the rtnl_lock held when there are no network 8278 * devices unregistering in any network namespace in net_list. 8279 */ 8280 struct net *net; 8281 bool unregistering; 8282 DEFINE_WAIT_FUNC(wait, woken_wake_function); 8283 8284 add_wait_queue(&netdev_unregistering_wq, &wait); 8285 for (;;) { 8286 unregistering = false; 8287 rtnl_lock(); 8288 list_for_each_entry(net, net_list, exit_list) { 8289 if (net->dev_unreg_count > 0) { 8290 unregistering = true; 8291 break; 8292 } 8293 } 8294 if (!unregistering) 8295 break; 8296 __rtnl_unlock(); 8297 8298 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); 8299 } 8300 remove_wait_queue(&netdev_unregistering_wq, &wait); 8301 } 8302 8303 static void __net_exit default_device_exit_batch(struct list_head *net_list) 8304 { 8305 /* At exit all network devices most be removed from a network 8306 * namespace. Do this in the reverse order of registration. 8307 * Do this across as many network namespaces as possible to 8308 * improve batching efficiency. 8309 */ 8310 struct net_device *dev; 8311 struct net *net; 8312 LIST_HEAD(dev_kill_list); 8313 8314 /* To prevent network device cleanup code from dereferencing 8315 * loopback devices or network devices that have been freed 8316 * wait here for all pending unregistrations to complete, 8317 * before unregistring the loopback device and allowing the 8318 * network namespace be freed. 8319 * 8320 * The netdev todo list containing all network devices 8321 * unregistrations that happen in default_device_exit_batch 8322 * will run in the rtnl_unlock() at the end of 8323 * default_device_exit_batch. 8324 */ 8325 rtnl_lock_unregistering(net_list); 8326 list_for_each_entry(net, net_list, exit_list) { 8327 for_each_netdev_reverse(net, dev) { 8328 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) 8329 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); 8330 else 8331 unregister_netdevice_queue(dev, &dev_kill_list); 8332 } 8333 } 8334 unregister_netdevice_many(&dev_kill_list); 8335 rtnl_unlock(); 8336 } 8337 8338 static struct pernet_operations __net_initdata default_device_ops = { 8339 .exit = default_device_exit, 8340 .exit_batch = default_device_exit_batch, 8341 }; 8342 8343 /* 8344 * Initialize the DEV module. At boot time this walks the device list and 8345 * unhooks any devices that fail to initialise (normally hardware not 8346 * present) and leaves us with a valid list of present and active devices. 8347 * 8348 */ 8349 8350 /* 8351 * This is called single threaded during boot, so no need 8352 * to take the rtnl semaphore. 8353 */ 8354 static int __init net_dev_init(void) 8355 { 8356 int i, rc = -ENOMEM; 8357 8358 BUG_ON(!dev_boot_phase); 8359 8360 if (dev_proc_init()) 8361 goto out; 8362 8363 if (netdev_kobject_init()) 8364 goto out; 8365 8366 INIT_LIST_HEAD(&ptype_all); 8367 for (i = 0; i < PTYPE_HASH_SIZE; i++) 8368 INIT_LIST_HEAD(&ptype_base[i]); 8369 8370 INIT_LIST_HEAD(&offload_base); 8371 8372 if (register_pernet_subsys(&netdev_net_ops)) 8373 goto out; 8374 8375 /* 8376 * Initialise the packet receive queues. 8377 */ 8378 8379 for_each_possible_cpu(i) { 8380 struct work_struct *flush = per_cpu_ptr(&flush_works, i); 8381 struct softnet_data *sd = &per_cpu(softnet_data, i); 8382 8383 INIT_WORK(flush, flush_backlog); 8384 8385 skb_queue_head_init(&sd->input_pkt_queue); 8386 skb_queue_head_init(&sd->process_queue); 8387 INIT_LIST_HEAD(&sd->poll_list); 8388 sd->output_queue_tailp = &sd->output_queue; 8389 #ifdef CONFIG_RPS 8390 sd->csd.func = rps_trigger_softirq; 8391 sd->csd.info = sd; 8392 sd->cpu = i; 8393 #endif 8394 8395 sd->backlog.poll = process_backlog; 8396 sd->backlog.weight = weight_p; 8397 } 8398 8399 dev_boot_phase = 0; 8400 8401 /* The loopback device is special if any other network devices 8402 * is present in a network namespace the loopback device must 8403 * be present. Since we now dynamically allocate and free the 8404 * loopback device ensure this invariant is maintained by 8405 * keeping the loopback device as the first device on the 8406 * list of network devices. Ensuring the loopback devices 8407 * is the first device that appears and the last network device 8408 * that disappears. 8409 */ 8410 if (register_pernet_device(&loopback_net_ops)) 8411 goto out; 8412 8413 if (register_pernet_device(&default_device_ops)) 8414 goto out; 8415 8416 open_softirq(NET_TX_SOFTIRQ, net_tx_action); 8417 open_softirq(NET_RX_SOFTIRQ, net_rx_action); 8418 8419 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead", 8420 NULL, dev_cpu_dead); 8421 WARN_ON(rc < 0); 8422 dst_subsys_init(); 8423 rc = 0; 8424 out: 8425 return rc; 8426 } 8427 8428 subsys_initcall(net_dev_init); 8429