1 /* 2 * NET3 Protocol independent device support routines. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 * 9 * Derived from the non IP parts of dev.c 1.0.19 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Mark Evans, <evansmp@uhura.aston.ac.uk> 13 * 14 * Additional Authors: 15 * Florian la Roche <rzsfl@rz.uni-sb.de> 16 * Alan Cox <gw4pts@gw4pts.ampr.org> 17 * David Hinds <dahinds@users.sourceforge.net> 18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 19 * Adam Sulmicki <adam@cfar.umd.edu> 20 * Pekka Riikonen <priikone@poesidon.pspt.fi> 21 * 22 * Changes: 23 * D.J. Barrow : Fixed bug where dev->refcnt gets set 24 * to 2 if register_netdev gets called 25 * before net_dev_init & also removed a 26 * few lines of code in the process. 27 * Alan Cox : device private ioctl copies fields back. 28 * Alan Cox : Transmit queue code does relevant 29 * stunts to keep the queue safe. 30 * Alan Cox : Fixed double lock. 31 * Alan Cox : Fixed promisc NULL pointer trap 32 * ???????? : Support the full private ioctl range 33 * Alan Cox : Moved ioctl permission check into 34 * drivers 35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI 36 * Alan Cox : 100 backlog just doesn't cut it when 37 * you start doing multicast video 8) 38 * Alan Cox : Rewrote net_bh and list manager. 39 * Alan Cox : Fix ETH_P_ALL echoback lengths. 40 * Alan Cox : Took out transmit every packet pass 41 * Saved a few bytes in the ioctl handler 42 * Alan Cox : Network driver sets packet type before 43 * calling netif_rx. Saves a function 44 * call a packet. 45 * Alan Cox : Hashed net_bh() 46 * Richard Kooijman: Timestamp fixes. 47 * Alan Cox : Wrong field in SIOCGIFDSTADDR 48 * Alan Cox : Device lock protection. 49 * Alan Cox : Fixed nasty side effect of device close 50 * changes. 51 * Rudi Cilibrasi : Pass the right thing to 52 * set_mac_address() 53 * Dave Miller : 32bit quantity for the device lock to 54 * make it work out on a Sparc. 55 * Bjorn Ekwall : Added KERNELD hack. 56 * Alan Cox : Cleaned up the backlog initialise. 57 * Craig Metz : SIOCGIFCONF fix if space for under 58 * 1 device. 59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there 60 * is no device open function. 61 * Andi Kleen : Fix error reporting for SIOCGIFCONF 62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF 63 * Cyrus Durgin : Cleaned for KMOD 64 * Adam Sulmicki : Bug Fix : Network Device Unload 65 * A network device unload needs to purge 66 * the backlog queue. 67 * Paul Rusty Russell : SIOCSIFNAME 68 * Pekka Riikonen : Netdev boot-time settings code 69 * Andrew Morton : Make unregister_netdevice wait 70 * indefinitely on dev->refcnt 71 * J Hadi Salim : - Backlog queue sampling 72 * - netif_rx() feedback 73 */ 74 75 #include <asm/uaccess.h> 76 #include <linux/bitops.h> 77 #include <linux/capability.h> 78 #include <linux/cpu.h> 79 #include <linux/types.h> 80 #include <linux/kernel.h> 81 #include <linux/hash.h> 82 #include <linux/slab.h> 83 #include <linux/sched.h> 84 #include <linux/mutex.h> 85 #include <linux/string.h> 86 #include <linux/mm.h> 87 #include <linux/socket.h> 88 #include <linux/sockios.h> 89 #include <linux/errno.h> 90 #include <linux/interrupt.h> 91 #include <linux/if_ether.h> 92 #include <linux/netdevice.h> 93 #include <linux/etherdevice.h> 94 #include <linux/ethtool.h> 95 #include <linux/notifier.h> 96 #include <linux/skbuff.h> 97 #include <net/net_namespace.h> 98 #include <net/sock.h> 99 #include <net/busy_poll.h> 100 #include <linux/rtnetlink.h> 101 #include <linux/stat.h> 102 #include <net/dst.h> 103 #include <net/dst_metadata.h> 104 #include <net/pkt_sched.h> 105 #include <net/checksum.h> 106 #include <net/xfrm.h> 107 #include <linux/highmem.h> 108 #include <linux/init.h> 109 #include <linux/module.h> 110 #include <linux/netpoll.h> 111 #include <linux/rcupdate.h> 112 #include <linux/delay.h> 113 #include <net/iw_handler.h> 114 #include <asm/current.h> 115 #include <linux/audit.h> 116 #include <linux/dmaengine.h> 117 #include <linux/err.h> 118 #include <linux/ctype.h> 119 #include <linux/if_arp.h> 120 #include <linux/if_vlan.h> 121 #include <linux/ip.h> 122 #include <net/ip.h> 123 #include <net/mpls.h> 124 #include <linux/ipv6.h> 125 #include <linux/in.h> 126 #include <linux/jhash.h> 127 #include <linux/random.h> 128 #include <trace/events/napi.h> 129 #include <trace/events/net.h> 130 #include <trace/events/skb.h> 131 #include <linux/pci.h> 132 #include <linux/inetdevice.h> 133 #include <linux/cpu_rmap.h> 134 #include <linux/static_key.h> 135 #include <linux/hashtable.h> 136 #include <linux/vmalloc.h> 137 #include <linux/if_macvlan.h> 138 #include <linux/errqueue.h> 139 #include <linux/hrtimer.h> 140 #include <linux/netfilter_ingress.h> 141 #include <linux/sctp.h> 142 143 #include "net-sysfs.h" 144 145 /* Instead of increasing this, you should create a hash table. */ 146 #define MAX_GRO_SKBS 8 147 148 /* This should be increased if a protocol with a bigger head is added. */ 149 #define GRO_MAX_HEAD (MAX_HEADER + 128) 150 151 static DEFINE_SPINLOCK(ptype_lock); 152 static DEFINE_SPINLOCK(offload_lock); 153 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; 154 struct list_head ptype_all __read_mostly; /* Taps */ 155 static struct list_head offload_base __read_mostly; 156 157 static int netif_rx_internal(struct sk_buff *skb); 158 static int call_netdevice_notifiers_info(unsigned long val, 159 struct net_device *dev, 160 struct netdev_notifier_info *info); 161 162 /* 163 * The @dev_base_head list is protected by @dev_base_lock and the rtnl 164 * semaphore. 165 * 166 * Pure readers hold dev_base_lock for reading, or rcu_read_lock() 167 * 168 * Writers must hold the rtnl semaphore while they loop through the 169 * dev_base_head list, and hold dev_base_lock for writing when they do the 170 * actual updates. This allows pure readers to access the list even 171 * while a writer is preparing to update it. 172 * 173 * To put it another way, dev_base_lock is held for writing only to 174 * protect against pure readers; the rtnl semaphore provides the 175 * protection against other writers. 176 * 177 * See, for example usages, register_netdevice() and 178 * unregister_netdevice(), which must be called with the rtnl 179 * semaphore held. 180 */ 181 DEFINE_RWLOCK(dev_base_lock); 182 EXPORT_SYMBOL(dev_base_lock); 183 184 /* protects napi_hash addition/deletion and napi_gen_id */ 185 static DEFINE_SPINLOCK(napi_hash_lock); 186 187 static unsigned int napi_gen_id = NR_CPUS; 188 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8); 189 190 static seqcount_t devnet_rename_seq; 191 192 static inline void dev_base_seq_inc(struct net *net) 193 { 194 while (++net->dev_base_seq == 0); 195 } 196 197 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name) 198 { 199 unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ)); 200 201 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; 202 } 203 204 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) 205 { 206 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; 207 } 208 209 static inline void rps_lock(struct softnet_data *sd) 210 { 211 #ifdef CONFIG_RPS 212 spin_lock(&sd->input_pkt_queue.lock); 213 #endif 214 } 215 216 static inline void rps_unlock(struct softnet_data *sd) 217 { 218 #ifdef CONFIG_RPS 219 spin_unlock(&sd->input_pkt_queue.lock); 220 #endif 221 } 222 223 /* Device list insertion */ 224 static void list_netdevice(struct net_device *dev) 225 { 226 struct net *net = dev_net(dev); 227 228 ASSERT_RTNL(); 229 230 write_lock_bh(&dev_base_lock); 231 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); 232 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); 233 hlist_add_head_rcu(&dev->index_hlist, 234 dev_index_hash(net, dev->ifindex)); 235 write_unlock_bh(&dev_base_lock); 236 237 dev_base_seq_inc(net); 238 } 239 240 /* Device list removal 241 * caller must respect a RCU grace period before freeing/reusing dev 242 */ 243 static void unlist_netdevice(struct net_device *dev) 244 { 245 ASSERT_RTNL(); 246 247 /* Unlink dev from the device chain */ 248 write_lock_bh(&dev_base_lock); 249 list_del_rcu(&dev->dev_list); 250 hlist_del_rcu(&dev->name_hlist); 251 hlist_del_rcu(&dev->index_hlist); 252 write_unlock_bh(&dev_base_lock); 253 254 dev_base_seq_inc(dev_net(dev)); 255 } 256 257 /* 258 * Our notifier list 259 */ 260 261 static RAW_NOTIFIER_HEAD(netdev_chain); 262 263 /* 264 * Device drivers call our routines to queue packets here. We empty the 265 * queue in the local softnet handler. 266 */ 267 268 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data); 269 EXPORT_PER_CPU_SYMBOL(softnet_data); 270 271 #ifdef CONFIG_LOCKDEP 272 /* 273 * register_netdevice() inits txq->_xmit_lock and sets lockdep class 274 * according to dev->type 275 */ 276 static const unsigned short netdev_lock_type[] = 277 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25, 278 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET, 279 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM, 280 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, 281 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, 282 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, 283 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, 284 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, 285 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, 286 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, 287 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, 288 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, 289 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM, 290 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE, 291 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE}; 292 293 static const char *const netdev_lock_name[] = 294 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", 295 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET", 296 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM", 297 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", 298 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", 299 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", 300 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", 301 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", 302 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", 303 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", 304 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", 305 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", 306 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM", 307 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE", 308 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"}; 309 310 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; 311 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; 312 313 static inline unsigned short netdev_lock_pos(unsigned short dev_type) 314 { 315 int i; 316 317 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++) 318 if (netdev_lock_type[i] == dev_type) 319 return i; 320 /* the last key is used by default */ 321 return ARRAY_SIZE(netdev_lock_type) - 1; 322 } 323 324 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 325 unsigned short dev_type) 326 { 327 int i; 328 329 i = netdev_lock_pos(dev_type); 330 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i], 331 netdev_lock_name[i]); 332 } 333 334 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 335 { 336 int i; 337 338 i = netdev_lock_pos(dev->type); 339 lockdep_set_class_and_name(&dev->addr_list_lock, 340 &netdev_addr_lock_key[i], 341 netdev_lock_name[i]); 342 } 343 #else 344 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 345 unsigned short dev_type) 346 { 347 } 348 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 349 { 350 } 351 #endif 352 353 /******************************************************************************* 354 355 Protocol management and registration routines 356 357 *******************************************************************************/ 358 359 /* 360 * Add a protocol ID to the list. Now that the input handler is 361 * smarter we can dispense with all the messy stuff that used to be 362 * here. 363 * 364 * BEWARE!!! Protocol handlers, mangling input packets, 365 * MUST BE last in hash buckets and checking protocol handlers 366 * MUST start from promiscuous ptype_all chain in net_bh. 367 * It is true now, do not change it. 368 * Explanation follows: if protocol handler, mangling packet, will 369 * be the first on list, it is not able to sense, that packet 370 * is cloned and should be copied-on-write, so that it will 371 * change it and subsequent readers will get broken packet. 372 * --ANK (980803) 373 */ 374 375 static inline struct list_head *ptype_head(const struct packet_type *pt) 376 { 377 if (pt->type == htons(ETH_P_ALL)) 378 return pt->dev ? &pt->dev->ptype_all : &ptype_all; 379 else 380 return pt->dev ? &pt->dev->ptype_specific : 381 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; 382 } 383 384 /** 385 * dev_add_pack - add packet handler 386 * @pt: packet type declaration 387 * 388 * Add a protocol handler to the networking stack. The passed &packet_type 389 * is linked into kernel lists and may not be freed until it has been 390 * removed from the kernel lists. 391 * 392 * This call does not sleep therefore it can not 393 * guarantee all CPU's that are in middle of receiving packets 394 * will see the new packet type (until the next received packet). 395 */ 396 397 void dev_add_pack(struct packet_type *pt) 398 { 399 struct list_head *head = ptype_head(pt); 400 401 spin_lock(&ptype_lock); 402 list_add_rcu(&pt->list, head); 403 spin_unlock(&ptype_lock); 404 } 405 EXPORT_SYMBOL(dev_add_pack); 406 407 /** 408 * __dev_remove_pack - remove packet handler 409 * @pt: packet type declaration 410 * 411 * Remove a protocol handler that was previously added to the kernel 412 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 413 * from the kernel lists and can be freed or reused once this function 414 * returns. 415 * 416 * The packet type might still be in use by receivers 417 * and must not be freed until after all the CPU's have gone 418 * through a quiescent state. 419 */ 420 void __dev_remove_pack(struct packet_type *pt) 421 { 422 struct list_head *head = ptype_head(pt); 423 struct packet_type *pt1; 424 425 spin_lock(&ptype_lock); 426 427 list_for_each_entry(pt1, head, list) { 428 if (pt == pt1) { 429 list_del_rcu(&pt->list); 430 goto out; 431 } 432 } 433 434 pr_warn("dev_remove_pack: %p not found\n", pt); 435 out: 436 spin_unlock(&ptype_lock); 437 } 438 EXPORT_SYMBOL(__dev_remove_pack); 439 440 /** 441 * dev_remove_pack - remove packet handler 442 * @pt: packet type declaration 443 * 444 * Remove a protocol handler that was previously added to the kernel 445 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 446 * from the kernel lists and can be freed or reused once this function 447 * returns. 448 * 449 * This call sleeps to guarantee that no CPU is looking at the packet 450 * type after return. 451 */ 452 void dev_remove_pack(struct packet_type *pt) 453 { 454 __dev_remove_pack(pt); 455 456 synchronize_net(); 457 } 458 EXPORT_SYMBOL(dev_remove_pack); 459 460 461 /** 462 * dev_add_offload - register offload handlers 463 * @po: protocol offload declaration 464 * 465 * Add protocol offload handlers to the networking stack. The passed 466 * &proto_offload is linked into kernel lists and may not be freed until 467 * it has been removed from the kernel lists. 468 * 469 * This call does not sleep therefore it can not 470 * guarantee all CPU's that are in middle of receiving packets 471 * will see the new offload handlers (until the next received packet). 472 */ 473 void dev_add_offload(struct packet_offload *po) 474 { 475 struct packet_offload *elem; 476 477 spin_lock(&offload_lock); 478 list_for_each_entry(elem, &offload_base, list) { 479 if (po->priority < elem->priority) 480 break; 481 } 482 list_add_rcu(&po->list, elem->list.prev); 483 spin_unlock(&offload_lock); 484 } 485 EXPORT_SYMBOL(dev_add_offload); 486 487 /** 488 * __dev_remove_offload - remove offload handler 489 * @po: packet offload declaration 490 * 491 * Remove a protocol offload handler that was previously added to the 492 * kernel offload handlers by dev_add_offload(). The passed &offload_type 493 * is removed from the kernel lists and can be freed or reused once this 494 * function returns. 495 * 496 * The packet type might still be in use by receivers 497 * and must not be freed until after all the CPU's have gone 498 * through a quiescent state. 499 */ 500 static void __dev_remove_offload(struct packet_offload *po) 501 { 502 struct list_head *head = &offload_base; 503 struct packet_offload *po1; 504 505 spin_lock(&offload_lock); 506 507 list_for_each_entry(po1, head, list) { 508 if (po == po1) { 509 list_del_rcu(&po->list); 510 goto out; 511 } 512 } 513 514 pr_warn("dev_remove_offload: %p not found\n", po); 515 out: 516 spin_unlock(&offload_lock); 517 } 518 519 /** 520 * dev_remove_offload - remove packet offload handler 521 * @po: packet offload declaration 522 * 523 * Remove a packet offload handler that was previously added to the kernel 524 * offload handlers by dev_add_offload(). The passed &offload_type is 525 * removed from the kernel lists and can be freed or reused once this 526 * function returns. 527 * 528 * This call sleeps to guarantee that no CPU is looking at the packet 529 * type after return. 530 */ 531 void dev_remove_offload(struct packet_offload *po) 532 { 533 __dev_remove_offload(po); 534 535 synchronize_net(); 536 } 537 EXPORT_SYMBOL(dev_remove_offload); 538 539 /****************************************************************************** 540 541 Device Boot-time Settings Routines 542 543 *******************************************************************************/ 544 545 /* Boot time configuration table */ 546 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX]; 547 548 /** 549 * netdev_boot_setup_add - add new setup entry 550 * @name: name of the device 551 * @map: configured settings for the device 552 * 553 * Adds new setup entry to the dev_boot_setup list. The function 554 * returns 0 on error and 1 on success. This is a generic routine to 555 * all netdevices. 556 */ 557 static int netdev_boot_setup_add(char *name, struct ifmap *map) 558 { 559 struct netdev_boot_setup *s; 560 int i; 561 562 s = dev_boot_setup; 563 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 564 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') { 565 memset(s[i].name, 0, sizeof(s[i].name)); 566 strlcpy(s[i].name, name, IFNAMSIZ); 567 memcpy(&s[i].map, map, sizeof(s[i].map)); 568 break; 569 } 570 } 571 572 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1; 573 } 574 575 /** 576 * netdev_boot_setup_check - check boot time settings 577 * @dev: the netdevice 578 * 579 * Check boot time settings for the device. 580 * The found settings are set for the device to be used 581 * later in the device probing. 582 * Returns 0 if no settings found, 1 if they are. 583 */ 584 int netdev_boot_setup_check(struct net_device *dev) 585 { 586 struct netdev_boot_setup *s = dev_boot_setup; 587 int i; 588 589 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 590 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' && 591 !strcmp(dev->name, s[i].name)) { 592 dev->irq = s[i].map.irq; 593 dev->base_addr = s[i].map.base_addr; 594 dev->mem_start = s[i].map.mem_start; 595 dev->mem_end = s[i].map.mem_end; 596 return 1; 597 } 598 } 599 return 0; 600 } 601 EXPORT_SYMBOL(netdev_boot_setup_check); 602 603 604 /** 605 * netdev_boot_base - get address from boot time settings 606 * @prefix: prefix for network device 607 * @unit: id for network device 608 * 609 * Check boot time settings for the base address of device. 610 * The found settings are set for the device to be used 611 * later in the device probing. 612 * Returns 0 if no settings found. 613 */ 614 unsigned long netdev_boot_base(const char *prefix, int unit) 615 { 616 const struct netdev_boot_setup *s = dev_boot_setup; 617 char name[IFNAMSIZ]; 618 int i; 619 620 sprintf(name, "%s%d", prefix, unit); 621 622 /* 623 * If device already registered then return base of 1 624 * to indicate not to probe for this interface 625 */ 626 if (__dev_get_by_name(&init_net, name)) 627 return 1; 628 629 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) 630 if (!strcmp(name, s[i].name)) 631 return s[i].map.base_addr; 632 return 0; 633 } 634 635 /* 636 * Saves at boot time configured settings for any netdevice. 637 */ 638 int __init netdev_boot_setup(char *str) 639 { 640 int ints[5]; 641 struct ifmap map; 642 643 str = get_options(str, ARRAY_SIZE(ints), ints); 644 if (!str || !*str) 645 return 0; 646 647 /* Save settings */ 648 memset(&map, 0, sizeof(map)); 649 if (ints[0] > 0) 650 map.irq = ints[1]; 651 if (ints[0] > 1) 652 map.base_addr = ints[2]; 653 if (ints[0] > 2) 654 map.mem_start = ints[3]; 655 if (ints[0] > 3) 656 map.mem_end = ints[4]; 657 658 /* Add new entry to the list */ 659 return netdev_boot_setup_add(str, &map); 660 } 661 662 __setup("netdev=", netdev_boot_setup); 663 664 /******************************************************************************* 665 666 Device Interface Subroutines 667 668 *******************************************************************************/ 669 670 /** 671 * dev_get_iflink - get 'iflink' value of a interface 672 * @dev: targeted interface 673 * 674 * Indicates the ifindex the interface is linked to. 675 * Physical interfaces have the same 'ifindex' and 'iflink' values. 676 */ 677 678 int dev_get_iflink(const struct net_device *dev) 679 { 680 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) 681 return dev->netdev_ops->ndo_get_iflink(dev); 682 683 return dev->ifindex; 684 } 685 EXPORT_SYMBOL(dev_get_iflink); 686 687 /** 688 * dev_fill_metadata_dst - Retrieve tunnel egress information. 689 * @dev: targeted interface 690 * @skb: The packet. 691 * 692 * For better visibility of tunnel traffic OVS needs to retrieve 693 * egress tunnel information for a packet. Following API allows 694 * user to get this info. 695 */ 696 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb) 697 { 698 struct ip_tunnel_info *info; 699 700 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst) 701 return -EINVAL; 702 703 info = skb_tunnel_info_unclone(skb); 704 if (!info) 705 return -ENOMEM; 706 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX))) 707 return -EINVAL; 708 709 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb); 710 } 711 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst); 712 713 /** 714 * __dev_get_by_name - find a device by its name 715 * @net: the applicable net namespace 716 * @name: name to find 717 * 718 * Find an interface by name. Must be called under RTNL semaphore 719 * or @dev_base_lock. If the name is found a pointer to the device 720 * is returned. If the name is not found then %NULL is returned. The 721 * reference counters are not incremented so the caller must be 722 * careful with locks. 723 */ 724 725 struct net_device *__dev_get_by_name(struct net *net, const char *name) 726 { 727 struct net_device *dev; 728 struct hlist_head *head = dev_name_hash(net, name); 729 730 hlist_for_each_entry(dev, head, name_hlist) 731 if (!strncmp(dev->name, name, IFNAMSIZ)) 732 return dev; 733 734 return NULL; 735 } 736 EXPORT_SYMBOL(__dev_get_by_name); 737 738 /** 739 * dev_get_by_name_rcu - find a device by its name 740 * @net: the applicable net namespace 741 * @name: name to find 742 * 743 * Find an interface by name. 744 * If the name is found a pointer to the device is returned. 745 * If the name is not found then %NULL is returned. 746 * The reference counters are not incremented so the caller must be 747 * careful with locks. The caller must hold RCU lock. 748 */ 749 750 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name) 751 { 752 struct net_device *dev; 753 struct hlist_head *head = dev_name_hash(net, name); 754 755 hlist_for_each_entry_rcu(dev, head, name_hlist) 756 if (!strncmp(dev->name, name, IFNAMSIZ)) 757 return dev; 758 759 return NULL; 760 } 761 EXPORT_SYMBOL(dev_get_by_name_rcu); 762 763 /** 764 * dev_get_by_name - find a device by its name 765 * @net: the applicable net namespace 766 * @name: name to find 767 * 768 * Find an interface by name. This can be called from any 769 * context and does its own locking. The returned handle has 770 * the usage count incremented and the caller must use dev_put() to 771 * release it when it is no longer needed. %NULL is returned if no 772 * matching device is found. 773 */ 774 775 struct net_device *dev_get_by_name(struct net *net, const char *name) 776 { 777 struct net_device *dev; 778 779 rcu_read_lock(); 780 dev = dev_get_by_name_rcu(net, name); 781 if (dev) 782 dev_hold(dev); 783 rcu_read_unlock(); 784 return dev; 785 } 786 EXPORT_SYMBOL(dev_get_by_name); 787 788 /** 789 * __dev_get_by_index - find a device by its ifindex 790 * @net: the applicable net namespace 791 * @ifindex: index of device 792 * 793 * Search for an interface by index. Returns %NULL if the device 794 * is not found or a pointer to the device. The device has not 795 * had its reference counter increased so the caller must be careful 796 * about locking. The caller must hold either the RTNL semaphore 797 * or @dev_base_lock. 798 */ 799 800 struct net_device *__dev_get_by_index(struct net *net, int ifindex) 801 { 802 struct net_device *dev; 803 struct hlist_head *head = dev_index_hash(net, ifindex); 804 805 hlist_for_each_entry(dev, head, index_hlist) 806 if (dev->ifindex == ifindex) 807 return dev; 808 809 return NULL; 810 } 811 EXPORT_SYMBOL(__dev_get_by_index); 812 813 /** 814 * dev_get_by_index_rcu - find a device by its ifindex 815 * @net: the applicable net namespace 816 * @ifindex: index of device 817 * 818 * Search for an interface by index. Returns %NULL if the device 819 * is not found or a pointer to the device. The device has not 820 * had its reference counter increased so the caller must be careful 821 * about locking. The caller must hold RCU lock. 822 */ 823 824 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex) 825 { 826 struct net_device *dev; 827 struct hlist_head *head = dev_index_hash(net, ifindex); 828 829 hlist_for_each_entry_rcu(dev, head, index_hlist) 830 if (dev->ifindex == ifindex) 831 return dev; 832 833 return NULL; 834 } 835 EXPORT_SYMBOL(dev_get_by_index_rcu); 836 837 838 /** 839 * dev_get_by_index - find a device by its ifindex 840 * @net: the applicable net namespace 841 * @ifindex: index of device 842 * 843 * Search for an interface by index. Returns NULL if the device 844 * is not found or a pointer to the device. The device returned has 845 * had a reference added and the pointer is safe until the user calls 846 * dev_put to indicate they have finished with it. 847 */ 848 849 struct net_device *dev_get_by_index(struct net *net, int ifindex) 850 { 851 struct net_device *dev; 852 853 rcu_read_lock(); 854 dev = dev_get_by_index_rcu(net, ifindex); 855 if (dev) 856 dev_hold(dev); 857 rcu_read_unlock(); 858 return dev; 859 } 860 EXPORT_SYMBOL(dev_get_by_index); 861 862 /** 863 * netdev_get_name - get a netdevice name, knowing its ifindex. 864 * @net: network namespace 865 * @name: a pointer to the buffer where the name will be stored. 866 * @ifindex: the ifindex of the interface to get the name from. 867 * 868 * The use of raw_seqcount_begin() and cond_resched() before 869 * retrying is required as we want to give the writers a chance 870 * to complete when CONFIG_PREEMPT is not set. 871 */ 872 int netdev_get_name(struct net *net, char *name, int ifindex) 873 { 874 struct net_device *dev; 875 unsigned int seq; 876 877 retry: 878 seq = raw_seqcount_begin(&devnet_rename_seq); 879 rcu_read_lock(); 880 dev = dev_get_by_index_rcu(net, ifindex); 881 if (!dev) { 882 rcu_read_unlock(); 883 return -ENODEV; 884 } 885 886 strcpy(name, dev->name); 887 rcu_read_unlock(); 888 if (read_seqcount_retry(&devnet_rename_seq, seq)) { 889 cond_resched(); 890 goto retry; 891 } 892 893 return 0; 894 } 895 896 /** 897 * dev_getbyhwaddr_rcu - find a device by its hardware address 898 * @net: the applicable net namespace 899 * @type: media type of device 900 * @ha: hardware address 901 * 902 * Search for an interface by MAC address. Returns NULL if the device 903 * is not found or a pointer to the device. 904 * The caller must hold RCU or RTNL. 905 * The returned device has not had its ref count increased 906 * and the caller must therefore be careful about locking 907 * 908 */ 909 910 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type, 911 const char *ha) 912 { 913 struct net_device *dev; 914 915 for_each_netdev_rcu(net, dev) 916 if (dev->type == type && 917 !memcmp(dev->dev_addr, ha, dev->addr_len)) 918 return dev; 919 920 return NULL; 921 } 922 EXPORT_SYMBOL(dev_getbyhwaddr_rcu); 923 924 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type) 925 { 926 struct net_device *dev; 927 928 ASSERT_RTNL(); 929 for_each_netdev(net, dev) 930 if (dev->type == type) 931 return dev; 932 933 return NULL; 934 } 935 EXPORT_SYMBOL(__dev_getfirstbyhwtype); 936 937 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) 938 { 939 struct net_device *dev, *ret = NULL; 940 941 rcu_read_lock(); 942 for_each_netdev_rcu(net, dev) 943 if (dev->type == type) { 944 dev_hold(dev); 945 ret = dev; 946 break; 947 } 948 rcu_read_unlock(); 949 return ret; 950 } 951 EXPORT_SYMBOL(dev_getfirstbyhwtype); 952 953 /** 954 * __dev_get_by_flags - find any device with given flags 955 * @net: the applicable net namespace 956 * @if_flags: IFF_* values 957 * @mask: bitmask of bits in if_flags to check 958 * 959 * Search for any interface with the given flags. Returns NULL if a device 960 * is not found or a pointer to the device. Must be called inside 961 * rtnl_lock(), and result refcount is unchanged. 962 */ 963 964 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags, 965 unsigned short mask) 966 { 967 struct net_device *dev, *ret; 968 969 ASSERT_RTNL(); 970 971 ret = NULL; 972 for_each_netdev(net, dev) { 973 if (((dev->flags ^ if_flags) & mask) == 0) { 974 ret = dev; 975 break; 976 } 977 } 978 return ret; 979 } 980 EXPORT_SYMBOL(__dev_get_by_flags); 981 982 /** 983 * dev_valid_name - check if name is okay for network device 984 * @name: name string 985 * 986 * Network device names need to be valid file names to 987 * to allow sysfs to work. We also disallow any kind of 988 * whitespace. 989 */ 990 bool dev_valid_name(const char *name) 991 { 992 if (*name == '\0') 993 return false; 994 if (strlen(name) >= IFNAMSIZ) 995 return false; 996 if (!strcmp(name, ".") || !strcmp(name, "..")) 997 return false; 998 999 while (*name) { 1000 if (*name == '/' || *name == ':' || isspace(*name)) 1001 return false; 1002 name++; 1003 } 1004 return true; 1005 } 1006 EXPORT_SYMBOL(dev_valid_name); 1007 1008 /** 1009 * __dev_alloc_name - allocate a name for a device 1010 * @net: network namespace to allocate the device name in 1011 * @name: name format string 1012 * @buf: scratch buffer and result name string 1013 * 1014 * Passed a format string - eg "lt%d" it will try and find a suitable 1015 * id. It scans list of devices to build up a free map, then chooses 1016 * the first empty slot. The caller must hold the dev_base or rtnl lock 1017 * while allocating the name and adding the device in order to avoid 1018 * duplicates. 1019 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1020 * Returns the number of the unit assigned or a negative errno code. 1021 */ 1022 1023 static int __dev_alloc_name(struct net *net, const char *name, char *buf) 1024 { 1025 int i = 0; 1026 const char *p; 1027 const int max_netdevices = 8*PAGE_SIZE; 1028 unsigned long *inuse; 1029 struct net_device *d; 1030 1031 p = strnchr(name, IFNAMSIZ-1, '%'); 1032 if (p) { 1033 /* 1034 * Verify the string as this thing may have come from 1035 * the user. There must be either one "%d" and no other "%" 1036 * characters. 1037 */ 1038 if (p[1] != 'd' || strchr(p + 2, '%')) 1039 return -EINVAL; 1040 1041 /* Use one page as a bit array of possible slots */ 1042 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC); 1043 if (!inuse) 1044 return -ENOMEM; 1045 1046 for_each_netdev(net, d) { 1047 if (!sscanf(d->name, name, &i)) 1048 continue; 1049 if (i < 0 || i >= max_netdevices) 1050 continue; 1051 1052 /* avoid cases where sscanf is not exact inverse of printf */ 1053 snprintf(buf, IFNAMSIZ, name, i); 1054 if (!strncmp(buf, d->name, IFNAMSIZ)) 1055 set_bit(i, inuse); 1056 } 1057 1058 i = find_first_zero_bit(inuse, max_netdevices); 1059 free_page((unsigned long) inuse); 1060 } 1061 1062 if (buf != name) 1063 snprintf(buf, IFNAMSIZ, name, i); 1064 if (!__dev_get_by_name(net, buf)) 1065 return i; 1066 1067 /* It is possible to run out of possible slots 1068 * when the name is long and there isn't enough space left 1069 * for the digits, or if all bits are used. 1070 */ 1071 return -ENFILE; 1072 } 1073 1074 /** 1075 * dev_alloc_name - allocate a name for a device 1076 * @dev: device 1077 * @name: name format string 1078 * 1079 * Passed a format string - eg "lt%d" it will try and find a suitable 1080 * id. It scans list of devices to build up a free map, then chooses 1081 * the first empty slot. The caller must hold the dev_base or rtnl lock 1082 * while allocating the name and adding the device in order to avoid 1083 * duplicates. 1084 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1085 * Returns the number of the unit assigned or a negative errno code. 1086 */ 1087 1088 int dev_alloc_name(struct net_device *dev, const char *name) 1089 { 1090 char buf[IFNAMSIZ]; 1091 struct net *net; 1092 int ret; 1093 1094 BUG_ON(!dev_net(dev)); 1095 net = dev_net(dev); 1096 ret = __dev_alloc_name(net, name, buf); 1097 if (ret >= 0) 1098 strlcpy(dev->name, buf, IFNAMSIZ); 1099 return ret; 1100 } 1101 EXPORT_SYMBOL(dev_alloc_name); 1102 1103 static int dev_alloc_name_ns(struct net *net, 1104 struct net_device *dev, 1105 const char *name) 1106 { 1107 char buf[IFNAMSIZ]; 1108 int ret; 1109 1110 ret = __dev_alloc_name(net, name, buf); 1111 if (ret >= 0) 1112 strlcpy(dev->name, buf, IFNAMSIZ); 1113 return ret; 1114 } 1115 1116 static int dev_get_valid_name(struct net *net, 1117 struct net_device *dev, 1118 const char *name) 1119 { 1120 BUG_ON(!net); 1121 1122 if (!dev_valid_name(name)) 1123 return -EINVAL; 1124 1125 if (strchr(name, '%')) 1126 return dev_alloc_name_ns(net, dev, name); 1127 else if (__dev_get_by_name(net, name)) 1128 return -EEXIST; 1129 else if (dev->name != name) 1130 strlcpy(dev->name, name, IFNAMSIZ); 1131 1132 return 0; 1133 } 1134 1135 /** 1136 * dev_change_name - change name of a device 1137 * @dev: device 1138 * @newname: name (or format string) must be at least IFNAMSIZ 1139 * 1140 * Change name of a device, can pass format strings "eth%d". 1141 * for wildcarding. 1142 */ 1143 int dev_change_name(struct net_device *dev, const char *newname) 1144 { 1145 unsigned char old_assign_type; 1146 char oldname[IFNAMSIZ]; 1147 int err = 0; 1148 int ret; 1149 struct net *net; 1150 1151 ASSERT_RTNL(); 1152 BUG_ON(!dev_net(dev)); 1153 1154 net = dev_net(dev); 1155 if (dev->flags & IFF_UP) 1156 return -EBUSY; 1157 1158 write_seqcount_begin(&devnet_rename_seq); 1159 1160 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) { 1161 write_seqcount_end(&devnet_rename_seq); 1162 return 0; 1163 } 1164 1165 memcpy(oldname, dev->name, IFNAMSIZ); 1166 1167 err = dev_get_valid_name(net, dev, newname); 1168 if (err < 0) { 1169 write_seqcount_end(&devnet_rename_seq); 1170 return err; 1171 } 1172 1173 if (oldname[0] && !strchr(oldname, '%')) 1174 netdev_info(dev, "renamed from %s\n", oldname); 1175 1176 old_assign_type = dev->name_assign_type; 1177 dev->name_assign_type = NET_NAME_RENAMED; 1178 1179 rollback: 1180 ret = device_rename(&dev->dev, dev->name); 1181 if (ret) { 1182 memcpy(dev->name, oldname, IFNAMSIZ); 1183 dev->name_assign_type = old_assign_type; 1184 write_seqcount_end(&devnet_rename_seq); 1185 return ret; 1186 } 1187 1188 write_seqcount_end(&devnet_rename_seq); 1189 1190 netdev_adjacent_rename_links(dev, oldname); 1191 1192 write_lock_bh(&dev_base_lock); 1193 hlist_del_rcu(&dev->name_hlist); 1194 write_unlock_bh(&dev_base_lock); 1195 1196 synchronize_rcu(); 1197 1198 write_lock_bh(&dev_base_lock); 1199 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); 1200 write_unlock_bh(&dev_base_lock); 1201 1202 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); 1203 ret = notifier_to_errno(ret); 1204 1205 if (ret) { 1206 /* err >= 0 after dev_alloc_name() or stores the first errno */ 1207 if (err >= 0) { 1208 err = ret; 1209 write_seqcount_begin(&devnet_rename_seq); 1210 memcpy(dev->name, oldname, IFNAMSIZ); 1211 memcpy(oldname, newname, IFNAMSIZ); 1212 dev->name_assign_type = old_assign_type; 1213 old_assign_type = NET_NAME_RENAMED; 1214 goto rollback; 1215 } else { 1216 pr_err("%s: name change rollback failed: %d\n", 1217 dev->name, ret); 1218 } 1219 } 1220 1221 return err; 1222 } 1223 1224 /** 1225 * dev_set_alias - change ifalias of a device 1226 * @dev: device 1227 * @alias: name up to IFALIASZ 1228 * @len: limit of bytes to copy from info 1229 * 1230 * Set ifalias for a device, 1231 */ 1232 int dev_set_alias(struct net_device *dev, const char *alias, size_t len) 1233 { 1234 char *new_ifalias; 1235 1236 ASSERT_RTNL(); 1237 1238 if (len >= IFALIASZ) 1239 return -EINVAL; 1240 1241 if (!len) { 1242 kfree(dev->ifalias); 1243 dev->ifalias = NULL; 1244 return 0; 1245 } 1246 1247 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL); 1248 if (!new_ifalias) 1249 return -ENOMEM; 1250 dev->ifalias = new_ifalias; 1251 1252 strlcpy(dev->ifalias, alias, len+1); 1253 return len; 1254 } 1255 1256 1257 /** 1258 * netdev_features_change - device changes features 1259 * @dev: device to cause notification 1260 * 1261 * Called to indicate a device has changed features. 1262 */ 1263 void netdev_features_change(struct net_device *dev) 1264 { 1265 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); 1266 } 1267 EXPORT_SYMBOL(netdev_features_change); 1268 1269 /** 1270 * netdev_state_change - device changes state 1271 * @dev: device to cause notification 1272 * 1273 * Called to indicate a device has changed state. This function calls 1274 * the notifier chains for netdev_chain and sends a NEWLINK message 1275 * to the routing socket. 1276 */ 1277 void netdev_state_change(struct net_device *dev) 1278 { 1279 if (dev->flags & IFF_UP) { 1280 struct netdev_notifier_change_info change_info; 1281 1282 change_info.flags_changed = 0; 1283 call_netdevice_notifiers_info(NETDEV_CHANGE, dev, 1284 &change_info.info); 1285 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL); 1286 } 1287 } 1288 EXPORT_SYMBOL(netdev_state_change); 1289 1290 /** 1291 * netdev_notify_peers - notify network peers about existence of @dev 1292 * @dev: network device 1293 * 1294 * Generate traffic such that interested network peers are aware of 1295 * @dev, such as by generating a gratuitous ARP. This may be used when 1296 * a device wants to inform the rest of the network about some sort of 1297 * reconfiguration such as a failover event or virtual machine 1298 * migration. 1299 */ 1300 void netdev_notify_peers(struct net_device *dev) 1301 { 1302 rtnl_lock(); 1303 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev); 1304 rtnl_unlock(); 1305 } 1306 EXPORT_SYMBOL(netdev_notify_peers); 1307 1308 static int __dev_open(struct net_device *dev) 1309 { 1310 const struct net_device_ops *ops = dev->netdev_ops; 1311 int ret; 1312 1313 ASSERT_RTNL(); 1314 1315 if (!netif_device_present(dev)) 1316 return -ENODEV; 1317 1318 /* Block netpoll from trying to do any rx path servicing. 1319 * If we don't do this there is a chance ndo_poll_controller 1320 * or ndo_poll may be running while we open the device 1321 */ 1322 netpoll_poll_disable(dev); 1323 1324 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev); 1325 ret = notifier_to_errno(ret); 1326 if (ret) 1327 return ret; 1328 1329 set_bit(__LINK_STATE_START, &dev->state); 1330 1331 if (ops->ndo_validate_addr) 1332 ret = ops->ndo_validate_addr(dev); 1333 1334 if (!ret && ops->ndo_open) 1335 ret = ops->ndo_open(dev); 1336 1337 netpoll_poll_enable(dev); 1338 1339 if (ret) 1340 clear_bit(__LINK_STATE_START, &dev->state); 1341 else { 1342 dev->flags |= IFF_UP; 1343 dev_set_rx_mode(dev); 1344 dev_activate(dev); 1345 add_device_randomness(dev->dev_addr, dev->addr_len); 1346 } 1347 1348 return ret; 1349 } 1350 1351 /** 1352 * dev_open - prepare an interface for use. 1353 * @dev: device to open 1354 * 1355 * Takes a device from down to up state. The device's private open 1356 * function is invoked and then the multicast lists are loaded. Finally 1357 * the device is moved into the up state and a %NETDEV_UP message is 1358 * sent to the netdev notifier chain. 1359 * 1360 * Calling this function on an active interface is a nop. On a failure 1361 * a negative errno code is returned. 1362 */ 1363 int dev_open(struct net_device *dev) 1364 { 1365 int ret; 1366 1367 if (dev->flags & IFF_UP) 1368 return 0; 1369 1370 ret = __dev_open(dev); 1371 if (ret < 0) 1372 return ret; 1373 1374 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); 1375 call_netdevice_notifiers(NETDEV_UP, dev); 1376 1377 return ret; 1378 } 1379 EXPORT_SYMBOL(dev_open); 1380 1381 static int __dev_close_many(struct list_head *head) 1382 { 1383 struct net_device *dev; 1384 1385 ASSERT_RTNL(); 1386 might_sleep(); 1387 1388 list_for_each_entry(dev, head, close_list) { 1389 /* Temporarily disable netpoll until the interface is down */ 1390 netpoll_poll_disable(dev); 1391 1392 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); 1393 1394 clear_bit(__LINK_STATE_START, &dev->state); 1395 1396 /* Synchronize to scheduled poll. We cannot touch poll list, it 1397 * can be even on different cpu. So just clear netif_running(). 1398 * 1399 * dev->stop() will invoke napi_disable() on all of it's 1400 * napi_struct instances on this device. 1401 */ 1402 smp_mb__after_atomic(); /* Commit netif_running(). */ 1403 } 1404 1405 dev_deactivate_many(head); 1406 1407 list_for_each_entry(dev, head, close_list) { 1408 const struct net_device_ops *ops = dev->netdev_ops; 1409 1410 /* 1411 * Call the device specific close. This cannot fail. 1412 * Only if device is UP 1413 * 1414 * We allow it to be called even after a DETACH hot-plug 1415 * event. 1416 */ 1417 if (ops->ndo_stop) 1418 ops->ndo_stop(dev); 1419 1420 dev->flags &= ~IFF_UP; 1421 netpoll_poll_enable(dev); 1422 } 1423 1424 return 0; 1425 } 1426 1427 static int __dev_close(struct net_device *dev) 1428 { 1429 int retval; 1430 LIST_HEAD(single); 1431 1432 list_add(&dev->close_list, &single); 1433 retval = __dev_close_many(&single); 1434 list_del(&single); 1435 1436 return retval; 1437 } 1438 1439 int dev_close_many(struct list_head *head, bool unlink) 1440 { 1441 struct net_device *dev, *tmp; 1442 1443 /* Remove the devices that don't need to be closed */ 1444 list_for_each_entry_safe(dev, tmp, head, close_list) 1445 if (!(dev->flags & IFF_UP)) 1446 list_del_init(&dev->close_list); 1447 1448 __dev_close_many(head); 1449 1450 list_for_each_entry_safe(dev, tmp, head, close_list) { 1451 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); 1452 call_netdevice_notifiers(NETDEV_DOWN, dev); 1453 if (unlink) 1454 list_del_init(&dev->close_list); 1455 } 1456 1457 return 0; 1458 } 1459 EXPORT_SYMBOL(dev_close_many); 1460 1461 /** 1462 * dev_close - shutdown an interface. 1463 * @dev: device to shutdown 1464 * 1465 * This function moves an active device into down state. A 1466 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device 1467 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier 1468 * chain. 1469 */ 1470 int dev_close(struct net_device *dev) 1471 { 1472 if (dev->flags & IFF_UP) { 1473 LIST_HEAD(single); 1474 1475 list_add(&dev->close_list, &single); 1476 dev_close_many(&single, true); 1477 list_del(&single); 1478 } 1479 return 0; 1480 } 1481 EXPORT_SYMBOL(dev_close); 1482 1483 1484 /** 1485 * dev_disable_lro - disable Large Receive Offload on a device 1486 * @dev: device 1487 * 1488 * Disable Large Receive Offload (LRO) on a net device. Must be 1489 * called under RTNL. This is needed if received packets may be 1490 * forwarded to another interface. 1491 */ 1492 void dev_disable_lro(struct net_device *dev) 1493 { 1494 struct net_device *lower_dev; 1495 struct list_head *iter; 1496 1497 dev->wanted_features &= ~NETIF_F_LRO; 1498 netdev_update_features(dev); 1499 1500 if (unlikely(dev->features & NETIF_F_LRO)) 1501 netdev_WARN(dev, "failed to disable LRO!\n"); 1502 1503 netdev_for_each_lower_dev(dev, lower_dev, iter) 1504 dev_disable_lro(lower_dev); 1505 } 1506 EXPORT_SYMBOL(dev_disable_lro); 1507 1508 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val, 1509 struct net_device *dev) 1510 { 1511 struct netdev_notifier_info info; 1512 1513 netdev_notifier_info_init(&info, dev); 1514 return nb->notifier_call(nb, val, &info); 1515 } 1516 1517 static int dev_boot_phase = 1; 1518 1519 /** 1520 * register_netdevice_notifier - register a network notifier block 1521 * @nb: notifier 1522 * 1523 * Register a notifier to be called when network device events occur. 1524 * The notifier passed is linked into the kernel structures and must 1525 * not be reused until it has been unregistered. A negative errno code 1526 * is returned on a failure. 1527 * 1528 * When registered all registration and up events are replayed 1529 * to the new notifier to allow device to have a race free 1530 * view of the network device list. 1531 */ 1532 1533 int register_netdevice_notifier(struct notifier_block *nb) 1534 { 1535 struct net_device *dev; 1536 struct net_device *last; 1537 struct net *net; 1538 int err; 1539 1540 rtnl_lock(); 1541 err = raw_notifier_chain_register(&netdev_chain, nb); 1542 if (err) 1543 goto unlock; 1544 if (dev_boot_phase) 1545 goto unlock; 1546 for_each_net(net) { 1547 for_each_netdev(net, dev) { 1548 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev); 1549 err = notifier_to_errno(err); 1550 if (err) 1551 goto rollback; 1552 1553 if (!(dev->flags & IFF_UP)) 1554 continue; 1555 1556 call_netdevice_notifier(nb, NETDEV_UP, dev); 1557 } 1558 } 1559 1560 unlock: 1561 rtnl_unlock(); 1562 return err; 1563 1564 rollback: 1565 last = dev; 1566 for_each_net(net) { 1567 for_each_netdev(net, dev) { 1568 if (dev == last) 1569 goto outroll; 1570 1571 if (dev->flags & IFF_UP) { 1572 call_netdevice_notifier(nb, NETDEV_GOING_DOWN, 1573 dev); 1574 call_netdevice_notifier(nb, NETDEV_DOWN, dev); 1575 } 1576 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); 1577 } 1578 } 1579 1580 outroll: 1581 raw_notifier_chain_unregister(&netdev_chain, nb); 1582 goto unlock; 1583 } 1584 EXPORT_SYMBOL(register_netdevice_notifier); 1585 1586 /** 1587 * unregister_netdevice_notifier - unregister a network notifier block 1588 * @nb: notifier 1589 * 1590 * Unregister a notifier previously registered by 1591 * register_netdevice_notifier(). The notifier is unlinked into the 1592 * kernel structures and may then be reused. A negative errno code 1593 * is returned on a failure. 1594 * 1595 * After unregistering unregister and down device events are synthesized 1596 * for all devices on the device list to the removed notifier to remove 1597 * the need for special case cleanup code. 1598 */ 1599 1600 int unregister_netdevice_notifier(struct notifier_block *nb) 1601 { 1602 struct net_device *dev; 1603 struct net *net; 1604 int err; 1605 1606 rtnl_lock(); 1607 err = raw_notifier_chain_unregister(&netdev_chain, nb); 1608 if (err) 1609 goto unlock; 1610 1611 for_each_net(net) { 1612 for_each_netdev(net, dev) { 1613 if (dev->flags & IFF_UP) { 1614 call_netdevice_notifier(nb, NETDEV_GOING_DOWN, 1615 dev); 1616 call_netdevice_notifier(nb, NETDEV_DOWN, dev); 1617 } 1618 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); 1619 } 1620 } 1621 unlock: 1622 rtnl_unlock(); 1623 return err; 1624 } 1625 EXPORT_SYMBOL(unregister_netdevice_notifier); 1626 1627 /** 1628 * call_netdevice_notifiers_info - call all network notifier blocks 1629 * @val: value passed unmodified to notifier function 1630 * @dev: net_device pointer passed unmodified to notifier function 1631 * @info: notifier information data 1632 * 1633 * Call all network notifier blocks. Parameters and return value 1634 * are as for raw_notifier_call_chain(). 1635 */ 1636 1637 static int call_netdevice_notifiers_info(unsigned long val, 1638 struct net_device *dev, 1639 struct netdev_notifier_info *info) 1640 { 1641 ASSERT_RTNL(); 1642 netdev_notifier_info_init(info, dev); 1643 return raw_notifier_call_chain(&netdev_chain, val, info); 1644 } 1645 1646 /** 1647 * call_netdevice_notifiers - call all network notifier blocks 1648 * @val: value passed unmodified to notifier function 1649 * @dev: net_device pointer passed unmodified to notifier function 1650 * 1651 * Call all network notifier blocks. Parameters and return value 1652 * are as for raw_notifier_call_chain(). 1653 */ 1654 1655 int call_netdevice_notifiers(unsigned long val, struct net_device *dev) 1656 { 1657 struct netdev_notifier_info info; 1658 1659 return call_netdevice_notifiers_info(val, dev, &info); 1660 } 1661 EXPORT_SYMBOL(call_netdevice_notifiers); 1662 1663 #ifdef CONFIG_NET_INGRESS 1664 static struct static_key ingress_needed __read_mostly; 1665 1666 void net_inc_ingress_queue(void) 1667 { 1668 static_key_slow_inc(&ingress_needed); 1669 } 1670 EXPORT_SYMBOL_GPL(net_inc_ingress_queue); 1671 1672 void net_dec_ingress_queue(void) 1673 { 1674 static_key_slow_dec(&ingress_needed); 1675 } 1676 EXPORT_SYMBOL_GPL(net_dec_ingress_queue); 1677 #endif 1678 1679 #ifdef CONFIG_NET_EGRESS 1680 static struct static_key egress_needed __read_mostly; 1681 1682 void net_inc_egress_queue(void) 1683 { 1684 static_key_slow_inc(&egress_needed); 1685 } 1686 EXPORT_SYMBOL_GPL(net_inc_egress_queue); 1687 1688 void net_dec_egress_queue(void) 1689 { 1690 static_key_slow_dec(&egress_needed); 1691 } 1692 EXPORT_SYMBOL_GPL(net_dec_egress_queue); 1693 #endif 1694 1695 static struct static_key netstamp_needed __read_mostly; 1696 #ifdef HAVE_JUMP_LABEL 1697 /* We are not allowed to call static_key_slow_dec() from irq context 1698 * If net_disable_timestamp() is called from irq context, defer the 1699 * static_key_slow_dec() calls. 1700 */ 1701 static atomic_t netstamp_needed_deferred; 1702 #endif 1703 1704 void net_enable_timestamp(void) 1705 { 1706 #ifdef HAVE_JUMP_LABEL 1707 int deferred = atomic_xchg(&netstamp_needed_deferred, 0); 1708 1709 if (deferred) { 1710 while (--deferred) 1711 static_key_slow_dec(&netstamp_needed); 1712 return; 1713 } 1714 #endif 1715 static_key_slow_inc(&netstamp_needed); 1716 } 1717 EXPORT_SYMBOL(net_enable_timestamp); 1718 1719 void net_disable_timestamp(void) 1720 { 1721 #ifdef HAVE_JUMP_LABEL 1722 if (in_interrupt()) { 1723 atomic_inc(&netstamp_needed_deferred); 1724 return; 1725 } 1726 #endif 1727 static_key_slow_dec(&netstamp_needed); 1728 } 1729 EXPORT_SYMBOL(net_disable_timestamp); 1730 1731 static inline void net_timestamp_set(struct sk_buff *skb) 1732 { 1733 skb->tstamp.tv64 = 0; 1734 if (static_key_false(&netstamp_needed)) 1735 __net_timestamp(skb); 1736 } 1737 1738 #define net_timestamp_check(COND, SKB) \ 1739 if (static_key_false(&netstamp_needed)) { \ 1740 if ((COND) && !(SKB)->tstamp.tv64) \ 1741 __net_timestamp(SKB); \ 1742 } \ 1743 1744 bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb) 1745 { 1746 unsigned int len; 1747 1748 if (!(dev->flags & IFF_UP)) 1749 return false; 1750 1751 len = dev->mtu + dev->hard_header_len + VLAN_HLEN; 1752 if (skb->len <= len) 1753 return true; 1754 1755 /* if TSO is enabled, we don't care about the length as the packet 1756 * could be forwarded without being segmented before 1757 */ 1758 if (skb_is_gso(skb)) 1759 return true; 1760 1761 return false; 1762 } 1763 EXPORT_SYMBOL_GPL(is_skb_forwardable); 1764 1765 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 1766 { 1767 if (skb_orphan_frags(skb, GFP_ATOMIC) || 1768 unlikely(!is_skb_forwardable(dev, skb))) { 1769 atomic_long_inc(&dev->rx_dropped); 1770 kfree_skb(skb); 1771 return NET_RX_DROP; 1772 } 1773 1774 skb_scrub_packet(skb, true); 1775 skb->priority = 0; 1776 skb->protocol = eth_type_trans(skb, dev); 1777 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); 1778 1779 return 0; 1780 } 1781 EXPORT_SYMBOL_GPL(__dev_forward_skb); 1782 1783 /** 1784 * dev_forward_skb - loopback an skb to another netif 1785 * 1786 * @dev: destination network device 1787 * @skb: buffer to forward 1788 * 1789 * return values: 1790 * NET_RX_SUCCESS (no congestion) 1791 * NET_RX_DROP (packet was dropped, but freed) 1792 * 1793 * dev_forward_skb can be used for injecting an skb from the 1794 * start_xmit function of one device into the receive queue 1795 * of another device. 1796 * 1797 * The receiving device may be in another namespace, so 1798 * we have to clear all information in the skb that could 1799 * impact namespace isolation. 1800 */ 1801 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 1802 { 1803 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb); 1804 } 1805 EXPORT_SYMBOL_GPL(dev_forward_skb); 1806 1807 static inline int deliver_skb(struct sk_buff *skb, 1808 struct packet_type *pt_prev, 1809 struct net_device *orig_dev) 1810 { 1811 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) 1812 return -ENOMEM; 1813 atomic_inc(&skb->users); 1814 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 1815 } 1816 1817 static inline void deliver_ptype_list_skb(struct sk_buff *skb, 1818 struct packet_type **pt, 1819 struct net_device *orig_dev, 1820 __be16 type, 1821 struct list_head *ptype_list) 1822 { 1823 struct packet_type *ptype, *pt_prev = *pt; 1824 1825 list_for_each_entry_rcu(ptype, ptype_list, list) { 1826 if (ptype->type != type) 1827 continue; 1828 if (pt_prev) 1829 deliver_skb(skb, pt_prev, orig_dev); 1830 pt_prev = ptype; 1831 } 1832 *pt = pt_prev; 1833 } 1834 1835 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) 1836 { 1837 if (!ptype->af_packet_priv || !skb->sk) 1838 return false; 1839 1840 if (ptype->id_match) 1841 return ptype->id_match(ptype, skb->sk); 1842 else if ((struct sock *)ptype->af_packet_priv == skb->sk) 1843 return true; 1844 1845 return false; 1846 } 1847 1848 /* 1849 * Support routine. Sends outgoing frames to any network 1850 * taps currently in use. 1851 */ 1852 1853 static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) 1854 { 1855 struct packet_type *ptype; 1856 struct sk_buff *skb2 = NULL; 1857 struct packet_type *pt_prev = NULL; 1858 struct list_head *ptype_list = &ptype_all; 1859 1860 rcu_read_lock(); 1861 again: 1862 list_for_each_entry_rcu(ptype, ptype_list, list) { 1863 /* Never send packets back to the socket 1864 * they originated from - MvS (miquels@drinkel.ow.org) 1865 */ 1866 if (skb_loop_sk(ptype, skb)) 1867 continue; 1868 1869 if (pt_prev) { 1870 deliver_skb(skb2, pt_prev, skb->dev); 1871 pt_prev = ptype; 1872 continue; 1873 } 1874 1875 /* need to clone skb, done only once */ 1876 skb2 = skb_clone(skb, GFP_ATOMIC); 1877 if (!skb2) 1878 goto out_unlock; 1879 1880 net_timestamp_set(skb2); 1881 1882 /* skb->nh should be correctly 1883 * set by sender, so that the second statement is 1884 * just protection against buggy protocols. 1885 */ 1886 skb_reset_mac_header(skb2); 1887 1888 if (skb_network_header(skb2) < skb2->data || 1889 skb_network_header(skb2) > skb_tail_pointer(skb2)) { 1890 net_crit_ratelimited("protocol %04x is buggy, dev %s\n", 1891 ntohs(skb2->protocol), 1892 dev->name); 1893 skb_reset_network_header(skb2); 1894 } 1895 1896 skb2->transport_header = skb2->network_header; 1897 skb2->pkt_type = PACKET_OUTGOING; 1898 pt_prev = ptype; 1899 } 1900 1901 if (ptype_list == &ptype_all) { 1902 ptype_list = &dev->ptype_all; 1903 goto again; 1904 } 1905 out_unlock: 1906 if (pt_prev) 1907 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); 1908 rcu_read_unlock(); 1909 } 1910 1911 /** 1912 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change 1913 * @dev: Network device 1914 * @txq: number of queues available 1915 * 1916 * If real_num_tx_queues is changed the tc mappings may no longer be 1917 * valid. To resolve this verify the tc mapping remains valid and if 1918 * not NULL the mapping. With no priorities mapping to this 1919 * offset/count pair it will no longer be used. In the worst case TC0 1920 * is invalid nothing can be done so disable priority mappings. If is 1921 * expected that drivers will fix this mapping if they can before 1922 * calling netif_set_real_num_tx_queues. 1923 */ 1924 static void netif_setup_tc(struct net_device *dev, unsigned int txq) 1925 { 1926 int i; 1927 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 1928 1929 /* If TC0 is invalidated disable TC mapping */ 1930 if (tc->offset + tc->count > txq) { 1931 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n"); 1932 dev->num_tc = 0; 1933 return; 1934 } 1935 1936 /* Invalidated prio to tc mappings set to TC0 */ 1937 for (i = 1; i < TC_BITMASK + 1; i++) { 1938 int q = netdev_get_prio_tc_map(dev, i); 1939 1940 tc = &dev->tc_to_txq[q]; 1941 if (tc->offset + tc->count > txq) { 1942 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n", 1943 i, q); 1944 netdev_set_prio_tc_map(dev, i, 0); 1945 } 1946 } 1947 } 1948 1949 #ifdef CONFIG_XPS 1950 static DEFINE_MUTEX(xps_map_mutex); 1951 #define xmap_dereference(P) \ 1952 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex)) 1953 1954 static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps, 1955 int cpu, u16 index) 1956 { 1957 struct xps_map *map = NULL; 1958 int pos; 1959 1960 if (dev_maps) 1961 map = xmap_dereference(dev_maps->cpu_map[cpu]); 1962 1963 for (pos = 0; map && pos < map->len; pos++) { 1964 if (map->queues[pos] == index) { 1965 if (map->len > 1) { 1966 map->queues[pos] = map->queues[--map->len]; 1967 } else { 1968 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL); 1969 kfree_rcu(map, rcu); 1970 map = NULL; 1971 } 1972 break; 1973 } 1974 } 1975 1976 return map; 1977 } 1978 1979 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index) 1980 { 1981 struct xps_dev_maps *dev_maps; 1982 int cpu, i; 1983 bool active = false; 1984 1985 mutex_lock(&xps_map_mutex); 1986 dev_maps = xmap_dereference(dev->xps_maps); 1987 1988 if (!dev_maps) 1989 goto out_no_maps; 1990 1991 for_each_possible_cpu(cpu) { 1992 for (i = index; i < dev->num_tx_queues; i++) { 1993 if (!remove_xps_queue(dev_maps, cpu, i)) 1994 break; 1995 } 1996 if (i == dev->num_tx_queues) 1997 active = true; 1998 } 1999 2000 if (!active) { 2001 RCU_INIT_POINTER(dev->xps_maps, NULL); 2002 kfree_rcu(dev_maps, rcu); 2003 } 2004 2005 for (i = index; i < dev->num_tx_queues; i++) 2006 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i), 2007 NUMA_NO_NODE); 2008 2009 out_no_maps: 2010 mutex_unlock(&xps_map_mutex); 2011 } 2012 2013 static struct xps_map *expand_xps_map(struct xps_map *map, 2014 int cpu, u16 index) 2015 { 2016 struct xps_map *new_map; 2017 int alloc_len = XPS_MIN_MAP_ALLOC; 2018 int i, pos; 2019 2020 for (pos = 0; map && pos < map->len; pos++) { 2021 if (map->queues[pos] != index) 2022 continue; 2023 return map; 2024 } 2025 2026 /* Need to add queue to this CPU's existing map */ 2027 if (map) { 2028 if (pos < map->alloc_len) 2029 return map; 2030 2031 alloc_len = map->alloc_len * 2; 2032 } 2033 2034 /* Need to allocate new map to store queue on this CPU's map */ 2035 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL, 2036 cpu_to_node(cpu)); 2037 if (!new_map) 2038 return NULL; 2039 2040 for (i = 0; i < pos; i++) 2041 new_map->queues[i] = map->queues[i]; 2042 new_map->alloc_len = alloc_len; 2043 new_map->len = pos; 2044 2045 return new_map; 2046 } 2047 2048 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask, 2049 u16 index) 2050 { 2051 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL; 2052 struct xps_map *map, *new_map; 2053 int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES); 2054 int cpu, numa_node_id = -2; 2055 bool active = false; 2056 2057 mutex_lock(&xps_map_mutex); 2058 2059 dev_maps = xmap_dereference(dev->xps_maps); 2060 2061 /* allocate memory for queue storage */ 2062 for_each_online_cpu(cpu) { 2063 if (!cpumask_test_cpu(cpu, mask)) 2064 continue; 2065 2066 if (!new_dev_maps) 2067 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL); 2068 if (!new_dev_maps) { 2069 mutex_unlock(&xps_map_mutex); 2070 return -ENOMEM; 2071 } 2072 2073 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) : 2074 NULL; 2075 2076 map = expand_xps_map(map, cpu, index); 2077 if (!map) 2078 goto error; 2079 2080 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map); 2081 } 2082 2083 if (!new_dev_maps) 2084 goto out_no_new_maps; 2085 2086 for_each_possible_cpu(cpu) { 2087 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) { 2088 /* add queue to CPU maps */ 2089 int pos = 0; 2090 2091 map = xmap_dereference(new_dev_maps->cpu_map[cpu]); 2092 while ((pos < map->len) && (map->queues[pos] != index)) 2093 pos++; 2094 2095 if (pos == map->len) 2096 map->queues[map->len++] = index; 2097 #ifdef CONFIG_NUMA 2098 if (numa_node_id == -2) 2099 numa_node_id = cpu_to_node(cpu); 2100 else if (numa_node_id != cpu_to_node(cpu)) 2101 numa_node_id = -1; 2102 #endif 2103 } else if (dev_maps) { 2104 /* fill in the new device map from the old device map */ 2105 map = xmap_dereference(dev_maps->cpu_map[cpu]); 2106 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map); 2107 } 2108 2109 } 2110 2111 rcu_assign_pointer(dev->xps_maps, new_dev_maps); 2112 2113 /* Cleanup old maps */ 2114 if (dev_maps) { 2115 for_each_possible_cpu(cpu) { 2116 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]); 2117 map = xmap_dereference(dev_maps->cpu_map[cpu]); 2118 if (map && map != new_map) 2119 kfree_rcu(map, rcu); 2120 } 2121 2122 kfree_rcu(dev_maps, rcu); 2123 } 2124 2125 dev_maps = new_dev_maps; 2126 active = true; 2127 2128 out_no_new_maps: 2129 /* update Tx queue numa node */ 2130 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index), 2131 (numa_node_id >= 0) ? numa_node_id : 2132 NUMA_NO_NODE); 2133 2134 if (!dev_maps) 2135 goto out_no_maps; 2136 2137 /* removes queue from unused CPUs */ 2138 for_each_possible_cpu(cpu) { 2139 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) 2140 continue; 2141 2142 if (remove_xps_queue(dev_maps, cpu, index)) 2143 active = true; 2144 } 2145 2146 /* free map if not active */ 2147 if (!active) { 2148 RCU_INIT_POINTER(dev->xps_maps, NULL); 2149 kfree_rcu(dev_maps, rcu); 2150 } 2151 2152 out_no_maps: 2153 mutex_unlock(&xps_map_mutex); 2154 2155 return 0; 2156 error: 2157 /* remove any maps that we added */ 2158 for_each_possible_cpu(cpu) { 2159 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]); 2160 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) : 2161 NULL; 2162 if (new_map && new_map != map) 2163 kfree(new_map); 2164 } 2165 2166 mutex_unlock(&xps_map_mutex); 2167 2168 kfree(new_dev_maps); 2169 return -ENOMEM; 2170 } 2171 EXPORT_SYMBOL(netif_set_xps_queue); 2172 2173 #endif 2174 /* 2175 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues 2176 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed. 2177 */ 2178 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) 2179 { 2180 int rc; 2181 2182 if (txq < 1 || txq > dev->num_tx_queues) 2183 return -EINVAL; 2184 2185 if (dev->reg_state == NETREG_REGISTERED || 2186 dev->reg_state == NETREG_UNREGISTERING) { 2187 ASSERT_RTNL(); 2188 2189 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, 2190 txq); 2191 if (rc) 2192 return rc; 2193 2194 if (dev->num_tc) 2195 netif_setup_tc(dev, txq); 2196 2197 if (txq < dev->real_num_tx_queues) { 2198 qdisc_reset_all_tx_gt(dev, txq); 2199 #ifdef CONFIG_XPS 2200 netif_reset_xps_queues_gt(dev, txq); 2201 #endif 2202 } 2203 } 2204 2205 dev->real_num_tx_queues = txq; 2206 return 0; 2207 } 2208 EXPORT_SYMBOL(netif_set_real_num_tx_queues); 2209 2210 #ifdef CONFIG_SYSFS 2211 /** 2212 * netif_set_real_num_rx_queues - set actual number of RX queues used 2213 * @dev: Network device 2214 * @rxq: Actual number of RX queues 2215 * 2216 * This must be called either with the rtnl_lock held or before 2217 * registration of the net device. Returns 0 on success, or a 2218 * negative error code. If called before registration, it always 2219 * succeeds. 2220 */ 2221 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) 2222 { 2223 int rc; 2224 2225 if (rxq < 1 || rxq > dev->num_rx_queues) 2226 return -EINVAL; 2227 2228 if (dev->reg_state == NETREG_REGISTERED) { 2229 ASSERT_RTNL(); 2230 2231 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, 2232 rxq); 2233 if (rc) 2234 return rc; 2235 } 2236 2237 dev->real_num_rx_queues = rxq; 2238 return 0; 2239 } 2240 EXPORT_SYMBOL(netif_set_real_num_rx_queues); 2241 #endif 2242 2243 /** 2244 * netif_get_num_default_rss_queues - default number of RSS queues 2245 * 2246 * This routine should set an upper limit on the number of RSS queues 2247 * used by default by multiqueue devices. 2248 */ 2249 int netif_get_num_default_rss_queues(void) 2250 { 2251 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus()); 2252 } 2253 EXPORT_SYMBOL(netif_get_num_default_rss_queues); 2254 2255 static inline void __netif_reschedule(struct Qdisc *q) 2256 { 2257 struct softnet_data *sd; 2258 unsigned long flags; 2259 2260 local_irq_save(flags); 2261 sd = this_cpu_ptr(&softnet_data); 2262 q->next_sched = NULL; 2263 *sd->output_queue_tailp = q; 2264 sd->output_queue_tailp = &q->next_sched; 2265 raise_softirq_irqoff(NET_TX_SOFTIRQ); 2266 local_irq_restore(flags); 2267 } 2268 2269 void __netif_schedule(struct Qdisc *q) 2270 { 2271 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) 2272 __netif_reschedule(q); 2273 } 2274 EXPORT_SYMBOL(__netif_schedule); 2275 2276 struct dev_kfree_skb_cb { 2277 enum skb_free_reason reason; 2278 }; 2279 2280 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb) 2281 { 2282 return (struct dev_kfree_skb_cb *)skb->cb; 2283 } 2284 2285 void netif_schedule_queue(struct netdev_queue *txq) 2286 { 2287 rcu_read_lock(); 2288 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) { 2289 struct Qdisc *q = rcu_dereference(txq->qdisc); 2290 2291 __netif_schedule(q); 2292 } 2293 rcu_read_unlock(); 2294 } 2295 EXPORT_SYMBOL(netif_schedule_queue); 2296 2297 /** 2298 * netif_wake_subqueue - allow sending packets on subqueue 2299 * @dev: network device 2300 * @queue_index: sub queue index 2301 * 2302 * Resume individual transmit queue of a device with multiple transmit queues. 2303 */ 2304 void netif_wake_subqueue(struct net_device *dev, u16 queue_index) 2305 { 2306 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index); 2307 2308 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) { 2309 struct Qdisc *q; 2310 2311 rcu_read_lock(); 2312 q = rcu_dereference(txq->qdisc); 2313 __netif_schedule(q); 2314 rcu_read_unlock(); 2315 } 2316 } 2317 EXPORT_SYMBOL(netif_wake_subqueue); 2318 2319 void netif_tx_wake_queue(struct netdev_queue *dev_queue) 2320 { 2321 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { 2322 struct Qdisc *q; 2323 2324 rcu_read_lock(); 2325 q = rcu_dereference(dev_queue->qdisc); 2326 __netif_schedule(q); 2327 rcu_read_unlock(); 2328 } 2329 } 2330 EXPORT_SYMBOL(netif_tx_wake_queue); 2331 2332 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason) 2333 { 2334 unsigned long flags; 2335 2336 if (likely(atomic_read(&skb->users) == 1)) { 2337 smp_rmb(); 2338 atomic_set(&skb->users, 0); 2339 } else if (likely(!atomic_dec_and_test(&skb->users))) { 2340 return; 2341 } 2342 get_kfree_skb_cb(skb)->reason = reason; 2343 local_irq_save(flags); 2344 skb->next = __this_cpu_read(softnet_data.completion_queue); 2345 __this_cpu_write(softnet_data.completion_queue, skb); 2346 raise_softirq_irqoff(NET_TX_SOFTIRQ); 2347 local_irq_restore(flags); 2348 } 2349 EXPORT_SYMBOL(__dev_kfree_skb_irq); 2350 2351 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason) 2352 { 2353 if (in_irq() || irqs_disabled()) 2354 __dev_kfree_skb_irq(skb, reason); 2355 else 2356 dev_kfree_skb(skb); 2357 } 2358 EXPORT_SYMBOL(__dev_kfree_skb_any); 2359 2360 2361 /** 2362 * netif_device_detach - mark device as removed 2363 * @dev: network device 2364 * 2365 * Mark device as removed from system and therefore no longer available. 2366 */ 2367 void netif_device_detach(struct net_device *dev) 2368 { 2369 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && 2370 netif_running(dev)) { 2371 netif_tx_stop_all_queues(dev); 2372 } 2373 } 2374 EXPORT_SYMBOL(netif_device_detach); 2375 2376 /** 2377 * netif_device_attach - mark device as attached 2378 * @dev: network device 2379 * 2380 * Mark device as attached from system and restart if needed. 2381 */ 2382 void netif_device_attach(struct net_device *dev) 2383 { 2384 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && 2385 netif_running(dev)) { 2386 netif_tx_wake_all_queues(dev); 2387 __netdev_watchdog_up(dev); 2388 } 2389 } 2390 EXPORT_SYMBOL(netif_device_attach); 2391 2392 /* 2393 * Returns a Tx hash based on the given packet descriptor a Tx queues' number 2394 * to be used as a distribution range. 2395 */ 2396 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb, 2397 unsigned int num_tx_queues) 2398 { 2399 u32 hash; 2400 u16 qoffset = 0; 2401 u16 qcount = num_tx_queues; 2402 2403 if (skb_rx_queue_recorded(skb)) { 2404 hash = skb_get_rx_queue(skb); 2405 while (unlikely(hash >= num_tx_queues)) 2406 hash -= num_tx_queues; 2407 return hash; 2408 } 2409 2410 if (dev->num_tc) { 2411 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); 2412 qoffset = dev->tc_to_txq[tc].offset; 2413 qcount = dev->tc_to_txq[tc].count; 2414 } 2415 2416 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset; 2417 } 2418 EXPORT_SYMBOL(__skb_tx_hash); 2419 2420 static void skb_warn_bad_offload(const struct sk_buff *skb) 2421 { 2422 static const netdev_features_t null_features = 0; 2423 struct net_device *dev = skb->dev; 2424 const char *name = ""; 2425 2426 if (!net_ratelimit()) 2427 return; 2428 2429 if (dev) { 2430 if (dev->dev.parent) 2431 name = dev_driver_string(dev->dev.parent); 2432 else 2433 name = netdev_name(dev); 2434 } 2435 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d " 2436 "gso_type=%d ip_summed=%d\n", 2437 name, dev ? &dev->features : &null_features, 2438 skb->sk ? &skb->sk->sk_route_caps : &null_features, 2439 skb->len, skb->data_len, skb_shinfo(skb)->gso_size, 2440 skb_shinfo(skb)->gso_type, skb->ip_summed); 2441 } 2442 2443 /* 2444 * Invalidate hardware checksum when packet is to be mangled, and 2445 * complete checksum manually on outgoing path. 2446 */ 2447 int skb_checksum_help(struct sk_buff *skb) 2448 { 2449 __wsum csum; 2450 int ret = 0, offset; 2451 2452 if (skb->ip_summed == CHECKSUM_COMPLETE) 2453 goto out_set_summed; 2454 2455 if (unlikely(skb_shinfo(skb)->gso_size)) { 2456 skb_warn_bad_offload(skb); 2457 return -EINVAL; 2458 } 2459 2460 /* Before computing a checksum, we should make sure no frag could 2461 * be modified by an external entity : checksum could be wrong. 2462 */ 2463 if (skb_has_shared_frag(skb)) { 2464 ret = __skb_linearize(skb); 2465 if (ret) 2466 goto out; 2467 } 2468 2469 offset = skb_checksum_start_offset(skb); 2470 BUG_ON(offset >= skb_headlen(skb)); 2471 csum = skb_checksum(skb, offset, skb->len - offset, 0); 2472 2473 offset += skb->csum_offset; 2474 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb)); 2475 2476 if (skb_cloned(skb) && 2477 !skb_clone_writable(skb, offset + sizeof(__sum16))) { 2478 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2479 if (ret) 2480 goto out; 2481 } 2482 2483 *(__sum16 *)(skb->data + offset) = csum_fold(csum); 2484 out_set_summed: 2485 skb->ip_summed = CHECKSUM_NONE; 2486 out: 2487 return ret; 2488 } 2489 EXPORT_SYMBOL(skb_checksum_help); 2490 2491 /* skb_csum_offload_check - Driver helper function to determine if a device 2492 * with limited checksum offload capabilities is able to offload the checksum 2493 * for a given packet. 2494 * 2495 * Arguments: 2496 * skb - sk_buff for the packet in question 2497 * spec - contains the description of what device can offload 2498 * csum_encapped - returns true if the checksum being offloaded is 2499 * encpasulated. That is it is checksum for the transport header 2500 * in the inner headers. 2501 * checksum_help - when set indicates that helper function should 2502 * call skb_checksum_help if offload checks fail 2503 * 2504 * Returns: 2505 * true: Packet has passed the checksum checks and should be offloadable to 2506 * the device (a driver may still need to check for additional 2507 * restrictions of its device) 2508 * false: Checksum is not offloadable. If checksum_help was set then 2509 * skb_checksum_help was called to resolve checksum for non-GSO 2510 * packets and when IP protocol is not SCTP 2511 */ 2512 bool __skb_csum_offload_chk(struct sk_buff *skb, 2513 const struct skb_csum_offl_spec *spec, 2514 bool *csum_encapped, 2515 bool csum_help) 2516 { 2517 struct iphdr *iph; 2518 struct ipv6hdr *ipv6; 2519 void *nhdr; 2520 int protocol; 2521 u8 ip_proto; 2522 2523 if (skb->protocol == htons(ETH_P_8021Q) || 2524 skb->protocol == htons(ETH_P_8021AD)) { 2525 if (!spec->vlan_okay) 2526 goto need_help; 2527 } 2528 2529 /* We check whether the checksum refers to a transport layer checksum in 2530 * the outermost header or an encapsulated transport layer checksum that 2531 * corresponds to the inner headers of the skb. If the checksum is for 2532 * something else in the packet we need help. 2533 */ 2534 if (skb_checksum_start_offset(skb) == skb_transport_offset(skb)) { 2535 /* Non-encapsulated checksum */ 2536 protocol = eproto_to_ipproto(vlan_get_protocol(skb)); 2537 nhdr = skb_network_header(skb); 2538 *csum_encapped = false; 2539 if (spec->no_not_encapped) 2540 goto need_help; 2541 } else if (skb->encapsulation && spec->encap_okay && 2542 skb_checksum_start_offset(skb) == 2543 skb_inner_transport_offset(skb)) { 2544 /* Encapsulated checksum */ 2545 *csum_encapped = true; 2546 switch (skb->inner_protocol_type) { 2547 case ENCAP_TYPE_ETHER: 2548 protocol = eproto_to_ipproto(skb->inner_protocol); 2549 break; 2550 case ENCAP_TYPE_IPPROTO: 2551 protocol = skb->inner_protocol; 2552 break; 2553 } 2554 nhdr = skb_inner_network_header(skb); 2555 } else { 2556 goto need_help; 2557 } 2558 2559 switch (protocol) { 2560 case IPPROTO_IP: 2561 if (!spec->ipv4_okay) 2562 goto need_help; 2563 iph = nhdr; 2564 ip_proto = iph->protocol; 2565 if (iph->ihl != 5 && !spec->ip_options_okay) 2566 goto need_help; 2567 break; 2568 case IPPROTO_IPV6: 2569 if (!spec->ipv6_okay) 2570 goto need_help; 2571 if (spec->no_encapped_ipv6 && *csum_encapped) 2572 goto need_help; 2573 ipv6 = nhdr; 2574 nhdr += sizeof(*ipv6); 2575 ip_proto = ipv6->nexthdr; 2576 break; 2577 default: 2578 goto need_help; 2579 } 2580 2581 ip_proto_again: 2582 switch (ip_proto) { 2583 case IPPROTO_TCP: 2584 if (!spec->tcp_okay || 2585 skb->csum_offset != offsetof(struct tcphdr, check)) 2586 goto need_help; 2587 break; 2588 case IPPROTO_UDP: 2589 if (!spec->udp_okay || 2590 skb->csum_offset != offsetof(struct udphdr, check)) 2591 goto need_help; 2592 break; 2593 case IPPROTO_SCTP: 2594 if (!spec->sctp_okay || 2595 skb->csum_offset != offsetof(struct sctphdr, checksum)) 2596 goto cant_help; 2597 break; 2598 case NEXTHDR_HOP: 2599 case NEXTHDR_ROUTING: 2600 case NEXTHDR_DEST: { 2601 u8 *opthdr = nhdr; 2602 2603 if (protocol != IPPROTO_IPV6 || !spec->ext_hdrs_okay) 2604 goto need_help; 2605 2606 ip_proto = opthdr[0]; 2607 nhdr += (opthdr[1] + 1) << 3; 2608 2609 goto ip_proto_again; 2610 } 2611 default: 2612 goto need_help; 2613 } 2614 2615 /* Passed the tests for offloading checksum */ 2616 return true; 2617 2618 need_help: 2619 if (csum_help && !skb_shinfo(skb)->gso_size) 2620 skb_checksum_help(skb); 2621 cant_help: 2622 return false; 2623 } 2624 EXPORT_SYMBOL(__skb_csum_offload_chk); 2625 2626 __be16 skb_network_protocol(struct sk_buff *skb, int *depth) 2627 { 2628 __be16 type = skb->protocol; 2629 2630 /* Tunnel gso handlers can set protocol to ethernet. */ 2631 if (type == htons(ETH_P_TEB)) { 2632 struct ethhdr *eth; 2633 2634 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr)))) 2635 return 0; 2636 2637 eth = (struct ethhdr *)skb_mac_header(skb); 2638 type = eth->h_proto; 2639 } 2640 2641 return __vlan_get_protocol(skb, type, depth); 2642 } 2643 2644 /** 2645 * skb_mac_gso_segment - mac layer segmentation handler. 2646 * @skb: buffer to segment 2647 * @features: features for the output path (see dev->features) 2648 */ 2649 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, 2650 netdev_features_t features) 2651 { 2652 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); 2653 struct packet_offload *ptype; 2654 int vlan_depth = skb->mac_len; 2655 __be16 type = skb_network_protocol(skb, &vlan_depth); 2656 2657 if (unlikely(!type)) 2658 return ERR_PTR(-EINVAL); 2659 2660 __skb_pull(skb, vlan_depth); 2661 2662 rcu_read_lock(); 2663 list_for_each_entry_rcu(ptype, &offload_base, list) { 2664 if (ptype->type == type && ptype->callbacks.gso_segment) { 2665 segs = ptype->callbacks.gso_segment(skb, features); 2666 break; 2667 } 2668 } 2669 rcu_read_unlock(); 2670 2671 __skb_push(skb, skb->data - skb_mac_header(skb)); 2672 2673 return segs; 2674 } 2675 EXPORT_SYMBOL(skb_mac_gso_segment); 2676 2677 2678 /* openvswitch calls this on rx path, so we need a different check. 2679 */ 2680 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) 2681 { 2682 if (tx_path) 2683 return skb->ip_summed != CHECKSUM_PARTIAL; 2684 else 2685 return skb->ip_summed == CHECKSUM_NONE; 2686 } 2687 2688 /** 2689 * __skb_gso_segment - Perform segmentation on skb. 2690 * @skb: buffer to segment 2691 * @features: features for the output path (see dev->features) 2692 * @tx_path: whether it is called in TX path 2693 * 2694 * This function segments the given skb and returns a list of segments. 2695 * 2696 * It may return NULL if the skb requires no segmentation. This is 2697 * only possible when GSO is used for verifying header integrity. 2698 * 2699 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb. 2700 */ 2701 struct sk_buff *__skb_gso_segment(struct sk_buff *skb, 2702 netdev_features_t features, bool tx_path) 2703 { 2704 if (unlikely(skb_needs_check(skb, tx_path))) { 2705 int err; 2706 2707 skb_warn_bad_offload(skb); 2708 2709 err = skb_cow_head(skb, 0); 2710 if (err < 0) 2711 return ERR_PTR(err); 2712 } 2713 2714 BUILD_BUG_ON(SKB_SGO_CB_OFFSET + 2715 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb)); 2716 2717 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb); 2718 SKB_GSO_CB(skb)->encap_level = 0; 2719 2720 skb_reset_mac_header(skb); 2721 skb_reset_mac_len(skb); 2722 2723 return skb_mac_gso_segment(skb, features); 2724 } 2725 EXPORT_SYMBOL(__skb_gso_segment); 2726 2727 /* Take action when hardware reception checksum errors are detected. */ 2728 #ifdef CONFIG_BUG 2729 void netdev_rx_csum_fault(struct net_device *dev) 2730 { 2731 if (net_ratelimit()) { 2732 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>"); 2733 dump_stack(); 2734 } 2735 } 2736 EXPORT_SYMBOL(netdev_rx_csum_fault); 2737 #endif 2738 2739 /* Actually, we should eliminate this check as soon as we know, that: 2740 * 1. IOMMU is present and allows to map all the memory. 2741 * 2. No high memory really exists on this machine. 2742 */ 2743 2744 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) 2745 { 2746 #ifdef CONFIG_HIGHMEM 2747 int i; 2748 if (!(dev->features & NETIF_F_HIGHDMA)) { 2749 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2750 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2751 if (PageHighMem(skb_frag_page(frag))) 2752 return 1; 2753 } 2754 } 2755 2756 if (PCI_DMA_BUS_IS_PHYS) { 2757 struct device *pdev = dev->dev.parent; 2758 2759 if (!pdev) 2760 return 0; 2761 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2762 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2763 dma_addr_t addr = page_to_phys(skb_frag_page(frag)); 2764 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask) 2765 return 1; 2766 } 2767 } 2768 #endif 2769 return 0; 2770 } 2771 2772 /* If MPLS offload request, verify we are testing hardware MPLS features 2773 * instead of standard features for the netdev. 2774 */ 2775 #if IS_ENABLED(CONFIG_NET_MPLS_GSO) 2776 static netdev_features_t net_mpls_features(struct sk_buff *skb, 2777 netdev_features_t features, 2778 __be16 type) 2779 { 2780 if (eth_p_mpls(type)) 2781 features &= skb->dev->mpls_features; 2782 2783 return features; 2784 } 2785 #else 2786 static netdev_features_t net_mpls_features(struct sk_buff *skb, 2787 netdev_features_t features, 2788 __be16 type) 2789 { 2790 return features; 2791 } 2792 #endif 2793 2794 static netdev_features_t harmonize_features(struct sk_buff *skb, 2795 netdev_features_t features) 2796 { 2797 int tmp; 2798 __be16 type; 2799 2800 type = skb_network_protocol(skb, &tmp); 2801 features = net_mpls_features(skb, features, type); 2802 2803 if (skb->ip_summed != CHECKSUM_NONE && 2804 !can_checksum_protocol(features, type)) { 2805 features &= ~NETIF_F_CSUM_MASK; 2806 } else if (illegal_highdma(skb->dev, skb)) { 2807 features &= ~NETIF_F_SG; 2808 } 2809 2810 return features; 2811 } 2812 2813 netdev_features_t passthru_features_check(struct sk_buff *skb, 2814 struct net_device *dev, 2815 netdev_features_t features) 2816 { 2817 return features; 2818 } 2819 EXPORT_SYMBOL(passthru_features_check); 2820 2821 static netdev_features_t dflt_features_check(const struct sk_buff *skb, 2822 struct net_device *dev, 2823 netdev_features_t features) 2824 { 2825 return vlan_features_check(skb, features); 2826 } 2827 2828 netdev_features_t netif_skb_features(struct sk_buff *skb) 2829 { 2830 struct net_device *dev = skb->dev; 2831 netdev_features_t features = dev->features; 2832 u16 gso_segs = skb_shinfo(skb)->gso_segs; 2833 2834 if (gso_segs > dev->gso_max_segs || gso_segs < dev->gso_min_segs) 2835 features &= ~NETIF_F_GSO_MASK; 2836 2837 /* If encapsulation offload request, verify we are testing 2838 * hardware encapsulation features instead of standard 2839 * features for the netdev 2840 */ 2841 if (skb->encapsulation) 2842 features &= dev->hw_enc_features; 2843 2844 if (skb_vlan_tagged(skb)) 2845 features = netdev_intersect_features(features, 2846 dev->vlan_features | 2847 NETIF_F_HW_VLAN_CTAG_TX | 2848 NETIF_F_HW_VLAN_STAG_TX); 2849 2850 if (dev->netdev_ops->ndo_features_check) 2851 features &= dev->netdev_ops->ndo_features_check(skb, dev, 2852 features); 2853 else 2854 features &= dflt_features_check(skb, dev, features); 2855 2856 return harmonize_features(skb, features); 2857 } 2858 EXPORT_SYMBOL(netif_skb_features); 2859 2860 static int xmit_one(struct sk_buff *skb, struct net_device *dev, 2861 struct netdev_queue *txq, bool more) 2862 { 2863 unsigned int len; 2864 int rc; 2865 2866 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all)) 2867 dev_queue_xmit_nit(skb, dev); 2868 2869 len = skb->len; 2870 trace_net_dev_start_xmit(skb, dev); 2871 rc = netdev_start_xmit(skb, dev, txq, more); 2872 trace_net_dev_xmit(skb, rc, dev, len); 2873 2874 return rc; 2875 } 2876 2877 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev, 2878 struct netdev_queue *txq, int *ret) 2879 { 2880 struct sk_buff *skb = first; 2881 int rc = NETDEV_TX_OK; 2882 2883 while (skb) { 2884 struct sk_buff *next = skb->next; 2885 2886 skb->next = NULL; 2887 rc = xmit_one(skb, dev, txq, next != NULL); 2888 if (unlikely(!dev_xmit_complete(rc))) { 2889 skb->next = next; 2890 goto out; 2891 } 2892 2893 skb = next; 2894 if (netif_xmit_stopped(txq) && skb) { 2895 rc = NETDEV_TX_BUSY; 2896 break; 2897 } 2898 } 2899 2900 out: 2901 *ret = rc; 2902 return skb; 2903 } 2904 2905 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, 2906 netdev_features_t features) 2907 { 2908 if (skb_vlan_tag_present(skb) && 2909 !vlan_hw_offload_capable(features, skb->vlan_proto)) 2910 skb = __vlan_hwaccel_push_inside(skb); 2911 return skb; 2912 } 2913 2914 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev) 2915 { 2916 netdev_features_t features; 2917 2918 if (skb->next) 2919 return skb; 2920 2921 features = netif_skb_features(skb); 2922 skb = validate_xmit_vlan(skb, features); 2923 if (unlikely(!skb)) 2924 goto out_null; 2925 2926 if (netif_needs_gso(skb, features)) { 2927 struct sk_buff *segs; 2928 2929 segs = skb_gso_segment(skb, features); 2930 if (IS_ERR(segs)) { 2931 goto out_kfree_skb; 2932 } else if (segs) { 2933 consume_skb(skb); 2934 skb = segs; 2935 } 2936 } else { 2937 if (skb_needs_linearize(skb, features) && 2938 __skb_linearize(skb)) 2939 goto out_kfree_skb; 2940 2941 /* If packet is not checksummed and device does not 2942 * support checksumming for this protocol, complete 2943 * checksumming here. 2944 */ 2945 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2946 if (skb->encapsulation) 2947 skb_set_inner_transport_header(skb, 2948 skb_checksum_start_offset(skb)); 2949 else 2950 skb_set_transport_header(skb, 2951 skb_checksum_start_offset(skb)); 2952 if (!(features & NETIF_F_CSUM_MASK) && 2953 skb_checksum_help(skb)) 2954 goto out_kfree_skb; 2955 } 2956 } 2957 2958 return skb; 2959 2960 out_kfree_skb: 2961 kfree_skb(skb); 2962 out_null: 2963 return NULL; 2964 } 2965 2966 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev) 2967 { 2968 struct sk_buff *next, *head = NULL, *tail; 2969 2970 for (; skb != NULL; skb = next) { 2971 next = skb->next; 2972 skb->next = NULL; 2973 2974 /* in case skb wont be segmented, point to itself */ 2975 skb->prev = skb; 2976 2977 skb = validate_xmit_skb(skb, dev); 2978 if (!skb) 2979 continue; 2980 2981 if (!head) 2982 head = skb; 2983 else 2984 tail->next = skb; 2985 /* If skb was segmented, skb->prev points to 2986 * the last segment. If not, it still contains skb. 2987 */ 2988 tail = skb->prev; 2989 } 2990 return head; 2991 } 2992 2993 static void qdisc_pkt_len_init(struct sk_buff *skb) 2994 { 2995 const struct skb_shared_info *shinfo = skb_shinfo(skb); 2996 2997 qdisc_skb_cb(skb)->pkt_len = skb->len; 2998 2999 /* To get more precise estimation of bytes sent on wire, 3000 * we add to pkt_len the headers size of all segments 3001 */ 3002 if (shinfo->gso_size) { 3003 unsigned int hdr_len; 3004 u16 gso_segs = shinfo->gso_segs; 3005 3006 /* mac layer + network layer */ 3007 hdr_len = skb_transport_header(skb) - skb_mac_header(skb); 3008 3009 /* + transport layer */ 3010 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) 3011 hdr_len += tcp_hdrlen(skb); 3012 else 3013 hdr_len += sizeof(struct udphdr); 3014 3015 if (shinfo->gso_type & SKB_GSO_DODGY) 3016 gso_segs = DIV_ROUND_UP(skb->len - hdr_len, 3017 shinfo->gso_size); 3018 3019 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; 3020 } 3021 } 3022 3023 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 3024 struct net_device *dev, 3025 struct netdev_queue *txq) 3026 { 3027 spinlock_t *root_lock = qdisc_lock(q); 3028 bool contended; 3029 int rc; 3030 3031 qdisc_calculate_pkt_len(skb, q); 3032 /* 3033 * Heuristic to force contended enqueues to serialize on a 3034 * separate lock before trying to get qdisc main lock. 3035 * This permits __QDISC___STATE_RUNNING owner to get the lock more 3036 * often and dequeue packets faster. 3037 */ 3038 contended = qdisc_is_running(q); 3039 if (unlikely(contended)) 3040 spin_lock(&q->busylock); 3041 3042 spin_lock(root_lock); 3043 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { 3044 kfree_skb(skb); 3045 rc = NET_XMIT_DROP; 3046 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && 3047 qdisc_run_begin(q)) { 3048 /* 3049 * This is a work-conserving queue; there are no old skbs 3050 * waiting to be sent out; and the qdisc is not running - 3051 * xmit the skb directly. 3052 */ 3053 3054 qdisc_bstats_update(q, skb); 3055 3056 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) { 3057 if (unlikely(contended)) { 3058 spin_unlock(&q->busylock); 3059 contended = false; 3060 } 3061 __qdisc_run(q); 3062 } else 3063 qdisc_run_end(q); 3064 3065 rc = NET_XMIT_SUCCESS; 3066 } else { 3067 rc = q->enqueue(skb, q) & NET_XMIT_MASK; 3068 if (qdisc_run_begin(q)) { 3069 if (unlikely(contended)) { 3070 spin_unlock(&q->busylock); 3071 contended = false; 3072 } 3073 __qdisc_run(q); 3074 } 3075 } 3076 spin_unlock(root_lock); 3077 if (unlikely(contended)) 3078 spin_unlock(&q->busylock); 3079 return rc; 3080 } 3081 3082 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO) 3083 static void skb_update_prio(struct sk_buff *skb) 3084 { 3085 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap); 3086 3087 if (!skb->priority && skb->sk && map) { 3088 unsigned int prioidx = 3089 sock_cgroup_prioidx(&skb->sk->sk_cgrp_data); 3090 3091 if (prioidx < map->priomap_len) 3092 skb->priority = map->priomap[prioidx]; 3093 } 3094 } 3095 #else 3096 #define skb_update_prio(skb) 3097 #endif 3098 3099 DEFINE_PER_CPU(int, xmit_recursion); 3100 EXPORT_SYMBOL(xmit_recursion); 3101 3102 #define RECURSION_LIMIT 10 3103 3104 /** 3105 * dev_loopback_xmit - loop back @skb 3106 * @net: network namespace this loopback is happening in 3107 * @sk: sk needed to be a netfilter okfn 3108 * @skb: buffer to transmit 3109 */ 3110 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb) 3111 { 3112 skb_reset_mac_header(skb); 3113 __skb_pull(skb, skb_network_offset(skb)); 3114 skb->pkt_type = PACKET_LOOPBACK; 3115 skb->ip_summed = CHECKSUM_UNNECESSARY; 3116 WARN_ON(!skb_dst(skb)); 3117 skb_dst_force(skb); 3118 netif_rx_ni(skb); 3119 return 0; 3120 } 3121 EXPORT_SYMBOL(dev_loopback_xmit); 3122 3123 #ifdef CONFIG_NET_EGRESS 3124 static struct sk_buff * 3125 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) 3126 { 3127 struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list); 3128 struct tcf_result cl_res; 3129 3130 if (!cl) 3131 return skb; 3132 3133 /* skb->tc_verd and qdisc_skb_cb(skb)->pkt_len were already set 3134 * earlier by the caller. 3135 */ 3136 qdisc_bstats_cpu_update(cl->q, skb); 3137 3138 switch (tc_classify(skb, cl, &cl_res, false)) { 3139 case TC_ACT_OK: 3140 case TC_ACT_RECLASSIFY: 3141 skb->tc_index = TC_H_MIN(cl_res.classid); 3142 break; 3143 case TC_ACT_SHOT: 3144 qdisc_qstats_cpu_drop(cl->q); 3145 *ret = NET_XMIT_DROP; 3146 goto drop; 3147 case TC_ACT_STOLEN: 3148 case TC_ACT_QUEUED: 3149 *ret = NET_XMIT_SUCCESS; 3150 drop: 3151 kfree_skb(skb); 3152 return NULL; 3153 case TC_ACT_REDIRECT: 3154 /* No need to push/pop skb's mac_header here on egress! */ 3155 skb_do_redirect(skb); 3156 *ret = NET_XMIT_SUCCESS; 3157 return NULL; 3158 default: 3159 break; 3160 } 3161 3162 return skb; 3163 } 3164 #endif /* CONFIG_NET_EGRESS */ 3165 3166 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb) 3167 { 3168 #ifdef CONFIG_XPS 3169 struct xps_dev_maps *dev_maps; 3170 struct xps_map *map; 3171 int queue_index = -1; 3172 3173 rcu_read_lock(); 3174 dev_maps = rcu_dereference(dev->xps_maps); 3175 if (dev_maps) { 3176 map = rcu_dereference( 3177 dev_maps->cpu_map[skb->sender_cpu - 1]); 3178 if (map) { 3179 if (map->len == 1) 3180 queue_index = map->queues[0]; 3181 else 3182 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb), 3183 map->len)]; 3184 if (unlikely(queue_index >= dev->real_num_tx_queues)) 3185 queue_index = -1; 3186 } 3187 } 3188 rcu_read_unlock(); 3189 3190 return queue_index; 3191 #else 3192 return -1; 3193 #endif 3194 } 3195 3196 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb) 3197 { 3198 struct sock *sk = skb->sk; 3199 int queue_index = sk_tx_queue_get(sk); 3200 3201 if (queue_index < 0 || skb->ooo_okay || 3202 queue_index >= dev->real_num_tx_queues) { 3203 int new_index = get_xps_queue(dev, skb); 3204 if (new_index < 0) 3205 new_index = skb_tx_hash(dev, skb); 3206 3207 if (queue_index != new_index && sk && 3208 sk_fullsock(sk) && 3209 rcu_access_pointer(sk->sk_dst_cache)) 3210 sk_tx_queue_set(sk, new_index); 3211 3212 queue_index = new_index; 3213 } 3214 3215 return queue_index; 3216 } 3217 3218 struct netdev_queue *netdev_pick_tx(struct net_device *dev, 3219 struct sk_buff *skb, 3220 void *accel_priv) 3221 { 3222 int queue_index = 0; 3223 3224 #ifdef CONFIG_XPS 3225 u32 sender_cpu = skb->sender_cpu - 1; 3226 3227 if (sender_cpu >= (u32)NR_CPUS) 3228 skb->sender_cpu = raw_smp_processor_id() + 1; 3229 #endif 3230 3231 if (dev->real_num_tx_queues != 1) { 3232 const struct net_device_ops *ops = dev->netdev_ops; 3233 if (ops->ndo_select_queue) 3234 queue_index = ops->ndo_select_queue(dev, skb, accel_priv, 3235 __netdev_pick_tx); 3236 else 3237 queue_index = __netdev_pick_tx(dev, skb); 3238 3239 if (!accel_priv) 3240 queue_index = netdev_cap_txqueue(dev, queue_index); 3241 } 3242 3243 skb_set_queue_mapping(skb, queue_index); 3244 return netdev_get_tx_queue(dev, queue_index); 3245 } 3246 3247 /** 3248 * __dev_queue_xmit - transmit a buffer 3249 * @skb: buffer to transmit 3250 * @accel_priv: private data used for L2 forwarding offload 3251 * 3252 * Queue a buffer for transmission to a network device. The caller must 3253 * have set the device and priority and built the buffer before calling 3254 * this function. The function can be called from an interrupt. 3255 * 3256 * A negative errno code is returned on a failure. A success does not 3257 * guarantee the frame will be transmitted as it may be dropped due 3258 * to congestion or traffic shaping. 3259 * 3260 * ----------------------------------------------------------------------------------- 3261 * I notice this method can also return errors from the queue disciplines, 3262 * including NET_XMIT_DROP, which is a positive value. So, errors can also 3263 * be positive. 3264 * 3265 * Regardless of the return value, the skb is consumed, so it is currently 3266 * difficult to retry a send to this method. (You can bump the ref count 3267 * before sending to hold a reference for retry if you are careful.) 3268 * 3269 * When calling this method, interrupts MUST be enabled. This is because 3270 * the BH enable code must have IRQs enabled so that it will not deadlock. 3271 * --BLG 3272 */ 3273 static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv) 3274 { 3275 struct net_device *dev = skb->dev; 3276 struct netdev_queue *txq; 3277 struct Qdisc *q; 3278 int rc = -ENOMEM; 3279 3280 skb_reset_mac_header(skb); 3281 3282 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP)) 3283 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED); 3284 3285 /* Disable soft irqs for various locks below. Also 3286 * stops preemption for RCU. 3287 */ 3288 rcu_read_lock_bh(); 3289 3290 skb_update_prio(skb); 3291 3292 qdisc_pkt_len_init(skb); 3293 #ifdef CONFIG_NET_CLS_ACT 3294 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS); 3295 # ifdef CONFIG_NET_EGRESS 3296 if (static_key_false(&egress_needed)) { 3297 skb = sch_handle_egress(skb, &rc, dev); 3298 if (!skb) 3299 goto out; 3300 } 3301 # endif 3302 #endif 3303 /* If device/qdisc don't need skb->dst, release it right now while 3304 * its hot in this cpu cache. 3305 */ 3306 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) 3307 skb_dst_drop(skb); 3308 else 3309 skb_dst_force(skb); 3310 3311 #ifdef CONFIG_NET_SWITCHDEV 3312 /* Don't forward if offload device already forwarded */ 3313 if (skb->offload_fwd_mark && 3314 skb->offload_fwd_mark == dev->offload_fwd_mark) { 3315 consume_skb(skb); 3316 rc = NET_XMIT_SUCCESS; 3317 goto out; 3318 } 3319 #endif 3320 3321 txq = netdev_pick_tx(dev, skb, accel_priv); 3322 q = rcu_dereference_bh(txq->qdisc); 3323 3324 trace_net_dev_queue(skb); 3325 if (q->enqueue) { 3326 rc = __dev_xmit_skb(skb, q, dev, txq); 3327 goto out; 3328 } 3329 3330 /* The device has no queue. Common case for software devices: 3331 loopback, all the sorts of tunnels... 3332 3333 Really, it is unlikely that netif_tx_lock protection is necessary 3334 here. (f.e. loopback and IP tunnels are clean ignoring statistics 3335 counters.) 3336 However, it is possible, that they rely on protection 3337 made by us here. 3338 3339 Check this and shot the lock. It is not prone from deadlocks. 3340 Either shot noqueue qdisc, it is even simpler 8) 3341 */ 3342 if (dev->flags & IFF_UP) { 3343 int cpu = smp_processor_id(); /* ok because BHs are off */ 3344 3345 if (txq->xmit_lock_owner != cpu) { 3346 3347 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT) 3348 goto recursion_alert; 3349 3350 skb = validate_xmit_skb(skb, dev); 3351 if (!skb) 3352 goto drop; 3353 3354 HARD_TX_LOCK(dev, txq, cpu); 3355 3356 if (!netif_xmit_stopped(txq)) { 3357 __this_cpu_inc(xmit_recursion); 3358 skb = dev_hard_start_xmit(skb, dev, txq, &rc); 3359 __this_cpu_dec(xmit_recursion); 3360 if (dev_xmit_complete(rc)) { 3361 HARD_TX_UNLOCK(dev, txq); 3362 goto out; 3363 } 3364 } 3365 HARD_TX_UNLOCK(dev, txq); 3366 net_crit_ratelimited("Virtual device %s asks to queue packet!\n", 3367 dev->name); 3368 } else { 3369 /* Recursion is detected! It is possible, 3370 * unfortunately 3371 */ 3372 recursion_alert: 3373 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", 3374 dev->name); 3375 } 3376 } 3377 3378 rc = -ENETDOWN; 3379 drop: 3380 rcu_read_unlock_bh(); 3381 3382 atomic_long_inc(&dev->tx_dropped); 3383 kfree_skb_list(skb); 3384 return rc; 3385 out: 3386 rcu_read_unlock_bh(); 3387 return rc; 3388 } 3389 3390 int dev_queue_xmit(struct sk_buff *skb) 3391 { 3392 return __dev_queue_xmit(skb, NULL); 3393 } 3394 EXPORT_SYMBOL(dev_queue_xmit); 3395 3396 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv) 3397 { 3398 return __dev_queue_xmit(skb, accel_priv); 3399 } 3400 EXPORT_SYMBOL(dev_queue_xmit_accel); 3401 3402 3403 /*======================================================================= 3404 Receiver routines 3405 =======================================================================*/ 3406 3407 int netdev_max_backlog __read_mostly = 1000; 3408 EXPORT_SYMBOL(netdev_max_backlog); 3409 3410 int netdev_tstamp_prequeue __read_mostly = 1; 3411 int netdev_budget __read_mostly = 300; 3412 int weight_p __read_mostly = 64; /* old backlog weight */ 3413 3414 /* Called with irq disabled */ 3415 static inline void ____napi_schedule(struct softnet_data *sd, 3416 struct napi_struct *napi) 3417 { 3418 list_add_tail(&napi->poll_list, &sd->poll_list); 3419 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 3420 } 3421 3422 #ifdef CONFIG_RPS 3423 3424 /* One global table that all flow-based protocols share. */ 3425 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; 3426 EXPORT_SYMBOL(rps_sock_flow_table); 3427 u32 rps_cpu_mask __read_mostly; 3428 EXPORT_SYMBOL(rps_cpu_mask); 3429 3430 struct static_key rps_needed __read_mostly; 3431 3432 static struct rps_dev_flow * 3433 set_rps_cpu(struct net_device *dev, struct sk_buff *skb, 3434 struct rps_dev_flow *rflow, u16 next_cpu) 3435 { 3436 if (next_cpu < nr_cpu_ids) { 3437 #ifdef CONFIG_RFS_ACCEL 3438 struct netdev_rx_queue *rxqueue; 3439 struct rps_dev_flow_table *flow_table; 3440 struct rps_dev_flow *old_rflow; 3441 u32 flow_id; 3442 u16 rxq_index; 3443 int rc; 3444 3445 /* Should we steer this flow to a different hardware queue? */ 3446 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || 3447 !(dev->features & NETIF_F_NTUPLE)) 3448 goto out; 3449 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); 3450 if (rxq_index == skb_get_rx_queue(skb)) 3451 goto out; 3452 3453 rxqueue = dev->_rx + rxq_index; 3454 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3455 if (!flow_table) 3456 goto out; 3457 flow_id = skb_get_hash(skb) & flow_table->mask; 3458 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, 3459 rxq_index, flow_id); 3460 if (rc < 0) 3461 goto out; 3462 old_rflow = rflow; 3463 rflow = &flow_table->flows[flow_id]; 3464 rflow->filter = rc; 3465 if (old_rflow->filter == rflow->filter) 3466 old_rflow->filter = RPS_NO_FILTER; 3467 out: 3468 #endif 3469 rflow->last_qtail = 3470 per_cpu(softnet_data, next_cpu).input_queue_head; 3471 } 3472 3473 rflow->cpu = next_cpu; 3474 return rflow; 3475 } 3476 3477 /* 3478 * get_rps_cpu is called from netif_receive_skb and returns the target 3479 * CPU from the RPS map of the receiving queue for a given skb. 3480 * rcu_read_lock must be held on entry. 3481 */ 3482 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, 3483 struct rps_dev_flow **rflowp) 3484 { 3485 const struct rps_sock_flow_table *sock_flow_table; 3486 struct netdev_rx_queue *rxqueue = dev->_rx; 3487 struct rps_dev_flow_table *flow_table; 3488 struct rps_map *map; 3489 int cpu = -1; 3490 u32 tcpu; 3491 u32 hash; 3492 3493 if (skb_rx_queue_recorded(skb)) { 3494 u16 index = skb_get_rx_queue(skb); 3495 3496 if (unlikely(index >= dev->real_num_rx_queues)) { 3497 WARN_ONCE(dev->real_num_rx_queues > 1, 3498 "%s received packet on queue %u, but number " 3499 "of RX queues is %u\n", 3500 dev->name, index, dev->real_num_rx_queues); 3501 goto done; 3502 } 3503 rxqueue += index; 3504 } 3505 3506 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */ 3507 3508 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3509 map = rcu_dereference(rxqueue->rps_map); 3510 if (!flow_table && !map) 3511 goto done; 3512 3513 skb_reset_network_header(skb); 3514 hash = skb_get_hash(skb); 3515 if (!hash) 3516 goto done; 3517 3518 sock_flow_table = rcu_dereference(rps_sock_flow_table); 3519 if (flow_table && sock_flow_table) { 3520 struct rps_dev_flow *rflow; 3521 u32 next_cpu; 3522 u32 ident; 3523 3524 /* First check into global flow table if there is a match */ 3525 ident = sock_flow_table->ents[hash & sock_flow_table->mask]; 3526 if ((ident ^ hash) & ~rps_cpu_mask) 3527 goto try_rps; 3528 3529 next_cpu = ident & rps_cpu_mask; 3530 3531 /* OK, now we know there is a match, 3532 * we can look at the local (per receive queue) flow table 3533 */ 3534 rflow = &flow_table->flows[hash & flow_table->mask]; 3535 tcpu = rflow->cpu; 3536 3537 /* 3538 * If the desired CPU (where last recvmsg was done) is 3539 * different from current CPU (one in the rx-queue flow 3540 * table entry), switch if one of the following holds: 3541 * - Current CPU is unset (>= nr_cpu_ids). 3542 * - Current CPU is offline. 3543 * - The current CPU's queue tail has advanced beyond the 3544 * last packet that was enqueued using this table entry. 3545 * This guarantees that all previous packets for the flow 3546 * have been dequeued, thus preserving in order delivery. 3547 */ 3548 if (unlikely(tcpu != next_cpu) && 3549 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) || 3550 ((int)(per_cpu(softnet_data, tcpu).input_queue_head - 3551 rflow->last_qtail)) >= 0)) { 3552 tcpu = next_cpu; 3553 rflow = set_rps_cpu(dev, skb, rflow, next_cpu); 3554 } 3555 3556 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { 3557 *rflowp = rflow; 3558 cpu = tcpu; 3559 goto done; 3560 } 3561 } 3562 3563 try_rps: 3564 3565 if (map) { 3566 tcpu = map->cpus[reciprocal_scale(hash, map->len)]; 3567 if (cpu_online(tcpu)) { 3568 cpu = tcpu; 3569 goto done; 3570 } 3571 } 3572 3573 done: 3574 return cpu; 3575 } 3576 3577 #ifdef CONFIG_RFS_ACCEL 3578 3579 /** 3580 * rps_may_expire_flow - check whether an RFS hardware filter may be removed 3581 * @dev: Device on which the filter was set 3582 * @rxq_index: RX queue index 3583 * @flow_id: Flow ID passed to ndo_rx_flow_steer() 3584 * @filter_id: Filter ID returned by ndo_rx_flow_steer() 3585 * 3586 * Drivers that implement ndo_rx_flow_steer() should periodically call 3587 * this function for each installed filter and remove the filters for 3588 * which it returns %true. 3589 */ 3590 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, 3591 u32 flow_id, u16 filter_id) 3592 { 3593 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; 3594 struct rps_dev_flow_table *flow_table; 3595 struct rps_dev_flow *rflow; 3596 bool expire = true; 3597 unsigned int cpu; 3598 3599 rcu_read_lock(); 3600 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3601 if (flow_table && flow_id <= flow_table->mask) { 3602 rflow = &flow_table->flows[flow_id]; 3603 cpu = ACCESS_ONCE(rflow->cpu); 3604 if (rflow->filter == filter_id && cpu < nr_cpu_ids && 3605 ((int)(per_cpu(softnet_data, cpu).input_queue_head - 3606 rflow->last_qtail) < 3607 (int)(10 * flow_table->mask))) 3608 expire = false; 3609 } 3610 rcu_read_unlock(); 3611 return expire; 3612 } 3613 EXPORT_SYMBOL(rps_may_expire_flow); 3614 3615 #endif /* CONFIG_RFS_ACCEL */ 3616 3617 /* Called from hardirq (IPI) context */ 3618 static void rps_trigger_softirq(void *data) 3619 { 3620 struct softnet_data *sd = data; 3621 3622 ____napi_schedule(sd, &sd->backlog); 3623 sd->received_rps++; 3624 } 3625 3626 #endif /* CONFIG_RPS */ 3627 3628 /* 3629 * Check if this softnet_data structure is another cpu one 3630 * If yes, queue it to our IPI list and return 1 3631 * If no, return 0 3632 */ 3633 static int rps_ipi_queued(struct softnet_data *sd) 3634 { 3635 #ifdef CONFIG_RPS 3636 struct softnet_data *mysd = this_cpu_ptr(&softnet_data); 3637 3638 if (sd != mysd) { 3639 sd->rps_ipi_next = mysd->rps_ipi_list; 3640 mysd->rps_ipi_list = sd; 3641 3642 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 3643 return 1; 3644 } 3645 #endif /* CONFIG_RPS */ 3646 return 0; 3647 } 3648 3649 #ifdef CONFIG_NET_FLOW_LIMIT 3650 int netdev_flow_limit_table_len __read_mostly = (1 << 12); 3651 #endif 3652 3653 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) 3654 { 3655 #ifdef CONFIG_NET_FLOW_LIMIT 3656 struct sd_flow_limit *fl; 3657 struct softnet_data *sd; 3658 unsigned int old_flow, new_flow; 3659 3660 if (qlen < (netdev_max_backlog >> 1)) 3661 return false; 3662 3663 sd = this_cpu_ptr(&softnet_data); 3664 3665 rcu_read_lock(); 3666 fl = rcu_dereference(sd->flow_limit); 3667 if (fl) { 3668 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1); 3669 old_flow = fl->history[fl->history_head]; 3670 fl->history[fl->history_head] = new_flow; 3671 3672 fl->history_head++; 3673 fl->history_head &= FLOW_LIMIT_HISTORY - 1; 3674 3675 if (likely(fl->buckets[old_flow])) 3676 fl->buckets[old_flow]--; 3677 3678 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { 3679 fl->count++; 3680 rcu_read_unlock(); 3681 return true; 3682 } 3683 } 3684 rcu_read_unlock(); 3685 #endif 3686 return false; 3687 } 3688 3689 /* 3690 * enqueue_to_backlog is called to queue an skb to a per CPU backlog 3691 * queue (may be a remote CPU queue). 3692 */ 3693 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, 3694 unsigned int *qtail) 3695 { 3696 struct softnet_data *sd; 3697 unsigned long flags; 3698 unsigned int qlen; 3699 3700 sd = &per_cpu(softnet_data, cpu); 3701 3702 local_irq_save(flags); 3703 3704 rps_lock(sd); 3705 if (!netif_running(skb->dev)) 3706 goto drop; 3707 qlen = skb_queue_len(&sd->input_pkt_queue); 3708 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) { 3709 if (qlen) { 3710 enqueue: 3711 __skb_queue_tail(&sd->input_pkt_queue, skb); 3712 input_queue_tail_incr_save(sd, qtail); 3713 rps_unlock(sd); 3714 local_irq_restore(flags); 3715 return NET_RX_SUCCESS; 3716 } 3717 3718 /* Schedule NAPI for backlog device 3719 * We can use non atomic operation since we own the queue lock 3720 */ 3721 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) { 3722 if (!rps_ipi_queued(sd)) 3723 ____napi_schedule(sd, &sd->backlog); 3724 } 3725 goto enqueue; 3726 } 3727 3728 drop: 3729 sd->dropped++; 3730 rps_unlock(sd); 3731 3732 local_irq_restore(flags); 3733 3734 atomic_long_inc(&skb->dev->rx_dropped); 3735 kfree_skb(skb); 3736 return NET_RX_DROP; 3737 } 3738 3739 static int netif_rx_internal(struct sk_buff *skb) 3740 { 3741 int ret; 3742 3743 net_timestamp_check(netdev_tstamp_prequeue, skb); 3744 3745 trace_netif_rx(skb); 3746 #ifdef CONFIG_RPS 3747 if (static_key_false(&rps_needed)) { 3748 struct rps_dev_flow voidflow, *rflow = &voidflow; 3749 int cpu; 3750 3751 preempt_disable(); 3752 rcu_read_lock(); 3753 3754 cpu = get_rps_cpu(skb->dev, skb, &rflow); 3755 if (cpu < 0) 3756 cpu = smp_processor_id(); 3757 3758 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 3759 3760 rcu_read_unlock(); 3761 preempt_enable(); 3762 } else 3763 #endif 3764 { 3765 unsigned int qtail; 3766 ret = enqueue_to_backlog(skb, get_cpu(), &qtail); 3767 put_cpu(); 3768 } 3769 return ret; 3770 } 3771 3772 /** 3773 * netif_rx - post buffer to the network code 3774 * @skb: buffer to post 3775 * 3776 * This function receives a packet from a device driver and queues it for 3777 * the upper (protocol) levels to process. It always succeeds. The buffer 3778 * may be dropped during processing for congestion control or by the 3779 * protocol layers. 3780 * 3781 * return values: 3782 * NET_RX_SUCCESS (no congestion) 3783 * NET_RX_DROP (packet was dropped) 3784 * 3785 */ 3786 3787 int netif_rx(struct sk_buff *skb) 3788 { 3789 trace_netif_rx_entry(skb); 3790 3791 return netif_rx_internal(skb); 3792 } 3793 EXPORT_SYMBOL(netif_rx); 3794 3795 int netif_rx_ni(struct sk_buff *skb) 3796 { 3797 int err; 3798 3799 trace_netif_rx_ni_entry(skb); 3800 3801 preempt_disable(); 3802 err = netif_rx_internal(skb); 3803 if (local_softirq_pending()) 3804 do_softirq(); 3805 preempt_enable(); 3806 3807 return err; 3808 } 3809 EXPORT_SYMBOL(netif_rx_ni); 3810 3811 static void net_tx_action(struct softirq_action *h) 3812 { 3813 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 3814 3815 if (sd->completion_queue) { 3816 struct sk_buff *clist; 3817 3818 local_irq_disable(); 3819 clist = sd->completion_queue; 3820 sd->completion_queue = NULL; 3821 local_irq_enable(); 3822 3823 while (clist) { 3824 struct sk_buff *skb = clist; 3825 clist = clist->next; 3826 3827 WARN_ON(atomic_read(&skb->users)); 3828 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED)) 3829 trace_consume_skb(skb); 3830 else 3831 trace_kfree_skb(skb, net_tx_action); 3832 3833 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) 3834 __kfree_skb(skb); 3835 else 3836 __kfree_skb_defer(skb); 3837 } 3838 3839 __kfree_skb_flush(); 3840 } 3841 3842 if (sd->output_queue) { 3843 struct Qdisc *head; 3844 3845 local_irq_disable(); 3846 head = sd->output_queue; 3847 sd->output_queue = NULL; 3848 sd->output_queue_tailp = &sd->output_queue; 3849 local_irq_enable(); 3850 3851 while (head) { 3852 struct Qdisc *q = head; 3853 spinlock_t *root_lock; 3854 3855 head = head->next_sched; 3856 3857 root_lock = qdisc_lock(q); 3858 if (spin_trylock(root_lock)) { 3859 smp_mb__before_atomic(); 3860 clear_bit(__QDISC_STATE_SCHED, 3861 &q->state); 3862 qdisc_run(q); 3863 spin_unlock(root_lock); 3864 } else { 3865 if (!test_bit(__QDISC_STATE_DEACTIVATED, 3866 &q->state)) { 3867 __netif_reschedule(q); 3868 } else { 3869 smp_mb__before_atomic(); 3870 clear_bit(__QDISC_STATE_SCHED, 3871 &q->state); 3872 } 3873 } 3874 } 3875 } 3876 } 3877 3878 #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \ 3879 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE)) 3880 /* This hook is defined here for ATM LANE */ 3881 int (*br_fdb_test_addr_hook)(struct net_device *dev, 3882 unsigned char *addr) __read_mostly; 3883 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); 3884 #endif 3885 3886 static inline struct sk_buff * 3887 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, 3888 struct net_device *orig_dev) 3889 { 3890 #ifdef CONFIG_NET_CLS_ACT 3891 struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list); 3892 struct tcf_result cl_res; 3893 3894 /* If there's at least one ingress present somewhere (so 3895 * we get here via enabled static key), remaining devices 3896 * that are not configured with an ingress qdisc will bail 3897 * out here. 3898 */ 3899 if (!cl) 3900 return skb; 3901 if (*pt_prev) { 3902 *ret = deliver_skb(skb, *pt_prev, orig_dev); 3903 *pt_prev = NULL; 3904 } 3905 3906 qdisc_skb_cb(skb)->pkt_len = skb->len; 3907 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS); 3908 qdisc_bstats_cpu_update(cl->q, skb); 3909 3910 switch (tc_classify(skb, cl, &cl_res, false)) { 3911 case TC_ACT_OK: 3912 case TC_ACT_RECLASSIFY: 3913 skb->tc_index = TC_H_MIN(cl_res.classid); 3914 break; 3915 case TC_ACT_SHOT: 3916 qdisc_qstats_cpu_drop(cl->q); 3917 case TC_ACT_STOLEN: 3918 case TC_ACT_QUEUED: 3919 kfree_skb(skb); 3920 return NULL; 3921 case TC_ACT_REDIRECT: 3922 /* skb_mac_header check was done by cls/act_bpf, so 3923 * we can safely push the L2 header back before 3924 * redirecting to another netdev 3925 */ 3926 __skb_push(skb, skb->mac_len); 3927 skb_do_redirect(skb); 3928 return NULL; 3929 default: 3930 break; 3931 } 3932 #endif /* CONFIG_NET_CLS_ACT */ 3933 return skb; 3934 } 3935 3936 /** 3937 * netdev_rx_handler_register - register receive handler 3938 * @dev: device to register a handler for 3939 * @rx_handler: receive handler to register 3940 * @rx_handler_data: data pointer that is used by rx handler 3941 * 3942 * Register a receive handler for a device. This handler will then be 3943 * called from __netif_receive_skb. A negative errno code is returned 3944 * on a failure. 3945 * 3946 * The caller must hold the rtnl_mutex. 3947 * 3948 * For a general description of rx_handler, see enum rx_handler_result. 3949 */ 3950 int netdev_rx_handler_register(struct net_device *dev, 3951 rx_handler_func_t *rx_handler, 3952 void *rx_handler_data) 3953 { 3954 ASSERT_RTNL(); 3955 3956 if (dev->rx_handler) 3957 return -EBUSY; 3958 3959 /* Note: rx_handler_data must be set before rx_handler */ 3960 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); 3961 rcu_assign_pointer(dev->rx_handler, rx_handler); 3962 3963 return 0; 3964 } 3965 EXPORT_SYMBOL_GPL(netdev_rx_handler_register); 3966 3967 /** 3968 * netdev_rx_handler_unregister - unregister receive handler 3969 * @dev: device to unregister a handler from 3970 * 3971 * Unregister a receive handler from a device. 3972 * 3973 * The caller must hold the rtnl_mutex. 3974 */ 3975 void netdev_rx_handler_unregister(struct net_device *dev) 3976 { 3977 3978 ASSERT_RTNL(); 3979 RCU_INIT_POINTER(dev->rx_handler, NULL); 3980 /* a reader seeing a non NULL rx_handler in a rcu_read_lock() 3981 * section has a guarantee to see a non NULL rx_handler_data 3982 * as well. 3983 */ 3984 synchronize_net(); 3985 RCU_INIT_POINTER(dev->rx_handler_data, NULL); 3986 } 3987 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); 3988 3989 /* 3990 * Limit the use of PFMEMALLOC reserves to those protocols that implement 3991 * the special handling of PFMEMALLOC skbs. 3992 */ 3993 static bool skb_pfmemalloc_protocol(struct sk_buff *skb) 3994 { 3995 switch (skb->protocol) { 3996 case htons(ETH_P_ARP): 3997 case htons(ETH_P_IP): 3998 case htons(ETH_P_IPV6): 3999 case htons(ETH_P_8021Q): 4000 case htons(ETH_P_8021AD): 4001 return true; 4002 default: 4003 return false; 4004 } 4005 } 4006 4007 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev, 4008 int *ret, struct net_device *orig_dev) 4009 { 4010 #ifdef CONFIG_NETFILTER_INGRESS 4011 if (nf_hook_ingress_active(skb)) { 4012 if (*pt_prev) { 4013 *ret = deliver_skb(skb, *pt_prev, orig_dev); 4014 *pt_prev = NULL; 4015 } 4016 4017 return nf_hook_ingress(skb); 4018 } 4019 #endif /* CONFIG_NETFILTER_INGRESS */ 4020 return 0; 4021 } 4022 4023 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc) 4024 { 4025 struct packet_type *ptype, *pt_prev; 4026 rx_handler_func_t *rx_handler; 4027 struct net_device *orig_dev; 4028 bool deliver_exact = false; 4029 int ret = NET_RX_DROP; 4030 __be16 type; 4031 4032 net_timestamp_check(!netdev_tstamp_prequeue, skb); 4033 4034 trace_netif_receive_skb(skb); 4035 4036 orig_dev = skb->dev; 4037 4038 skb_reset_network_header(skb); 4039 if (!skb_transport_header_was_set(skb)) 4040 skb_reset_transport_header(skb); 4041 skb_reset_mac_len(skb); 4042 4043 pt_prev = NULL; 4044 4045 another_round: 4046 skb->skb_iif = skb->dev->ifindex; 4047 4048 __this_cpu_inc(softnet_data.processed); 4049 4050 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 4051 skb->protocol == cpu_to_be16(ETH_P_8021AD)) { 4052 skb = skb_vlan_untag(skb); 4053 if (unlikely(!skb)) 4054 goto out; 4055 } 4056 4057 #ifdef CONFIG_NET_CLS_ACT 4058 if (skb->tc_verd & TC_NCLS) { 4059 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd); 4060 goto ncls; 4061 } 4062 #endif 4063 4064 if (pfmemalloc) 4065 goto skip_taps; 4066 4067 list_for_each_entry_rcu(ptype, &ptype_all, list) { 4068 if (pt_prev) 4069 ret = deliver_skb(skb, pt_prev, orig_dev); 4070 pt_prev = ptype; 4071 } 4072 4073 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { 4074 if (pt_prev) 4075 ret = deliver_skb(skb, pt_prev, orig_dev); 4076 pt_prev = ptype; 4077 } 4078 4079 skip_taps: 4080 #ifdef CONFIG_NET_INGRESS 4081 if (static_key_false(&ingress_needed)) { 4082 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev); 4083 if (!skb) 4084 goto out; 4085 4086 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0) 4087 goto out; 4088 } 4089 #endif 4090 #ifdef CONFIG_NET_CLS_ACT 4091 skb->tc_verd = 0; 4092 ncls: 4093 #endif 4094 if (pfmemalloc && !skb_pfmemalloc_protocol(skb)) 4095 goto drop; 4096 4097 if (skb_vlan_tag_present(skb)) { 4098 if (pt_prev) { 4099 ret = deliver_skb(skb, pt_prev, orig_dev); 4100 pt_prev = NULL; 4101 } 4102 if (vlan_do_receive(&skb)) 4103 goto another_round; 4104 else if (unlikely(!skb)) 4105 goto out; 4106 } 4107 4108 rx_handler = rcu_dereference(skb->dev->rx_handler); 4109 if (rx_handler) { 4110 if (pt_prev) { 4111 ret = deliver_skb(skb, pt_prev, orig_dev); 4112 pt_prev = NULL; 4113 } 4114 switch (rx_handler(&skb)) { 4115 case RX_HANDLER_CONSUMED: 4116 ret = NET_RX_SUCCESS; 4117 goto out; 4118 case RX_HANDLER_ANOTHER: 4119 goto another_round; 4120 case RX_HANDLER_EXACT: 4121 deliver_exact = true; 4122 case RX_HANDLER_PASS: 4123 break; 4124 default: 4125 BUG(); 4126 } 4127 } 4128 4129 if (unlikely(skb_vlan_tag_present(skb))) { 4130 if (skb_vlan_tag_get_id(skb)) 4131 skb->pkt_type = PACKET_OTHERHOST; 4132 /* Note: we might in the future use prio bits 4133 * and set skb->priority like in vlan_do_receive() 4134 * For the time being, just ignore Priority Code Point 4135 */ 4136 skb->vlan_tci = 0; 4137 } 4138 4139 type = skb->protocol; 4140 4141 /* deliver only exact match when indicated */ 4142 if (likely(!deliver_exact)) { 4143 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4144 &ptype_base[ntohs(type) & 4145 PTYPE_HASH_MASK]); 4146 } 4147 4148 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4149 &orig_dev->ptype_specific); 4150 4151 if (unlikely(skb->dev != orig_dev)) { 4152 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4153 &skb->dev->ptype_specific); 4154 } 4155 4156 if (pt_prev) { 4157 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) 4158 goto drop; 4159 else 4160 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 4161 } else { 4162 drop: 4163 if (!deliver_exact) 4164 atomic_long_inc(&skb->dev->rx_dropped); 4165 else 4166 atomic_long_inc(&skb->dev->rx_nohandler); 4167 kfree_skb(skb); 4168 /* Jamal, now you will not able to escape explaining 4169 * me how you were going to use this. :-) 4170 */ 4171 ret = NET_RX_DROP; 4172 } 4173 4174 out: 4175 return ret; 4176 } 4177 4178 static int __netif_receive_skb(struct sk_buff *skb) 4179 { 4180 int ret; 4181 4182 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) { 4183 unsigned long pflags = current->flags; 4184 4185 /* 4186 * PFMEMALLOC skbs are special, they should 4187 * - be delivered to SOCK_MEMALLOC sockets only 4188 * - stay away from userspace 4189 * - have bounded memory usage 4190 * 4191 * Use PF_MEMALLOC as this saves us from propagating the allocation 4192 * context down to all allocation sites. 4193 */ 4194 current->flags |= PF_MEMALLOC; 4195 ret = __netif_receive_skb_core(skb, true); 4196 tsk_restore_flags(current, pflags, PF_MEMALLOC); 4197 } else 4198 ret = __netif_receive_skb_core(skb, false); 4199 4200 return ret; 4201 } 4202 4203 static int netif_receive_skb_internal(struct sk_buff *skb) 4204 { 4205 int ret; 4206 4207 net_timestamp_check(netdev_tstamp_prequeue, skb); 4208 4209 if (skb_defer_rx_timestamp(skb)) 4210 return NET_RX_SUCCESS; 4211 4212 rcu_read_lock(); 4213 4214 #ifdef CONFIG_RPS 4215 if (static_key_false(&rps_needed)) { 4216 struct rps_dev_flow voidflow, *rflow = &voidflow; 4217 int cpu = get_rps_cpu(skb->dev, skb, &rflow); 4218 4219 if (cpu >= 0) { 4220 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 4221 rcu_read_unlock(); 4222 return ret; 4223 } 4224 } 4225 #endif 4226 ret = __netif_receive_skb(skb); 4227 rcu_read_unlock(); 4228 return ret; 4229 } 4230 4231 /** 4232 * netif_receive_skb - process receive buffer from network 4233 * @skb: buffer to process 4234 * 4235 * netif_receive_skb() is the main receive data processing function. 4236 * It always succeeds. The buffer may be dropped during processing 4237 * for congestion control or by the protocol layers. 4238 * 4239 * This function may only be called from softirq context and interrupts 4240 * should be enabled. 4241 * 4242 * Return values (usually ignored): 4243 * NET_RX_SUCCESS: no congestion 4244 * NET_RX_DROP: packet was dropped 4245 */ 4246 int netif_receive_skb(struct sk_buff *skb) 4247 { 4248 trace_netif_receive_skb_entry(skb); 4249 4250 return netif_receive_skb_internal(skb); 4251 } 4252 EXPORT_SYMBOL(netif_receive_skb); 4253 4254 /* Network device is going away, flush any packets still pending 4255 * Called with irqs disabled. 4256 */ 4257 static void flush_backlog(void *arg) 4258 { 4259 struct net_device *dev = arg; 4260 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 4261 struct sk_buff *skb, *tmp; 4262 4263 rps_lock(sd); 4264 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { 4265 if (skb->dev == dev) { 4266 __skb_unlink(skb, &sd->input_pkt_queue); 4267 kfree_skb(skb); 4268 input_queue_head_incr(sd); 4269 } 4270 } 4271 rps_unlock(sd); 4272 4273 skb_queue_walk_safe(&sd->process_queue, skb, tmp) { 4274 if (skb->dev == dev) { 4275 __skb_unlink(skb, &sd->process_queue); 4276 kfree_skb(skb); 4277 input_queue_head_incr(sd); 4278 } 4279 } 4280 } 4281 4282 static int napi_gro_complete(struct sk_buff *skb) 4283 { 4284 struct packet_offload *ptype; 4285 __be16 type = skb->protocol; 4286 struct list_head *head = &offload_base; 4287 int err = -ENOENT; 4288 4289 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); 4290 4291 if (NAPI_GRO_CB(skb)->count == 1) { 4292 skb_shinfo(skb)->gso_size = 0; 4293 goto out; 4294 } 4295 4296 rcu_read_lock(); 4297 list_for_each_entry_rcu(ptype, head, list) { 4298 if (ptype->type != type || !ptype->callbacks.gro_complete) 4299 continue; 4300 4301 err = ptype->callbacks.gro_complete(skb, 0); 4302 break; 4303 } 4304 rcu_read_unlock(); 4305 4306 if (err) { 4307 WARN_ON(&ptype->list == head); 4308 kfree_skb(skb); 4309 return NET_RX_SUCCESS; 4310 } 4311 4312 out: 4313 return netif_receive_skb_internal(skb); 4314 } 4315 4316 /* napi->gro_list contains packets ordered by age. 4317 * youngest packets at the head of it. 4318 * Complete skbs in reverse order to reduce latencies. 4319 */ 4320 void napi_gro_flush(struct napi_struct *napi, bool flush_old) 4321 { 4322 struct sk_buff *skb, *prev = NULL; 4323 4324 /* scan list and build reverse chain */ 4325 for (skb = napi->gro_list; skb != NULL; skb = skb->next) { 4326 skb->prev = prev; 4327 prev = skb; 4328 } 4329 4330 for (skb = prev; skb; skb = prev) { 4331 skb->next = NULL; 4332 4333 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) 4334 return; 4335 4336 prev = skb->prev; 4337 napi_gro_complete(skb); 4338 napi->gro_count--; 4339 } 4340 4341 napi->gro_list = NULL; 4342 } 4343 EXPORT_SYMBOL(napi_gro_flush); 4344 4345 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb) 4346 { 4347 struct sk_buff *p; 4348 unsigned int maclen = skb->dev->hard_header_len; 4349 u32 hash = skb_get_hash_raw(skb); 4350 4351 for (p = napi->gro_list; p; p = p->next) { 4352 unsigned long diffs; 4353 4354 NAPI_GRO_CB(p)->flush = 0; 4355 4356 if (hash != skb_get_hash_raw(p)) { 4357 NAPI_GRO_CB(p)->same_flow = 0; 4358 continue; 4359 } 4360 4361 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; 4362 diffs |= p->vlan_tci ^ skb->vlan_tci; 4363 diffs |= skb_metadata_dst_cmp(p, skb); 4364 if (maclen == ETH_HLEN) 4365 diffs |= compare_ether_header(skb_mac_header(p), 4366 skb_mac_header(skb)); 4367 else if (!diffs) 4368 diffs = memcmp(skb_mac_header(p), 4369 skb_mac_header(skb), 4370 maclen); 4371 NAPI_GRO_CB(p)->same_flow = !diffs; 4372 } 4373 } 4374 4375 static void skb_gro_reset_offset(struct sk_buff *skb) 4376 { 4377 const struct skb_shared_info *pinfo = skb_shinfo(skb); 4378 const skb_frag_t *frag0 = &pinfo->frags[0]; 4379 4380 NAPI_GRO_CB(skb)->data_offset = 0; 4381 NAPI_GRO_CB(skb)->frag0 = NULL; 4382 NAPI_GRO_CB(skb)->frag0_len = 0; 4383 4384 if (skb_mac_header(skb) == skb_tail_pointer(skb) && 4385 pinfo->nr_frags && 4386 !PageHighMem(skb_frag_page(frag0))) { 4387 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); 4388 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0); 4389 } 4390 } 4391 4392 static void gro_pull_from_frag0(struct sk_buff *skb, int grow) 4393 { 4394 struct skb_shared_info *pinfo = skb_shinfo(skb); 4395 4396 BUG_ON(skb->end - skb->tail < grow); 4397 4398 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); 4399 4400 skb->data_len -= grow; 4401 skb->tail += grow; 4402 4403 pinfo->frags[0].page_offset += grow; 4404 skb_frag_size_sub(&pinfo->frags[0], grow); 4405 4406 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { 4407 skb_frag_unref(skb, 0); 4408 memmove(pinfo->frags, pinfo->frags + 1, 4409 --pinfo->nr_frags * sizeof(pinfo->frags[0])); 4410 } 4411 } 4412 4413 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 4414 { 4415 struct sk_buff **pp = NULL; 4416 struct packet_offload *ptype; 4417 __be16 type = skb->protocol; 4418 struct list_head *head = &offload_base; 4419 int same_flow; 4420 enum gro_result ret; 4421 int grow; 4422 4423 if (!(skb->dev->features & NETIF_F_GRO)) 4424 goto normal; 4425 4426 if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad) 4427 goto normal; 4428 4429 gro_list_prepare(napi, skb); 4430 4431 rcu_read_lock(); 4432 list_for_each_entry_rcu(ptype, head, list) { 4433 if (ptype->type != type || !ptype->callbacks.gro_receive) 4434 continue; 4435 4436 skb_set_network_header(skb, skb_gro_offset(skb)); 4437 skb_reset_mac_len(skb); 4438 NAPI_GRO_CB(skb)->same_flow = 0; 4439 NAPI_GRO_CB(skb)->flush = 0; 4440 NAPI_GRO_CB(skb)->free = 0; 4441 NAPI_GRO_CB(skb)->encap_mark = 0; 4442 NAPI_GRO_CB(skb)->gro_remcsum_start = 0; 4443 4444 /* Setup for GRO checksum validation */ 4445 switch (skb->ip_summed) { 4446 case CHECKSUM_COMPLETE: 4447 NAPI_GRO_CB(skb)->csum = skb->csum; 4448 NAPI_GRO_CB(skb)->csum_valid = 1; 4449 NAPI_GRO_CB(skb)->csum_cnt = 0; 4450 break; 4451 case CHECKSUM_UNNECESSARY: 4452 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; 4453 NAPI_GRO_CB(skb)->csum_valid = 0; 4454 break; 4455 default: 4456 NAPI_GRO_CB(skb)->csum_cnt = 0; 4457 NAPI_GRO_CB(skb)->csum_valid = 0; 4458 } 4459 4460 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb); 4461 break; 4462 } 4463 rcu_read_unlock(); 4464 4465 if (&ptype->list == head) 4466 goto normal; 4467 4468 same_flow = NAPI_GRO_CB(skb)->same_flow; 4469 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; 4470 4471 if (pp) { 4472 struct sk_buff *nskb = *pp; 4473 4474 *pp = nskb->next; 4475 nskb->next = NULL; 4476 napi_gro_complete(nskb); 4477 napi->gro_count--; 4478 } 4479 4480 if (same_flow) 4481 goto ok; 4482 4483 if (NAPI_GRO_CB(skb)->flush) 4484 goto normal; 4485 4486 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) { 4487 struct sk_buff *nskb = napi->gro_list; 4488 4489 /* locate the end of the list to select the 'oldest' flow */ 4490 while (nskb->next) { 4491 pp = &nskb->next; 4492 nskb = *pp; 4493 } 4494 *pp = NULL; 4495 nskb->next = NULL; 4496 napi_gro_complete(nskb); 4497 } else { 4498 napi->gro_count++; 4499 } 4500 NAPI_GRO_CB(skb)->count = 1; 4501 NAPI_GRO_CB(skb)->age = jiffies; 4502 NAPI_GRO_CB(skb)->last = skb; 4503 skb_shinfo(skb)->gso_size = skb_gro_len(skb); 4504 skb->next = napi->gro_list; 4505 napi->gro_list = skb; 4506 ret = GRO_HELD; 4507 4508 pull: 4509 grow = skb_gro_offset(skb) - skb_headlen(skb); 4510 if (grow > 0) 4511 gro_pull_from_frag0(skb, grow); 4512 ok: 4513 return ret; 4514 4515 normal: 4516 ret = GRO_NORMAL; 4517 goto pull; 4518 } 4519 4520 struct packet_offload *gro_find_receive_by_type(__be16 type) 4521 { 4522 struct list_head *offload_head = &offload_base; 4523 struct packet_offload *ptype; 4524 4525 list_for_each_entry_rcu(ptype, offload_head, list) { 4526 if (ptype->type != type || !ptype->callbacks.gro_receive) 4527 continue; 4528 return ptype; 4529 } 4530 return NULL; 4531 } 4532 EXPORT_SYMBOL(gro_find_receive_by_type); 4533 4534 struct packet_offload *gro_find_complete_by_type(__be16 type) 4535 { 4536 struct list_head *offload_head = &offload_base; 4537 struct packet_offload *ptype; 4538 4539 list_for_each_entry_rcu(ptype, offload_head, list) { 4540 if (ptype->type != type || !ptype->callbacks.gro_complete) 4541 continue; 4542 return ptype; 4543 } 4544 return NULL; 4545 } 4546 EXPORT_SYMBOL(gro_find_complete_by_type); 4547 4548 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) 4549 { 4550 switch (ret) { 4551 case GRO_NORMAL: 4552 if (netif_receive_skb_internal(skb)) 4553 ret = GRO_DROP; 4554 break; 4555 4556 case GRO_DROP: 4557 kfree_skb(skb); 4558 break; 4559 4560 case GRO_MERGED_FREE: 4561 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) { 4562 skb_dst_drop(skb); 4563 kmem_cache_free(skbuff_head_cache, skb); 4564 } else { 4565 __kfree_skb(skb); 4566 } 4567 break; 4568 4569 case GRO_HELD: 4570 case GRO_MERGED: 4571 break; 4572 } 4573 4574 return ret; 4575 } 4576 4577 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 4578 { 4579 skb_mark_napi_id(skb, napi); 4580 trace_napi_gro_receive_entry(skb); 4581 4582 skb_gro_reset_offset(skb); 4583 4584 return napi_skb_finish(dev_gro_receive(napi, skb), skb); 4585 } 4586 EXPORT_SYMBOL(napi_gro_receive); 4587 4588 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) 4589 { 4590 if (unlikely(skb->pfmemalloc)) { 4591 consume_skb(skb); 4592 return; 4593 } 4594 __skb_pull(skb, skb_headlen(skb)); 4595 /* restore the reserve we had after netdev_alloc_skb_ip_align() */ 4596 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); 4597 skb->vlan_tci = 0; 4598 skb->dev = napi->dev; 4599 skb->skb_iif = 0; 4600 skb->encapsulation = 0; 4601 skb_shinfo(skb)->gso_type = 0; 4602 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 4603 4604 napi->skb = skb; 4605 } 4606 4607 struct sk_buff *napi_get_frags(struct napi_struct *napi) 4608 { 4609 struct sk_buff *skb = napi->skb; 4610 4611 if (!skb) { 4612 skb = napi_alloc_skb(napi, GRO_MAX_HEAD); 4613 if (skb) { 4614 napi->skb = skb; 4615 skb_mark_napi_id(skb, napi); 4616 } 4617 } 4618 return skb; 4619 } 4620 EXPORT_SYMBOL(napi_get_frags); 4621 4622 static gro_result_t napi_frags_finish(struct napi_struct *napi, 4623 struct sk_buff *skb, 4624 gro_result_t ret) 4625 { 4626 switch (ret) { 4627 case GRO_NORMAL: 4628 case GRO_HELD: 4629 __skb_push(skb, ETH_HLEN); 4630 skb->protocol = eth_type_trans(skb, skb->dev); 4631 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb)) 4632 ret = GRO_DROP; 4633 break; 4634 4635 case GRO_DROP: 4636 case GRO_MERGED_FREE: 4637 napi_reuse_skb(napi, skb); 4638 break; 4639 4640 case GRO_MERGED: 4641 break; 4642 } 4643 4644 return ret; 4645 } 4646 4647 /* Upper GRO stack assumes network header starts at gro_offset=0 4648 * Drivers could call both napi_gro_frags() and napi_gro_receive() 4649 * We copy ethernet header into skb->data to have a common layout. 4650 */ 4651 static struct sk_buff *napi_frags_skb(struct napi_struct *napi) 4652 { 4653 struct sk_buff *skb = napi->skb; 4654 const struct ethhdr *eth; 4655 unsigned int hlen = sizeof(*eth); 4656 4657 napi->skb = NULL; 4658 4659 skb_reset_mac_header(skb); 4660 skb_gro_reset_offset(skb); 4661 4662 eth = skb_gro_header_fast(skb, 0); 4663 if (unlikely(skb_gro_header_hard(skb, hlen))) { 4664 eth = skb_gro_header_slow(skb, hlen, 0); 4665 if (unlikely(!eth)) { 4666 napi_reuse_skb(napi, skb); 4667 return NULL; 4668 } 4669 } else { 4670 gro_pull_from_frag0(skb, hlen); 4671 NAPI_GRO_CB(skb)->frag0 += hlen; 4672 NAPI_GRO_CB(skb)->frag0_len -= hlen; 4673 } 4674 __skb_pull(skb, hlen); 4675 4676 /* 4677 * This works because the only protocols we care about don't require 4678 * special handling. 4679 * We'll fix it up properly in napi_frags_finish() 4680 */ 4681 skb->protocol = eth->h_proto; 4682 4683 return skb; 4684 } 4685 4686 gro_result_t napi_gro_frags(struct napi_struct *napi) 4687 { 4688 struct sk_buff *skb = napi_frags_skb(napi); 4689 4690 if (!skb) 4691 return GRO_DROP; 4692 4693 trace_napi_gro_frags_entry(skb); 4694 4695 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); 4696 } 4697 EXPORT_SYMBOL(napi_gro_frags); 4698 4699 /* Compute the checksum from gro_offset and return the folded value 4700 * after adding in any pseudo checksum. 4701 */ 4702 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb) 4703 { 4704 __wsum wsum; 4705 __sum16 sum; 4706 4707 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); 4708 4709 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ 4710 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); 4711 if (likely(!sum)) { 4712 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 4713 !skb->csum_complete_sw) 4714 netdev_rx_csum_fault(skb->dev); 4715 } 4716 4717 NAPI_GRO_CB(skb)->csum = wsum; 4718 NAPI_GRO_CB(skb)->csum_valid = 1; 4719 4720 return sum; 4721 } 4722 EXPORT_SYMBOL(__skb_gro_checksum_complete); 4723 4724 /* 4725 * net_rps_action_and_irq_enable sends any pending IPI's for rps. 4726 * Note: called with local irq disabled, but exits with local irq enabled. 4727 */ 4728 static void net_rps_action_and_irq_enable(struct softnet_data *sd) 4729 { 4730 #ifdef CONFIG_RPS 4731 struct softnet_data *remsd = sd->rps_ipi_list; 4732 4733 if (remsd) { 4734 sd->rps_ipi_list = NULL; 4735 4736 local_irq_enable(); 4737 4738 /* Send pending IPI's to kick RPS processing on remote cpus. */ 4739 while (remsd) { 4740 struct softnet_data *next = remsd->rps_ipi_next; 4741 4742 if (cpu_online(remsd->cpu)) 4743 smp_call_function_single_async(remsd->cpu, 4744 &remsd->csd); 4745 remsd = next; 4746 } 4747 } else 4748 #endif 4749 local_irq_enable(); 4750 } 4751 4752 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) 4753 { 4754 #ifdef CONFIG_RPS 4755 return sd->rps_ipi_list != NULL; 4756 #else 4757 return false; 4758 #endif 4759 } 4760 4761 static int process_backlog(struct napi_struct *napi, int quota) 4762 { 4763 int work = 0; 4764 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); 4765 4766 /* Check if we have pending ipi, its better to send them now, 4767 * not waiting net_rx_action() end. 4768 */ 4769 if (sd_has_rps_ipi_waiting(sd)) { 4770 local_irq_disable(); 4771 net_rps_action_and_irq_enable(sd); 4772 } 4773 4774 napi->weight = weight_p; 4775 local_irq_disable(); 4776 while (1) { 4777 struct sk_buff *skb; 4778 4779 while ((skb = __skb_dequeue(&sd->process_queue))) { 4780 rcu_read_lock(); 4781 local_irq_enable(); 4782 __netif_receive_skb(skb); 4783 rcu_read_unlock(); 4784 local_irq_disable(); 4785 input_queue_head_incr(sd); 4786 if (++work >= quota) { 4787 local_irq_enable(); 4788 return work; 4789 } 4790 } 4791 4792 rps_lock(sd); 4793 if (skb_queue_empty(&sd->input_pkt_queue)) { 4794 /* 4795 * Inline a custom version of __napi_complete(). 4796 * only current cpu owns and manipulates this napi, 4797 * and NAPI_STATE_SCHED is the only possible flag set 4798 * on backlog. 4799 * We can use a plain write instead of clear_bit(), 4800 * and we dont need an smp_mb() memory barrier. 4801 */ 4802 napi->state = 0; 4803 rps_unlock(sd); 4804 4805 break; 4806 } 4807 4808 skb_queue_splice_tail_init(&sd->input_pkt_queue, 4809 &sd->process_queue); 4810 rps_unlock(sd); 4811 } 4812 local_irq_enable(); 4813 4814 return work; 4815 } 4816 4817 /** 4818 * __napi_schedule - schedule for receive 4819 * @n: entry to schedule 4820 * 4821 * The entry's receive function will be scheduled to run. 4822 * Consider using __napi_schedule_irqoff() if hard irqs are masked. 4823 */ 4824 void __napi_schedule(struct napi_struct *n) 4825 { 4826 unsigned long flags; 4827 4828 local_irq_save(flags); 4829 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 4830 local_irq_restore(flags); 4831 } 4832 EXPORT_SYMBOL(__napi_schedule); 4833 4834 /** 4835 * __napi_schedule_irqoff - schedule for receive 4836 * @n: entry to schedule 4837 * 4838 * Variant of __napi_schedule() assuming hard irqs are masked 4839 */ 4840 void __napi_schedule_irqoff(struct napi_struct *n) 4841 { 4842 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 4843 } 4844 EXPORT_SYMBOL(__napi_schedule_irqoff); 4845 4846 void __napi_complete(struct napi_struct *n) 4847 { 4848 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state)); 4849 4850 list_del_init(&n->poll_list); 4851 smp_mb__before_atomic(); 4852 clear_bit(NAPI_STATE_SCHED, &n->state); 4853 } 4854 EXPORT_SYMBOL(__napi_complete); 4855 4856 void napi_complete_done(struct napi_struct *n, int work_done) 4857 { 4858 unsigned long flags; 4859 4860 /* 4861 * don't let napi dequeue from the cpu poll list 4862 * just in case its running on a different cpu 4863 */ 4864 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state))) 4865 return; 4866 4867 if (n->gro_list) { 4868 unsigned long timeout = 0; 4869 4870 if (work_done) 4871 timeout = n->dev->gro_flush_timeout; 4872 4873 if (timeout) 4874 hrtimer_start(&n->timer, ns_to_ktime(timeout), 4875 HRTIMER_MODE_REL_PINNED); 4876 else 4877 napi_gro_flush(n, false); 4878 } 4879 if (likely(list_empty(&n->poll_list))) { 4880 WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state)); 4881 } else { 4882 /* If n->poll_list is not empty, we need to mask irqs */ 4883 local_irq_save(flags); 4884 __napi_complete(n); 4885 local_irq_restore(flags); 4886 } 4887 } 4888 EXPORT_SYMBOL(napi_complete_done); 4889 4890 /* must be called under rcu_read_lock(), as we dont take a reference */ 4891 static struct napi_struct *napi_by_id(unsigned int napi_id) 4892 { 4893 unsigned int hash = napi_id % HASH_SIZE(napi_hash); 4894 struct napi_struct *napi; 4895 4896 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node) 4897 if (napi->napi_id == napi_id) 4898 return napi; 4899 4900 return NULL; 4901 } 4902 4903 #if defined(CONFIG_NET_RX_BUSY_POLL) 4904 #define BUSY_POLL_BUDGET 8 4905 bool sk_busy_loop(struct sock *sk, int nonblock) 4906 { 4907 unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0; 4908 int (*busy_poll)(struct napi_struct *dev); 4909 struct napi_struct *napi; 4910 int rc = false; 4911 4912 rcu_read_lock(); 4913 4914 napi = napi_by_id(sk->sk_napi_id); 4915 if (!napi) 4916 goto out; 4917 4918 /* Note: ndo_busy_poll method is optional in linux-4.5 */ 4919 busy_poll = napi->dev->netdev_ops->ndo_busy_poll; 4920 4921 do { 4922 rc = 0; 4923 local_bh_disable(); 4924 if (busy_poll) { 4925 rc = busy_poll(napi); 4926 } else if (napi_schedule_prep(napi)) { 4927 void *have = netpoll_poll_lock(napi); 4928 4929 if (test_bit(NAPI_STATE_SCHED, &napi->state)) { 4930 rc = napi->poll(napi, BUSY_POLL_BUDGET); 4931 trace_napi_poll(napi); 4932 if (rc == BUSY_POLL_BUDGET) { 4933 napi_complete_done(napi, rc); 4934 napi_schedule(napi); 4935 } 4936 } 4937 netpoll_poll_unlock(have); 4938 } 4939 if (rc > 0) 4940 NET_ADD_STATS_BH(sock_net(sk), 4941 LINUX_MIB_BUSYPOLLRXPACKETS, rc); 4942 local_bh_enable(); 4943 4944 if (rc == LL_FLUSH_FAILED) 4945 break; /* permanent failure */ 4946 4947 cpu_relax(); 4948 } while (!nonblock && skb_queue_empty(&sk->sk_receive_queue) && 4949 !need_resched() && !busy_loop_timeout(end_time)); 4950 4951 rc = !skb_queue_empty(&sk->sk_receive_queue); 4952 out: 4953 rcu_read_unlock(); 4954 return rc; 4955 } 4956 EXPORT_SYMBOL(sk_busy_loop); 4957 4958 #endif /* CONFIG_NET_RX_BUSY_POLL */ 4959 4960 void napi_hash_add(struct napi_struct *napi) 4961 { 4962 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) || 4963 test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) 4964 return; 4965 4966 spin_lock(&napi_hash_lock); 4967 4968 /* 0..NR_CPUS+1 range is reserved for sender_cpu use */ 4969 do { 4970 if (unlikely(++napi_gen_id < NR_CPUS + 1)) 4971 napi_gen_id = NR_CPUS + 1; 4972 } while (napi_by_id(napi_gen_id)); 4973 napi->napi_id = napi_gen_id; 4974 4975 hlist_add_head_rcu(&napi->napi_hash_node, 4976 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); 4977 4978 spin_unlock(&napi_hash_lock); 4979 } 4980 EXPORT_SYMBOL_GPL(napi_hash_add); 4981 4982 /* Warning : caller is responsible to make sure rcu grace period 4983 * is respected before freeing memory containing @napi 4984 */ 4985 bool napi_hash_del(struct napi_struct *napi) 4986 { 4987 bool rcu_sync_needed = false; 4988 4989 spin_lock(&napi_hash_lock); 4990 4991 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) { 4992 rcu_sync_needed = true; 4993 hlist_del_rcu(&napi->napi_hash_node); 4994 } 4995 spin_unlock(&napi_hash_lock); 4996 return rcu_sync_needed; 4997 } 4998 EXPORT_SYMBOL_GPL(napi_hash_del); 4999 5000 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer) 5001 { 5002 struct napi_struct *napi; 5003 5004 napi = container_of(timer, struct napi_struct, timer); 5005 if (napi->gro_list) 5006 napi_schedule(napi); 5007 5008 return HRTIMER_NORESTART; 5009 } 5010 5011 void netif_napi_add(struct net_device *dev, struct napi_struct *napi, 5012 int (*poll)(struct napi_struct *, int), int weight) 5013 { 5014 INIT_LIST_HEAD(&napi->poll_list); 5015 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); 5016 napi->timer.function = napi_watchdog; 5017 napi->gro_count = 0; 5018 napi->gro_list = NULL; 5019 napi->skb = NULL; 5020 napi->poll = poll; 5021 if (weight > NAPI_POLL_WEIGHT) 5022 pr_err_once("netif_napi_add() called with weight %d on device %s\n", 5023 weight, dev->name); 5024 napi->weight = weight; 5025 list_add(&napi->dev_list, &dev->napi_list); 5026 napi->dev = dev; 5027 #ifdef CONFIG_NETPOLL 5028 spin_lock_init(&napi->poll_lock); 5029 napi->poll_owner = -1; 5030 #endif 5031 set_bit(NAPI_STATE_SCHED, &napi->state); 5032 napi_hash_add(napi); 5033 } 5034 EXPORT_SYMBOL(netif_napi_add); 5035 5036 void napi_disable(struct napi_struct *n) 5037 { 5038 might_sleep(); 5039 set_bit(NAPI_STATE_DISABLE, &n->state); 5040 5041 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state)) 5042 msleep(1); 5043 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state)) 5044 msleep(1); 5045 5046 hrtimer_cancel(&n->timer); 5047 5048 clear_bit(NAPI_STATE_DISABLE, &n->state); 5049 } 5050 EXPORT_SYMBOL(napi_disable); 5051 5052 /* Must be called in process context */ 5053 void netif_napi_del(struct napi_struct *napi) 5054 { 5055 might_sleep(); 5056 if (napi_hash_del(napi)) 5057 synchronize_net(); 5058 list_del_init(&napi->dev_list); 5059 napi_free_frags(napi); 5060 5061 kfree_skb_list(napi->gro_list); 5062 napi->gro_list = NULL; 5063 napi->gro_count = 0; 5064 } 5065 EXPORT_SYMBOL(netif_napi_del); 5066 5067 static int napi_poll(struct napi_struct *n, struct list_head *repoll) 5068 { 5069 void *have; 5070 int work, weight; 5071 5072 list_del_init(&n->poll_list); 5073 5074 have = netpoll_poll_lock(n); 5075 5076 weight = n->weight; 5077 5078 /* This NAPI_STATE_SCHED test is for avoiding a race 5079 * with netpoll's poll_napi(). Only the entity which 5080 * obtains the lock and sees NAPI_STATE_SCHED set will 5081 * actually make the ->poll() call. Therefore we avoid 5082 * accidentally calling ->poll() when NAPI is not scheduled. 5083 */ 5084 work = 0; 5085 if (test_bit(NAPI_STATE_SCHED, &n->state)) { 5086 work = n->poll(n, weight); 5087 trace_napi_poll(n); 5088 } 5089 5090 WARN_ON_ONCE(work > weight); 5091 5092 if (likely(work < weight)) 5093 goto out_unlock; 5094 5095 /* Drivers must not modify the NAPI state if they 5096 * consume the entire weight. In such cases this code 5097 * still "owns" the NAPI instance and therefore can 5098 * move the instance around on the list at-will. 5099 */ 5100 if (unlikely(napi_disable_pending(n))) { 5101 napi_complete(n); 5102 goto out_unlock; 5103 } 5104 5105 if (n->gro_list) { 5106 /* flush too old packets 5107 * If HZ < 1000, flush all packets. 5108 */ 5109 napi_gro_flush(n, HZ >= 1000); 5110 } 5111 5112 /* Some drivers may have called napi_schedule 5113 * prior to exhausting their budget. 5114 */ 5115 if (unlikely(!list_empty(&n->poll_list))) { 5116 pr_warn_once("%s: Budget exhausted after napi rescheduled\n", 5117 n->dev ? n->dev->name : "backlog"); 5118 goto out_unlock; 5119 } 5120 5121 list_add_tail(&n->poll_list, repoll); 5122 5123 out_unlock: 5124 netpoll_poll_unlock(have); 5125 5126 return work; 5127 } 5128 5129 static void net_rx_action(struct softirq_action *h) 5130 { 5131 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 5132 unsigned long time_limit = jiffies + 2; 5133 int budget = netdev_budget; 5134 LIST_HEAD(list); 5135 LIST_HEAD(repoll); 5136 5137 local_irq_disable(); 5138 list_splice_init(&sd->poll_list, &list); 5139 local_irq_enable(); 5140 5141 for (;;) { 5142 struct napi_struct *n; 5143 5144 if (list_empty(&list)) { 5145 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll)) 5146 return; 5147 break; 5148 } 5149 5150 n = list_first_entry(&list, struct napi_struct, poll_list); 5151 budget -= napi_poll(n, &repoll); 5152 5153 /* If softirq window is exhausted then punt. 5154 * Allow this to run for 2 jiffies since which will allow 5155 * an average latency of 1.5/HZ. 5156 */ 5157 if (unlikely(budget <= 0 || 5158 time_after_eq(jiffies, time_limit))) { 5159 sd->time_squeeze++; 5160 break; 5161 } 5162 } 5163 5164 __kfree_skb_flush(); 5165 local_irq_disable(); 5166 5167 list_splice_tail_init(&sd->poll_list, &list); 5168 list_splice_tail(&repoll, &list); 5169 list_splice(&list, &sd->poll_list); 5170 if (!list_empty(&sd->poll_list)) 5171 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 5172 5173 net_rps_action_and_irq_enable(sd); 5174 } 5175 5176 struct netdev_adjacent { 5177 struct net_device *dev; 5178 5179 /* upper master flag, there can only be one master device per list */ 5180 bool master; 5181 5182 /* counter for the number of times this device was added to us */ 5183 u16 ref_nr; 5184 5185 /* private field for the users */ 5186 void *private; 5187 5188 struct list_head list; 5189 struct rcu_head rcu; 5190 }; 5191 5192 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev, 5193 struct list_head *adj_list) 5194 { 5195 struct netdev_adjacent *adj; 5196 5197 list_for_each_entry(adj, adj_list, list) { 5198 if (adj->dev == adj_dev) 5199 return adj; 5200 } 5201 return NULL; 5202 } 5203 5204 /** 5205 * netdev_has_upper_dev - Check if device is linked to an upper device 5206 * @dev: device 5207 * @upper_dev: upper device to check 5208 * 5209 * Find out if a device is linked to specified upper device and return true 5210 * in case it is. Note that this checks only immediate upper device, 5211 * not through a complete stack of devices. The caller must hold the RTNL lock. 5212 */ 5213 bool netdev_has_upper_dev(struct net_device *dev, 5214 struct net_device *upper_dev) 5215 { 5216 ASSERT_RTNL(); 5217 5218 return __netdev_find_adj(upper_dev, &dev->all_adj_list.upper); 5219 } 5220 EXPORT_SYMBOL(netdev_has_upper_dev); 5221 5222 /** 5223 * netdev_has_any_upper_dev - Check if device is linked to some device 5224 * @dev: device 5225 * 5226 * Find out if a device is linked to an upper device and return true in case 5227 * it is. The caller must hold the RTNL lock. 5228 */ 5229 static bool netdev_has_any_upper_dev(struct net_device *dev) 5230 { 5231 ASSERT_RTNL(); 5232 5233 return !list_empty(&dev->all_adj_list.upper); 5234 } 5235 5236 /** 5237 * netdev_master_upper_dev_get - Get master upper device 5238 * @dev: device 5239 * 5240 * Find a master upper device and return pointer to it or NULL in case 5241 * it's not there. The caller must hold the RTNL lock. 5242 */ 5243 struct net_device *netdev_master_upper_dev_get(struct net_device *dev) 5244 { 5245 struct netdev_adjacent *upper; 5246 5247 ASSERT_RTNL(); 5248 5249 if (list_empty(&dev->adj_list.upper)) 5250 return NULL; 5251 5252 upper = list_first_entry(&dev->adj_list.upper, 5253 struct netdev_adjacent, list); 5254 if (likely(upper->master)) 5255 return upper->dev; 5256 return NULL; 5257 } 5258 EXPORT_SYMBOL(netdev_master_upper_dev_get); 5259 5260 void *netdev_adjacent_get_private(struct list_head *adj_list) 5261 { 5262 struct netdev_adjacent *adj; 5263 5264 adj = list_entry(adj_list, struct netdev_adjacent, list); 5265 5266 return adj->private; 5267 } 5268 EXPORT_SYMBOL(netdev_adjacent_get_private); 5269 5270 /** 5271 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list 5272 * @dev: device 5273 * @iter: list_head ** of the current position 5274 * 5275 * Gets the next device from the dev's upper list, starting from iter 5276 * position. The caller must hold RCU read lock. 5277 */ 5278 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev, 5279 struct list_head **iter) 5280 { 5281 struct netdev_adjacent *upper; 5282 5283 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 5284 5285 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5286 5287 if (&upper->list == &dev->adj_list.upper) 5288 return NULL; 5289 5290 *iter = &upper->list; 5291 5292 return upper->dev; 5293 } 5294 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu); 5295 5296 /** 5297 * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list 5298 * @dev: device 5299 * @iter: list_head ** of the current position 5300 * 5301 * Gets the next device from the dev's upper list, starting from iter 5302 * position. The caller must hold RCU read lock. 5303 */ 5304 struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev, 5305 struct list_head **iter) 5306 { 5307 struct netdev_adjacent *upper; 5308 5309 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 5310 5311 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5312 5313 if (&upper->list == &dev->all_adj_list.upper) 5314 return NULL; 5315 5316 *iter = &upper->list; 5317 5318 return upper->dev; 5319 } 5320 EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu); 5321 5322 /** 5323 * netdev_lower_get_next_private - Get the next ->private from the 5324 * lower neighbour list 5325 * @dev: device 5326 * @iter: list_head ** of the current position 5327 * 5328 * Gets the next netdev_adjacent->private from the dev's lower neighbour 5329 * list, starting from iter position. The caller must hold either hold the 5330 * RTNL lock or its own locking that guarantees that the neighbour lower 5331 * list will remain unchanged. 5332 */ 5333 void *netdev_lower_get_next_private(struct net_device *dev, 5334 struct list_head **iter) 5335 { 5336 struct netdev_adjacent *lower; 5337 5338 lower = list_entry(*iter, struct netdev_adjacent, list); 5339 5340 if (&lower->list == &dev->adj_list.lower) 5341 return NULL; 5342 5343 *iter = lower->list.next; 5344 5345 return lower->private; 5346 } 5347 EXPORT_SYMBOL(netdev_lower_get_next_private); 5348 5349 /** 5350 * netdev_lower_get_next_private_rcu - Get the next ->private from the 5351 * lower neighbour list, RCU 5352 * variant 5353 * @dev: device 5354 * @iter: list_head ** of the current position 5355 * 5356 * Gets the next netdev_adjacent->private from the dev's lower neighbour 5357 * list, starting from iter position. The caller must hold RCU read lock. 5358 */ 5359 void *netdev_lower_get_next_private_rcu(struct net_device *dev, 5360 struct list_head **iter) 5361 { 5362 struct netdev_adjacent *lower; 5363 5364 WARN_ON_ONCE(!rcu_read_lock_held()); 5365 5366 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5367 5368 if (&lower->list == &dev->adj_list.lower) 5369 return NULL; 5370 5371 *iter = &lower->list; 5372 5373 return lower->private; 5374 } 5375 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu); 5376 5377 /** 5378 * netdev_lower_get_next - Get the next device from the lower neighbour 5379 * list 5380 * @dev: device 5381 * @iter: list_head ** of the current position 5382 * 5383 * Gets the next netdev_adjacent from the dev's lower neighbour 5384 * list, starting from iter position. The caller must hold RTNL lock or 5385 * its own locking that guarantees that the neighbour lower 5386 * list will remain unchanged. 5387 */ 5388 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter) 5389 { 5390 struct netdev_adjacent *lower; 5391 5392 lower = list_entry(*iter, struct netdev_adjacent, list); 5393 5394 if (&lower->list == &dev->adj_list.lower) 5395 return NULL; 5396 5397 *iter = lower->list.next; 5398 5399 return lower->dev; 5400 } 5401 EXPORT_SYMBOL(netdev_lower_get_next); 5402 5403 /** 5404 * netdev_lower_get_first_private_rcu - Get the first ->private from the 5405 * lower neighbour list, RCU 5406 * variant 5407 * @dev: device 5408 * 5409 * Gets the first netdev_adjacent->private from the dev's lower neighbour 5410 * list. The caller must hold RCU read lock. 5411 */ 5412 void *netdev_lower_get_first_private_rcu(struct net_device *dev) 5413 { 5414 struct netdev_adjacent *lower; 5415 5416 lower = list_first_or_null_rcu(&dev->adj_list.lower, 5417 struct netdev_adjacent, list); 5418 if (lower) 5419 return lower->private; 5420 return NULL; 5421 } 5422 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu); 5423 5424 /** 5425 * netdev_master_upper_dev_get_rcu - Get master upper device 5426 * @dev: device 5427 * 5428 * Find a master upper device and return pointer to it or NULL in case 5429 * it's not there. The caller must hold the RCU read lock. 5430 */ 5431 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev) 5432 { 5433 struct netdev_adjacent *upper; 5434 5435 upper = list_first_or_null_rcu(&dev->adj_list.upper, 5436 struct netdev_adjacent, list); 5437 if (upper && likely(upper->master)) 5438 return upper->dev; 5439 return NULL; 5440 } 5441 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu); 5442 5443 static int netdev_adjacent_sysfs_add(struct net_device *dev, 5444 struct net_device *adj_dev, 5445 struct list_head *dev_list) 5446 { 5447 char linkname[IFNAMSIZ+7]; 5448 sprintf(linkname, dev_list == &dev->adj_list.upper ? 5449 "upper_%s" : "lower_%s", adj_dev->name); 5450 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), 5451 linkname); 5452 } 5453 static void netdev_adjacent_sysfs_del(struct net_device *dev, 5454 char *name, 5455 struct list_head *dev_list) 5456 { 5457 char linkname[IFNAMSIZ+7]; 5458 sprintf(linkname, dev_list == &dev->adj_list.upper ? 5459 "upper_%s" : "lower_%s", name); 5460 sysfs_remove_link(&(dev->dev.kobj), linkname); 5461 } 5462 5463 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev, 5464 struct net_device *adj_dev, 5465 struct list_head *dev_list) 5466 { 5467 return (dev_list == &dev->adj_list.upper || 5468 dev_list == &dev->adj_list.lower) && 5469 net_eq(dev_net(dev), dev_net(adj_dev)); 5470 } 5471 5472 static int __netdev_adjacent_dev_insert(struct net_device *dev, 5473 struct net_device *adj_dev, 5474 struct list_head *dev_list, 5475 void *private, bool master) 5476 { 5477 struct netdev_adjacent *adj; 5478 int ret; 5479 5480 adj = __netdev_find_adj(adj_dev, dev_list); 5481 5482 if (adj) { 5483 adj->ref_nr++; 5484 return 0; 5485 } 5486 5487 adj = kmalloc(sizeof(*adj), GFP_KERNEL); 5488 if (!adj) 5489 return -ENOMEM; 5490 5491 adj->dev = adj_dev; 5492 adj->master = master; 5493 adj->ref_nr = 1; 5494 adj->private = private; 5495 dev_hold(adj_dev); 5496 5497 pr_debug("dev_hold for %s, because of link added from %s to %s\n", 5498 adj_dev->name, dev->name, adj_dev->name); 5499 5500 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) { 5501 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list); 5502 if (ret) 5503 goto free_adj; 5504 } 5505 5506 /* Ensure that master link is always the first item in list. */ 5507 if (master) { 5508 ret = sysfs_create_link(&(dev->dev.kobj), 5509 &(adj_dev->dev.kobj), "master"); 5510 if (ret) 5511 goto remove_symlinks; 5512 5513 list_add_rcu(&adj->list, dev_list); 5514 } else { 5515 list_add_tail_rcu(&adj->list, dev_list); 5516 } 5517 5518 return 0; 5519 5520 remove_symlinks: 5521 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 5522 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 5523 free_adj: 5524 kfree(adj); 5525 dev_put(adj_dev); 5526 5527 return ret; 5528 } 5529 5530 static void __netdev_adjacent_dev_remove(struct net_device *dev, 5531 struct net_device *adj_dev, 5532 struct list_head *dev_list) 5533 { 5534 struct netdev_adjacent *adj; 5535 5536 adj = __netdev_find_adj(adj_dev, dev_list); 5537 5538 if (!adj) { 5539 pr_err("tried to remove device %s from %s\n", 5540 dev->name, adj_dev->name); 5541 BUG(); 5542 } 5543 5544 if (adj->ref_nr > 1) { 5545 pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name, 5546 adj->ref_nr-1); 5547 adj->ref_nr--; 5548 return; 5549 } 5550 5551 if (adj->master) 5552 sysfs_remove_link(&(dev->dev.kobj), "master"); 5553 5554 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 5555 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 5556 5557 list_del_rcu(&adj->list); 5558 pr_debug("dev_put for %s, because link removed from %s to %s\n", 5559 adj_dev->name, dev->name, adj_dev->name); 5560 dev_put(adj_dev); 5561 kfree_rcu(adj, rcu); 5562 } 5563 5564 static int __netdev_adjacent_dev_link_lists(struct net_device *dev, 5565 struct net_device *upper_dev, 5566 struct list_head *up_list, 5567 struct list_head *down_list, 5568 void *private, bool master) 5569 { 5570 int ret; 5571 5572 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private, 5573 master); 5574 if (ret) 5575 return ret; 5576 5577 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private, 5578 false); 5579 if (ret) { 5580 __netdev_adjacent_dev_remove(dev, upper_dev, up_list); 5581 return ret; 5582 } 5583 5584 return 0; 5585 } 5586 5587 static int __netdev_adjacent_dev_link(struct net_device *dev, 5588 struct net_device *upper_dev) 5589 { 5590 return __netdev_adjacent_dev_link_lists(dev, upper_dev, 5591 &dev->all_adj_list.upper, 5592 &upper_dev->all_adj_list.lower, 5593 NULL, false); 5594 } 5595 5596 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev, 5597 struct net_device *upper_dev, 5598 struct list_head *up_list, 5599 struct list_head *down_list) 5600 { 5601 __netdev_adjacent_dev_remove(dev, upper_dev, up_list); 5602 __netdev_adjacent_dev_remove(upper_dev, dev, down_list); 5603 } 5604 5605 static void __netdev_adjacent_dev_unlink(struct net_device *dev, 5606 struct net_device *upper_dev) 5607 { 5608 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 5609 &dev->all_adj_list.upper, 5610 &upper_dev->all_adj_list.lower); 5611 } 5612 5613 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev, 5614 struct net_device *upper_dev, 5615 void *private, bool master) 5616 { 5617 int ret = __netdev_adjacent_dev_link(dev, upper_dev); 5618 5619 if (ret) 5620 return ret; 5621 5622 ret = __netdev_adjacent_dev_link_lists(dev, upper_dev, 5623 &dev->adj_list.upper, 5624 &upper_dev->adj_list.lower, 5625 private, master); 5626 if (ret) { 5627 __netdev_adjacent_dev_unlink(dev, upper_dev); 5628 return ret; 5629 } 5630 5631 return 0; 5632 } 5633 5634 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev, 5635 struct net_device *upper_dev) 5636 { 5637 __netdev_adjacent_dev_unlink(dev, upper_dev); 5638 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 5639 &dev->adj_list.upper, 5640 &upper_dev->adj_list.lower); 5641 } 5642 5643 static int __netdev_upper_dev_link(struct net_device *dev, 5644 struct net_device *upper_dev, bool master, 5645 void *upper_priv, void *upper_info) 5646 { 5647 struct netdev_notifier_changeupper_info changeupper_info; 5648 struct netdev_adjacent *i, *j, *to_i, *to_j; 5649 int ret = 0; 5650 5651 ASSERT_RTNL(); 5652 5653 if (dev == upper_dev) 5654 return -EBUSY; 5655 5656 /* To prevent loops, check if dev is not upper device to upper_dev. */ 5657 if (__netdev_find_adj(dev, &upper_dev->all_adj_list.upper)) 5658 return -EBUSY; 5659 5660 if (__netdev_find_adj(upper_dev, &dev->adj_list.upper)) 5661 return -EEXIST; 5662 5663 if (master && netdev_master_upper_dev_get(dev)) 5664 return -EBUSY; 5665 5666 changeupper_info.upper_dev = upper_dev; 5667 changeupper_info.master = master; 5668 changeupper_info.linking = true; 5669 changeupper_info.upper_info = upper_info; 5670 5671 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev, 5672 &changeupper_info.info); 5673 ret = notifier_to_errno(ret); 5674 if (ret) 5675 return ret; 5676 5677 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv, 5678 master); 5679 if (ret) 5680 return ret; 5681 5682 /* Now that we linked these devs, make all the upper_dev's 5683 * all_adj_list.upper visible to every dev's all_adj_list.lower an 5684 * versa, and don't forget the devices itself. All of these 5685 * links are non-neighbours. 5686 */ 5687 list_for_each_entry(i, &dev->all_adj_list.lower, list) { 5688 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) { 5689 pr_debug("Interlinking %s with %s, non-neighbour\n", 5690 i->dev->name, j->dev->name); 5691 ret = __netdev_adjacent_dev_link(i->dev, j->dev); 5692 if (ret) 5693 goto rollback_mesh; 5694 } 5695 } 5696 5697 /* add dev to every upper_dev's upper device */ 5698 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) { 5699 pr_debug("linking %s's upper device %s with %s\n", 5700 upper_dev->name, i->dev->name, dev->name); 5701 ret = __netdev_adjacent_dev_link(dev, i->dev); 5702 if (ret) 5703 goto rollback_upper_mesh; 5704 } 5705 5706 /* add upper_dev to every dev's lower device */ 5707 list_for_each_entry(i, &dev->all_adj_list.lower, list) { 5708 pr_debug("linking %s's lower device %s with %s\n", dev->name, 5709 i->dev->name, upper_dev->name); 5710 ret = __netdev_adjacent_dev_link(i->dev, upper_dev); 5711 if (ret) 5712 goto rollback_lower_mesh; 5713 } 5714 5715 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev, 5716 &changeupper_info.info); 5717 ret = notifier_to_errno(ret); 5718 if (ret) 5719 goto rollback_lower_mesh; 5720 5721 return 0; 5722 5723 rollback_lower_mesh: 5724 to_i = i; 5725 list_for_each_entry(i, &dev->all_adj_list.lower, list) { 5726 if (i == to_i) 5727 break; 5728 __netdev_adjacent_dev_unlink(i->dev, upper_dev); 5729 } 5730 5731 i = NULL; 5732 5733 rollback_upper_mesh: 5734 to_i = i; 5735 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) { 5736 if (i == to_i) 5737 break; 5738 __netdev_adjacent_dev_unlink(dev, i->dev); 5739 } 5740 5741 i = j = NULL; 5742 5743 rollback_mesh: 5744 to_i = i; 5745 to_j = j; 5746 list_for_each_entry(i, &dev->all_adj_list.lower, list) { 5747 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) { 5748 if (i == to_i && j == to_j) 5749 break; 5750 __netdev_adjacent_dev_unlink(i->dev, j->dev); 5751 } 5752 if (i == to_i) 5753 break; 5754 } 5755 5756 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 5757 5758 return ret; 5759 } 5760 5761 /** 5762 * netdev_upper_dev_link - Add a link to the upper device 5763 * @dev: device 5764 * @upper_dev: new upper device 5765 * 5766 * Adds a link to device which is upper to this one. The caller must hold 5767 * the RTNL lock. On a failure a negative errno code is returned. 5768 * On success the reference counts are adjusted and the function 5769 * returns zero. 5770 */ 5771 int netdev_upper_dev_link(struct net_device *dev, 5772 struct net_device *upper_dev) 5773 { 5774 return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL); 5775 } 5776 EXPORT_SYMBOL(netdev_upper_dev_link); 5777 5778 /** 5779 * netdev_master_upper_dev_link - Add a master link to the upper device 5780 * @dev: device 5781 * @upper_dev: new upper device 5782 * @upper_priv: upper device private 5783 * @upper_info: upper info to be passed down via notifier 5784 * 5785 * Adds a link to device which is upper to this one. In this case, only 5786 * one master upper device can be linked, although other non-master devices 5787 * might be linked as well. The caller must hold the RTNL lock. 5788 * On a failure a negative errno code is returned. On success the reference 5789 * counts are adjusted and the function returns zero. 5790 */ 5791 int netdev_master_upper_dev_link(struct net_device *dev, 5792 struct net_device *upper_dev, 5793 void *upper_priv, void *upper_info) 5794 { 5795 return __netdev_upper_dev_link(dev, upper_dev, true, 5796 upper_priv, upper_info); 5797 } 5798 EXPORT_SYMBOL(netdev_master_upper_dev_link); 5799 5800 /** 5801 * netdev_upper_dev_unlink - Removes a link to upper device 5802 * @dev: device 5803 * @upper_dev: new upper device 5804 * 5805 * Removes a link to device which is upper to this one. The caller must hold 5806 * the RTNL lock. 5807 */ 5808 void netdev_upper_dev_unlink(struct net_device *dev, 5809 struct net_device *upper_dev) 5810 { 5811 struct netdev_notifier_changeupper_info changeupper_info; 5812 struct netdev_adjacent *i, *j; 5813 ASSERT_RTNL(); 5814 5815 changeupper_info.upper_dev = upper_dev; 5816 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev; 5817 changeupper_info.linking = false; 5818 5819 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev, 5820 &changeupper_info.info); 5821 5822 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 5823 5824 /* Here is the tricky part. We must remove all dev's lower 5825 * devices from all upper_dev's upper devices and vice 5826 * versa, to maintain the graph relationship. 5827 */ 5828 list_for_each_entry(i, &dev->all_adj_list.lower, list) 5829 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) 5830 __netdev_adjacent_dev_unlink(i->dev, j->dev); 5831 5832 /* remove also the devices itself from lower/upper device 5833 * list 5834 */ 5835 list_for_each_entry(i, &dev->all_adj_list.lower, list) 5836 __netdev_adjacent_dev_unlink(i->dev, upper_dev); 5837 5838 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) 5839 __netdev_adjacent_dev_unlink(dev, i->dev); 5840 5841 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev, 5842 &changeupper_info.info); 5843 } 5844 EXPORT_SYMBOL(netdev_upper_dev_unlink); 5845 5846 /** 5847 * netdev_bonding_info_change - Dispatch event about slave change 5848 * @dev: device 5849 * @bonding_info: info to dispatch 5850 * 5851 * Send NETDEV_BONDING_INFO to netdev notifiers with info. 5852 * The caller must hold the RTNL lock. 5853 */ 5854 void netdev_bonding_info_change(struct net_device *dev, 5855 struct netdev_bonding_info *bonding_info) 5856 { 5857 struct netdev_notifier_bonding_info info; 5858 5859 memcpy(&info.bonding_info, bonding_info, 5860 sizeof(struct netdev_bonding_info)); 5861 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev, 5862 &info.info); 5863 } 5864 EXPORT_SYMBOL(netdev_bonding_info_change); 5865 5866 static void netdev_adjacent_add_links(struct net_device *dev) 5867 { 5868 struct netdev_adjacent *iter; 5869 5870 struct net *net = dev_net(dev); 5871 5872 list_for_each_entry(iter, &dev->adj_list.upper, list) { 5873 if (!net_eq(net,dev_net(iter->dev))) 5874 continue; 5875 netdev_adjacent_sysfs_add(iter->dev, dev, 5876 &iter->dev->adj_list.lower); 5877 netdev_adjacent_sysfs_add(dev, iter->dev, 5878 &dev->adj_list.upper); 5879 } 5880 5881 list_for_each_entry(iter, &dev->adj_list.lower, list) { 5882 if (!net_eq(net,dev_net(iter->dev))) 5883 continue; 5884 netdev_adjacent_sysfs_add(iter->dev, dev, 5885 &iter->dev->adj_list.upper); 5886 netdev_adjacent_sysfs_add(dev, iter->dev, 5887 &dev->adj_list.lower); 5888 } 5889 } 5890 5891 static void netdev_adjacent_del_links(struct net_device *dev) 5892 { 5893 struct netdev_adjacent *iter; 5894 5895 struct net *net = dev_net(dev); 5896 5897 list_for_each_entry(iter, &dev->adj_list.upper, list) { 5898 if (!net_eq(net,dev_net(iter->dev))) 5899 continue; 5900 netdev_adjacent_sysfs_del(iter->dev, dev->name, 5901 &iter->dev->adj_list.lower); 5902 netdev_adjacent_sysfs_del(dev, iter->dev->name, 5903 &dev->adj_list.upper); 5904 } 5905 5906 list_for_each_entry(iter, &dev->adj_list.lower, list) { 5907 if (!net_eq(net,dev_net(iter->dev))) 5908 continue; 5909 netdev_adjacent_sysfs_del(iter->dev, dev->name, 5910 &iter->dev->adj_list.upper); 5911 netdev_adjacent_sysfs_del(dev, iter->dev->name, 5912 &dev->adj_list.lower); 5913 } 5914 } 5915 5916 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname) 5917 { 5918 struct netdev_adjacent *iter; 5919 5920 struct net *net = dev_net(dev); 5921 5922 list_for_each_entry(iter, &dev->adj_list.upper, list) { 5923 if (!net_eq(net,dev_net(iter->dev))) 5924 continue; 5925 netdev_adjacent_sysfs_del(iter->dev, oldname, 5926 &iter->dev->adj_list.lower); 5927 netdev_adjacent_sysfs_add(iter->dev, dev, 5928 &iter->dev->adj_list.lower); 5929 } 5930 5931 list_for_each_entry(iter, &dev->adj_list.lower, list) { 5932 if (!net_eq(net,dev_net(iter->dev))) 5933 continue; 5934 netdev_adjacent_sysfs_del(iter->dev, oldname, 5935 &iter->dev->adj_list.upper); 5936 netdev_adjacent_sysfs_add(iter->dev, dev, 5937 &iter->dev->adj_list.upper); 5938 } 5939 } 5940 5941 void *netdev_lower_dev_get_private(struct net_device *dev, 5942 struct net_device *lower_dev) 5943 { 5944 struct netdev_adjacent *lower; 5945 5946 if (!lower_dev) 5947 return NULL; 5948 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower); 5949 if (!lower) 5950 return NULL; 5951 5952 return lower->private; 5953 } 5954 EXPORT_SYMBOL(netdev_lower_dev_get_private); 5955 5956 5957 int dev_get_nest_level(struct net_device *dev, 5958 bool (*type_check)(const struct net_device *dev)) 5959 { 5960 struct net_device *lower = NULL; 5961 struct list_head *iter; 5962 int max_nest = -1; 5963 int nest; 5964 5965 ASSERT_RTNL(); 5966 5967 netdev_for_each_lower_dev(dev, lower, iter) { 5968 nest = dev_get_nest_level(lower, type_check); 5969 if (max_nest < nest) 5970 max_nest = nest; 5971 } 5972 5973 if (type_check(dev)) 5974 max_nest++; 5975 5976 return max_nest; 5977 } 5978 EXPORT_SYMBOL(dev_get_nest_level); 5979 5980 /** 5981 * netdev_lower_change - Dispatch event about lower device state change 5982 * @lower_dev: device 5983 * @lower_state_info: state to dispatch 5984 * 5985 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info. 5986 * The caller must hold the RTNL lock. 5987 */ 5988 void netdev_lower_state_changed(struct net_device *lower_dev, 5989 void *lower_state_info) 5990 { 5991 struct netdev_notifier_changelowerstate_info changelowerstate_info; 5992 5993 ASSERT_RTNL(); 5994 changelowerstate_info.lower_state_info = lower_state_info; 5995 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev, 5996 &changelowerstate_info.info); 5997 } 5998 EXPORT_SYMBOL(netdev_lower_state_changed); 5999 6000 static void dev_change_rx_flags(struct net_device *dev, int flags) 6001 { 6002 const struct net_device_ops *ops = dev->netdev_ops; 6003 6004 if (ops->ndo_change_rx_flags) 6005 ops->ndo_change_rx_flags(dev, flags); 6006 } 6007 6008 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify) 6009 { 6010 unsigned int old_flags = dev->flags; 6011 kuid_t uid; 6012 kgid_t gid; 6013 6014 ASSERT_RTNL(); 6015 6016 dev->flags |= IFF_PROMISC; 6017 dev->promiscuity += inc; 6018 if (dev->promiscuity == 0) { 6019 /* 6020 * Avoid overflow. 6021 * If inc causes overflow, untouch promisc and return error. 6022 */ 6023 if (inc < 0) 6024 dev->flags &= ~IFF_PROMISC; 6025 else { 6026 dev->promiscuity -= inc; 6027 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n", 6028 dev->name); 6029 return -EOVERFLOW; 6030 } 6031 } 6032 if (dev->flags != old_flags) { 6033 pr_info("device %s %s promiscuous mode\n", 6034 dev->name, 6035 dev->flags & IFF_PROMISC ? "entered" : "left"); 6036 if (audit_enabled) { 6037 current_uid_gid(&uid, &gid); 6038 audit_log(current->audit_context, GFP_ATOMIC, 6039 AUDIT_ANOM_PROMISCUOUS, 6040 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u", 6041 dev->name, (dev->flags & IFF_PROMISC), 6042 (old_flags & IFF_PROMISC), 6043 from_kuid(&init_user_ns, audit_get_loginuid(current)), 6044 from_kuid(&init_user_ns, uid), 6045 from_kgid(&init_user_ns, gid), 6046 audit_get_sessionid(current)); 6047 } 6048 6049 dev_change_rx_flags(dev, IFF_PROMISC); 6050 } 6051 if (notify) 6052 __dev_notify_flags(dev, old_flags, IFF_PROMISC); 6053 return 0; 6054 } 6055 6056 /** 6057 * dev_set_promiscuity - update promiscuity count on a device 6058 * @dev: device 6059 * @inc: modifier 6060 * 6061 * Add or remove promiscuity from a device. While the count in the device 6062 * remains above zero the interface remains promiscuous. Once it hits zero 6063 * the device reverts back to normal filtering operation. A negative inc 6064 * value is used to drop promiscuity on the device. 6065 * Return 0 if successful or a negative errno code on error. 6066 */ 6067 int dev_set_promiscuity(struct net_device *dev, int inc) 6068 { 6069 unsigned int old_flags = dev->flags; 6070 int err; 6071 6072 err = __dev_set_promiscuity(dev, inc, true); 6073 if (err < 0) 6074 return err; 6075 if (dev->flags != old_flags) 6076 dev_set_rx_mode(dev); 6077 return err; 6078 } 6079 EXPORT_SYMBOL(dev_set_promiscuity); 6080 6081 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify) 6082 { 6083 unsigned int old_flags = dev->flags, old_gflags = dev->gflags; 6084 6085 ASSERT_RTNL(); 6086 6087 dev->flags |= IFF_ALLMULTI; 6088 dev->allmulti += inc; 6089 if (dev->allmulti == 0) { 6090 /* 6091 * Avoid overflow. 6092 * If inc causes overflow, untouch allmulti and return error. 6093 */ 6094 if (inc < 0) 6095 dev->flags &= ~IFF_ALLMULTI; 6096 else { 6097 dev->allmulti -= inc; 6098 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n", 6099 dev->name); 6100 return -EOVERFLOW; 6101 } 6102 } 6103 if (dev->flags ^ old_flags) { 6104 dev_change_rx_flags(dev, IFF_ALLMULTI); 6105 dev_set_rx_mode(dev); 6106 if (notify) 6107 __dev_notify_flags(dev, old_flags, 6108 dev->gflags ^ old_gflags); 6109 } 6110 return 0; 6111 } 6112 6113 /** 6114 * dev_set_allmulti - update allmulti count on a device 6115 * @dev: device 6116 * @inc: modifier 6117 * 6118 * Add or remove reception of all multicast frames to a device. While the 6119 * count in the device remains above zero the interface remains listening 6120 * to all interfaces. Once it hits zero the device reverts back to normal 6121 * filtering operation. A negative @inc value is used to drop the counter 6122 * when releasing a resource needing all multicasts. 6123 * Return 0 if successful or a negative errno code on error. 6124 */ 6125 6126 int dev_set_allmulti(struct net_device *dev, int inc) 6127 { 6128 return __dev_set_allmulti(dev, inc, true); 6129 } 6130 EXPORT_SYMBOL(dev_set_allmulti); 6131 6132 /* 6133 * Upload unicast and multicast address lists to device and 6134 * configure RX filtering. When the device doesn't support unicast 6135 * filtering it is put in promiscuous mode while unicast addresses 6136 * are present. 6137 */ 6138 void __dev_set_rx_mode(struct net_device *dev) 6139 { 6140 const struct net_device_ops *ops = dev->netdev_ops; 6141 6142 /* dev_open will call this function so the list will stay sane. */ 6143 if (!(dev->flags&IFF_UP)) 6144 return; 6145 6146 if (!netif_device_present(dev)) 6147 return; 6148 6149 if (!(dev->priv_flags & IFF_UNICAST_FLT)) { 6150 /* Unicast addresses changes may only happen under the rtnl, 6151 * therefore calling __dev_set_promiscuity here is safe. 6152 */ 6153 if (!netdev_uc_empty(dev) && !dev->uc_promisc) { 6154 __dev_set_promiscuity(dev, 1, false); 6155 dev->uc_promisc = true; 6156 } else if (netdev_uc_empty(dev) && dev->uc_promisc) { 6157 __dev_set_promiscuity(dev, -1, false); 6158 dev->uc_promisc = false; 6159 } 6160 } 6161 6162 if (ops->ndo_set_rx_mode) 6163 ops->ndo_set_rx_mode(dev); 6164 } 6165 6166 void dev_set_rx_mode(struct net_device *dev) 6167 { 6168 netif_addr_lock_bh(dev); 6169 __dev_set_rx_mode(dev); 6170 netif_addr_unlock_bh(dev); 6171 } 6172 6173 /** 6174 * dev_get_flags - get flags reported to userspace 6175 * @dev: device 6176 * 6177 * Get the combination of flag bits exported through APIs to userspace. 6178 */ 6179 unsigned int dev_get_flags(const struct net_device *dev) 6180 { 6181 unsigned int flags; 6182 6183 flags = (dev->flags & ~(IFF_PROMISC | 6184 IFF_ALLMULTI | 6185 IFF_RUNNING | 6186 IFF_LOWER_UP | 6187 IFF_DORMANT)) | 6188 (dev->gflags & (IFF_PROMISC | 6189 IFF_ALLMULTI)); 6190 6191 if (netif_running(dev)) { 6192 if (netif_oper_up(dev)) 6193 flags |= IFF_RUNNING; 6194 if (netif_carrier_ok(dev)) 6195 flags |= IFF_LOWER_UP; 6196 if (netif_dormant(dev)) 6197 flags |= IFF_DORMANT; 6198 } 6199 6200 return flags; 6201 } 6202 EXPORT_SYMBOL(dev_get_flags); 6203 6204 int __dev_change_flags(struct net_device *dev, unsigned int flags) 6205 { 6206 unsigned int old_flags = dev->flags; 6207 int ret; 6208 6209 ASSERT_RTNL(); 6210 6211 /* 6212 * Set the flags on our device. 6213 */ 6214 6215 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | 6216 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL | 6217 IFF_AUTOMEDIA)) | 6218 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | 6219 IFF_ALLMULTI)); 6220 6221 /* 6222 * Load in the correct multicast list now the flags have changed. 6223 */ 6224 6225 if ((old_flags ^ flags) & IFF_MULTICAST) 6226 dev_change_rx_flags(dev, IFF_MULTICAST); 6227 6228 dev_set_rx_mode(dev); 6229 6230 /* 6231 * Have we downed the interface. We handle IFF_UP ourselves 6232 * according to user attempts to set it, rather than blindly 6233 * setting it. 6234 */ 6235 6236 ret = 0; 6237 if ((old_flags ^ flags) & IFF_UP) 6238 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev); 6239 6240 if ((flags ^ dev->gflags) & IFF_PROMISC) { 6241 int inc = (flags & IFF_PROMISC) ? 1 : -1; 6242 unsigned int old_flags = dev->flags; 6243 6244 dev->gflags ^= IFF_PROMISC; 6245 6246 if (__dev_set_promiscuity(dev, inc, false) >= 0) 6247 if (dev->flags != old_flags) 6248 dev_set_rx_mode(dev); 6249 } 6250 6251 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI 6252 is important. Some (broken) drivers set IFF_PROMISC, when 6253 IFF_ALLMULTI is requested not asking us and not reporting. 6254 */ 6255 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { 6256 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; 6257 6258 dev->gflags ^= IFF_ALLMULTI; 6259 __dev_set_allmulti(dev, inc, false); 6260 } 6261 6262 return ret; 6263 } 6264 6265 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, 6266 unsigned int gchanges) 6267 { 6268 unsigned int changes = dev->flags ^ old_flags; 6269 6270 if (gchanges) 6271 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC); 6272 6273 if (changes & IFF_UP) { 6274 if (dev->flags & IFF_UP) 6275 call_netdevice_notifiers(NETDEV_UP, dev); 6276 else 6277 call_netdevice_notifiers(NETDEV_DOWN, dev); 6278 } 6279 6280 if (dev->flags & IFF_UP && 6281 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) { 6282 struct netdev_notifier_change_info change_info; 6283 6284 change_info.flags_changed = changes; 6285 call_netdevice_notifiers_info(NETDEV_CHANGE, dev, 6286 &change_info.info); 6287 } 6288 } 6289 6290 /** 6291 * dev_change_flags - change device settings 6292 * @dev: device 6293 * @flags: device state flags 6294 * 6295 * Change settings on device based state flags. The flags are 6296 * in the userspace exported format. 6297 */ 6298 int dev_change_flags(struct net_device *dev, unsigned int flags) 6299 { 6300 int ret; 6301 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; 6302 6303 ret = __dev_change_flags(dev, flags); 6304 if (ret < 0) 6305 return ret; 6306 6307 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); 6308 __dev_notify_flags(dev, old_flags, changes); 6309 return ret; 6310 } 6311 EXPORT_SYMBOL(dev_change_flags); 6312 6313 static int __dev_set_mtu(struct net_device *dev, int new_mtu) 6314 { 6315 const struct net_device_ops *ops = dev->netdev_ops; 6316 6317 if (ops->ndo_change_mtu) 6318 return ops->ndo_change_mtu(dev, new_mtu); 6319 6320 dev->mtu = new_mtu; 6321 return 0; 6322 } 6323 6324 /** 6325 * dev_set_mtu - Change maximum transfer unit 6326 * @dev: device 6327 * @new_mtu: new transfer unit 6328 * 6329 * Change the maximum transfer size of the network device. 6330 */ 6331 int dev_set_mtu(struct net_device *dev, int new_mtu) 6332 { 6333 int err, orig_mtu; 6334 6335 if (new_mtu == dev->mtu) 6336 return 0; 6337 6338 /* MTU must be positive. */ 6339 if (new_mtu < 0) 6340 return -EINVAL; 6341 6342 if (!netif_device_present(dev)) 6343 return -ENODEV; 6344 6345 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev); 6346 err = notifier_to_errno(err); 6347 if (err) 6348 return err; 6349 6350 orig_mtu = dev->mtu; 6351 err = __dev_set_mtu(dev, new_mtu); 6352 6353 if (!err) { 6354 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); 6355 err = notifier_to_errno(err); 6356 if (err) { 6357 /* setting mtu back and notifying everyone again, 6358 * so that they have a chance to revert changes. 6359 */ 6360 __dev_set_mtu(dev, orig_mtu); 6361 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); 6362 } 6363 } 6364 return err; 6365 } 6366 EXPORT_SYMBOL(dev_set_mtu); 6367 6368 /** 6369 * dev_set_group - Change group this device belongs to 6370 * @dev: device 6371 * @new_group: group this device should belong to 6372 */ 6373 void dev_set_group(struct net_device *dev, int new_group) 6374 { 6375 dev->group = new_group; 6376 } 6377 EXPORT_SYMBOL(dev_set_group); 6378 6379 /** 6380 * dev_set_mac_address - Change Media Access Control Address 6381 * @dev: device 6382 * @sa: new address 6383 * 6384 * Change the hardware (MAC) address of the device 6385 */ 6386 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa) 6387 { 6388 const struct net_device_ops *ops = dev->netdev_ops; 6389 int err; 6390 6391 if (!ops->ndo_set_mac_address) 6392 return -EOPNOTSUPP; 6393 if (sa->sa_family != dev->type) 6394 return -EINVAL; 6395 if (!netif_device_present(dev)) 6396 return -ENODEV; 6397 err = ops->ndo_set_mac_address(dev, sa); 6398 if (err) 6399 return err; 6400 dev->addr_assign_type = NET_ADDR_SET; 6401 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 6402 add_device_randomness(dev->dev_addr, dev->addr_len); 6403 return 0; 6404 } 6405 EXPORT_SYMBOL(dev_set_mac_address); 6406 6407 /** 6408 * dev_change_carrier - Change device carrier 6409 * @dev: device 6410 * @new_carrier: new value 6411 * 6412 * Change device carrier 6413 */ 6414 int dev_change_carrier(struct net_device *dev, bool new_carrier) 6415 { 6416 const struct net_device_ops *ops = dev->netdev_ops; 6417 6418 if (!ops->ndo_change_carrier) 6419 return -EOPNOTSUPP; 6420 if (!netif_device_present(dev)) 6421 return -ENODEV; 6422 return ops->ndo_change_carrier(dev, new_carrier); 6423 } 6424 EXPORT_SYMBOL(dev_change_carrier); 6425 6426 /** 6427 * dev_get_phys_port_id - Get device physical port ID 6428 * @dev: device 6429 * @ppid: port ID 6430 * 6431 * Get device physical port ID 6432 */ 6433 int dev_get_phys_port_id(struct net_device *dev, 6434 struct netdev_phys_item_id *ppid) 6435 { 6436 const struct net_device_ops *ops = dev->netdev_ops; 6437 6438 if (!ops->ndo_get_phys_port_id) 6439 return -EOPNOTSUPP; 6440 return ops->ndo_get_phys_port_id(dev, ppid); 6441 } 6442 EXPORT_SYMBOL(dev_get_phys_port_id); 6443 6444 /** 6445 * dev_get_phys_port_name - Get device physical port name 6446 * @dev: device 6447 * @name: port name 6448 * @len: limit of bytes to copy to name 6449 * 6450 * Get device physical port name 6451 */ 6452 int dev_get_phys_port_name(struct net_device *dev, 6453 char *name, size_t len) 6454 { 6455 const struct net_device_ops *ops = dev->netdev_ops; 6456 6457 if (!ops->ndo_get_phys_port_name) 6458 return -EOPNOTSUPP; 6459 return ops->ndo_get_phys_port_name(dev, name, len); 6460 } 6461 EXPORT_SYMBOL(dev_get_phys_port_name); 6462 6463 /** 6464 * dev_change_proto_down - update protocol port state information 6465 * @dev: device 6466 * @proto_down: new value 6467 * 6468 * This info can be used by switch drivers to set the phys state of the 6469 * port. 6470 */ 6471 int dev_change_proto_down(struct net_device *dev, bool proto_down) 6472 { 6473 const struct net_device_ops *ops = dev->netdev_ops; 6474 6475 if (!ops->ndo_change_proto_down) 6476 return -EOPNOTSUPP; 6477 if (!netif_device_present(dev)) 6478 return -ENODEV; 6479 return ops->ndo_change_proto_down(dev, proto_down); 6480 } 6481 EXPORT_SYMBOL(dev_change_proto_down); 6482 6483 /** 6484 * dev_new_index - allocate an ifindex 6485 * @net: the applicable net namespace 6486 * 6487 * Returns a suitable unique value for a new device interface 6488 * number. The caller must hold the rtnl semaphore or the 6489 * dev_base_lock to be sure it remains unique. 6490 */ 6491 static int dev_new_index(struct net *net) 6492 { 6493 int ifindex = net->ifindex; 6494 for (;;) { 6495 if (++ifindex <= 0) 6496 ifindex = 1; 6497 if (!__dev_get_by_index(net, ifindex)) 6498 return net->ifindex = ifindex; 6499 } 6500 } 6501 6502 /* Delayed registration/unregisteration */ 6503 static LIST_HEAD(net_todo_list); 6504 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq); 6505 6506 static void net_set_todo(struct net_device *dev) 6507 { 6508 list_add_tail(&dev->todo_list, &net_todo_list); 6509 dev_net(dev)->dev_unreg_count++; 6510 } 6511 6512 static void rollback_registered_many(struct list_head *head) 6513 { 6514 struct net_device *dev, *tmp; 6515 LIST_HEAD(close_head); 6516 6517 BUG_ON(dev_boot_phase); 6518 ASSERT_RTNL(); 6519 6520 list_for_each_entry_safe(dev, tmp, head, unreg_list) { 6521 /* Some devices call without registering 6522 * for initialization unwind. Remove those 6523 * devices and proceed with the remaining. 6524 */ 6525 if (dev->reg_state == NETREG_UNINITIALIZED) { 6526 pr_debug("unregister_netdevice: device %s/%p never was registered\n", 6527 dev->name, dev); 6528 6529 WARN_ON(1); 6530 list_del(&dev->unreg_list); 6531 continue; 6532 } 6533 dev->dismantle = true; 6534 BUG_ON(dev->reg_state != NETREG_REGISTERED); 6535 } 6536 6537 /* If device is running, close it first. */ 6538 list_for_each_entry(dev, head, unreg_list) 6539 list_add_tail(&dev->close_list, &close_head); 6540 dev_close_many(&close_head, true); 6541 6542 list_for_each_entry(dev, head, unreg_list) { 6543 /* And unlink it from device chain. */ 6544 unlist_netdevice(dev); 6545 6546 dev->reg_state = NETREG_UNREGISTERING; 6547 on_each_cpu(flush_backlog, dev, 1); 6548 } 6549 6550 synchronize_net(); 6551 6552 list_for_each_entry(dev, head, unreg_list) { 6553 struct sk_buff *skb = NULL; 6554 6555 /* Shutdown queueing discipline. */ 6556 dev_shutdown(dev); 6557 6558 6559 /* Notify protocols, that we are about to destroy 6560 this device. They should clean all the things. 6561 */ 6562 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 6563 6564 if (!dev->rtnl_link_ops || 6565 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 6566 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 6567 GFP_KERNEL); 6568 6569 /* 6570 * Flush the unicast and multicast chains 6571 */ 6572 dev_uc_flush(dev); 6573 dev_mc_flush(dev); 6574 6575 if (dev->netdev_ops->ndo_uninit) 6576 dev->netdev_ops->ndo_uninit(dev); 6577 6578 if (skb) 6579 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL); 6580 6581 /* Notifier chain MUST detach us all upper devices. */ 6582 WARN_ON(netdev_has_any_upper_dev(dev)); 6583 6584 /* Remove entries from kobject tree */ 6585 netdev_unregister_kobject(dev); 6586 #ifdef CONFIG_XPS 6587 /* Remove XPS queueing entries */ 6588 netif_reset_xps_queues_gt(dev, 0); 6589 #endif 6590 } 6591 6592 synchronize_net(); 6593 6594 list_for_each_entry(dev, head, unreg_list) 6595 dev_put(dev); 6596 } 6597 6598 static void rollback_registered(struct net_device *dev) 6599 { 6600 LIST_HEAD(single); 6601 6602 list_add(&dev->unreg_list, &single); 6603 rollback_registered_many(&single); 6604 list_del(&single); 6605 } 6606 6607 static netdev_features_t netdev_sync_upper_features(struct net_device *lower, 6608 struct net_device *upper, netdev_features_t features) 6609 { 6610 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 6611 netdev_features_t feature; 6612 int feature_bit; 6613 6614 for_each_netdev_feature(&upper_disables, feature_bit) { 6615 feature = __NETIF_F_BIT(feature_bit); 6616 if (!(upper->wanted_features & feature) 6617 && (features & feature)) { 6618 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n", 6619 &feature, upper->name); 6620 features &= ~feature; 6621 } 6622 } 6623 6624 return features; 6625 } 6626 6627 static void netdev_sync_lower_features(struct net_device *upper, 6628 struct net_device *lower, netdev_features_t features) 6629 { 6630 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 6631 netdev_features_t feature; 6632 int feature_bit; 6633 6634 for_each_netdev_feature(&upper_disables, feature_bit) { 6635 feature = __NETIF_F_BIT(feature_bit); 6636 if (!(features & feature) && (lower->features & feature)) { 6637 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n", 6638 &feature, lower->name); 6639 lower->wanted_features &= ~feature; 6640 netdev_update_features(lower); 6641 6642 if (unlikely(lower->features & feature)) 6643 netdev_WARN(upper, "failed to disable %pNF on %s!\n", 6644 &feature, lower->name); 6645 } 6646 } 6647 } 6648 6649 static netdev_features_t netdev_fix_features(struct net_device *dev, 6650 netdev_features_t features) 6651 { 6652 /* Fix illegal checksum combinations */ 6653 if ((features & NETIF_F_HW_CSUM) && 6654 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { 6655 netdev_warn(dev, "mixed HW and IP checksum settings.\n"); 6656 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); 6657 } 6658 6659 /* TSO requires that SG is present as well. */ 6660 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) { 6661 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n"); 6662 features &= ~NETIF_F_ALL_TSO; 6663 } 6664 6665 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) && 6666 !(features & NETIF_F_IP_CSUM)) { 6667 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n"); 6668 features &= ~NETIF_F_TSO; 6669 features &= ~NETIF_F_TSO_ECN; 6670 } 6671 6672 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) && 6673 !(features & NETIF_F_IPV6_CSUM)) { 6674 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n"); 6675 features &= ~NETIF_F_TSO6; 6676 } 6677 6678 /* TSO ECN requires that TSO is present as well. */ 6679 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN) 6680 features &= ~NETIF_F_TSO_ECN; 6681 6682 /* Software GSO depends on SG. */ 6683 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) { 6684 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n"); 6685 features &= ~NETIF_F_GSO; 6686 } 6687 6688 /* UFO needs SG and checksumming */ 6689 if (features & NETIF_F_UFO) { 6690 /* maybe split UFO into V4 and V6? */ 6691 if (!(features & NETIF_F_HW_CSUM) && 6692 ((features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) != 6693 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) { 6694 netdev_dbg(dev, 6695 "Dropping NETIF_F_UFO since no checksum offload features.\n"); 6696 features &= ~NETIF_F_UFO; 6697 } 6698 6699 if (!(features & NETIF_F_SG)) { 6700 netdev_dbg(dev, 6701 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n"); 6702 features &= ~NETIF_F_UFO; 6703 } 6704 } 6705 6706 #ifdef CONFIG_NET_RX_BUSY_POLL 6707 if (dev->netdev_ops->ndo_busy_poll) 6708 features |= NETIF_F_BUSY_POLL; 6709 else 6710 #endif 6711 features &= ~NETIF_F_BUSY_POLL; 6712 6713 return features; 6714 } 6715 6716 int __netdev_update_features(struct net_device *dev) 6717 { 6718 struct net_device *upper, *lower; 6719 netdev_features_t features; 6720 struct list_head *iter; 6721 int err = -1; 6722 6723 ASSERT_RTNL(); 6724 6725 features = netdev_get_wanted_features(dev); 6726 6727 if (dev->netdev_ops->ndo_fix_features) 6728 features = dev->netdev_ops->ndo_fix_features(dev, features); 6729 6730 /* driver might be less strict about feature dependencies */ 6731 features = netdev_fix_features(dev, features); 6732 6733 /* some features can't be enabled if they're off an an upper device */ 6734 netdev_for_each_upper_dev_rcu(dev, upper, iter) 6735 features = netdev_sync_upper_features(dev, upper, features); 6736 6737 if (dev->features == features) 6738 goto sync_lower; 6739 6740 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", 6741 &dev->features, &features); 6742 6743 if (dev->netdev_ops->ndo_set_features) 6744 err = dev->netdev_ops->ndo_set_features(dev, features); 6745 else 6746 err = 0; 6747 6748 if (unlikely(err < 0)) { 6749 netdev_err(dev, 6750 "set_features() failed (%d); wanted %pNF, left %pNF\n", 6751 err, &features, &dev->features); 6752 /* return non-0 since some features might have changed and 6753 * it's better to fire a spurious notification than miss it 6754 */ 6755 return -1; 6756 } 6757 6758 sync_lower: 6759 /* some features must be disabled on lower devices when disabled 6760 * on an upper device (think: bonding master or bridge) 6761 */ 6762 netdev_for_each_lower_dev(dev, lower, iter) 6763 netdev_sync_lower_features(dev, lower, features); 6764 6765 if (!err) 6766 dev->features = features; 6767 6768 return err < 0 ? 0 : 1; 6769 } 6770 6771 /** 6772 * netdev_update_features - recalculate device features 6773 * @dev: the device to check 6774 * 6775 * Recalculate dev->features set and send notifications if it 6776 * has changed. Should be called after driver or hardware dependent 6777 * conditions might have changed that influence the features. 6778 */ 6779 void netdev_update_features(struct net_device *dev) 6780 { 6781 if (__netdev_update_features(dev)) 6782 netdev_features_change(dev); 6783 } 6784 EXPORT_SYMBOL(netdev_update_features); 6785 6786 /** 6787 * netdev_change_features - recalculate device features 6788 * @dev: the device to check 6789 * 6790 * Recalculate dev->features set and send notifications even 6791 * if they have not changed. Should be called instead of 6792 * netdev_update_features() if also dev->vlan_features might 6793 * have changed to allow the changes to be propagated to stacked 6794 * VLAN devices. 6795 */ 6796 void netdev_change_features(struct net_device *dev) 6797 { 6798 __netdev_update_features(dev); 6799 netdev_features_change(dev); 6800 } 6801 EXPORT_SYMBOL(netdev_change_features); 6802 6803 /** 6804 * netif_stacked_transfer_operstate - transfer operstate 6805 * @rootdev: the root or lower level device to transfer state from 6806 * @dev: the device to transfer operstate to 6807 * 6808 * Transfer operational state from root to device. This is normally 6809 * called when a stacking relationship exists between the root 6810 * device and the device(a leaf device). 6811 */ 6812 void netif_stacked_transfer_operstate(const struct net_device *rootdev, 6813 struct net_device *dev) 6814 { 6815 if (rootdev->operstate == IF_OPER_DORMANT) 6816 netif_dormant_on(dev); 6817 else 6818 netif_dormant_off(dev); 6819 6820 if (netif_carrier_ok(rootdev)) { 6821 if (!netif_carrier_ok(dev)) 6822 netif_carrier_on(dev); 6823 } else { 6824 if (netif_carrier_ok(dev)) 6825 netif_carrier_off(dev); 6826 } 6827 } 6828 EXPORT_SYMBOL(netif_stacked_transfer_operstate); 6829 6830 #ifdef CONFIG_SYSFS 6831 static int netif_alloc_rx_queues(struct net_device *dev) 6832 { 6833 unsigned int i, count = dev->num_rx_queues; 6834 struct netdev_rx_queue *rx; 6835 size_t sz = count * sizeof(*rx); 6836 6837 BUG_ON(count < 1); 6838 6839 rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 6840 if (!rx) { 6841 rx = vzalloc(sz); 6842 if (!rx) 6843 return -ENOMEM; 6844 } 6845 dev->_rx = rx; 6846 6847 for (i = 0; i < count; i++) 6848 rx[i].dev = dev; 6849 return 0; 6850 } 6851 #endif 6852 6853 static void netdev_init_one_queue(struct net_device *dev, 6854 struct netdev_queue *queue, void *_unused) 6855 { 6856 /* Initialize queue lock */ 6857 spin_lock_init(&queue->_xmit_lock); 6858 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); 6859 queue->xmit_lock_owner = -1; 6860 netdev_queue_numa_node_write(queue, NUMA_NO_NODE); 6861 queue->dev = dev; 6862 #ifdef CONFIG_BQL 6863 dql_init(&queue->dql, HZ); 6864 #endif 6865 } 6866 6867 static void netif_free_tx_queues(struct net_device *dev) 6868 { 6869 kvfree(dev->_tx); 6870 } 6871 6872 static int netif_alloc_netdev_queues(struct net_device *dev) 6873 { 6874 unsigned int count = dev->num_tx_queues; 6875 struct netdev_queue *tx; 6876 size_t sz = count * sizeof(*tx); 6877 6878 if (count < 1 || count > 0xffff) 6879 return -EINVAL; 6880 6881 tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 6882 if (!tx) { 6883 tx = vzalloc(sz); 6884 if (!tx) 6885 return -ENOMEM; 6886 } 6887 dev->_tx = tx; 6888 6889 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); 6890 spin_lock_init(&dev->tx_global_lock); 6891 6892 return 0; 6893 } 6894 6895 void netif_tx_stop_all_queues(struct net_device *dev) 6896 { 6897 unsigned int i; 6898 6899 for (i = 0; i < dev->num_tx_queues; i++) { 6900 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 6901 netif_tx_stop_queue(txq); 6902 } 6903 } 6904 EXPORT_SYMBOL(netif_tx_stop_all_queues); 6905 6906 /** 6907 * register_netdevice - register a network device 6908 * @dev: device to register 6909 * 6910 * Take a completed network device structure and add it to the kernel 6911 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 6912 * chain. 0 is returned on success. A negative errno code is returned 6913 * on a failure to set up the device, or if the name is a duplicate. 6914 * 6915 * Callers must hold the rtnl semaphore. You may want 6916 * register_netdev() instead of this. 6917 * 6918 * BUGS: 6919 * The locking appears insufficient to guarantee two parallel registers 6920 * will not get the same name. 6921 */ 6922 6923 int register_netdevice(struct net_device *dev) 6924 { 6925 int ret; 6926 struct net *net = dev_net(dev); 6927 6928 BUG_ON(dev_boot_phase); 6929 ASSERT_RTNL(); 6930 6931 might_sleep(); 6932 6933 /* When net_device's are persistent, this will be fatal. */ 6934 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); 6935 BUG_ON(!net); 6936 6937 spin_lock_init(&dev->addr_list_lock); 6938 netdev_set_addr_lockdep_class(dev); 6939 6940 ret = dev_get_valid_name(net, dev, dev->name); 6941 if (ret < 0) 6942 goto out; 6943 6944 /* Init, if this function is available */ 6945 if (dev->netdev_ops->ndo_init) { 6946 ret = dev->netdev_ops->ndo_init(dev); 6947 if (ret) { 6948 if (ret > 0) 6949 ret = -EIO; 6950 goto out; 6951 } 6952 } 6953 6954 if (((dev->hw_features | dev->features) & 6955 NETIF_F_HW_VLAN_CTAG_FILTER) && 6956 (!dev->netdev_ops->ndo_vlan_rx_add_vid || 6957 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { 6958 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n"); 6959 ret = -EINVAL; 6960 goto err_uninit; 6961 } 6962 6963 ret = -EBUSY; 6964 if (!dev->ifindex) 6965 dev->ifindex = dev_new_index(net); 6966 else if (__dev_get_by_index(net, dev->ifindex)) 6967 goto err_uninit; 6968 6969 /* Transfer changeable features to wanted_features and enable 6970 * software offloads (GSO and GRO). 6971 */ 6972 dev->hw_features |= NETIF_F_SOFT_FEATURES; 6973 dev->features |= NETIF_F_SOFT_FEATURES; 6974 dev->wanted_features = dev->features & dev->hw_features; 6975 6976 if (!(dev->flags & IFF_LOOPBACK)) { 6977 dev->hw_features |= NETIF_F_NOCACHE_COPY; 6978 } 6979 6980 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. 6981 */ 6982 dev->vlan_features |= NETIF_F_HIGHDMA; 6983 6984 /* Make NETIF_F_SG inheritable to tunnel devices. 6985 */ 6986 dev->hw_enc_features |= NETIF_F_SG; 6987 6988 /* Make NETIF_F_SG inheritable to MPLS. 6989 */ 6990 dev->mpls_features |= NETIF_F_SG; 6991 6992 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); 6993 ret = notifier_to_errno(ret); 6994 if (ret) 6995 goto err_uninit; 6996 6997 ret = netdev_register_kobject(dev); 6998 if (ret) 6999 goto err_uninit; 7000 dev->reg_state = NETREG_REGISTERED; 7001 7002 __netdev_update_features(dev); 7003 7004 /* 7005 * Default initial state at registry is that the 7006 * device is present. 7007 */ 7008 7009 set_bit(__LINK_STATE_PRESENT, &dev->state); 7010 7011 linkwatch_init_dev(dev); 7012 7013 dev_init_scheduler(dev); 7014 dev_hold(dev); 7015 list_netdevice(dev); 7016 add_device_randomness(dev->dev_addr, dev->addr_len); 7017 7018 /* If the device has permanent device address, driver should 7019 * set dev_addr and also addr_assign_type should be set to 7020 * NET_ADDR_PERM (default value). 7021 */ 7022 if (dev->addr_assign_type == NET_ADDR_PERM) 7023 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); 7024 7025 /* Notify protocols, that a new device appeared. */ 7026 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); 7027 ret = notifier_to_errno(ret); 7028 if (ret) { 7029 rollback_registered(dev); 7030 dev->reg_state = NETREG_UNREGISTERED; 7031 } 7032 /* 7033 * Prevent userspace races by waiting until the network 7034 * device is fully setup before sending notifications. 7035 */ 7036 if (!dev->rtnl_link_ops || 7037 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 7038 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); 7039 7040 out: 7041 return ret; 7042 7043 err_uninit: 7044 if (dev->netdev_ops->ndo_uninit) 7045 dev->netdev_ops->ndo_uninit(dev); 7046 goto out; 7047 } 7048 EXPORT_SYMBOL(register_netdevice); 7049 7050 /** 7051 * init_dummy_netdev - init a dummy network device for NAPI 7052 * @dev: device to init 7053 * 7054 * This takes a network device structure and initialize the minimum 7055 * amount of fields so it can be used to schedule NAPI polls without 7056 * registering a full blown interface. This is to be used by drivers 7057 * that need to tie several hardware interfaces to a single NAPI 7058 * poll scheduler due to HW limitations. 7059 */ 7060 int init_dummy_netdev(struct net_device *dev) 7061 { 7062 /* Clear everything. Note we don't initialize spinlocks 7063 * are they aren't supposed to be taken by any of the 7064 * NAPI code and this dummy netdev is supposed to be 7065 * only ever used for NAPI polls 7066 */ 7067 memset(dev, 0, sizeof(struct net_device)); 7068 7069 /* make sure we BUG if trying to hit standard 7070 * register/unregister code path 7071 */ 7072 dev->reg_state = NETREG_DUMMY; 7073 7074 /* NAPI wants this */ 7075 INIT_LIST_HEAD(&dev->napi_list); 7076 7077 /* a dummy interface is started by default */ 7078 set_bit(__LINK_STATE_PRESENT, &dev->state); 7079 set_bit(__LINK_STATE_START, &dev->state); 7080 7081 /* Note : We dont allocate pcpu_refcnt for dummy devices, 7082 * because users of this 'device' dont need to change 7083 * its refcount. 7084 */ 7085 7086 return 0; 7087 } 7088 EXPORT_SYMBOL_GPL(init_dummy_netdev); 7089 7090 7091 /** 7092 * register_netdev - register a network device 7093 * @dev: device to register 7094 * 7095 * Take a completed network device structure and add it to the kernel 7096 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 7097 * chain. 0 is returned on success. A negative errno code is returned 7098 * on a failure to set up the device, or if the name is a duplicate. 7099 * 7100 * This is a wrapper around register_netdevice that takes the rtnl semaphore 7101 * and expands the device name if you passed a format string to 7102 * alloc_netdev. 7103 */ 7104 int register_netdev(struct net_device *dev) 7105 { 7106 int err; 7107 7108 rtnl_lock(); 7109 err = register_netdevice(dev); 7110 rtnl_unlock(); 7111 return err; 7112 } 7113 EXPORT_SYMBOL(register_netdev); 7114 7115 int netdev_refcnt_read(const struct net_device *dev) 7116 { 7117 int i, refcnt = 0; 7118 7119 for_each_possible_cpu(i) 7120 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); 7121 return refcnt; 7122 } 7123 EXPORT_SYMBOL(netdev_refcnt_read); 7124 7125 /** 7126 * netdev_wait_allrefs - wait until all references are gone. 7127 * @dev: target net_device 7128 * 7129 * This is called when unregistering network devices. 7130 * 7131 * Any protocol or device that holds a reference should register 7132 * for netdevice notification, and cleanup and put back the 7133 * reference if they receive an UNREGISTER event. 7134 * We can get stuck here if buggy protocols don't correctly 7135 * call dev_put. 7136 */ 7137 static void netdev_wait_allrefs(struct net_device *dev) 7138 { 7139 unsigned long rebroadcast_time, warning_time; 7140 int refcnt; 7141 7142 linkwatch_forget_dev(dev); 7143 7144 rebroadcast_time = warning_time = jiffies; 7145 refcnt = netdev_refcnt_read(dev); 7146 7147 while (refcnt != 0) { 7148 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { 7149 rtnl_lock(); 7150 7151 /* Rebroadcast unregister notification */ 7152 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 7153 7154 __rtnl_unlock(); 7155 rcu_barrier(); 7156 rtnl_lock(); 7157 7158 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 7159 if (test_bit(__LINK_STATE_LINKWATCH_PENDING, 7160 &dev->state)) { 7161 /* We must not have linkwatch events 7162 * pending on unregister. If this 7163 * happens, we simply run the queue 7164 * unscheduled, resulting in a noop 7165 * for this device. 7166 */ 7167 linkwatch_run_queue(); 7168 } 7169 7170 __rtnl_unlock(); 7171 7172 rebroadcast_time = jiffies; 7173 } 7174 7175 msleep(250); 7176 7177 refcnt = netdev_refcnt_read(dev); 7178 7179 if (time_after(jiffies, warning_time + 10 * HZ)) { 7180 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", 7181 dev->name, refcnt); 7182 warning_time = jiffies; 7183 } 7184 } 7185 } 7186 7187 /* The sequence is: 7188 * 7189 * rtnl_lock(); 7190 * ... 7191 * register_netdevice(x1); 7192 * register_netdevice(x2); 7193 * ... 7194 * unregister_netdevice(y1); 7195 * unregister_netdevice(y2); 7196 * ... 7197 * rtnl_unlock(); 7198 * free_netdev(y1); 7199 * free_netdev(y2); 7200 * 7201 * We are invoked by rtnl_unlock(). 7202 * This allows us to deal with problems: 7203 * 1) We can delete sysfs objects which invoke hotplug 7204 * without deadlocking with linkwatch via keventd. 7205 * 2) Since we run with the RTNL semaphore not held, we can sleep 7206 * safely in order to wait for the netdev refcnt to drop to zero. 7207 * 7208 * We must not return until all unregister events added during 7209 * the interval the lock was held have been completed. 7210 */ 7211 void netdev_run_todo(void) 7212 { 7213 struct list_head list; 7214 7215 /* Snapshot list, allow later requests */ 7216 list_replace_init(&net_todo_list, &list); 7217 7218 __rtnl_unlock(); 7219 7220 7221 /* Wait for rcu callbacks to finish before next phase */ 7222 if (!list_empty(&list)) 7223 rcu_barrier(); 7224 7225 while (!list_empty(&list)) { 7226 struct net_device *dev 7227 = list_first_entry(&list, struct net_device, todo_list); 7228 list_del(&dev->todo_list); 7229 7230 rtnl_lock(); 7231 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 7232 __rtnl_unlock(); 7233 7234 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { 7235 pr_err("network todo '%s' but state %d\n", 7236 dev->name, dev->reg_state); 7237 dump_stack(); 7238 continue; 7239 } 7240 7241 dev->reg_state = NETREG_UNREGISTERED; 7242 7243 netdev_wait_allrefs(dev); 7244 7245 /* paranoia */ 7246 BUG_ON(netdev_refcnt_read(dev)); 7247 BUG_ON(!list_empty(&dev->ptype_all)); 7248 BUG_ON(!list_empty(&dev->ptype_specific)); 7249 WARN_ON(rcu_access_pointer(dev->ip_ptr)); 7250 WARN_ON(rcu_access_pointer(dev->ip6_ptr)); 7251 WARN_ON(dev->dn_ptr); 7252 7253 if (dev->destructor) 7254 dev->destructor(dev); 7255 7256 /* Report a network device has been unregistered */ 7257 rtnl_lock(); 7258 dev_net(dev)->dev_unreg_count--; 7259 __rtnl_unlock(); 7260 wake_up(&netdev_unregistering_wq); 7261 7262 /* Free network device */ 7263 kobject_put(&dev->dev.kobj); 7264 } 7265 } 7266 7267 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has 7268 * all the same fields in the same order as net_device_stats, with only 7269 * the type differing, but rtnl_link_stats64 may have additional fields 7270 * at the end for newer counters. 7271 */ 7272 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64, 7273 const struct net_device_stats *netdev_stats) 7274 { 7275 #if BITS_PER_LONG == 64 7276 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats)); 7277 memcpy(stats64, netdev_stats, sizeof(*stats64)); 7278 /* zero out counters that only exist in rtnl_link_stats64 */ 7279 memset((char *)stats64 + sizeof(*netdev_stats), 0, 7280 sizeof(*stats64) - sizeof(*netdev_stats)); 7281 #else 7282 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long); 7283 const unsigned long *src = (const unsigned long *)netdev_stats; 7284 u64 *dst = (u64 *)stats64; 7285 7286 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64)); 7287 for (i = 0; i < n; i++) 7288 dst[i] = src[i]; 7289 /* zero out counters that only exist in rtnl_link_stats64 */ 7290 memset((char *)stats64 + n * sizeof(u64), 0, 7291 sizeof(*stats64) - n * sizeof(u64)); 7292 #endif 7293 } 7294 EXPORT_SYMBOL(netdev_stats_to_stats64); 7295 7296 /** 7297 * dev_get_stats - get network device statistics 7298 * @dev: device to get statistics from 7299 * @storage: place to store stats 7300 * 7301 * Get network statistics from device. Return @storage. 7302 * The device driver may provide its own method by setting 7303 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats; 7304 * otherwise the internal statistics structure is used. 7305 */ 7306 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, 7307 struct rtnl_link_stats64 *storage) 7308 { 7309 const struct net_device_ops *ops = dev->netdev_ops; 7310 7311 if (ops->ndo_get_stats64) { 7312 memset(storage, 0, sizeof(*storage)); 7313 ops->ndo_get_stats64(dev, storage); 7314 } else if (ops->ndo_get_stats) { 7315 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); 7316 } else { 7317 netdev_stats_to_stats64(storage, &dev->stats); 7318 } 7319 storage->rx_dropped += atomic_long_read(&dev->rx_dropped); 7320 storage->tx_dropped += atomic_long_read(&dev->tx_dropped); 7321 storage->rx_nohandler += atomic_long_read(&dev->rx_nohandler); 7322 return storage; 7323 } 7324 EXPORT_SYMBOL(dev_get_stats); 7325 7326 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) 7327 { 7328 struct netdev_queue *queue = dev_ingress_queue(dev); 7329 7330 #ifdef CONFIG_NET_CLS_ACT 7331 if (queue) 7332 return queue; 7333 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 7334 if (!queue) 7335 return NULL; 7336 netdev_init_one_queue(dev, queue, NULL); 7337 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); 7338 queue->qdisc_sleeping = &noop_qdisc; 7339 rcu_assign_pointer(dev->ingress_queue, queue); 7340 #endif 7341 return queue; 7342 } 7343 7344 static const struct ethtool_ops default_ethtool_ops; 7345 7346 void netdev_set_default_ethtool_ops(struct net_device *dev, 7347 const struct ethtool_ops *ops) 7348 { 7349 if (dev->ethtool_ops == &default_ethtool_ops) 7350 dev->ethtool_ops = ops; 7351 } 7352 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops); 7353 7354 void netdev_freemem(struct net_device *dev) 7355 { 7356 char *addr = (char *)dev - dev->padded; 7357 7358 kvfree(addr); 7359 } 7360 7361 /** 7362 * alloc_netdev_mqs - allocate network device 7363 * @sizeof_priv: size of private data to allocate space for 7364 * @name: device name format string 7365 * @name_assign_type: origin of device name 7366 * @setup: callback to initialize device 7367 * @txqs: the number of TX subqueues to allocate 7368 * @rxqs: the number of RX subqueues to allocate 7369 * 7370 * Allocates a struct net_device with private data area for driver use 7371 * and performs basic initialization. Also allocates subqueue structs 7372 * for each queue on the device. 7373 */ 7374 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, 7375 unsigned char name_assign_type, 7376 void (*setup)(struct net_device *), 7377 unsigned int txqs, unsigned int rxqs) 7378 { 7379 struct net_device *dev; 7380 size_t alloc_size; 7381 struct net_device *p; 7382 7383 BUG_ON(strlen(name) >= sizeof(dev->name)); 7384 7385 if (txqs < 1) { 7386 pr_err("alloc_netdev: Unable to allocate device with zero queues\n"); 7387 return NULL; 7388 } 7389 7390 #ifdef CONFIG_SYSFS 7391 if (rxqs < 1) { 7392 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n"); 7393 return NULL; 7394 } 7395 #endif 7396 7397 alloc_size = sizeof(struct net_device); 7398 if (sizeof_priv) { 7399 /* ensure 32-byte alignment of private area */ 7400 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN); 7401 alloc_size += sizeof_priv; 7402 } 7403 /* ensure 32-byte alignment of whole construct */ 7404 alloc_size += NETDEV_ALIGN - 1; 7405 7406 p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 7407 if (!p) 7408 p = vzalloc(alloc_size); 7409 if (!p) 7410 return NULL; 7411 7412 dev = PTR_ALIGN(p, NETDEV_ALIGN); 7413 dev->padded = (char *)dev - (char *)p; 7414 7415 dev->pcpu_refcnt = alloc_percpu(int); 7416 if (!dev->pcpu_refcnt) 7417 goto free_dev; 7418 7419 if (dev_addr_init(dev)) 7420 goto free_pcpu; 7421 7422 dev_mc_init(dev); 7423 dev_uc_init(dev); 7424 7425 dev_net_set(dev, &init_net); 7426 7427 dev->gso_max_size = GSO_MAX_SIZE; 7428 dev->gso_max_segs = GSO_MAX_SEGS; 7429 dev->gso_min_segs = 0; 7430 7431 INIT_LIST_HEAD(&dev->napi_list); 7432 INIT_LIST_HEAD(&dev->unreg_list); 7433 INIT_LIST_HEAD(&dev->close_list); 7434 INIT_LIST_HEAD(&dev->link_watch_list); 7435 INIT_LIST_HEAD(&dev->adj_list.upper); 7436 INIT_LIST_HEAD(&dev->adj_list.lower); 7437 INIT_LIST_HEAD(&dev->all_adj_list.upper); 7438 INIT_LIST_HEAD(&dev->all_adj_list.lower); 7439 INIT_LIST_HEAD(&dev->ptype_all); 7440 INIT_LIST_HEAD(&dev->ptype_specific); 7441 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; 7442 setup(dev); 7443 7444 if (!dev->tx_queue_len) { 7445 dev->priv_flags |= IFF_NO_QUEUE; 7446 dev->tx_queue_len = 1; 7447 } 7448 7449 dev->num_tx_queues = txqs; 7450 dev->real_num_tx_queues = txqs; 7451 if (netif_alloc_netdev_queues(dev)) 7452 goto free_all; 7453 7454 #ifdef CONFIG_SYSFS 7455 dev->num_rx_queues = rxqs; 7456 dev->real_num_rx_queues = rxqs; 7457 if (netif_alloc_rx_queues(dev)) 7458 goto free_all; 7459 #endif 7460 7461 strcpy(dev->name, name); 7462 dev->name_assign_type = name_assign_type; 7463 dev->group = INIT_NETDEV_GROUP; 7464 if (!dev->ethtool_ops) 7465 dev->ethtool_ops = &default_ethtool_ops; 7466 7467 nf_hook_ingress_init(dev); 7468 7469 return dev; 7470 7471 free_all: 7472 free_netdev(dev); 7473 return NULL; 7474 7475 free_pcpu: 7476 free_percpu(dev->pcpu_refcnt); 7477 free_dev: 7478 netdev_freemem(dev); 7479 return NULL; 7480 } 7481 EXPORT_SYMBOL(alloc_netdev_mqs); 7482 7483 /** 7484 * free_netdev - free network device 7485 * @dev: device 7486 * 7487 * This function does the last stage of destroying an allocated device 7488 * interface. The reference to the device object is released. 7489 * If this is the last reference then it will be freed. 7490 * Must be called in process context. 7491 */ 7492 void free_netdev(struct net_device *dev) 7493 { 7494 struct napi_struct *p, *n; 7495 7496 might_sleep(); 7497 netif_free_tx_queues(dev); 7498 #ifdef CONFIG_SYSFS 7499 kvfree(dev->_rx); 7500 #endif 7501 7502 kfree(rcu_dereference_protected(dev->ingress_queue, 1)); 7503 7504 /* Flush device addresses */ 7505 dev_addr_flush(dev); 7506 7507 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) 7508 netif_napi_del(p); 7509 7510 free_percpu(dev->pcpu_refcnt); 7511 dev->pcpu_refcnt = NULL; 7512 7513 /* Compatibility with error handling in drivers */ 7514 if (dev->reg_state == NETREG_UNINITIALIZED) { 7515 netdev_freemem(dev); 7516 return; 7517 } 7518 7519 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); 7520 dev->reg_state = NETREG_RELEASED; 7521 7522 /* will free via device release */ 7523 put_device(&dev->dev); 7524 } 7525 EXPORT_SYMBOL(free_netdev); 7526 7527 /** 7528 * synchronize_net - Synchronize with packet receive processing 7529 * 7530 * Wait for packets currently being received to be done. 7531 * Does not block later packets from starting. 7532 */ 7533 void synchronize_net(void) 7534 { 7535 might_sleep(); 7536 if (rtnl_is_locked()) 7537 synchronize_rcu_expedited(); 7538 else 7539 synchronize_rcu(); 7540 } 7541 EXPORT_SYMBOL(synchronize_net); 7542 7543 /** 7544 * unregister_netdevice_queue - remove device from the kernel 7545 * @dev: device 7546 * @head: list 7547 * 7548 * This function shuts down a device interface and removes it 7549 * from the kernel tables. 7550 * If head not NULL, device is queued to be unregistered later. 7551 * 7552 * Callers must hold the rtnl semaphore. You may want 7553 * unregister_netdev() instead of this. 7554 */ 7555 7556 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) 7557 { 7558 ASSERT_RTNL(); 7559 7560 if (head) { 7561 list_move_tail(&dev->unreg_list, head); 7562 } else { 7563 rollback_registered(dev); 7564 /* Finish processing unregister after unlock */ 7565 net_set_todo(dev); 7566 } 7567 } 7568 EXPORT_SYMBOL(unregister_netdevice_queue); 7569 7570 /** 7571 * unregister_netdevice_many - unregister many devices 7572 * @head: list of devices 7573 * 7574 * Note: As most callers use a stack allocated list_head, 7575 * we force a list_del() to make sure stack wont be corrupted later. 7576 */ 7577 void unregister_netdevice_many(struct list_head *head) 7578 { 7579 struct net_device *dev; 7580 7581 if (!list_empty(head)) { 7582 rollback_registered_many(head); 7583 list_for_each_entry(dev, head, unreg_list) 7584 net_set_todo(dev); 7585 list_del(head); 7586 } 7587 } 7588 EXPORT_SYMBOL(unregister_netdevice_many); 7589 7590 /** 7591 * unregister_netdev - remove device from the kernel 7592 * @dev: device 7593 * 7594 * This function shuts down a device interface and removes it 7595 * from the kernel tables. 7596 * 7597 * This is just a wrapper for unregister_netdevice that takes 7598 * the rtnl semaphore. In general you want to use this and not 7599 * unregister_netdevice. 7600 */ 7601 void unregister_netdev(struct net_device *dev) 7602 { 7603 rtnl_lock(); 7604 unregister_netdevice(dev); 7605 rtnl_unlock(); 7606 } 7607 EXPORT_SYMBOL(unregister_netdev); 7608 7609 /** 7610 * dev_change_net_namespace - move device to different nethost namespace 7611 * @dev: device 7612 * @net: network namespace 7613 * @pat: If not NULL name pattern to try if the current device name 7614 * is already taken in the destination network namespace. 7615 * 7616 * This function shuts down a device interface and moves it 7617 * to a new network namespace. On success 0 is returned, on 7618 * a failure a netagive errno code is returned. 7619 * 7620 * Callers must hold the rtnl semaphore. 7621 */ 7622 7623 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat) 7624 { 7625 int err; 7626 7627 ASSERT_RTNL(); 7628 7629 /* Don't allow namespace local devices to be moved. */ 7630 err = -EINVAL; 7631 if (dev->features & NETIF_F_NETNS_LOCAL) 7632 goto out; 7633 7634 /* Ensure the device has been registrered */ 7635 if (dev->reg_state != NETREG_REGISTERED) 7636 goto out; 7637 7638 /* Get out if there is nothing todo */ 7639 err = 0; 7640 if (net_eq(dev_net(dev), net)) 7641 goto out; 7642 7643 /* Pick the destination device name, and ensure 7644 * we can use it in the destination network namespace. 7645 */ 7646 err = -EEXIST; 7647 if (__dev_get_by_name(net, dev->name)) { 7648 /* We get here if we can't use the current device name */ 7649 if (!pat) 7650 goto out; 7651 if (dev_get_valid_name(net, dev, pat) < 0) 7652 goto out; 7653 } 7654 7655 /* 7656 * And now a mini version of register_netdevice unregister_netdevice. 7657 */ 7658 7659 /* If device is running close it first. */ 7660 dev_close(dev); 7661 7662 /* And unlink it from device chain */ 7663 err = -ENODEV; 7664 unlist_netdevice(dev); 7665 7666 synchronize_net(); 7667 7668 /* Shutdown queueing discipline. */ 7669 dev_shutdown(dev); 7670 7671 /* Notify protocols, that we are about to destroy 7672 this device. They should clean all the things. 7673 7674 Note that dev->reg_state stays at NETREG_REGISTERED. 7675 This is wanted because this way 8021q and macvlan know 7676 the device is just moving and can keep their slaves up. 7677 */ 7678 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 7679 rcu_barrier(); 7680 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 7681 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL); 7682 7683 /* 7684 * Flush the unicast and multicast chains 7685 */ 7686 dev_uc_flush(dev); 7687 dev_mc_flush(dev); 7688 7689 /* Send a netdev-removed uevent to the old namespace */ 7690 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); 7691 netdev_adjacent_del_links(dev); 7692 7693 /* Actually switch the network namespace */ 7694 dev_net_set(dev, net); 7695 7696 /* If there is an ifindex conflict assign a new one */ 7697 if (__dev_get_by_index(net, dev->ifindex)) 7698 dev->ifindex = dev_new_index(net); 7699 7700 /* Send a netdev-add uevent to the new namespace */ 7701 kobject_uevent(&dev->dev.kobj, KOBJ_ADD); 7702 netdev_adjacent_add_links(dev); 7703 7704 /* Fixup kobjects */ 7705 err = device_rename(&dev->dev, dev->name); 7706 WARN_ON(err); 7707 7708 /* Add the device back in the hashes */ 7709 list_netdevice(dev); 7710 7711 /* Notify protocols, that a new device appeared. */ 7712 call_netdevice_notifiers(NETDEV_REGISTER, dev); 7713 7714 /* 7715 * Prevent userspace races by waiting until the network 7716 * device is fully setup before sending notifications. 7717 */ 7718 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); 7719 7720 synchronize_net(); 7721 err = 0; 7722 out: 7723 return err; 7724 } 7725 EXPORT_SYMBOL_GPL(dev_change_net_namespace); 7726 7727 static int dev_cpu_callback(struct notifier_block *nfb, 7728 unsigned long action, 7729 void *ocpu) 7730 { 7731 struct sk_buff **list_skb; 7732 struct sk_buff *skb; 7733 unsigned int cpu, oldcpu = (unsigned long)ocpu; 7734 struct softnet_data *sd, *oldsd; 7735 7736 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN) 7737 return NOTIFY_OK; 7738 7739 local_irq_disable(); 7740 cpu = smp_processor_id(); 7741 sd = &per_cpu(softnet_data, cpu); 7742 oldsd = &per_cpu(softnet_data, oldcpu); 7743 7744 /* Find end of our completion_queue. */ 7745 list_skb = &sd->completion_queue; 7746 while (*list_skb) 7747 list_skb = &(*list_skb)->next; 7748 /* Append completion queue from offline CPU. */ 7749 *list_skb = oldsd->completion_queue; 7750 oldsd->completion_queue = NULL; 7751 7752 /* Append output queue from offline CPU. */ 7753 if (oldsd->output_queue) { 7754 *sd->output_queue_tailp = oldsd->output_queue; 7755 sd->output_queue_tailp = oldsd->output_queue_tailp; 7756 oldsd->output_queue = NULL; 7757 oldsd->output_queue_tailp = &oldsd->output_queue; 7758 } 7759 /* Append NAPI poll list from offline CPU, with one exception : 7760 * process_backlog() must be called by cpu owning percpu backlog. 7761 * We properly handle process_queue & input_pkt_queue later. 7762 */ 7763 while (!list_empty(&oldsd->poll_list)) { 7764 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, 7765 struct napi_struct, 7766 poll_list); 7767 7768 list_del_init(&napi->poll_list); 7769 if (napi->poll == process_backlog) 7770 napi->state = 0; 7771 else 7772 ____napi_schedule(sd, napi); 7773 } 7774 7775 raise_softirq_irqoff(NET_TX_SOFTIRQ); 7776 local_irq_enable(); 7777 7778 /* Process offline CPU's input_pkt_queue */ 7779 while ((skb = __skb_dequeue(&oldsd->process_queue))) { 7780 netif_rx_ni(skb); 7781 input_queue_head_incr(oldsd); 7782 } 7783 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { 7784 netif_rx_ni(skb); 7785 input_queue_head_incr(oldsd); 7786 } 7787 7788 return NOTIFY_OK; 7789 } 7790 7791 7792 /** 7793 * netdev_increment_features - increment feature set by one 7794 * @all: current feature set 7795 * @one: new feature set 7796 * @mask: mask feature set 7797 * 7798 * Computes a new feature set after adding a device with feature set 7799 * @one to the master device with current feature set @all. Will not 7800 * enable anything that is off in @mask. Returns the new feature set. 7801 */ 7802 netdev_features_t netdev_increment_features(netdev_features_t all, 7803 netdev_features_t one, netdev_features_t mask) 7804 { 7805 if (mask & NETIF_F_HW_CSUM) 7806 mask |= NETIF_F_CSUM_MASK; 7807 mask |= NETIF_F_VLAN_CHALLENGED; 7808 7809 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask; 7810 all &= one | ~NETIF_F_ALL_FOR_ALL; 7811 7812 /* If one device supports hw checksumming, set for all. */ 7813 if (all & NETIF_F_HW_CSUM) 7814 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM); 7815 7816 return all; 7817 } 7818 EXPORT_SYMBOL(netdev_increment_features); 7819 7820 static struct hlist_head * __net_init netdev_create_hash(void) 7821 { 7822 int i; 7823 struct hlist_head *hash; 7824 7825 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL); 7826 if (hash != NULL) 7827 for (i = 0; i < NETDEV_HASHENTRIES; i++) 7828 INIT_HLIST_HEAD(&hash[i]); 7829 7830 return hash; 7831 } 7832 7833 /* Initialize per network namespace state */ 7834 static int __net_init netdev_init(struct net *net) 7835 { 7836 if (net != &init_net) 7837 INIT_LIST_HEAD(&net->dev_base_head); 7838 7839 net->dev_name_head = netdev_create_hash(); 7840 if (net->dev_name_head == NULL) 7841 goto err_name; 7842 7843 net->dev_index_head = netdev_create_hash(); 7844 if (net->dev_index_head == NULL) 7845 goto err_idx; 7846 7847 return 0; 7848 7849 err_idx: 7850 kfree(net->dev_name_head); 7851 err_name: 7852 return -ENOMEM; 7853 } 7854 7855 /** 7856 * netdev_drivername - network driver for the device 7857 * @dev: network device 7858 * 7859 * Determine network driver for device. 7860 */ 7861 const char *netdev_drivername(const struct net_device *dev) 7862 { 7863 const struct device_driver *driver; 7864 const struct device *parent; 7865 const char *empty = ""; 7866 7867 parent = dev->dev.parent; 7868 if (!parent) 7869 return empty; 7870 7871 driver = parent->driver; 7872 if (driver && driver->name) 7873 return driver->name; 7874 return empty; 7875 } 7876 7877 static void __netdev_printk(const char *level, const struct net_device *dev, 7878 struct va_format *vaf) 7879 { 7880 if (dev && dev->dev.parent) { 7881 dev_printk_emit(level[1] - '0', 7882 dev->dev.parent, 7883 "%s %s %s%s: %pV", 7884 dev_driver_string(dev->dev.parent), 7885 dev_name(dev->dev.parent), 7886 netdev_name(dev), netdev_reg_state(dev), 7887 vaf); 7888 } else if (dev) { 7889 printk("%s%s%s: %pV", 7890 level, netdev_name(dev), netdev_reg_state(dev), vaf); 7891 } else { 7892 printk("%s(NULL net_device): %pV", level, vaf); 7893 } 7894 } 7895 7896 void netdev_printk(const char *level, const struct net_device *dev, 7897 const char *format, ...) 7898 { 7899 struct va_format vaf; 7900 va_list args; 7901 7902 va_start(args, format); 7903 7904 vaf.fmt = format; 7905 vaf.va = &args; 7906 7907 __netdev_printk(level, dev, &vaf); 7908 7909 va_end(args); 7910 } 7911 EXPORT_SYMBOL(netdev_printk); 7912 7913 #define define_netdev_printk_level(func, level) \ 7914 void func(const struct net_device *dev, const char *fmt, ...) \ 7915 { \ 7916 struct va_format vaf; \ 7917 va_list args; \ 7918 \ 7919 va_start(args, fmt); \ 7920 \ 7921 vaf.fmt = fmt; \ 7922 vaf.va = &args; \ 7923 \ 7924 __netdev_printk(level, dev, &vaf); \ 7925 \ 7926 va_end(args); \ 7927 } \ 7928 EXPORT_SYMBOL(func); 7929 7930 define_netdev_printk_level(netdev_emerg, KERN_EMERG); 7931 define_netdev_printk_level(netdev_alert, KERN_ALERT); 7932 define_netdev_printk_level(netdev_crit, KERN_CRIT); 7933 define_netdev_printk_level(netdev_err, KERN_ERR); 7934 define_netdev_printk_level(netdev_warn, KERN_WARNING); 7935 define_netdev_printk_level(netdev_notice, KERN_NOTICE); 7936 define_netdev_printk_level(netdev_info, KERN_INFO); 7937 7938 static void __net_exit netdev_exit(struct net *net) 7939 { 7940 kfree(net->dev_name_head); 7941 kfree(net->dev_index_head); 7942 } 7943 7944 static struct pernet_operations __net_initdata netdev_net_ops = { 7945 .init = netdev_init, 7946 .exit = netdev_exit, 7947 }; 7948 7949 static void __net_exit default_device_exit(struct net *net) 7950 { 7951 struct net_device *dev, *aux; 7952 /* 7953 * Push all migratable network devices back to the 7954 * initial network namespace 7955 */ 7956 rtnl_lock(); 7957 for_each_netdev_safe(net, dev, aux) { 7958 int err; 7959 char fb_name[IFNAMSIZ]; 7960 7961 /* Ignore unmoveable devices (i.e. loopback) */ 7962 if (dev->features & NETIF_F_NETNS_LOCAL) 7963 continue; 7964 7965 /* Leave virtual devices for the generic cleanup */ 7966 if (dev->rtnl_link_ops) 7967 continue; 7968 7969 /* Push remaining network devices to init_net */ 7970 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); 7971 err = dev_change_net_namespace(dev, &init_net, fb_name); 7972 if (err) { 7973 pr_emerg("%s: failed to move %s to init_net: %d\n", 7974 __func__, dev->name, err); 7975 BUG(); 7976 } 7977 } 7978 rtnl_unlock(); 7979 } 7980 7981 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list) 7982 { 7983 /* Return with the rtnl_lock held when there are no network 7984 * devices unregistering in any network namespace in net_list. 7985 */ 7986 struct net *net; 7987 bool unregistering; 7988 DEFINE_WAIT_FUNC(wait, woken_wake_function); 7989 7990 add_wait_queue(&netdev_unregistering_wq, &wait); 7991 for (;;) { 7992 unregistering = false; 7993 rtnl_lock(); 7994 list_for_each_entry(net, net_list, exit_list) { 7995 if (net->dev_unreg_count > 0) { 7996 unregistering = true; 7997 break; 7998 } 7999 } 8000 if (!unregistering) 8001 break; 8002 __rtnl_unlock(); 8003 8004 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); 8005 } 8006 remove_wait_queue(&netdev_unregistering_wq, &wait); 8007 } 8008 8009 static void __net_exit default_device_exit_batch(struct list_head *net_list) 8010 { 8011 /* At exit all network devices most be removed from a network 8012 * namespace. Do this in the reverse order of registration. 8013 * Do this across as many network namespaces as possible to 8014 * improve batching efficiency. 8015 */ 8016 struct net_device *dev; 8017 struct net *net; 8018 LIST_HEAD(dev_kill_list); 8019 8020 /* To prevent network device cleanup code from dereferencing 8021 * loopback devices or network devices that have been freed 8022 * wait here for all pending unregistrations to complete, 8023 * before unregistring the loopback device and allowing the 8024 * network namespace be freed. 8025 * 8026 * The netdev todo list containing all network devices 8027 * unregistrations that happen in default_device_exit_batch 8028 * will run in the rtnl_unlock() at the end of 8029 * default_device_exit_batch. 8030 */ 8031 rtnl_lock_unregistering(net_list); 8032 list_for_each_entry(net, net_list, exit_list) { 8033 for_each_netdev_reverse(net, dev) { 8034 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) 8035 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); 8036 else 8037 unregister_netdevice_queue(dev, &dev_kill_list); 8038 } 8039 } 8040 unregister_netdevice_many(&dev_kill_list); 8041 rtnl_unlock(); 8042 } 8043 8044 static struct pernet_operations __net_initdata default_device_ops = { 8045 .exit = default_device_exit, 8046 .exit_batch = default_device_exit_batch, 8047 }; 8048 8049 /* 8050 * Initialize the DEV module. At boot time this walks the device list and 8051 * unhooks any devices that fail to initialise (normally hardware not 8052 * present) and leaves us with a valid list of present and active devices. 8053 * 8054 */ 8055 8056 /* 8057 * This is called single threaded during boot, so no need 8058 * to take the rtnl semaphore. 8059 */ 8060 static int __init net_dev_init(void) 8061 { 8062 int i, rc = -ENOMEM; 8063 8064 BUG_ON(!dev_boot_phase); 8065 8066 if (dev_proc_init()) 8067 goto out; 8068 8069 if (netdev_kobject_init()) 8070 goto out; 8071 8072 INIT_LIST_HEAD(&ptype_all); 8073 for (i = 0; i < PTYPE_HASH_SIZE; i++) 8074 INIT_LIST_HEAD(&ptype_base[i]); 8075 8076 INIT_LIST_HEAD(&offload_base); 8077 8078 if (register_pernet_subsys(&netdev_net_ops)) 8079 goto out; 8080 8081 /* 8082 * Initialise the packet receive queues. 8083 */ 8084 8085 for_each_possible_cpu(i) { 8086 struct softnet_data *sd = &per_cpu(softnet_data, i); 8087 8088 skb_queue_head_init(&sd->input_pkt_queue); 8089 skb_queue_head_init(&sd->process_queue); 8090 INIT_LIST_HEAD(&sd->poll_list); 8091 sd->output_queue_tailp = &sd->output_queue; 8092 #ifdef CONFIG_RPS 8093 sd->csd.func = rps_trigger_softirq; 8094 sd->csd.info = sd; 8095 sd->cpu = i; 8096 #endif 8097 8098 sd->backlog.poll = process_backlog; 8099 sd->backlog.weight = weight_p; 8100 } 8101 8102 dev_boot_phase = 0; 8103 8104 /* The loopback device is special if any other network devices 8105 * is present in a network namespace the loopback device must 8106 * be present. Since we now dynamically allocate and free the 8107 * loopback device ensure this invariant is maintained by 8108 * keeping the loopback device as the first device on the 8109 * list of network devices. Ensuring the loopback devices 8110 * is the first device that appears and the last network device 8111 * that disappears. 8112 */ 8113 if (register_pernet_device(&loopback_net_ops)) 8114 goto out; 8115 8116 if (register_pernet_device(&default_device_ops)) 8117 goto out; 8118 8119 open_softirq(NET_TX_SOFTIRQ, net_tx_action); 8120 open_softirq(NET_RX_SOFTIRQ, net_rx_action); 8121 8122 hotcpu_notifier(dev_cpu_callback, 0); 8123 dst_subsys_init(); 8124 rc = 0; 8125 out: 8126 return rc; 8127 } 8128 8129 subsys_initcall(net_dev_init); 8130