1 /* 2 * NET3 Protocol independent device support routines. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 * 9 * Derived from the non IP parts of dev.c 1.0.19 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Mark Evans, <evansmp@uhura.aston.ac.uk> 13 * 14 * Additional Authors: 15 * Florian la Roche <rzsfl@rz.uni-sb.de> 16 * Alan Cox <gw4pts@gw4pts.ampr.org> 17 * David Hinds <dahinds@users.sourceforge.net> 18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 19 * Adam Sulmicki <adam@cfar.umd.edu> 20 * Pekka Riikonen <priikone@poesidon.pspt.fi> 21 * 22 * Changes: 23 * D.J. Barrow : Fixed bug where dev->refcnt gets set 24 * to 2 if register_netdev gets called 25 * before net_dev_init & also removed a 26 * few lines of code in the process. 27 * Alan Cox : device private ioctl copies fields back. 28 * Alan Cox : Transmit queue code does relevant 29 * stunts to keep the queue safe. 30 * Alan Cox : Fixed double lock. 31 * Alan Cox : Fixed promisc NULL pointer trap 32 * ???????? : Support the full private ioctl range 33 * Alan Cox : Moved ioctl permission check into 34 * drivers 35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI 36 * Alan Cox : 100 backlog just doesn't cut it when 37 * you start doing multicast video 8) 38 * Alan Cox : Rewrote net_bh and list manager. 39 * Alan Cox : Fix ETH_P_ALL echoback lengths. 40 * Alan Cox : Took out transmit every packet pass 41 * Saved a few bytes in the ioctl handler 42 * Alan Cox : Network driver sets packet type before 43 * calling netif_rx. Saves a function 44 * call a packet. 45 * Alan Cox : Hashed net_bh() 46 * Richard Kooijman: Timestamp fixes. 47 * Alan Cox : Wrong field in SIOCGIFDSTADDR 48 * Alan Cox : Device lock protection. 49 * Alan Cox : Fixed nasty side effect of device close 50 * changes. 51 * Rudi Cilibrasi : Pass the right thing to 52 * set_mac_address() 53 * Dave Miller : 32bit quantity for the device lock to 54 * make it work out on a Sparc. 55 * Bjorn Ekwall : Added KERNELD hack. 56 * Alan Cox : Cleaned up the backlog initialise. 57 * Craig Metz : SIOCGIFCONF fix if space for under 58 * 1 device. 59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there 60 * is no device open function. 61 * Andi Kleen : Fix error reporting for SIOCGIFCONF 62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF 63 * Cyrus Durgin : Cleaned for KMOD 64 * Adam Sulmicki : Bug Fix : Network Device Unload 65 * A network device unload needs to purge 66 * the backlog queue. 67 * Paul Rusty Russell : SIOCSIFNAME 68 * Pekka Riikonen : Netdev boot-time settings code 69 * Andrew Morton : Make unregister_netdevice wait 70 * indefinitely on dev->refcnt 71 * J Hadi Salim : - Backlog queue sampling 72 * - netif_rx() feedback 73 */ 74 75 #include <linux/uaccess.h> 76 #include <linux/bitops.h> 77 #include <linux/capability.h> 78 #include <linux/cpu.h> 79 #include <linux/types.h> 80 #include <linux/kernel.h> 81 #include <linux/hash.h> 82 #include <linux/slab.h> 83 #include <linux/sched.h> 84 #include <linux/sched/mm.h> 85 #include <linux/mutex.h> 86 #include <linux/string.h> 87 #include <linux/mm.h> 88 #include <linux/socket.h> 89 #include <linux/sockios.h> 90 #include <linux/errno.h> 91 #include <linux/interrupt.h> 92 #include <linux/if_ether.h> 93 #include <linux/netdevice.h> 94 #include <linux/etherdevice.h> 95 #include <linux/ethtool.h> 96 #include <linux/notifier.h> 97 #include <linux/skbuff.h> 98 #include <linux/bpf.h> 99 #include <linux/bpf_trace.h> 100 #include <net/net_namespace.h> 101 #include <net/sock.h> 102 #include <net/busy_poll.h> 103 #include <linux/rtnetlink.h> 104 #include <linux/stat.h> 105 #include <net/dst.h> 106 #include <net/dst_metadata.h> 107 #include <net/pkt_sched.h> 108 #include <net/pkt_cls.h> 109 #include <net/checksum.h> 110 #include <net/xfrm.h> 111 #include <linux/highmem.h> 112 #include <linux/init.h> 113 #include <linux/module.h> 114 #include <linux/netpoll.h> 115 #include <linux/rcupdate.h> 116 #include <linux/delay.h> 117 #include <net/iw_handler.h> 118 #include <asm/current.h> 119 #include <linux/audit.h> 120 #include <linux/dmaengine.h> 121 #include <linux/err.h> 122 #include <linux/ctype.h> 123 #include <linux/if_arp.h> 124 #include <linux/if_vlan.h> 125 #include <linux/ip.h> 126 #include <net/ip.h> 127 #include <net/mpls.h> 128 #include <linux/ipv6.h> 129 #include <linux/in.h> 130 #include <linux/jhash.h> 131 #include <linux/random.h> 132 #include <trace/events/napi.h> 133 #include <trace/events/net.h> 134 #include <trace/events/skb.h> 135 #include <linux/pci.h> 136 #include <linux/inetdevice.h> 137 #include <linux/cpu_rmap.h> 138 #include <linux/static_key.h> 139 #include <linux/hashtable.h> 140 #include <linux/vmalloc.h> 141 #include <linux/if_macvlan.h> 142 #include <linux/errqueue.h> 143 #include <linux/hrtimer.h> 144 #include <linux/netfilter_ingress.h> 145 #include <linux/crash_dump.h> 146 #include <linux/sctp.h> 147 #include <net/udp_tunnel.h> 148 #include <linux/net_namespace.h> 149 150 #include "net-sysfs.h" 151 152 /* Instead of increasing this, you should create a hash table. */ 153 #define MAX_GRO_SKBS 8 154 155 /* This should be increased if a protocol with a bigger head is added. */ 156 #define GRO_MAX_HEAD (MAX_HEADER + 128) 157 158 static DEFINE_SPINLOCK(ptype_lock); 159 static DEFINE_SPINLOCK(offload_lock); 160 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; 161 struct list_head ptype_all __read_mostly; /* Taps */ 162 static struct list_head offload_base __read_mostly; 163 164 static int netif_rx_internal(struct sk_buff *skb); 165 static int call_netdevice_notifiers_info(unsigned long val, 166 struct netdev_notifier_info *info); 167 static struct napi_struct *napi_by_id(unsigned int napi_id); 168 169 /* 170 * The @dev_base_head list is protected by @dev_base_lock and the rtnl 171 * semaphore. 172 * 173 * Pure readers hold dev_base_lock for reading, or rcu_read_lock() 174 * 175 * Writers must hold the rtnl semaphore while they loop through the 176 * dev_base_head list, and hold dev_base_lock for writing when they do the 177 * actual updates. This allows pure readers to access the list even 178 * while a writer is preparing to update it. 179 * 180 * To put it another way, dev_base_lock is held for writing only to 181 * protect against pure readers; the rtnl semaphore provides the 182 * protection against other writers. 183 * 184 * See, for example usages, register_netdevice() and 185 * unregister_netdevice(), which must be called with the rtnl 186 * semaphore held. 187 */ 188 DEFINE_RWLOCK(dev_base_lock); 189 EXPORT_SYMBOL(dev_base_lock); 190 191 static DEFINE_MUTEX(ifalias_mutex); 192 193 /* protects napi_hash addition/deletion and napi_gen_id */ 194 static DEFINE_SPINLOCK(napi_hash_lock); 195 196 static unsigned int napi_gen_id = NR_CPUS; 197 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8); 198 199 static seqcount_t devnet_rename_seq; 200 201 static inline void dev_base_seq_inc(struct net *net) 202 { 203 while (++net->dev_base_seq == 0) 204 ; 205 } 206 207 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name) 208 { 209 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ)); 210 211 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; 212 } 213 214 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) 215 { 216 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; 217 } 218 219 static inline void rps_lock(struct softnet_data *sd) 220 { 221 #ifdef CONFIG_RPS 222 spin_lock(&sd->input_pkt_queue.lock); 223 #endif 224 } 225 226 static inline void rps_unlock(struct softnet_data *sd) 227 { 228 #ifdef CONFIG_RPS 229 spin_unlock(&sd->input_pkt_queue.lock); 230 #endif 231 } 232 233 /* Device list insertion */ 234 static void list_netdevice(struct net_device *dev) 235 { 236 struct net *net = dev_net(dev); 237 238 ASSERT_RTNL(); 239 240 write_lock_bh(&dev_base_lock); 241 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); 242 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); 243 hlist_add_head_rcu(&dev->index_hlist, 244 dev_index_hash(net, dev->ifindex)); 245 write_unlock_bh(&dev_base_lock); 246 247 dev_base_seq_inc(net); 248 } 249 250 /* Device list removal 251 * caller must respect a RCU grace period before freeing/reusing dev 252 */ 253 static void unlist_netdevice(struct net_device *dev) 254 { 255 ASSERT_RTNL(); 256 257 /* Unlink dev from the device chain */ 258 write_lock_bh(&dev_base_lock); 259 list_del_rcu(&dev->dev_list); 260 hlist_del_rcu(&dev->name_hlist); 261 hlist_del_rcu(&dev->index_hlist); 262 write_unlock_bh(&dev_base_lock); 263 264 dev_base_seq_inc(dev_net(dev)); 265 } 266 267 /* 268 * Our notifier list 269 */ 270 271 static RAW_NOTIFIER_HEAD(netdev_chain); 272 273 /* 274 * Device drivers call our routines to queue packets here. We empty the 275 * queue in the local softnet handler. 276 */ 277 278 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data); 279 EXPORT_PER_CPU_SYMBOL(softnet_data); 280 281 #ifdef CONFIG_LOCKDEP 282 /* 283 * register_netdevice() inits txq->_xmit_lock and sets lockdep class 284 * according to dev->type 285 */ 286 static const unsigned short netdev_lock_type[] = { 287 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25, 288 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET, 289 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM, 290 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, 291 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, 292 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, 293 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, 294 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, 295 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, 296 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, 297 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, 298 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, 299 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM, 300 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE, 301 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE}; 302 303 static const char *const netdev_lock_name[] = { 304 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", 305 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET", 306 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM", 307 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", 308 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", 309 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", 310 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", 311 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", 312 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", 313 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", 314 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", 315 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", 316 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM", 317 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE", 318 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"}; 319 320 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; 321 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; 322 323 static inline unsigned short netdev_lock_pos(unsigned short dev_type) 324 { 325 int i; 326 327 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++) 328 if (netdev_lock_type[i] == dev_type) 329 return i; 330 /* the last key is used by default */ 331 return ARRAY_SIZE(netdev_lock_type) - 1; 332 } 333 334 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 335 unsigned short dev_type) 336 { 337 int i; 338 339 i = netdev_lock_pos(dev_type); 340 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i], 341 netdev_lock_name[i]); 342 } 343 344 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 345 { 346 int i; 347 348 i = netdev_lock_pos(dev->type); 349 lockdep_set_class_and_name(&dev->addr_list_lock, 350 &netdev_addr_lock_key[i], 351 netdev_lock_name[i]); 352 } 353 #else 354 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 355 unsigned short dev_type) 356 { 357 } 358 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 359 { 360 } 361 #endif 362 363 /******************************************************************************* 364 * 365 * Protocol management and registration routines 366 * 367 *******************************************************************************/ 368 369 370 /* 371 * Add a protocol ID to the list. Now that the input handler is 372 * smarter we can dispense with all the messy stuff that used to be 373 * here. 374 * 375 * BEWARE!!! Protocol handlers, mangling input packets, 376 * MUST BE last in hash buckets and checking protocol handlers 377 * MUST start from promiscuous ptype_all chain in net_bh. 378 * It is true now, do not change it. 379 * Explanation follows: if protocol handler, mangling packet, will 380 * be the first on list, it is not able to sense, that packet 381 * is cloned and should be copied-on-write, so that it will 382 * change it and subsequent readers will get broken packet. 383 * --ANK (980803) 384 */ 385 386 static inline struct list_head *ptype_head(const struct packet_type *pt) 387 { 388 if (pt->type == htons(ETH_P_ALL)) 389 return pt->dev ? &pt->dev->ptype_all : &ptype_all; 390 else 391 return pt->dev ? &pt->dev->ptype_specific : 392 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; 393 } 394 395 /** 396 * dev_add_pack - add packet handler 397 * @pt: packet type declaration 398 * 399 * Add a protocol handler to the networking stack. The passed &packet_type 400 * is linked into kernel lists and may not be freed until it has been 401 * removed from the kernel lists. 402 * 403 * This call does not sleep therefore it can not 404 * guarantee all CPU's that are in middle of receiving packets 405 * will see the new packet type (until the next received packet). 406 */ 407 408 void dev_add_pack(struct packet_type *pt) 409 { 410 struct list_head *head = ptype_head(pt); 411 412 spin_lock(&ptype_lock); 413 list_add_rcu(&pt->list, head); 414 spin_unlock(&ptype_lock); 415 } 416 EXPORT_SYMBOL(dev_add_pack); 417 418 /** 419 * __dev_remove_pack - remove packet handler 420 * @pt: packet type declaration 421 * 422 * Remove a protocol handler that was previously added to the kernel 423 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 424 * from the kernel lists and can be freed or reused once this function 425 * returns. 426 * 427 * The packet type might still be in use by receivers 428 * and must not be freed until after all the CPU's have gone 429 * through a quiescent state. 430 */ 431 void __dev_remove_pack(struct packet_type *pt) 432 { 433 struct list_head *head = ptype_head(pt); 434 struct packet_type *pt1; 435 436 spin_lock(&ptype_lock); 437 438 list_for_each_entry(pt1, head, list) { 439 if (pt == pt1) { 440 list_del_rcu(&pt->list); 441 goto out; 442 } 443 } 444 445 pr_warn("dev_remove_pack: %p not found\n", pt); 446 out: 447 spin_unlock(&ptype_lock); 448 } 449 EXPORT_SYMBOL(__dev_remove_pack); 450 451 /** 452 * dev_remove_pack - remove packet handler 453 * @pt: packet type declaration 454 * 455 * Remove a protocol handler that was previously added to the kernel 456 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 457 * from the kernel lists and can be freed or reused once this function 458 * returns. 459 * 460 * This call sleeps to guarantee that no CPU is looking at the packet 461 * type after return. 462 */ 463 void dev_remove_pack(struct packet_type *pt) 464 { 465 __dev_remove_pack(pt); 466 467 synchronize_net(); 468 } 469 EXPORT_SYMBOL(dev_remove_pack); 470 471 472 /** 473 * dev_add_offload - register offload handlers 474 * @po: protocol offload declaration 475 * 476 * Add protocol offload handlers to the networking stack. The passed 477 * &proto_offload is linked into kernel lists and may not be freed until 478 * it has been removed from the kernel lists. 479 * 480 * This call does not sleep therefore it can not 481 * guarantee all CPU's that are in middle of receiving packets 482 * will see the new offload handlers (until the next received packet). 483 */ 484 void dev_add_offload(struct packet_offload *po) 485 { 486 struct packet_offload *elem; 487 488 spin_lock(&offload_lock); 489 list_for_each_entry(elem, &offload_base, list) { 490 if (po->priority < elem->priority) 491 break; 492 } 493 list_add_rcu(&po->list, elem->list.prev); 494 spin_unlock(&offload_lock); 495 } 496 EXPORT_SYMBOL(dev_add_offload); 497 498 /** 499 * __dev_remove_offload - remove offload handler 500 * @po: packet offload declaration 501 * 502 * Remove a protocol offload handler that was previously added to the 503 * kernel offload handlers by dev_add_offload(). The passed &offload_type 504 * is removed from the kernel lists and can be freed or reused once this 505 * function returns. 506 * 507 * The packet type might still be in use by receivers 508 * and must not be freed until after all the CPU's have gone 509 * through a quiescent state. 510 */ 511 static void __dev_remove_offload(struct packet_offload *po) 512 { 513 struct list_head *head = &offload_base; 514 struct packet_offload *po1; 515 516 spin_lock(&offload_lock); 517 518 list_for_each_entry(po1, head, list) { 519 if (po == po1) { 520 list_del_rcu(&po->list); 521 goto out; 522 } 523 } 524 525 pr_warn("dev_remove_offload: %p not found\n", po); 526 out: 527 spin_unlock(&offload_lock); 528 } 529 530 /** 531 * dev_remove_offload - remove packet offload handler 532 * @po: packet offload declaration 533 * 534 * Remove a packet offload handler that was previously added to the kernel 535 * offload handlers by dev_add_offload(). The passed &offload_type is 536 * removed from the kernel lists and can be freed or reused once this 537 * function returns. 538 * 539 * This call sleeps to guarantee that no CPU is looking at the packet 540 * type after return. 541 */ 542 void dev_remove_offload(struct packet_offload *po) 543 { 544 __dev_remove_offload(po); 545 546 synchronize_net(); 547 } 548 EXPORT_SYMBOL(dev_remove_offload); 549 550 /****************************************************************************** 551 * 552 * Device Boot-time Settings Routines 553 * 554 ******************************************************************************/ 555 556 /* Boot time configuration table */ 557 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX]; 558 559 /** 560 * netdev_boot_setup_add - add new setup entry 561 * @name: name of the device 562 * @map: configured settings for the device 563 * 564 * Adds new setup entry to the dev_boot_setup list. The function 565 * returns 0 on error and 1 on success. This is a generic routine to 566 * all netdevices. 567 */ 568 static int netdev_boot_setup_add(char *name, struct ifmap *map) 569 { 570 struct netdev_boot_setup *s; 571 int i; 572 573 s = dev_boot_setup; 574 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 575 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') { 576 memset(s[i].name, 0, sizeof(s[i].name)); 577 strlcpy(s[i].name, name, IFNAMSIZ); 578 memcpy(&s[i].map, map, sizeof(s[i].map)); 579 break; 580 } 581 } 582 583 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1; 584 } 585 586 /** 587 * netdev_boot_setup_check - check boot time settings 588 * @dev: the netdevice 589 * 590 * Check boot time settings for the device. 591 * The found settings are set for the device to be used 592 * later in the device probing. 593 * Returns 0 if no settings found, 1 if they are. 594 */ 595 int netdev_boot_setup_check(struct net_device *dev) 596 { 597 struct netdev_boot_setup *s = dev_boot_setup; 598 int i; 599 600 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 601 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' && 602 !strcmp(dev->name, s[i].name)) { 603 dev->irq = s[i].map.irq; 604 dev->base_addr = s[i].map.base_addr; 605 dev->mem_start = s[i].map.mem_start; 606 dev->mem_end = s[i].map.mem_end; 607 return 1; 608 } 609 } 610 return 0; 611 } 612 EXPORT_SYMBOL(netdev_boot_setup_check); 613 614 615 /** 616 * netdev_boot_base - get address from boot time settings 617 * @prefix: prefix for network device 618 * @unit: id for network device 619 * 620 * Check boot time settings for the base address of device. 621 * The found settings are set for the device to be used 622 * later in the device probing. 623 * Returns 0 if no settings found. 624 */ 625 unsigned long netdev_boot_base(const char *prefix, int unit) 626 { 627 const struct netdev_boot_setup *s = dev_boot_setup; 628 char name[IFNAMSIZ]; 629 int i; 630 631 sprintf(name, "%s%d", prefix, unit); 632 633 /* 634 * If device already registered then return base of 1 635 * to indicate not to probe for this interface 636 */ 637 if (__dev_get_by_name(&init_net, name)) 638 return 1; 639 640 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) 641 if (!strcmp(name, s[i].name)) 642 return s[i].map.base_addr; 643 return 0; 644 } 645 646 /* 647 * Saves at boot time configured settings for any netdevice. 648 */ 649 int __init netdev_boot_setup(char *str) 650 { 651 int ints[5]; 652 struct ifmap map; 653 654 str = get_options(str, ARRAY_SIZE(ints), ints); 655 if (!str || !*str) 656 return 0; 657 658 /* Save settings */ 659 memset(&map, 0, sizeof(map)); 660 if (ints[0] > 0) 661 map.irq = ints[1]; 662 if (ints[0] > 1) 663 map.base_addr = ints[2]; 664 if (ints[0] > 2) 665 map.mem_start = ints[3]; 666 if (ints[0] > 3) 667 map.mem_end = ints[4]; 668 669 /* Add new entry to the list */ 670 return netdev_boot_setup_add(str, &map); 671 } 672 673 __setup("netdev=", netdev_boot_setup); 674 675 /******************************************************************************* 676 * 677 * Device Interface Subroutines 678 * 679 *******************************************************************************/ 680 681 /** 682 * dev_get_iflink - get 'iflink' value of a interface 683 * @dev: targeted interface 684 * 685 * Indicates the ifindex the interface is linked to. 686 * Physical interfaces have the same 'ifindex' and 'iflink' values. 687 */ 688 689 int dev_get_iflink(const struct net_device *dev) 690 { 691 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) 692 return dev->netdev_ops->ndo_get_iflink(dev); 693 694 return dev->ifindex; 695 } 696 EXPORT_SYMBOL(dev_get_iflink); 697 698 /** 699 * dev_fill_metadata_dst - Retrieve tunnel egress information. 700 * @dev: targeted interface 701 * @skb: The packet. 702 * 703 * For better visibility of tunnel traffic OVS needs to retrieve 704 * egress tunnel information for a packet. Following API allows 705 * user to get this info. 706 */ 707 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb) 708 { 709 struct ip_tunnel_info *info; 710 711 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst) 712 return -EINVAL; 713 714 info = skb_tunnel_info_unclone(skb); 715 if (!info) 716 return -ENOMEM; 717 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX))) 718 return -EINVAL; 719 720 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb); 721 } 722 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst); 723 724 /** 725 * __dev_get_by_name - find a device by its name 726 * @net: the applicable net namespace 727 * @name: name to find 728 * 729 * Find an interface by name. Must be called under RTNL semaphore 730 * or @dev_base_lock. If the name is found a pointer to the device 731 * is returned. If the name is not found then %NULL is returned. The 732 * reference counters are not incremented so the caller must be 733 * careful with locks. 734 */ 735 736 struct net_device *__dev_get_by_name(struct net *net, const char *name) 737 { 738 struct net_device *dev; 739 struct hlist_head *head = dev_name_hash(net, name); 740 741 hlist_for_each_entry(dev, head, name_hlist) 742 if (!strncmp(dev->name, name, IFNAMSIZ)) 743 return dev; 744 745 return NULL; 746 } 747 EXPORT_SYMBOL(__dev_get_by_name); 748 749 /** 750 * dev_get_by_name_rcu - find a device by its name 751 * @net: the applicable net namespace 752 * @name: name to find 753 * 754 * Find an interface by name. 755 * If the name is found a pointer to the device is returned. 756 * If the name is not found then %NULL is returned. 757 * The reference counters are not incremented so the caller must be 758 * careful with locks. The caller must hold RCU lock. 759 */ 760 761 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name) 762 { 763 struct net_device *dev; 764 struct hlist_head *head = dev_name_hash(net, name); 765 766 hlist_for_each_entry_rcu(dev, head, name_hlist) 767 if (!strncmp(dev->name, name, IFNAMSIZ)) 768 return dev; 769 770 return NULL; 771 } 772 EXPORT_SYMBOL(dev_get_by_name_rcu); 773 774 /** 775 * dev_get_by_name - find a device by its name 776 * @net: the applicable net namespace 777 * @name: name to find 778 * 779 * Find an interface by name. This can be called from any 780 * context and does its own locking. The returned handle has 781 * the usage count incremented and the caller must use dev_put() to 782 * release it when it is no longer needed. %NULL is returned if no 783 * matching device is found. 784 */ 785 786 struct net_device *dev_get_by_name(struct net *net, const char *name) 787 { 788 struct net_device *dev; 789 790 rcu_read_lock(); 791 dev = dev_get_by_name_rcu(net, name); 792 if (dev) 793 dev_hold(dev); 794 rcu_read_unlock(); 795 return dev; 796 } 797 EXPORT_SYMBOL(dev_get_by_name); 798 799 /** 800 * __dev_get_by_index - find a device by its ifindex 801 * @net: the applicable net namespace 802 * @ifindex: index of device 803 * 804 * Search for an interface by index. Returns %NULL if the device 805 * is not found or a pointer to the device. The device has not 806 * had its reference counter increased so the caller must be careful 807 * about locking. The caller must hold either the RTNL semaphore 808 * or @dev_base_lock. 809 */ 810 811 struct net_device *__dev_get_by_index(struct net *net, int ifindex) 812 { 813 struct net_device *dev; 814 struct hlist_head *head = dev_index_hash(net, ifindex); 815 816 hlist_for_each_entry(dev, head, index_hlist) 817 if (dev->ifindex == ifindex) 818 return dev; 819 820 return NULL; 821 } 822 EXPORT_SYMBOL(__dev_get_by_index); 823 824 /** 825 * dev_get_by_index_rcu - find a device by its ifindex 826 * @net: the applicable net namespace 827 * @ifindex: index of device 828 * 829 * Search for an interface by index. Returns %NULL if the device 830 * is not found or a pointer to the device. The device has not 831 * had its reference counter increased so the caller must be careful 832 * about locking. The caller must hold RCU lock. 833 */ 834 835 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex) 836 { 837 struct net_device *dev; 838 struct hlist_head *head = dev_index_hash(net, ifindex); 839 840 hlist_for_each_entry_rcu(dev, head, index_hlist) 841 if (dev->ifindex == ifindex) 842 return dev; 843 844 return NULL; 845 } 846 EXPORT_SYMBOL(dev_get_by_index_rcu); 847 848 849 /** 850 * dev_get_by_index - find a device by its ifindex 851 * @net: the applicable net namespace 852 * @ifindex: index of device 853 * 854 * Search for an interface by index. Returns NULL if the device 855 * is not found or a pointer to the device. The device returned has 856 * had a reference added and the pointer is safe until the user calls 857 * dev_put to indicate they have finished with it. 858 */ 859 860 struct net_device *dev_get_by_index(struct net *net, int ifindex) 861 { 862 struct net_device *dev; 863 864 rcu_read_lock(); 865 dev = dev_get_by_index_rcu(net, ifindex); 866 if (dev) 867 dev_hold(dev); 868 rcu_read_unlock(); 869 return dev; 870 } 871 EXPORT_SYMBOL(dev_get_by_index); 872 873 /** 874 * dev_get_by_napi_id - find a device by napi_id 875 * @napi_id: ID of the NAPI struct 876 * 877 * Search for an interface by NAPI ID. Returns %NULL if the device 878 * is not found or a pointer to the device. The device has not had 879 * its reference counter increased so the caller must be careful 880 * about locking. The caller must hold RCU lock. 881 */ 882 883 struct net_device *dev_get_by_napi_id(unsigned int napi_id) 884 { 885 struct napi_struct *napi; 886 887 WARN_ON_ONCE(!rcu_read_lock_held()); 888 889 if (napi_id < MIN_NAPI_ID) 890 return NULL; 891 892 napi = napi_by_id(napi_id); 893 894 return napi ? napi->dev : NULL; 895 } 896 EXPORT_SYMBOL(dev_get_by_napi_id); 897 898 /** 899 * netdev_get_name - get a netdevice name, knowing its ifindex. 900 * @net: network namespace 901 * @name: a pointer to the buffer where the name will be stored. 902 * @ifindex: the ifindex of the interface to get the name from. 903 * 904 * The use of raw_seqcount_begin() and cond_resched() before 905 * retrying is required as we want to give the writers a chance 906 * to complete when CONFIG_PREEMPT is not set. 907 */ 908 int netdev_get_name(struct net *net, char *name, int ifindex) 909 { 910 struct net_device *dev; 911 unsigned int seq; 912 913 retry: 914 seq = raw_seqcount_begin(&devnet_rename_seq); 915 rcu_read_lock(); 916 dev = dev_get_by_index_rcu(net, ifindex); 917 if (!dev) { 918 rcu_read_unlock(); 919 return -ENODEV; 920 } 921 922 strcpy(name, dev->name); 923 rcu_read_unlock(); 924 if (read_seqcount_retry(&devnet_rename_seq, seq)) { 925 cond_resched(); 926 goto retry; 927 } 928 929 return 0; 930 } 931 932 /** 933 * dev_getbyhwaddr_rcu - find a device by its hardware address 934 * @net: the applicable net namespace 935 * @type: media type of device 936 * @ha: hardware address 937 * 938 * Search for an interface by MAC address. Returns NULL if the device 939 * is not found or a pointer to the device. 940 * The caller must hold RCU or RTNL. 941 * The returned device has not had its ref count increased 942 * and the caller must therefore be careful about locking 943 * 944 */ 945 946 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type, 947 const char *ha) 948 { 949 struct net_device *dev; 950 951 for_each_netdev_rcu(net, dev) 952 if (dev->type == type && 953 !memcmp(dev->dev_addr, ha, dev->addr_len)) 954 return dev; 955 956 return NULL; 957 } 958 EXPORT_SYMBOL(dev_getbyhwaddr_rcu); 959 960 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type) 961 { 962 struct net_device *dev; 963 964 ASSERT_RTNL(); 965 for_each_netdev(net, dev) 966 if (dev->type == type) 967 return dev; 968 969 return NULL; 970 } 971 EXPORT_SYMBOL(__dev_getfirstbyhwtype); 972 973 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) 974 { 975 struct net_device *dev, *ret = NULL; 976 977 rcu_read_lock(); 978 for_each_netdev_rcu(net, dev) 979 if (dev->type == type) { 980 dev_hold(dev); 981 ret = dev; 982 break; 983 } 984 rcu_read_unlock(); 985 return ret; 986 } 987 EXPORT_SYMBOL(dev_getfirstbyhwtype); 988 989 /** 990 * __dev_get_by_flags - find any device with given flags 991 * @net: the applicable net namespace 992 * @if_flags: IFF_* values 993 * @mask: bitmask of bits in if_flags to check 994 * 995 * Search for any interface with the given flags. Returns NULL if a device 996 * is not found or a pointer to the device. Must be called inside 997 * rtnl_lock(), and result refcount is unchanged. 998 */ 999 1000 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags, 1001 unsigned short mask) 1002 { 1003 struct net_device *dev, *ret; 1004 1005 ASSERT_RTNL(); 1006 1007 ret = NULL; 1008 for_each_netdev(net, dev) { 1009 if (((dev->flags ^ if_flags) & mask) == 0) { 1010 ret = dev; 1011 break; 1012 } 1013 } 1014 return ret; 1015 } 1016 EXPORT_SYMBOL(__dev_get_by_flags); 1017 1018 /** 1019 * dev_valid_name - check if name is okay for network device 1020 * @name: name string 1021 * 1022 * Network device names need to be valid file names to 1023 * to allow sysfs to work. We also disallow any kind of 1024 * whitespace. 1025 */ 1026 bool dev_valid_name(const char *name) 1027 { 1028 if (*name == '\0') 1029 return false; 1030 if (strlen(name) >= IFNAMSIZ) 1031 return false; 1032 if (!strcmp(name, ".") || !strcmp(name, "..")) 1033 return false; 1034 1035 while (*name) { 1036 if (*name == '/' || *name == ':' || isspace(*name)) 1037 return false; 1038 name++; 1039 } 1040 return true; 1041 } 1042 EXPORT_SYMBOL(dev_valid_name); 1043 1044 /** 1045 * __dev_alloc_name - allocate a name for a device 1046 * @net: network namespace to allocate the device name in 1047 * @name: name format string 1048 * @buf: scratch buffer and result name string 1049 * 1050 * Passed a format string - eg "lt%d" it will try and find a suitable 1051 * id. It scans list of devices to build up a free map, then chooses 1052 * the first empty slot. The caller must hold the dev_base or rtnl lock 1053 * while allocating the name and adding the device in order to avoid 1054 * duplicates. 1055 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1056 * Returns the number of the unit assigned or a negative errno code. 1057 */ 1058 1059 static int __dev_alloc_name(struct net *net, const char *name, char *buf) 1060 { 1061 int i = 0; 1062 const char *p; 1063 const int max_netdevices = 8*PAGE_SIZE; 1064 unsigned long *inuse; 1065 struct net_device *d; 1066 1067 if (!dev_valid_name(name)) 1068 return -EINVAL; 1069 1070 p = strchr(name, '%'); 1071 if (p) { 1072 /* 1073 * Verify the string as this thing may have come from 1074 * the user. There must be either one "%d" and no other "%" 1075 * characters. 1076 */ 1077 if (p[1] != 'd' || strchr(p + 2, '%')) 1078 return -EINVAL; 1079 1080 /* Use one page as a bit array of possible slots */ 1081 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC); 1082 if (!inuse) 1083 return -ENOMEM; 1084 1085 for_each_netdev(net, d) { 1086 if (!sscanf(d->name, name, &i)) 1087 continue; 1088 if (i < 0 || i >= max_netdevices) 1089 continue; 1090 1091 /* avoid cases where sscanf is not exact inverse of printf */ 1092 snprintf(buf, IFNAMSIZ, name, i); 1093 if (!strncmp(buf, d->name, IFNAMSIZ)) 1094 set_bit(i, inuse); 1095 } 1096 1097 i = find_first_zero_bit(inuse, max_netdevices); 1098 free_page((unsigned long) inuse); 1099 } 1100 1101 snprintf(buf, IFNAMSIZ, name, i); 1102 if (!__dev_get_by_name(net, buf)) 1103 return i; 1104 1105 /* It is possible to run out of possible slots 1106 * when the name is long and there isn't enough space left 1107 * for the digits, or if all bits are used. 1108 */ 1109 return -ENFILE; 1110 } 1111 1112 static int dev_alloc_name_ns(struct net *net, 1113 struct net_device *dev, 1114 const char *name) 1115 { 1116 char buf[IFNAMSIZ]; 1117 int ret; 1118 1119 BUG_ON(!net); 1120 ret = __dev_alloc_name(net, name, buf); 1121 if (ret >= 0) 1122 strlcpy(dev->name, buf, IFNAMSIZ); 1123 return ret; 1124 } 1125 1126 /** 1127 * dev_alloc_name - allocate a name for a device 1128 * @dev: device 1129 * @name: name format string 1130 * 1131 * Passed a format string - eg "lt%d" it will try and find a suitable 1132 * id. It scans list of devices to build up a free map, then chooses 1133 * the first empty slot. The caller must hold the dev_base or rtnl lock 1134 * while allocating the name and adding the device in order to avoid 1135 * duplicates. 1136 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1137 * Returns the number of the unit assigned or a negative errno code. 1138 */ 1139 1140 int dev_alloc_name(struct net_device *dev, const char *name) 1141 { 1142 return dev_alloc_name_ns(dev_net(dev), dev, name); 1143 } 1144 EXPORT_SYMBOL(dev_alloc_name); 1145 1146 int dev_get_valid_name(struct net *net, struct net_device *dev, 1147 const char *name) 1148 { 1149 return dev_alloc_name_ns(net, dev, name); 1150 } 1151 EXPORT_SYMBOL(dev_get_valid_name); 1152 1153 /** 1154 * dev_change_name - change name of a device 1155 * @dev: device 1156 * @newname: name (or format string) must be at least IFNAMSIZ 1157 * 1158 * Change name of a device, can pass format strings "eth%d". 1159 * for wildcarding. 1160 */ 1161 int dev_change_name(struct net_device *dev, const char *newname) 1162 { 1163 unsigned char old_assign_type; 1164 char oldname[IFNAMSIZ]; 1165 int err = 0; 1166 int ret; 1167 struct net *net; 1168 1169 ASSERT_RTNL(); 1170 BUG_ON(!dev_net(dev)); 1171 1172 net = dev_net(dev); 1173 if (dev->flags & IFF_UP) 1174 return -EBUSY; 1175 1176 write_seqcount_begin(&devnet_rename_seq); 1177 1178 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) { 1179 write_seqcount_end(&devnet_rename_seq); 1180 return 0; 1181 } 1182 1183 memcpy(oldname, dev->name, IFNAMSIZ); 1184 1185 err = dev_get_valid_name(net, dev, newname); 1186 if (err < 0) { 1187 write_seqcount_end(&devnet_rename_seq); 1188 return err; 1189 } 1190 1191 if (oldname[0] && !strchr(oldname, '%')) 1192 netdev_info(dev, "renamed from %s\n", oldname); 1193 1194 old_assign_type = dev->name_assign_type; 1195 dev->name_assign_type = NET_NAME_RENAMED; 1196 1197 rollback: 1198 ret = device_rename(&dev->dev, dev->name); 1199 if (ret) { 1200 memcpy(dev->name, oldname, IFNAMSIZ); 1201 dev->name_assign_type = old_assign_type; 1202 write_seqcount_end(&devnet_rename_seq); 1203 return ret; 1204 } 1205 1206 write_seqcount_end(&devnet_rename_seq); 1207 1208 netdev_adjacent_rename_links(dev, oldname); 1209 1210 write_lock_bh(&dev_base_lock); 1211 hlist_del_rcu(&dev->name_hlist); 1212 write_unlock_bh(&dev_base_lock); 1213 1214 synchronize_rcu(); 1215 1216 write_lock_bh(&dev_base_lock); 1217 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); 1218 write_unlock_bh(&dev_base_lock); 1219 1220 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); 1221 ret = notifier_to_errno(ret); 1222 1223 if (ret) { 1224 /* err >= 0 after dev_alloc_name() or stores the first errno */ 1225 if (err >= 0) { 1226 err = ret; 1227 write_seqcount_begin(&devnet_rename_seq); 1228 memcpy(dev->name, oldname, IFNAMSIZ); 1229 memcpy(oldname, newname, IFNAMSIZ); 1230 dev->name_assign_type = old_assign_type; 1231 old_assign_type = NET_NAME_RENAMED; 1232 goto rollback; 1233 } else { 1234 pr_err("%s: name change rollback failed: %d\n", 1235 dev->name, ret); 1236 } 1237 } 1238 1239 return err; 1240 } 1241 1242 /** 1243 * dev_set_alias - change ifalias of a device 1244 * @dev: device 1245 * @alias: name up to IFALIASZ 1246 * @len: limit of bytes to copy from info 1247 * 1248 * Set ifalias for a device, 1249 */ 1250 int dev_set_alias(struct net_device *dev, const char *alias, size_t len) 1251 { 1252 struct dev_ifalias *new_alias = NULL; 1253 1254 if (len >= IFALIASZ) 1255 return -EINVAL; 1256 1257 if (len) { 1258 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL); 1259 if (!new_alias) 1260 return -ENOMEM; 1261 1262 memcpy(new_alias->ifalias, alias, len); 1263 new_alias->ifalias[len] = 0; 1264 } 1265 1266 mutex_lock(&ifalias_mutex); 1267 rcu_swap_protected(dev->ifalias, new_alias, 1268 mutex_is_locked(&ifalias_mutex)); 1269 mutex_unlock(&ifalias_mutex); 1270 1271 if (new_alias) 1272 kfree_rcu(new_alias, rcuhead); 1273 1274 return len; 1275 } 1276 1277 /** 1278 * dev_get_alias - get ifalias of a device 1279 * @dev: device 1280 * @name: buffer to store name of ifalias 1281 * @len: size of buffer 1282 * 1283 * get ifalias for a device. Caller must make sure dev cannot go 1284 * away, e.g. rcu read lock or own a reference count to device. 1285 */ 1286 int dev_get_alias(const struct net_device *dev, char *name, size_t len) 1287 { 1288 const struct dev_ifalias *alias; 1289 int ret = 0; 1290 1291 rcu_read_lock(); 1292 alias = rcu_dereference(dev->ifalias); 1293 if (alias) 1294 ret = snprintf(name, len, "%s", alias->ifalias); 1295 rcu_read_unlock(); 1296 1297 return ret; 1298 } 1299 1300 /** 1301 * netdev_features_change - device changes features 1302 * @dev: device to cause notification 1303 * 1304 * Called to indicate a device has changed features. 1305 */ 1306 void netdev_features_change(struct net_device *dev) 1307 { 1308 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); 1309 } 1310 EXPORT_SYMBOL(netdev_features_change); 1311 1312 /** 1313 * netdev_state_change - device changes state 1314 * @dev: device to cause notification 1315 * 1316 * Called to indicate a device has changed state. This function calls 1317 * the notifier chains for netdev_chain and sends a NEWLINK message 1318 * to the routing socket. 1319 */ 1320 void netdev_state_change(struct net_device *dev) 1321 { 1322 if (dev->flags & IFF_UP) { 1323 struct netdev_notifier_change_info change_info = { 1324 .info.dev = dev, 1325 }; 1326 1327 call_netdevice_notifiers_info(NETDEV_CHANGE, 1328 &change_info.info); 1329 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL); 1330 } 1331 } 1332 EXPORT_SYMBOL(netdev_state_change); 1333 1334 /** 1335 * netdev_notify_peers - notify network peers about existence of @dev 1336 * @dev: network device 1337 * 1338 * Generate traffic such that interested network peers are aware of 1339 * @dev, such as by generating a gratuitous ARP. This may be used when 1340 * a device wants to inform the rest of the network about some sort of 1341 * reconfiguration such as a failover event or virtual machine 1342 * migration. 1343 */ 1344 void netdev_notify_peers(struct net_device *dev) 1345 { 1346 rtnl_lock(); 1347 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev); 1348 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev); 1349 rtnl_unlock(); 1350 } 1351 EXPORT_SYMBOL(netdev_notify_peers); 1352 1353 static int __dev_open(struct net_device *dev) 1354 { 1355 const struct net_device_ops *ops = dev->netdev_ops; 1356 int ret; 1357 1358 ASSERT_RTNL(); 1359 1360 if (!netif_device_present(dev)) 1361 return -ENODEV; 1362 1363 /* Block netpoll from trying to do any rx path servicing. 1364 * If we don't do this there is a chance ndo_poll_controller 1365 * or ndo_poll may be running while we open the device 1366 */ 1367 netpoll_poll_disable(dev); 1368 1369 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev); 1370 ret = notifier_to_errno(ret); 1371 if (ret) 1372 return ret; 1373 1374 set_bit(__LINK_STATE_START, &dev->state); 1375 1376 if (ops->ndo_validate_addr) 1377 ret = ops->ndo_validate_addr(dev); 1378 1379 if (!ret && ops->ndo_open) 1380 ret = ops->ndo_open(dev); 1381 1382 netpoll_poll_enable(dev); 1383 1384 if (ret) 1385 clear_bit(__LINK_STATE_START, &dev->state); 1386 else { 1387 dev->flags |= IFF_UP; 1388 dev_set_rx_mode(dev); 1389 dev_activate(dev); 1390 add_device_randomness(dev->dev_addr, dev->addr_len); 1391 } 1392 1393 return ret; 1394 } 1395 1396 /** 1397 * dev_open - prepare an interface for use. 1398 * @dev: device to open 1399 * 1400 * Takes a device from down to up state. The device's private open 1401 * function is invoked and then the multicast lists are loaded. Finally 1402 * the device is moved into the up state and a %NETDEV_UP message is 1403 * sent to the netdev notifier chain. 1404 * 1405 * Calling this function on an active interface is a nop. On a failure 1406 * a negative errno code is returned. 1407 */ 1408 int dev_open(struct net_device *dev) 1409 { 1410 int ret; 1411 1412 if (dev->flags & IFF_UP) 1413 return 0; 1414 1415 ret = __dev_open(dev); 1416 if (ret < 0) 1417 return ret; 1418 1419 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); 1420 call_netdevice_notifiers(NETDEV_UP, dev); 1421 1422 return ret; 1423 } 1424 EXPORT_SYMBOL(dev_open); 1425 1426 static void __dev_close_many(struct list_head *head) 1427 { 1428 struct net_device *dev; 1429 1430 ASSERT_RTNL(); 1431 might_sleep(); 1432 1433 list_for_each_entry(dev, head, close_list) { 1434 /* Temporarily disable netpoll until the interface is down */ 1435 netpoll_poll_disable(dev); 1436 1437 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); 1438 1439 clear_bit(__LINK_STATE_START, &dev->state); 1440 1441 /* Synchronize to scheduled poll. We cannot touch poll list, it 1442 * can be even on different cpu. So just clear netif_running(). 1443 * 1444 * dev->stop() will invoke napi_disable() on all of it's 1445 * napi_struct instances on this device. 1446 */ 1447 smp_mb__after_atomic(); /* Commit netif_running(). */ 1448 } 1449 1450 dev_deactivate_many(head); 1451 1452 list_for_each_entry(dev, head, close_list) { 1453 const struct net_device_ops *ops = dev->netdev_ops; 1454 1455 /* 1456 * Call the device specific close. This cannot fail. 1457 * Only if device is UP 1458 * 1459 * We allow it to be called even after a DETACH hot-plug 1460 * event. 1461 */ 1462 if (ops->ndo_stop) 1463 ops->ndo_stop(dev); 1464 1465 dev->flags &= ~IFF_UP; 1466 netpoll_poll_enable(dev); 1467 } 1468 } 1469 1470 static void __dev_close(struct net_device *dev) 1471 { 1472 LIST_HEAD(single); 1473 1474 list_add(&dev->close_list, &single); 1475 __dev_close_many(&single); 1476 list_del(&single); 1477 } 1478 1479 void dev_close_many(struct list_head *head, bool unlink) 1480 { 1481 struct net_device *dev, *tmp; 1482 1483 /* Remove the devices that don't need to be closed */ 1484 list_for_each_entry_safe(dev, tmp, head, close_list) 1485 if (!(dev->flags & IFF_UP)) 1486 list_del_init(&dev->close_list); 1487 1488 __dev_close_many(head); 1489 1490 list_for_each_entry_safe(dev, tmp, head, close_list) { 1491 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); 1492 call_netdevice_notifiers(NETDEV_DOWN, dev); 1493 if (unlink) 1494 list_del_init(&dev->close_list); 1495 } 1496 } 1497 EXPORT_SYMBOL(dev_close_many); 1498 1499 /** 1500 * dev_close - shutdown an interface. 1501 * @dev: device to shutdown 1502 * 1503 * This function moves an active device into down state. A 1504 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device 1505 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier 1506 * chain. 1507 */ 1508 void dev_close(struct net_device *dev) 1509 { 1510 if (dev->flags & IFF_UP) { 1511 LIST_HEAD(single); 1512 1513 list_add(&dev->close_list, &single); 1514 dev_close_many(&single, true); 1515 list_del(&single); 1516 } 1517 } 1518 EXPORT_SYMBOL(dev_close); 1519 1520 1521 /** 1522 * dev_disable_lro - disable Large Receive Offload on a device 1523 * @dev: device 1524 * 1525 * Disable Large Receive Offload (LRO) on a net device. Must be 1526 * called under RTNL. This is needed if received packets may be 1527 * forwarded to another interface. 1528 */ 1529 void dev_disable_lro(struct net_device *dev) 1530 { 1531 struct net_device *lower_dev; 1532 struct list_head *iter; 1533 1534 dev->wanted_features &= ~NETIF_F_LRO; 1535 netdev_update_features(dev); 1536 1537 if (unlikely(dev->features & NETIF_F_LRO)) 1538 netdev_WARN(dev, "failed to disable LRO!\n"); 1539 1540 netdev_for_each_lower_dev(dev, lower_dev, iter) 1541 dev_disable_lro(lower_dev); 1542 } 1543 EXPORT_SYMBOL(dev_disable_lro); 1544 1545 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val, 1546 struct net_device *dev) 1547 { 1548 struct netdev_notifier_info info = { 1549 .dev = dev, 1550 }; 1551 1552 return nb->notifier_call(nb, val, &info); 1553 } 1554 1555 static int dev_boot_phase = 1; 1556 1557 /** 1558 * register_netdevice_notifier - register a network notifier block 1559 * @nb: notifier 1560 * 1561 * Register a notifier to be called when network device events occur. 1562 * The notifier passed is linked into the kernel structures and must 1563 * not be reused until it has been unregistered. A negative errno code 1564 * is returned on a failure. 1565 * 1566 * When registered all registration and up events are replayed 1567 * to the new notifier to allow device to have a race free 1568 * view of the network device list. 1569 */ 1570 1571 int register_netdevice_notifier(struct notifier_block *nb) 1572 { 1573 struct net_device *dev; 1574 struct net_device *last; 1575 struct net *net; 1576 int err; 1577 1578 rtnl_lock(); 1579 err = raw_notifier_chain_register(&netdev_chain, nb); 1580 if (err) 1581 goto unlock; 1582 if (dev_boot_phase) 1583 goto unlock; 1584 for_each_net(net) { 1585 for_each_netdev(net, dev) { 1586 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev); 1587 err = notifier_to_errno(err); 1588 if (err) 1589 goto rollback; 1590 1591 if (!(dev->flags & IFF_UP)) 1592 continue; 1593 1594 call_netdevice_notifier(nb, NETDEV_UP, dev); 1595 } 1596 } 1597 1598 unlock: 1599 rtnl_unlock(); 1600 return err; 1601 1602 rollback: 1603 last = dev; 1604 for_each_net(net) { 1605 for_each_netdev(net, dev) { 1606 if (dev == last) 1607 goto outroll; 1608 1609 if (dev->flags & IFF_UP) { 1610 call_netdevice_notifier(nb, NETDEV_GOING_DOWN, 1611 dev); 1612 call_netdevice_notifier(nb, NETDEV_DOWN, dev); 1613 } 1614 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); 1615 } 1616 } 1617 1618 outroll: 1619 raw_notifier_chain_unregister(&netdev_chain, nb); 1620 goto unlock; 1621 } 1622 EXPORT_SYMBOL(register_netdevice_notifier); 1623 1624 /** 1625 * unregister_netdevice_notifier - unregister a network notifier block 1626 * @nb: notifier 1627 * 1628 * Unregister a notifier previously registered by 1629 * register_netdevice_notifier(). The notifier is unlinked into the 1630 * kernel structures and may then be reused. A negative errno code 1631 * is returned on a failure. 1632 * 1633 * After unregistering unregister and down device events are synthesized 1634 * for all devices on the device list to the removed notifier to remove 1635 * the need for special case cleanup code. 1636 */ 1637 1638 int unregister_netdevice_notifier(struct notifier_block *nb) 1639 { 1640 struct net_device *dev; 1641 struct net *net; 1642 int err; 1643 1644 rtnl_lock(); 1645 err = raw_notifier_chain_unregister(&netdev_chain, nb); 1646 if (err) 1647 goto unlock; 1648 1649 for_each_net(net) { 1650 for_each_netdev(net, dev) { 1651 if (dev->flags & IFF_UP) { 1652 call_netdevice_notifier(nb, NETDEV_GOING_DOWN, 1653 dev); 1654 call_netdevice_notifier(nb, NETDEV_DOWN, dev); 1655 } 1656 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); 1657 } 1658 } 1659 unlock: 1660 rtnl_unlock(); 1661 return err; 1662 } 1663 EXPORT_SYMBOL(unregister_netdevice_notifier); 1664 1665 /** 1666 * call_netdevice_notifiers_info - call all network notifier blocks 1667 * @val: value passed unmodified to notifier function 1668 * @dev: net_device pointer passed unmodified to notifier function 1669 * @info: notifier information data 1670 * 1671 * Call all network notifier blocks. Parameters and return value 1672 * are as for raw_notifier_call_chain(). 1673 */ 1674 1675 static int call_netdevice_notifiers_info(unsigned long val, 1676 struct netdev_notifier_info *info) 1677 { 1678 ASSERT_RTNL(); 1679 return raw_notifier_call_chain(&netdev_chain, val, info); 1680 } 1681 1682 /** 1683 * call_netdevice_notifiers - call all network notifier blocks 1684 * @val: value passed unmodified to notifier function 1685 * @dev: net_device pointer passed unmodified to notifier function 1686 * 1687 * Call all network notifier blocks. Parameters and return value 1688 * are as for raw_notifier_call_chain(). 1689 */ 1690 1691 int call_netdevice_notifiers(unsigned long val, struct net_device *dev) 1692 { 1693 struct netdev_notifier_info info = { 1694 .dev = dev, 1695 }; 1696 1697 return call_netdevice_notifiers_info(val, &info); 1698 } 1699 EXPORT_SYMBOL(call_netdevice_notifiers); 1700 1701 #ifdef CONFIG_NET_INGRESS 1702 static struct static_key ingress_needed __read_mostly; 1703 1704 void net_inc_ingress_queue(void) 1705 { 1706 static_key_slow_inc(&ingress_needed); 1707 } 1708 EXPORT_SYMBOL_GPL(net_inc_ingress_queue); 1709 1710 void net_dec_ingress_queue(void) 1711 { 1712 static_key_slow_dec(&ingress_needed); 1713 } 1714 EXPORT_SYMBOL_GPL(net_dec_ingress_queue); 1715 #endif 1716 1717 #ifdef CONFIG_NET_EGRESS 1718 static struct static_key egress_needed __read_mostly; 1719 1720 void net_inc_egress_queue(void) 1721 { 1722 static_key_slow_inc(&egress_needed); 1723 } 1724 EXPORT_SYMBOL_GPL(net_inc_egress_queue); 1725 1726 void net_dec_egress_queue(void) 1727 { 1728 static_key_slow_dec(&egress_needed); 1729 } 1730 EXPORT_SYMBOL_GPL(net_dec_egress_queue); 1731 #endif 1732 1733 static struct static_key netstamp_needed __read_mostly; 1734 #ifdef HAVE_JUMP_LABEL 1735 static atomic_t netstamp_needed_deferred; 1736 static atomic_t netstamp_wanted; 1737 static void netstamp_clear(struct work_struct *work) 1738 { 1739 int deferred = atomic_xchg(&netstamp_needed_deferred, 0); 1740 int wanted; 1741 1742 wanted = atomic_add_return(deferred, &netstamp_wanted); 1743 if (wanted > 0) 1744 static_key_enable(&netstamp_needed); 1745 else 1746 static_key_disable(&netstamp_needed); 1747 } 1748 static DECLARE_WORK(netstamp_work, netstamp_clear); 1749 #endif 1750 1751 void net_enable_timestamp(void) 1752 { 1753 #ifdef HAVE_JUMP_LABEL 1754 int wanted; 1755 1756 while (1) { 1757 wanted = atomic_read(&netstamp_wanted); 1758 if (wanted <= 0) 1759 break; 1760 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted) 1761 return; 1762 } 1763 atomic_inc(&netstamp_needed_deferred); 1764 schedule_work(&netstamp_work); 1765 #else 1766 static_key_slow_inc(&netstamp_needed); 1767 #endif 1768 } 1769 EXPORT_SYMBOL(net_enable_timestamp); 1770 1771 void net_disable_timestamp(void) 1772 { 1773 #ifdef HAVE_JUMP_LABEL 1774 int wanted; 1775 1776 while (1) { 1777 wanted = atomic_read(&netstamp_wanted); 1778 if (wanted <= 1) 1779 break; 1780 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted) 1781 return; 1782 } 1783 atomic_dec(&netstamp_needed_deferred); 1784 schedule_work(&netstamp_work); 1785 #else 1786 static_key_slow_dec(&netstamp_needed); 1787 #endif 1788 } 1789 EXPORT_SYMBOL(net_disable_timestamp); 1790 1791 static inline void net_timestamp_set(struct sk_buff *skb) 1792 { 1793 skb->tstamp = 0; 1794 if (static_key_false(&netstamp_needed)) 1795 __net_timestamp(skb); 1796 } 1797 1798 #define net_timestamp_check(COND, SKB) \ 1799 if (static_key_false(&netstamp_needed)) { \ 1800 if ((COND) && !(SKB)->tstamp) \ 1801 __net_timestamp(SKB); \ 1802 } \ 1803 1804 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb) 1805 { 1806 unsigned int len; 1807 1808 if (!(dev->flags & IFF_UP)) 1809 return false; 1810 1811 len = dev->mtu + dev->hard_header_len + VLAN_HLEN; 1812 if (skb->len <= len) 1813 return true; 1814 1815 /* if TSO is enabled, we don't care about the length as the packet 1816 * could be forwarded without being segmented before 1817 */ 1818 if (skb_is_gso(skb)) 1819 return true; 1820 1821 return false; 1822 } 1823 EXPORT_SYMBOL_GPL(is_skb_forwardable); 1824 1825 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 1826 { 1827 int ret = ____dev_forward_skb(dev, skb); 1828 1829 if (likely(!ret)) { 1830 skb->protocol = eth_type_trans(skb, dev); 1831 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); 1832 } 1833 1834 return ret; 1835 } 1836 EXPORT_SYMBOL_GPL(__dev_forward_skb); 1837 1838 /** 1839 * dev_forward_skb - loopback an skb to another netif 1840 * 1841 * @dev: destination network device 1842 * @skb: buffer to forward 1843 * 1844 * return values: 1845 * NET_RX_SUCCESS (no congestion) 1846 * NET_RX_DROP (packet was dropped, but freed) 1847 * 1848 * dev_forward_skb can be used for injecting an skb from the 1849 * start_xmit function of one device into the receive queue 1850 * of another device. 1851 * 1852 * The receiving device may be in another namespace, so 1853 * we have to clear all information in the skb that could 1854 * impact namespace isolation. 1855 */ 1856 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 1857 { 1858 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb); 1859 } 1860 EXPORT_SYMBOL_GPL(dev_forward_skb); 1861 1862 static inline int deliver_skb(struct sk_buff *skb, 1863 struct packet_type *pt_prev, 1864 struct net_device *orig_dev) 1865 { 1866 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) 1867 return -ENOMEM; 1868 refcount_inc(&skb->users); 1869 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 1870 } 1871 1872 static inline void deliver_ptype_list_skb(struct sk_buff *skb, 1873 struct packet_type **pt, 1874 struct net_device *orig_dev, 1875 __be16 type, 1876 struct list_head *ptype_list) 1877 { 1878 struct packet_type *ptype, *pt_prev = *pt; 1879 1880 list_for_each_entry_rcu(ptype, ptype_list, list) { 1881 if (ptype->type != type) 1882 continue; 1883 if (pt_prev) 1884 deliver_skb(skb, pt_prev, orig_dev); 1885 pt_prev = ptype; 1886 } 1887 *pt = pt_prev; 1888 } 1889 1890 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) 1891 { 1892 if (!ptype->af_packet_priv || !skb->sk) 1893 return false; 1894 1895 if (ptype->id_match) 1896 return ptype->id_match(ptype, skb->sk); 1897 else if ((struct sock *)ptype->af_packet_priv == skb->sk) 1898 return true; 1899 1900 return false; 1901 } 1902 1903 /* 1904 * Support routine. Sends outgoing frames to any network 1905 * taps currently in use. 1906 */ 1907 1908 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) 1909 { 1910 struct packet_type *ptype; 1911 struct sk_buff *skb2 = NULL; 1912 struct packet_type *pt_prev = NULL; 1913 struct list_head *ptype_list = &ptype_all; 1914 1915 rcu_read_lock(); 1916 again: 1917 list_for_each_entry_rcu(ptype, ptype_list, list) { 1918 /* Never send packets back to the socket 1919 * they originated from - MvS (miquels@drinkel.ow.org) 1920 */ 1921 if (skb_loop_sk(ptype, skb)) 1922 continue; 1923 1924 if (pt_prev) { 1925 deliver_skb(skb2, pt_prev, skb->dev); 1926 pt_prev = ptype; 1927 continue; 1928 } 1929 1930 /* need to clone skb, done only once */ 1931 skb2 = skb_clone(skb, GFP_ATOMIC); 1932 if (!skb2) 1933 goto out_unlock; 1934 1935 net_timestamp_set(skb2); 1936 1937 /* skb->nh should be correctly 1938 * set by sender, so that the second statement is 1939 * just protection against buggy protocols. 1940 */ 1941 skb_reset_mac_header(skb2); 1942 1943 if (skb_network_header(skb2) < skb2->data || 1944 skb_network_header(skb2) > skb_tail_pointer(skb2)) { 1945 net_crit_ratelimited("protocol %04x is buggy, dev %s\n", 1946 ntohs(skb2->protocol), 1947 dev->name); 1948 skb_reset_network_header(skb2); 1949 } 1950 1951 skb2->transport_header = skb2->network_header; 1952 skb2->pkt_type = PACKET_OUTGOING; 1953 pt_prev = ptype; 1954 } 1955 1956 if (ptype_list == &ptype_all) { 1957 ptype_list = &dev->ptype_all; 1958 goto again; 1959 } 1960 out_unlock: 1961 if (pt_prev) { 1962 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC)) 1963 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); 1964 else 1965 kfree_skb(skb2); 1966 } 1967 rcu_read_unlock(); 1968 } 1969 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit); 1970 1971 /** 1972 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change 1973 * @dev: Network device 1974 * @txq: number of queues available 1975 * 1976 * If real_num_tx_queues is changed the tc mappings may no longer be 1977 * valid. To resolve this verify the tc mapping remains valid and if 1978 * not NULL the mapping. With no priorities mapping to this 1979 * offset/count pair it will no longer be used. In the worst case TC0 1980 * is invalid nothing can be done so disable priority mappings. If is 1981 * expected that drivers will fix this mapping if they can before 1982 * calling netif_set_real_num_tx_queues. 1983 */ 1984 static void netif_setup_tc(struct net_device *dev, unsigned int txq) 1985 { 1986 int i; 1987 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 1988 1989 /* If TC0 is invalidated disable TC mapping */ 1990 if (tc->offset + tc->count > txq) { 1991 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n"); 1992 dev->num_tc = 0; 1993 return; 1994 } 1995 1996 /* Invalidated prio to tc mappings set to TC0 */ 1997 for (i = 1; i < TC_BITMASK + 1; i++) { 1998 int q = netdev_get_prio_tc_map(dev, i); 1999 2000 tc = &dev->tc_to_txq[q]; 2001 if (tc->offset + tc->count > txq) { 2002 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n", 2003 i, q); 2004 netdev_set_prio_tc_map(dev, i, 0); 2005 } 2006 } 2007 } 2008 2009 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq) 2010 { 2011 if (dev->num_tc) { 2012 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 2013 int i; 2014 2015 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) { 2016 if ((txq - tc->offset) < tc->count) 2017 return i; 2018 } 2019 2020 return -1; 2021 } 2022 2023 return 0; 2024 } 2025 EXPORT_SYMBOL(netdev_txq_to_tc); 2026 2027 #ifdef CONFIG_XPS 2028 static DEFINE_MUTEX(xps_map_mutex); 2029 #define xmap_dereference(P) \ 2030 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex)) 2031 2032 static bool remove_xps_queue(struct xps_dev_maps *dev_maps, 2033 int tci, u16 index) 2034 { 2035 struct xps_map *map = NULL; 2036 int pos; 2037 2038 if (dev_maps) 2039 map = xmap_dereference(dev_maps->cpu_map[tci]); 2040 if (!map) 2041 return false; 2042 2043 for (pos = map->len; pos--;) { 2044 if (map->queues[pos] != index) 2045 continue; 2046 2047 if (map->len > 1) { 2048 map->queues[pos] = map->queues[--map->len]; 2049 break; 2050 } 2051 2052 RCU_INIT_POINTER(dev_maps->cpu_map[tci], NULL); 2053 kfree_rcu(map, rcu); 2054 return false; 2055 } 2056 2057 return true; 2058 } 2059 2060 static bool remove_xps_queue_cpu(struct net_device *dev, 2061 struct xps_dev_maps *dev_maps, 2062 int cpu, u16 offset, u16 count) 2063 { 2064 int num_tc = dev->num_tc ? : 1; 2065 bool active = false; 2066 int tci; 2067 2068 for (tci = cpu * num_tc; num_tc--; tci++) { 2069 int i, j; 2070 2071 for (i = count, j = offset; i--; j++) { 2072 if (!remove_xps_queue(dev_maps, cpu, j)) 2073 break; 2074 } 2075 2076 active |= i < 0; 2077 } 2078 2079 return active; 2080 } 2081 2082 static void netif_reset_xps_queues(struct net_device *dev, u16 offset, 2083 u16 count) 2084 { 2085 struct xps_dev_maps *dev_maps; 2086 int cpu, i; 2087 bool active = false; 2088 2089 mutex_lock(&xps_map_mutex); 2090 dev_maps = xmap_dereference(dev->xps_maps); 2091 2092 if (!dev_maps) 2093 goto out_no_maps; 2094 2095 for_each_possible_cpu(cpu) 2096 active |= remove_xps_queue_cpu(dev, dev_maps, cpu, 2097 offset, count); 2098 2099 if (!active) { 2100 RCU_INIT_POINTER(dev->xps_maps, NULL); 2101 kfree_rcu(dev_maps, rcu); 2102 } 2103 2104 for (i = offset + (count - 1); count--; i--) 2105 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i), 2106 NUMA_NO_NODE); 2107 2108 out_no_maps: 2109 mutex_unlock(&xps_map_mutex); 2110 } 2111 2112 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index) 2113 { 2114 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index); 2115 } 2116 2117 static struct xps_map *expand_xps_map(struct xps_map *map, 2118 int cpu, u16 index) 2119 { 2120 struct xps_map *new_map; 2121 int alloc_len = XPS_MIN_MAP_ALLOC; 2122 int i, pos; 2123 2124 for (pos = 0; map && pos < map->len; pos++) { 2125 if (map->queues[pos] != index) 2126 continue; 2127 return map; 2128 } 2129 2130 /* Need to add queue to this CPU's existing map */ 2131 if (map) { 2132 if (pos < map->alloc_len) 2133 return map; 2134 2135 alloc_len = map->alloc_len * 2; 2136 } 2137 2138 /* Need to allocate new map to store queue on this CPU's map */ 2139 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL, 2140 cpu_to_node(cpu)); 2141 if (!new_map) 2142 return NULL; 2143 2144 for (i = 0; i < pos; i++) 2145 new_map->queues[i] = map->queues[i]; 2146 new_map->alloc_len = alloc_len; 2147 new_map->len = pos; 2148 2149 return new_map; 2150 } 2151 2152 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask, 2153 u16 index) 2154 { 2155 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL; 2156 int i, cpu, tci, numa_node_id = -2; 2157 int maps_sz, num_tc = 1, tc = 0; 2158 struct xps_map *map, *new_map; 2159 bool active = false; 2160 2161 if (dev->num_tc) { 2162 num_tc = dev->num_tc; 2163 tc = netdev_txq_to_tc(dev, index); 2164 if (tc < 0) 2165 return -EINVAL; 2166 } 2167 2168 maps_sz = XPS_DEV_MAPS_SIZE(num_tc); 2169 if (maps_sz < L1_CACHE_BYTES) 2170 maps_sz = L1_CACHE_BYTES; 2171 2172 mutex_lock(&xps_map_mutex); 2173 2174 dev_maps = xmap_dereference(dev->xps_maps); 2175 2176 /* allocate memory for queue storage */ 2177 for_each_cpu_and(cpu, cpu_online_mask, mask) { 2178 if (!new_dev_maps) 2179 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL); 2180 if (!new_dev_maps) { 2181 mutex_unlock(&xps_map_mutex); 2182 return -ENOMEM; 2183 } 2184 2185 tci = cpu * num_tc + tc; 2186 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[tci]) : 2187 NULL; 2188 2189 map = expand_xps_map(map, cpu, index); 2190 if (!map) 2191 goto error; 2192 2193 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map); 2194 } 2195 2196 if (!new_dev_maps) 2197 goto out_no_new_maps; 2198 2199 for_each_possible_cpu(cpu) { 2200 /* copy maps belonging to foreign traffic classes */ 2201 for (i = tc, tci = cpu * num_tc; dev_maps && i--; tci++) { 2202 /* fill in the new device map from the old device map */ 2203 map = xmap_dereference(dev_maps->cpu_map[tci]); 2204 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map); 2205 } 2206 2207 /* We need to explicitly update tci as prevous loop 2208 * could break out early if dev_maps is NULL. 2209 */ 2210 tci = cpu * num_tc + tc; 2211 2212 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) { 2213 /* add queue to CPU maps */ 2214 int pos = 0; 2215 2216 map = xmap_dereference(new_dev_maps->cpu_map[tci]); 2217 while ((pos < map->len) && (map->queues[pos] != index)) 2218 pos++; 2219 2220 if (pos == map->len) 2221 map->queues[map->len++] = index; 2222 #ifdef CONFIG_NUMA 2223 if (numa_node_id == -2) 2224 numa_node_id = cpu_to_node(cpu); 2225 else if (numa_node_id != cpu_to_node(cpu)) 2226 numa_node_id = -1; 2227 #endif 2228 } else if (dev_maps) { 2229 /* fill in the new device map from the old device map */ 2230 map = xmap_dereference(dev_maps->cpu_map[tci]); 2231 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map); 2232 } 2233 2234 /* copy maps belonging to foreign traffic classes */ 2235 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) { 2236 /* fill in the new device map from the old device map */ 2237 map = xmap_dereference(dev_maps->cpu_map[tci]); 2238 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map); 2239 } 2240 } 2241 2242 rcu_assign_pointer(dev->xps_maps, new_dev_maps); 2243 2244 /* Cleanup old maps */ 2245 if (!dev_maps) 2246 goto out_no_old_maps; 2247 2248 for_each_possible_cpu(cpu) { 2249 for (i = num_tc, tci = cpu * num_tc; i--; tci++) { 2250 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]); 2251 map = xmap_dereference(dev_maps->cpu_map[tci]); 2252 if (map && map != new_map) 2253 kfree_rcu(map, rcu); 2254 } 2255 } 2256 2257 kfree_rcu(dev_maps, rcu); 2258 2259 out_no_old_maps: 2260 dev_maps = new_dev_maps; 2261 active = true; 2262 2263 out_no_new_maps: 2264 /* update Tx queue numa node */ 2265 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index), 2266 (numa_node_id >= 0) ? numa_node_id : 2267 NUMA_NO_NODE); 2268 2269 if (!dev_maps) 2270 goto out_no_maps; 2271 2272 /* removes queue from unused CPUs */ 2273 for_each_possible_cpu(cpu) { 2274 for (i = tc, tci = cpu * num_tc; i--; tci++) 2275 active |= remove_xps_queue(dev_maps, tci, index); 2276 if (!cpumask_test_cpu(cpu, mask) || !cpu_online(cpu)) 2277 active |= remove_xps_queue(dev_maps, tci, index); 2278 for (i = num_tc - tc, tci++; --i; tci++) 2279 active |= remove_xps_queue(dev_maps, tci, index); 2280 } 2281 2282 /* free map if not active */ 2283 if (!active) { 2284 RCU_INIT_POINTER(dev->xps_maps, NULL); 2285 kfree_rcu(dev_maps, rcu); 2286 } 2287 2288 out_no_maps: 2289 mutex_unlock(&xps_map_mutex); 2290 2291 return 0; 2292 error: 2293 /* remove any maps that we added */ 2294 for_each_possible_cpu(cpu) { 2295 for (i = num_tc, tci = cpu * num_tc; i--; tci++) { 2296 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]); 2297 map = dev_maps ? 2298 xmap_dereference(dev_maps->cpu_map[tci]) : 2299 NULL; 2300 if (new_map && new_map != map) 2301 kfree(new_map); 2302 } 2303 } 2304 2305 mutex_unlock(&xps_map_mutex); 2306 2307 kfree(new_dev_maps); 2308 return -ENOMEM; 2309 } 2310 EXPORT_SYMBOL(netif_set_xps_queue); 2311 2312 #endif 2313 void netdev_reset_tc(struct net_device *dev) 2314 { 2315 #ifdef CONFIG_XPS 2316 netif_reset_xps_queues_gt(dev, 0); 2317 #endif 2318 dev->num_tc = 0; 2319 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq)); 2320 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map)); 2321 } 2322 EXPORT_SYMBOL(netdev_reset_tc); 2323 2324 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset) 2325 { 2326 if (tc >= dev->num_tc) 2327 return -EINVAL; 2328 2329 #ifdef CONFIG_XPS 2330 netif_reset_xps_queues(dev, offset, count); 2331 #endif 2332 dev->tc_to_txq[tc].count = count; 2333 dev->tc_to_txq[tc].offset = offset; 2334 return 0; 2335 } 2336 EXPORT_SYMBOL(netdev_set_tc_queue); 2337 2338 int netdev_set_num_tc(struct net_device *dev, u8 num_tc) 2339 { 2340 if (num_tc > TC_MAX_QUEUE) 2341 return -EINVAL; 2342 2343 #ifdef CONFIG_XPS 2344 netif_reset_xps_queues_gt(dev, 0); 2345 #endif 2346 dev->num_tc = num_tc; 2347 return 0; 2348 } 2349 EXPORT_SYMBOL(netdev_set_num_tc); 2350 2351 /* 2352 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues 2353 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed. 2354 */ 2355 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) 2356 { 2357 int rc; 2358 2359 if (txq < 1 || txq > dev->num_tx_queues) 2360 return -EINVAL; 2361 2362 if (dev->reg_state == NETREG_REGISTERED || 2363 dev->reg_state == NETREG_UNREGISTERING) { 2364 ASSERT_RTNL(); 2365 2366 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, 2367 txq); 2368 if (rc) 2369 return rc; 2370 2371 if (dev->num_tc) 2372 netif_setup_tc(dev, txq); 2373 2374 if (txq < dev->real_num_tx_queues) { 2375 qdisc_reset_all_tx_gt(dev, txq); 2376 #ifdef CONFIG_XPS 2377 netif_reset_xps_queues_gt(dev, txq); 2378 #endif 2379 } 2380 } 2381 2382 dev->real_num_tx_queues = txq; 2383 return 0; 2384 } 2385 EXPORT_SYMBOL(netif_set_real_num_tx_queues); 2386 2387 #ifdef CONFIG_SYSFS 2388 /** 2389 * netif_set_real_num_rx_queues - set actual number of RX queues used 2390 * @dev: Network device 2391 * @rxq: Actual number of RX queues 2392 * 2393 * This must be called either with the rtnl_lock held or before 2394 * registration of the net device. Returns 0 on success, or a 2395 * negative error code. If called before registration, it always 2396 * succeeds. 2397 */ 2398 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) 2399 { 2400 int rc; 2401 2402 if (rxq < 1 || rxq > dev->num_rx_queues) 2403 return -EINVAL; 2404 2405 if (dev->reg_state == NETREG_REGISTERED) { 2406 ASSERT_RTNL(); 2407 2408 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, 2409 rxq); 2410 if (rc) 2411 return rc; 2412 } 2413 2414 dev->real_num_rx_queues = rxq; 2415 return 0; 2416 } 2417 EXPORT_SYMBOL(netif_set_real_num_rx_queues); 2418 #endif 2419 2420 /** 2421 * netif_get_num_default_rss_queues - default number of RSS queues 2422 * 2423 * This routine should set an upper limit on the number of RSS queues 2424 * used by default by multiqueue devices. 2425 */ 2426 int netif_get_num_default_rss_queues(void) 2427 { 2428 return is_kdump_kernel() ? 2429 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus()); 2430 } 2431 EXPORT_SYMBOL(netif_get_num_default_rss_queues); 2432 2433 static void __netif_reschedule(struct Qdisc *q) 2434 { 2435 struct softnet_data *sd; 2436 unsigned long flags; 2437 2438 local_irq_save(flags); 2439 sd = this_cpu_ptr(&softnet_data); 2440 q->next_sched = NULL; 2441 *sd->output_queue_tailp = q; 2442 sd->output_queue_tailp = &q->next_sched; 2443 raise_softirq_irqoff(NET_TX_SOFTIRQ); 2444 local_irq_restore(flags); 2445 } 2446 2447 void __netif_schedule(struct Qdisc *q) 2448 { 2449 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) 2450 __netif_reschedule(q); 2451 } 2452 EXPORT_SYMBOL(__netif_schedule); 2453 2454 struct dev_kfree_skb_cb { 2455 enum skb_free_reason reason; 2456 }; 2457 2458 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb) 2459 { 2460 return (struct dev_kfree_skb_cb *)skb->cb; 2461 } 2462 2463 void netif_schedule_queue(struct netdev_queue *txq) 2464 { 2465 rcu_read_lock(); 2466 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) { 2467 struct Qdisc *q = rcu_dereference(txq->qdisc); 2468 2469 __netif_schedule(q); 2470 } 2471 rcu_read_unlock(); 2472 } 2473 EXPORT_SYMBOL(netif_schedule_queue); 2474 2475 void netif_tx_wake_queue(struct netdev_queue *dev_queue) 2476 { 2477 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { 2478 struct Qdisc *q; 2479 2480 rcu_read_lock(); 2481 q = rcu_dereference(dev_queue->qdisc); 2482 __netif_schedule(q); 2483 rcu_read_unlock(); 2484 } 2485 } 2486 EXPORT_SYMBOL(netif_tx_wake_queue); 2487 2488 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason) 2489 { 2490 unsigned long flags; 2491 2492 if (unlikely(!skb)) 2493 return; 2494 2495 if (likely(refcount_read(&skb->users) == 1)) { 2496 smp_rmb(); 2497 refcount_set(&skb->users, 0); 2498 } else if (likely(!refcount_dec_and_test(&skb->users))) { 2499 return; 2500 } 2501 get_kfree_skb_cb(skb)->reason = reason; 2502 local_irq_save(flags); 2503 skb->next = __this_cpu_read(softnet_data.completion_queue); 2504 __this_cpu_write(softnet_data.completion_queue, skb); 2505 raise_softirq_irqoff(NET_TX_SOFTIRQ); 2506 local_irq_restore(flags); 2507 } 2508 EXPORT_SYMBOL(__dev_kfree_skb_irq); 2509 2510 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason) 2511 { 2512 if (in_irq() || irqs_disabled()) 2513 __dev_kfree_skb_irq(skb, reason); 2514 else 2515 dev_kfree_skb(skb); 2516 } 2517 EXPORT_SYMBOL(__dev_kfree_skb_any); 2518 2519 2520 /** 2521 * netif_device_detach - mark device as removed 2522 * @dev: network device 2523 * 2524 * Mark device as removed from system and therefore no longer available. 2525 */ 2526 void netif_device_detach(struct net_device *dev) 2527 { 2528 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && 2529 netif_running(dev)) { 2530 netif_tx_stop_all_queues(dev); 2531 } 2532 } 2533 EXPORT_SYMBOL(netif_device_detach); 2534 2535 /** 2536 * netif_device_attach - mark device as attached 2537 * @dev: network device 2538 * 2539 * Mark device as attached from system and restart if needed. 2540 */ 2541 void netif_device_attach(struct net_device *dev) 2542 { 2543 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && 2544 netif_running(dev)) { 2545 netif_tx_wake_all_queues(dev); 2546 __netdev_watchdog_up(dev); 2547 } 2548 } 2549 EXPORT_SYMBOL(netif_device_attach); 2550 2551 /* 2552 * Returns a Tx hash based on the given packet descriptor a Tx queues' number 2553 * to be used as a distribution range. 2554 */ 2555 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb, 2556 unsigned int num_tx_queues) 2557 { 2558 u32 hash; 2559 u16 qoffset = 0; 2560 u16 qcount = num_tx_queues; 2561 2562 if (skb_rx_queue_recorded(skb)) { 2563 hash = skb_get_rx_queue(skb); 2564 while (unlikely(hash >= num_tx_queues)) 2565 hash -= num_tx_queues; 2566 return hash; 2567 } 2568 2569 if (dev->num_tc) { 2570 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); 2571 2572 qoffset = dev->tc_to_txq[tc].offset; 2573 qcount = dev->tc_to_txq[tc].count; 2574 } 2575 2576 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset; 2577 } 2578 EXPORT_SYMBOL(__skb_tx_hash); 2579 2580 static void skb_warn_bad_offload(const struct sk_buff *skb) 2581 { 2582 static const netdev_features_t null_features; 2583 struct net_device *dev = skb->dev; 2584 const char *name = ""; 2585 2586 if (!net_ratelimit()) 2587 return; 2588 2589 if (dev) { 2590 if (dev->dev.parent) 2591 name = dev_driver_string(dev->dev.parent); 2592 else 2593 name = netdev_name(dev); 2594 } 2595 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d " 2596 "gso_type=%d ip_summed=%d\n", 2597 name, dev ? &dev->features : &null_features, 2598 skb->sk ? &skb->sk->sk_route_caps : &null_features, 2599 skb->len, skb->data_len, skb_shinfo(skb)->gso_size, 2600 skb_shinfo(skb)->gso_type, skb->ip_summed); 2601 } 2602 2603 /* 2604 * Invalidate hardware checksum when packet is to be mangled, and 2605 * complete checksum manually on outgoing path. 2606 */ 2607 int skb_checksum_help(struct sk_buff *skb) 2608 { 2609 __wsum csum; 2610 int ret = 0, offset; 2611 2612 if (skb->ip_summed == CHECKSUM_COMPLETE) 2613 goto out_set_summed; 2614 2615 if (unlikely(skb_shinfo(skb)->gso_size)) { 2616 skb_warn_bad_offload(skb); 2617 return -EINVAL; 2618 } 2619 2620 /* Before computing a checksum, we should make sure no frag could 2621 * be modified by an external entity : checksum could be wrong. 2622 */ 2623 if (skb_has_shared_frag(skb)) { 2624 ret = __skb_linearize(skb); 2625 if (ret) 2626 goto out; 2627 } 2628 2629 offset = skb_checksum_start_offset(skb); 2630 BUG_ON(offset >= skb_headlen(skb)); 2631 csum = skb_checksum(skb, offset, skb->len - offset, 0); 2632 2633 offset += skb->csum_offset; 2634 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb)); 2635 2636 if (skb_cloned(skb) && 2637 !skb_clone_writable(skb, offset + sizeof(__sum16))) { 2638 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2639 if (ret) 2640 goto out; 2641 } 2642 2643 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0; 2644 out_set_summed: 2645 skb->ip_summed = CHECKSUM_NONE; 2646 out: 2647 return ret; 2648 } 2649 EXPORT_SYMBOL(skb_checksum_help); 2650 2651 int skb_crc32c_csum_help(struct sk_buff *skb) 2652 { 2653 __le32 crc32c_csum; 2654 int ret = 0, offset, start; 2655 2656 if (skb->ip_summed != CHECKSUM_PARTIAL) 2657 goto out; 2658 2659 if (unlikely(skb_is_gso(skb))) 2660 goto out; 2661 2662 /* Before computing a checksum, we should make sure no frag could 2663 * be modified by an external entity : checksum could be wrong. 2664 */ 2665 if (unlikely(skb_has_shared_frag(skb))) { 2666 ret = __skb_linearize(skb); 2667 if (ret) 2668 goto out; 2669 } 2670 start = skb_checksum_start_offset(skb); 2671 offset = start + offsetof(struct sctphdr, checksum); 2672 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) { 2673 ret = -EINVAL; 2674 goto out; 2675 } 2676 if (skb_cloned(skb) && 2677 !skb_clone_writable(skb, offset + sizeof(__le32))) { 2678 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2679 if (ret) 2680 goto out; 2681 } 2682 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start, 2683 skb->len - start, ~(__u32)0, 2684 crc32c_csum_stub)); 2685 *(__le32 *)(skb->data + offset) = crc32c_csum; 2686 skb->ip_summed = CHECKSUM_NONE; 2687 skb->csum_not_inet = 0; 2688 out: 2689 return ret; 2690 } 2691 2692 __be16 skb_network_protocol(struct sk_buff *skb, int *depth) 2693 { 2694 __be16 type = skb->protocol; 2695 2696 /* Tunnel gso handlers can set protocol to ethernet. */ 2697 if (type == htons(ETH_P_TEB)) { 2698 struct ethhdr *eth; 2699 2700 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr)))) 2701 return 0; 2702 2703 eth = (struct ethhdr *)skb_mac_header(skb); 2704 type = eth->h_proto; 2705 } 2706 2707 return __vlan_get_protocol(skb, type, depth); 2708 } 2709 2710 /** 2711 * skb_mac_gso_segment - mac layer segmentation handler. 2712 * @skb: buffer to segment 2713 * @features: features for the output path (see dev->features) 2714 */ 2715 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, 2716 netdev_features_t features) 2717 { 2718 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); 2719 struct packet_offload *ptype; 2720 int vlan_depth = skb->mac_len; 2721 __be16 type = skb_network_protocol(skb, &vlan_depth); 2722 2723 if (unlikely(!type)) 2724 return ERR_PTR(-EINVAL); 2725 2726 __skb_pull(skb, vlan_depth); 2727 2728 rcu_read_lock(); 2729 list_for_each_entry_rcu(ptype, &offload_base, list) { 2730 if (ptype->type == type && ptype->callbacks.gso_segment) { 2731 segs = ptype->callbacks.gso_segment(skb, features); 2732 break; 2733 } 2734 } 2735 rcu_read_unlock(); 2736 2737 __skb_push(skb, skb->data - skb_mac_header(skb)); 2738 2739 return segs; 2740 } 2741 EXPORT_SYMBOL(skb_mac_gso_segment); 2742 2743 2744 /* openvswitch calls this on rx path, so we need a different check. 2745 */ 2746 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) 2747 { 2748 if (tx_path) 2749 return skb->ip_summed != CHECKSUM_PARTIAL && 2750 skb->ip_summed != CHECKSUM_UNNECESSARY; 2751 2752 return skb->ip_summed == CHECKSUM_NONE; 2753 } 2754 2755 /** 2756 * __skb_gso_segment - Perform segmentation on skb. 2757 * @skb: buffer to segment 2758 * @features: features for the output path (see dev->features) 2759 * @tx_path: whether it is called in TX path 2760 * 2761 * This function segments the given skb and returns a list of segments. 2762 * 2763 * It may return NULL if the skb requires no segmentation. This is 2764 * only possible when GSO is used for verifying header integrity. 2765 * 2766 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb. 2767 */ 2768 struct sk_buff *__skb_gso_segment(struct sk_buff *skb, 2769 netdev_features_t features, bool tx_path) 2770 { 2771 struct sk_buff *segs; 2772 2773 if (unlikely(skb_needs_check(skb, tx_path))) { 2774 int err; 2775 2776 /* We're going to init ->check field in TCP or UDP header */ 2777 err = skb_cow_head(skb, 0); 2778 if (err < 0) 2779 return ERR_PTR(err); 2780 } 2781 2782 /* Only report GSO partial support if it will enable us to 2783 * support segmentation on this frame without needing additional 2784 * work. 2785 */ 2786 if (features & NETIF_F_GSO_PARTIAL) { 2787 netdev_features_t partial_features = NETIF_F_GSO_ROBUST; 2788 struct net_device *dev = skb->dev; 2789 2790 partial_features |= dev->features & dev->gso_partial_features; 2791 if (!skb_gso_ok(skb, features | partial_features)) 2792 features &= ~NETIF_F_GSO_PARTIAL; 2793 } 2794 2795 BUILD_BUG_ON(SKB_SGO_CB_OFFSET + 2796 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb)); 2797 2798 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb); 2799 SKB_GSO_CB(skb)->encap_level = 0; 2800 2801 skb_reset_mac_header(skb); 2802 skb_reset_mac_len(skb); 2803 2804 segs = skb_mac_gso_segment(skb, features); 2805 2806 if (unlikely(skb_needs_check(skb, tx_path))) 2807 skb_warn_bad_offload(skb); 2808 2809 return segs; 2810 } 2811 EXPORT_SYMBOL(__skb_gso_segment); 2812 2813 /* Take action when hardware reception checksum errors are detected. */ 2814 #ifdef CONFIG_BUG 2815 void netdev_rx_csum_fault(struct net_device *dev) 2816 { 2817 if (net_ratelimit()) { 2818 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>"); 2819 dump_stack(); 2820 } 2821 } 2822 EXPORT_SYMBOL(netdev_rx_csum_fault); 2823 #endif 2824 2825 /* Actually, we should eliminate this check as soon as we know, that: 2826 * 1. IOMMU is present and allows to map all the memory. 2827 * 2. No high memory really exists on this machine. 2828 */ 2829 2830 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) 2831 { 2832 #ifdef CONFIG_HIGHMEM 2833 int i; 2834 2835 if (!(dev->features & NETIF_F_HIGHDMA)) { 2836 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2837 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2838 2839 if (PageHighMem(skb_frag_page(frag))) 2840 return 1; 2841 } 2842 } 2843 2844 if (PCI_DMA_BUS_IS_PHYS) { 2845 struct device *pdev = dev->dev.parent; 2846 2847 if (!pdev) 2848 return 0; 2849 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2850 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2851 dma_addr_t addr = page_to_phys(skb_frag_page(frag)); 2852 2853 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask) 2854 return 1; 2855 } 2856 } 2857 #endif 2858 return 0; 2859 } 2860 2861 /* If MPLS offload request, verify we are testing hardware MPLS features 2862 * instead of standard features for the netdev. 2863 */ 2864 #if IS_ENABLED(CONFIG_NET_MPLS_GSO) 2865 static netdev_features_t net_mpls_features(struct sk_buff *skb, 2866 netdev_features_t features, 2867 __be16 type) 2868 { 2869 if (eth_p_mpls(type)) 2870 features &= skb->dev->mpls_features; 2871 2872 return features; 2873 } 2874 #else 2875 static netdev_features_t net_mpls_features(struct sk_buff *skb, 2876 netdev_features_t features, 2877 __be16 type) 2878 { 2879 return features; 2880 } 2881 #endif 2882 2883 static netdev_features_t harmonize_features(struct sk_buff *skb, 2884 netdev_features_t features) 2885 { 2886 int tmp; 2887 __be16 type; 2888 2889 type = skb_network_protocol(skb, &tmp); 2890 features = net_mpls_features(skb, features, type); 2891 2892 if (skb->ip_summed != CHECKSUM_NONE && 2893 !can_checksum_protocol(features, type)) { 2894 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 2895 } 2896 if (illegal_highdma(skb->dev, skb)) 2897 features &= ~NETIF_F_SG; 2898 2899 return features; 2900 } 2901 2902 netdev_features_t passthru_features_check(struct sk_buff *skb, 2903 struct net_device *dev, 2904 netdev_features_t features) 2905 { 2906 return features; 2907 } 2908 EXPORT_SYMBOL(passthru_features_check); 2909 2910 static netdev_features_t dflt_features_check(const struct sk_buff *skb, 2911 struct net_device *dev, 2912 netdev_features_t features) 2913 { 2914 return vlan_features_check(skb, features); 2915 } 2916 2917 static netdev_features_t gso_features_check(const struct sk_buff *skb, 2918 struct net_device *dev, 2919 netdev_features_t features) 2920 { 2921 u16 gso_segs = skb_shinfo(skb)->gso_segs; 2922 2923 if (gso_segs > dev->gso_max_segs) 2924 return features & ~NETIF_F_GSO_MASK; 2925 2926 /* Support for GSO partial features requires software 2927 * intervention before we can actually process the packets 2928 * so we need to strip support for any partial features now 2929 * and we can pull them back in after we have partially 2930 * segmented the frame. 2931 */ 2932 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)) 2933 features &= ~dev->gso_partial_features; 2934 2935 /* Make sure to clear the IPv4 ID mangling feature if the 2936 * IPv4 header has the potential to be fragmented. 2937 */ 2938 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { 2939 struct iphdr *iph = skb->encapsulation ? 2940 inner_ip_hdr(skb) : ip_hdr(skb); 2941 2942 if (!(iph->frag_off & htons(IP_DF))) 2943 features &= ~NETIF_F_TSO_MANGLEID; 2944 } 2945 2946 return features; 2947 } 2948 2949 netdev_features_t netif_skb_features(struct sk_buff *skb) 2950 { 2951 struct net_device *dev = skb->dev; 2952 netdev_features_t features = dev->features; 2953 2954 if (skb_is_gso(skb)) 2955 features = gso_features_check(skb, dev, features); 2956 2957 /* If encapsulation offload request, verify we are testing 2958 * hardware encapsulation features instead of standard 2959 * features for the netdev 2960 */ 2961 if (skb->encapsulation) 2962 features &= dev->hw_enc_features; 2963 2964 if (skb_vlan_tagged(skb)) 2965 features = netdev_intersect_features(features, 2966 dev->vlan_features | 2967 NETIF_F_HW_VLAN_CTAG_TX | 2968 NETIF_F_HW_VLAN_STAG_TX); 2969 2970 if (dev->netdev_ops->ndo_features_check) 2971 features &= dev->netdev_ops->ndo_features_check(skb, dev, 2972 features); 2973 else 2974 features &= dflt_features_check(skb, dev, features); 2975 2976 return harmonize_features(skb, features); 2977 } 2978 EXPORT_SYMBOL(netif_skb_features); 2979 2980 static int xmit_one(struct sk_buff *skb, struct net_device *dev, 2981 struct netdev_queue *txq, bool more) 2982 { 2983 unsigned int len; 2984 int rc; 2985 2986 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all)) 2987 dev_queue_xmit_nit(skb, dev); 2988 2989 len = skb->len; 2990 trace_net_dev_start_xmit(skb, dev); 2991 rc = netdev_start_xmit(skb, dev, txq, more); 2992 trace_net_dev_xmit(skb, rc, dev, len); 2993 2994 return rc; 2995 } 2996 2997 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev, 2998 struct netdev_queue *txq, int *ret) 2999 { 3000 struct sk_buff *skb = first; 3001 int rc = NETDEV_TX_OK; 3002 3003 while (skb) { 3004 struct sk_buff *next = skb->next; 3005 3006 skb->next = NULL; 3007 rc = xmit_one(skb, dev, txq, next != NULL); 3008 if (unlikely(!dev_xmit_complete(rc))) { 3009 skb->next = next; 3010 goto out; 3011 } 3012 3013 skb = next; 3014 if (netif_xmit_stopped(txq) && skb) { 3015 rc = NETDEV_TX_BUSY; 3016 break; 3017 } 3018 } 3019 3020 out: 3021 *ret = rc; 3022 return skb; 3023 } 3024 3025 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, 3026 netdev_features_t features) 3027 { 3028 if (skb_vlan_tag_present(skb) && 3029 !vlan_hw_offload_capable(features, skb->vlan_proto)) 3030 skb = __vlan_hwaccel_push_inside(skb); 3031 return skb; 3032 } 3033 3034 int skb_csum_hwoffload_help(struct sk_buff *skb, 3035 const netdev_features_t features) 3036 { 3037 if (unlikely(skb->csum_not_inet)) 3038 return !!(features & NETIF_F_SCTP_CRC) ? 0 : 3039 skb_crc32c_csum_help(skb); 3040 3041 return !!(features & NETIF_F_CSUM_MASK) ? 0 : skb_checksum_help(skb); 3042 } 3043 EXPORT_SYMBOL(skb_csum_hwoffload_help); 3044 3045 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev) 3046 { 3047 netdev_features_t features; 3048 3049 features = netif_skb_features(skb); 3050 skb = validate_xmit_vlan(skb, features); 3051 if (unlikely(!skb)) 3052 goto out_null; 3053 3054 if (netif_needs_gso(skb, features)) { 3055 struct sk_buff *segs; 3056 3057 segs = skb_gso_segment(skb, features); 3058 if (IS_ERR(segs)) { 3059 goto out_kfree_skb; 3060 } else if (segs) { 3061 consume_skb(skb); 3062 skb = segs; 3063 } 3064 } else { 3065 if (skb_needs_linearize(skb, features) && 3066 __skb_linearize(skb)) 3067 goto out_kfree_skb; 3068 3069 if (validate_xmit_xfrm(skb, features)) 3070 goto out_kfree_skb; 3071 3072 /* If packet is not checksummed and device does not 3073 * support checksumming for this protocol, complete 3074 * checksumming here. 3075 */ 3076 if (skb->ip_summed == CHECKSUM_PARTIAL) { 3077 if (skb->encapsulation) 3078 skb_set_inner_transport_header(skb, 3079 skb_checksum_start_offset(skb)); 3080 else 3081 skb_set_transport_header(skb, 3082 skb_checksum_start_offset(skb)); 3083 if (skb_csum_hwoffload_help(skb, features)) 3084 goto out_kfree_skb; 3085 } 3086 } 3087 3088 return skb; 3089 3090 out_kfree_skb: 3091 kfree_skb(skb); 3092 out_null: 3093 atomic_long_inc(&dev->tx_dropped); 3094 return NULL; 3095 } 3096 3097 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev) 3098 { 3099 struct sk_buff *next, *head = NULL, *tail; 3100 3101 for (; skb != NULL; skb = next) { 3102 next = skb->next; 3103 skb->next = NULL; 3104 3105 /* in case skb wont be segmented, point to itself */ 3106 skb->prev = skb; 3107 3108 skb = validate_xmit_skb(skb, dev); 3109 if (!skb) 3110 continue; 3111 3112 if (!head) 3113 head = skb; 3114 else 3115 tail->next = skb; 3116 /* If skb was segmented, skb->prev points to 3117 * the last segment. If not, it still contains skb. 3118 */ 3119 tail = skb->prev; 3120 } 3121 return head; 3122 } 3123 EXPORT_SYMBOL_GPL(validate_xmit_skb_list); 3124 3125 static void qdisc_pkt_len_init(struct sk_buff *skb) 3126 { 3127 const struct skb_shared_info *shinfo = skb_shinfo(skb); 3128 3129 qdisc_skb_cb(skb)->pkt_len = skb->len; 3130 3131 /* To get more precise estimation of bytes sent on wire, 3132 * we add to pkt_len the headers size of all segments 3133 */ 3134 if (shinfo->gso_size) { 3135 unsigned int hdr_len; 3136 u16 gso_segs = shinfo->gso_segs; 3137 3138 /* mac layer + network layer */ 3139 hdr_len = skb_transport_header(skb) - skb_mac_header(skb); 3140 3141 /* + transport layer */ 3142 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) 3143 hdr_len += tcp_hdrlen(skb); 3144 else 3145 hdr_len += sizeof(struct udphdr); 3146 3147 if (shinfo->gso_type & SKB_GSO_DODGY) 3148 gso_segs = DIV_ROUND_UP(skb->len - hdr_len, 3149 shinfo->gso_size); 3150 3151 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; 3152 } 3153 } 3154 3155 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 3156 struct net_device *dev, 3157 struct netdev_queue *txq) 3158 { 3159 spinlock_t *root_lock = qdisc_lock(q); 3160 struct sk_buff *to_free = NULL; 3161 bool contended; 3162 int rc; 3163 3164 qdisc_calculate_pkt_len(skb, q); 3165 /* 3166 * Heuristic to force contended enqueues to serialize on a 3167 * separate lock before trying to get qdisc main lock. 3168 * This permits qdisc->running owner to get the lock more 3169 * often and dequeue packets faster. 3170 */ 3171 contended = qdisc_is_running(q); 3172 if (unlikely(contended)) 3173 spin_lock(&q->busylock); 3174 3175 spin_lock(root_lock); 3176 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { 3177 __qdisc_drop(skb, &to_free); 3178 rc = NET_XMIT_DROP; 3179 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && 3180 qdisc_run_begin(q)) { 3181 /* 3182 * This is a work-conserving queue; there are no old skbs 3183 * waiting to be sent out; and the qdisc is not running - 3184 * xmit the skb directly. 3185 */ 3186 3187 qdisc_bstats_update(q, skb); 3188 3189 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) { 3190 if (unlikely(contended)) { 3191 spin_unlock(&q->busylock); 3192 contended = false; 3193 } 3194 __qdisc_run(q); 3195 } else 3196 qdisc_run_end(q); 3197 3198 rc = NET_XMIT_SUCCESS; 3199 } else { 3200 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK; 3201 if (qdisc_run_begin(q)) { 3202 if (unlikely(contended)) { 3203 spin_unlock(&q->busylock); 3204 contended = false; 3205 } 3206 __qdisc_run(q); 3207 } 3208 } 3209 spin_unlock(root_lock); 3210 if (unlikely(to_free)) 3211 kfree_skb_list(to_free); 3212 if (unlikely(contended)) 3213 spin_unlock(&q->busylock); 3214 return rc; 3215 } 3216 3217 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO) 3218 static void skb_update_prio(struct sk_buff *skb) 3219 { 3220 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap); 3221 3222 if (!skb->priority && skb->sk && map) { 3223 unsigned int prioidx = 3224 sock_cgroup_prioidx(&skb->sk->sk_cgrp_data); 3225 3226 if (prioidx < map->priomap_len) 3227 skb->priority = map->priomap[prioidx]; 3228 } 3229 } 3230 #else 3231 #define skb_update_prio(skb) 3232 #endif 3233 3234 DEFINE_PER_CPU(int, xmit_recursion); 3235 EXPORT_SYMBOL(xmit_recursion); 3236 3237 /** 3238 * dev_loopback_xmit - loop back @skb 3239 * @net: network namespace this loopback is happening in 3240 * @sk: sk needed to be a netfilter okfn 3241 * @skb: buffer to transmit 3242 */ 3243 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb) 3244 { 3245 skb_reset_mac_header(skb); 3246 __skb_pull(skb, skb_network_offset(skb)); 3247 skb->pkt_type = PACKET_LOOPBACK; 3248 skb->ip_summed = CHECKSUM_UNNECESSARY; 3249 WARN_ON(!skb_dst(skb)); 3250 skb_dst_force(skb); 3251 netif_rx_ni(skb); 3252 return 0; 3253 } 3254 EXPORT_SYMBOL(dev_loopback_xmit); 3255 3256 #ifdef CONFIG_NET_EGRESS 3257 static struct sk_buff * 3258 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) 3259 { 3260 struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress); 3261 struct tcf_result cl_res; 3262 3263 if (!miniq) 3264 return skb; 3265 3266 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */ 3267 mini_qdisc_bstats_cpu_update(miniq, skb); 3268 3269 switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) { 3270 case TC_ACT_OK: 3271 case TC_ACT_RECLASSIFY: 3272 skb->tc_index = TC_H_MIN(cl_res.classid); 3273 break; 3274 case TC_ACT_SHOT: 3275 mini_qdisc_qstats_cpu_drop(miniq); 3276 *ret = NET_XMIT_DROP; 3277 kfree_skb(skb); 3278 return NULL; 3279 case TC_ACT_STOLEN: 3280 case TC_ACT_QUEUED: 3281 case TC_ACT_TRAP: 3282 *ret = NET_XMIT_SUCCESS; 3283 consume_skb(skb); 3284 return NULL; 3285 case TC_ACT_REDIRECT: 3286 /* No need to push/pop skb's mac_header here on egress! */ 3287 skb_do_redirect(skb); 3288 *ret = NET_XMIT_SUCCESS; 3289 return NULL; 3290 default: 3291 break; 3292 } 3293 3294 return skb; 3295 } 3296 #endif /* CONFIG_NET_EGRESS */ 3297 3298 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb) 3299 { 3300 #ifdef CONFIG_XPS 3301 struct xps_dev_maps *dev_maps; 3302 struct xps_map *map; 3303 int queue_index = -1; 3304 3305 rcu_read_lock(); 3306 dev_maps = rcu_dereference(dev->xps_maps); 3307 if (dev_maps) { 3308 unsigned int tci = skb->sender_cpu - 1; 3309 3310 if (dev->num_tc) { 3311 tci *= dev->num_tc; 3312 tci += netdev_get_prio_tc_map(dev, skb->priority); 3313 } 3314 3315 map = rcu_dereference(dev_maps->cpu_map[tci]); 3316 if (map) { 3317 if (map->len == 1) 3318 queue_index = map->queues[0]; 3319 else 3320 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb), 3321 map->len)]; 3322 if (unlikely(queue_index >= dev->real_num_tx_queues)) 3323 queue_index = -1; 3324 } 3325 } 3326 rcu_read_unlock(); 3327 3328 return queue_index; 3329 #else 3330 return -1; 3331 #endif 3332 } 3333 3334 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb) 3335 { 3336 struct sock *sk = skb->sk; 3337 int queue_index = sk_tx_queue_get(sk); 3338 3339 if (queue_index < 0 || skb->ooo_okay || 3340 queue_index >= dev->real_num_tx_queues) { 3341 int new_index = get_xps_queue(dev, skb); 3342 3343 if (new_index < 0) 3344 new_index = skb_tx_hash(dev, skb); 3345 3346 if (queue_index != new_index && sk && 3347 sk_fullsock(sk) && 3348 rcu_access_pointer(sk->sk_dst_cache)) 3349 sk_tx_queue_set(sk, new_index); 3350 3351 queue_index = new_index; 3352 } 3353 3354 return queue_index; 3355 } 3356 3357 struct netdev_queue *netdev_pick_tx(struct net_device *dev, 3358 struct sk_buff *skb, 3359 void *accel_priv) 3360 { 3361 int queue_index = 0; 3362 3363 #ifdef CONFIG_XPS 3364 u32 sender_cpu = skb->sender_cpu - 1; 3365 3366 if (sender_cpu >= (u32)NR_CPUS) 3367 skb->sender_cpu = raw_smp_processor_id() + 1; 3368 #endif 3369 3370 if (dev->real_num_tx_queues != 1) { 3371 const struct net_device_ops *ops = dev->netdev_ops; 3372 3373 if (ops->ndo_select_queue) 3374 queue_index = ops->ndo_select_queue(dev, skb, accel_priv, 3375 __netdev_pick_tx); 3376 else 3377 queue_index = __netdev_pick_tx(dev, skb); 3378 3379 if (!accel_priv) 3380 queue_index = netdev_cap_txqueue(dev, queue_index); 3381 } 3382 3383 skb_set_queue_mapping(skb, queue_index); 3384 return netdev_get_tx_queue(dev, queue_index); 3385 } 3386 3387 /** 3388 * __dev_queue_xmit - transmit a buffer 3389 * @skb: buffer to transmit 3390 * @accel_priv: private data used for L2 forwarding offload 3391 * 3392 * Queue a buffer for transmission to a network device. The caller must 3393 * have set the device and priority and built the buffer before calling 3394 * this function. The function can be called from an interrupt. 3395 * 3396 * A negative errno code is returned on a failure. A success does not 3397 * guarantee the frame will be transmitted as it may be dropped due 3398 * to congestion or traffic shaping. 3399 * 3400 * ----------------------------------------------------------------------------------- 3401 * I notice this method can also return errors from the queue disciplines, 3402 * including NET_XMIT_DROP, which is a positive value. So, errors can also 3403 * be positive. 3404 * 3405 * Regardless of the return value, the skb is consumed, so it is currently 3406 * difficult to retry a send to this method. (You can bump the ref count 3407 * before sending to hold a reference for retry if you are careful.) 3408 * 3409 * When calling this method, interrupts MUST be enabled. This is because 3410 * the BH enable code must have IRQs enabled so that it will not deadlock. 3411 * --BLG 3412 */ 3413 static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv) 3414 { 3415 struct net_device *dev = skb->dev; 3416 struct netdev_queue *txq; 3417 struct Qdisc *q; 3418 int rc = -ENOMEM; 3419 3420 skb_reset_mac_header(skb); 3421 3422 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP)) 3423 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED); 3424 3425 /* Disable soft irqs for various locks below. Also 3426 * stops preemption for RCU. 3427 */ 3428 rcu_read_lock_bh(); 3429 3430 skb_update_prio(skb); 3431 3432 qdisc_pkt_len_init(skb); 3433 #ifdef CONFIG_NET_CLS_ACT 3434 skb->tc_at_ingress = 0; 3435 # ifdef CONFIG_NET_EGRESS 3436 if (static_key_false(&egress_needed)) { 3437 skb = sch_handle_egress(skb, &rc, dev); 3438 if (!skb) 3439 goto out; 3440 } 3441 # endif 3442 #endif 3443 /* If device/qdisc don't need skb->dst, release it right now while 3444 * its hot in this cpu cache. 3445 */ 3446 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) 3447 skb_dst_drop(skb); 3448 else 3449 skb_dst_force(skb); 3450 3451 txq = netdev_pick_tx(dev, skb, accel_priv); 3452 q = rcu_dereference_bh(txq->qdisc); 3453 3454 trace_net_dev_queue(skb); 3455 if (q->enqueue) { 3456 rc = __dev_xmit_skb(skb, q, dev, txq); 3457 goto out; 3458 } 3459 3460 /* The device has no queue. Common case for software devices: 3461 * loopback, all the sorts of tunnels... 3462 3463 * Really, it is unlikely that netif_tx_lock protection is necessary 3464 * here. (f.e. loopback and IP tunnels are clean ignoring statistics 3465 * counters.) 3466 * However, it is possible, that they rely on protection 3467 * made by us here. 3468 3469 * Check this and shot the lock. It is not prone from deadlocks. 3470 *Either shot noqueue qdisc, it is even simpler 8) 3471 */ 3472 if (dev->flags & IFF_UP) { 3473 int cpu = smp_processor_id(); /* ok because BHs are off */ 3474 3475 if (txq->xmit_lock_owner != cpu) { 3476 if (unlikely(__this_cpu_read(xmit_recursion) > 3477 XMIT_RECURSION_LIMIT)) 3478 goto recursion_alert; 3479 3480 skb = validate_xmit_skb(skb, dev); 3481 if (!skb) 3482 goto out; 3483 3484 HARD_TX_LOCK(dev, txq, cpu); 3485 3486 if (!netif_xmit_stopped(txq)) { 3487 __this_cpu_inc(xmit_recursion); 3488 skb = dev_hard_start_xmit(skb, dev, txq, &rc); 3489 __this_cpu_dec(xmit_recursion); 3490 if (dev_xmit_complete(rc)) { 3491 HARD_TX_UNLOCK(dev, txq); 3492 goto out; 3493 } 3494 } 3495 HARD_TX_UNLOCK(dev, txq); 3496 net_crit_ratelimited("Virtual device %s asks to queue packet!\n", 3497 dev->name); 3498 } else { 3499 /* Recursion is detected! It is possible, 3500 * unfortunately 3501 */ 3502 recursion_alert: 3503 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", 3504 dev->name); 3505 } 3506 } 3507 3508 rc = -ENETDOWN; 3509 rcu_read_unlock_bh(); 3510 3511 atomic_long_inc(&dev->tx_dropped); 3512 kfree_skb_list(skb); 3513 return rc; 3514 out: 3515 rcu_read_unlock_bh(); 3516 return rc; 3517 } 3518 3519 int dev_queue_xmit(struct sk_buff *skb) 3520 { 3521 return __dev_queue_xmit(skb, NULL); 3522 } 3523 EXPORT_SYMBOL(dev_queue_xmit); 3524 3525 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv) 3526 { 3527 return __dev_queue_xmit(skb, accel_priv); 3528 } 3529 EXPORT_SYMBOL(dev_queue_xmit_accel); 3530 3531 3532 /************************************************************************* 3533 * Receiver routines 3534 *************************************************************************/ 3535 3536 int netdev_max_backlog __read_mostly = 1000; 3537 EXPORT_SYMBOL(netdev_max_backlog); 3538 3539 int netdev_tstamp_prequeue __read_mostly = 1; 3540 int netdev_budget __read_mostly = 300; 3541 unsigned int __read_mostly netdev_budget_usecs = 2000; 3542 int weight_p __read_mostly = 64; /* old backlog weight */ 3543 int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */ 3544 int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */ 3545 int dev_rx_weight __read_mostly = 64; 3546 int dev_tx_weight __read_mostly = 64; 3547 3548 /* Called with irq disabled */ 3549 static inline void ____napi_schedule(struct softnet_data *sd, 3550 struct napi_struct *napi) 3551 { 3552 list_add_tail(&napi->poll_list, &sd->poll_list); 3553 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 3554 } 3555 3556 #ifdef CONFIG_RPS 3557 3558 /* One global table that all flow-based protocols share. */ 3559 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; 3560 EXPORT_SYMBOL(rps_sock_flow_table); 3561 u32 rps_cpu_mask __read_mostly; 3562 EXPORT_SYMBOL(rps_cpu_mask); 3563 3564 struct static_key rps_needed __read_mostly; 3565 EXPORT_SYMBOL(rps_needed); 3566 struct static_key rfs_needed __read_mostly; 3567 EXPORT_SYMBOL(rfs_needed); 3568 3569 static struct rps_dev_flow * 3570 set_rps_cpu(struct net_device *dev, struct sk_buff *skb, 3571 struct rps_dev_flow *rflow, u16 next_cpu) 3572 { 3573 if (next_cpu < nr_cpu_ids) { 3574 #ifdef CONFIG_RFS_ACCEL 3575 struct netdev_rx_queue *rxqueue; 3576 struct rps_dev_flow_table *flow_table; 3577 struct rps_dev_flow *old_rflow; 3578 u32 flow_id; 3579 u16 rxq_index; 3580 int rc; 3581 3582 /* Should we steer this flow to a different hardware queue? */ 3583 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || 3584 !(dev->features & NETIF_F_NTUPLE)) 3585 goto out; 3586 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); 3587 if (rxq_index == skb_get_rx_queue(skb)) 3588 goto out; 3589 3590 rxqueue = dev->_rx + rxq_index; 3591 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3592 if (!flow_table) 3593 goto out; 3594 flow_id = skb_get_hash(skb) & flow_table->mask; 3595 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, 3596 rxq_index, flow_id); 3597 if (rc < 0) 3598 goto out; 3599 old_rflow = rflow; 3600 rflow = &flow_table->flows[flow_id]; 3601 rflow->filter = rc; 3602 if (old_rflow->filter == rflow->filter) 3603 old_rflow->filter = RPS_NO_FILTER; 3604 out: 3605 #endif 3606 rflow->last_qtail = 3607 per_cpu(softnet_data, next_cpu).input_queue_head; 3608 } 3609 3610 rflow->cpu = next_cpu; 3611 return rflow; 3612 } 3613 3614 /* 3615 * get_rps_cpu is called from netif_receive_skb and returns the target 3616 * CPU from the RPS map of the receiving queue for a given skb. 3617 * rcu_read_lock must be held on entry. 3618 */ 3619 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, 3620 struct rps_dev_flow **rflowp) 3621 { 3622 const struct rps_sock_flow_table *sock_flow_table; 3623 struct netdev_rx_queue *rxqueue = dev->_rx; 3624 struct rps_dev_flow_table *flow_table; 3625 struct rps_map *map; 3626 int cpu = -1; 3627 u32 tcpu; 3628 u32 hash; 3629 3630 if (skb_rx_queue_recorded(skb)) { 3631 u16 index = skb_get_rx_queue(skb); 3632 3633 if (unlikely(index >= dev->real_num_rx_queues)) { 3634 WARN_ONCE(dev->real_num_rx_queues > 1, 3635 "%s received packet on queue %u, but number " 3636 "of RX queues is %u\n", 3637 dev->name, index, dev->real_num_rx_queues); 3638 goto done; 3639 } 3640 rxqueue += index; 3641 } 3642 3643 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */ 3644 3645 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3646 map = rcu_dereference(rxqueue->rps_map); 3647 if (!flow_table && !map) 3648 goto done; 3649 3650 skb_reset_network_header(skb); 3651 hash = skb_get_hash(skb); 3652 if (!hash) 3653 goto done; 3654 3655 sock_flow_table = rcu_dereference(rps_sock_flow_table); 3656 if (flow_table && sock_flow_table) { 3657 struct rps_dev_flow *rflow; 3658 u32 next_cpu; 3659 u32 ident; 3660 3661 /* First check into global flow table if there is a match */ 3662 ident = sock_flow_table->ents[hash & sock_flow_table->mask]; 3663 if ((ident ^ hash) & ~rps_cpu_mask) 3664 goto try_rps; 3665 3666 next_cpu = ident & rps_cpu_mask; 3667 3668 /* OK, now we know there is a match, 3669 * we can look at the local (per receive queue) flow table 3670 */ 3671 rflow = &flow_table->flows[hash & flow_table->mask]; 3672 tcpu = rflow->cpu; 3673 3674 /* 3675 * If the desired CPU (where last recvmsg was done) is 3676 * different from current CPU (one in the rx-queue flow 3677 * table entry), switch if one of the following holds: 3678 * - Current CPU is unset (>= nr_cpu_ids). 3679 * - Current CPU is offline. 3680 * - The current CPU's queue tail has advanced beyond the 3681 * last packet that was enqueued using this table entry. 3682 * This guarantees that all previous packets for the flow 3683 * have been dequeued, thus preserving in order delivery. 3684 */ 3685 if (unlikely(tcpu != next_cpu) && 3686 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) || 3687 ((int)(per_cpu(softnet_data, tcpu).input_queue_head - 3688 rflow->last_qtail)) >= 0)) { 3689 tcpu = next_cpu; 3690 rflow = set_rps_cpu(dev, skb, rflow, next_cpu); 3691 } 3692 3693 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { 3694 *rflowp = rflow; 3695 cpu = tcpu; 3696 goto done; 3697 } 3698 } 3699 3700 try_rps: 3701 3702 if (map) { 3703 tcpu = map->cpus[reciprocal_scale(hash, map->len)]; 3704 if (cpu_online(tcpu)) { 3705 cpu = tcpu; 3706 goto done; 3707 } 3708 } 3709 3710 done: 3711 return cpu; 3712 } 3713 3714 #ifdef CONFIG_RFS_ACCEL 3715 3716 /** 3717 * rps_may_expire_flow - check whether an RFS hardware filter may be removed 3718 * @dev: Device on which the filter was set 3719 * @rxq_index: RX queue index 3720 * @flow_id: Flow ID passed to ndo_rx_flow_steer() 3721 * @filter_id: Filter ID returned by ndo_rx_flow_steer() 3722 * 3723 * Drivers that implement ndo_rx_flow_steer() should periodically call 3724 * this function for each installed filter and remove the filters for 3725 * which it returns %true. 3726 */ 3727 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, 3728 u32 flow_id, u16 filter_id) 3729 { 3730 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; 3731 struct rps_dev_flow_table *flow_table; 3732 struct rps_dev_flow *rflow; 3733 bool expire = true; 3734 unsigned int cpu; 3735 3736 rcu_read_lock(); 3737 flow_table = rcu_dereference(rxqueue->rps_flow_table); 3738 if (flow_table && flow_id <= flow_table->mask) { 3739 rflow = &flow_table->flows[flow_id]; 3740 cpu = READ_ONCE(rflow->cpu); 3741 if (rflow->filter == filter_id && cpu < nr_cpu_ids && 3742 ((int)(per_cpu(softnet_data, cpu).input_queue_head - 3743 rflow->last_qtail) < 3744 (int)(10 * flow_table->mask))) 3745 expire = false; 3746 } 3747 rcu_read_unlock(); 3748 return expire; 3749 } 3750 EXPORT_SYMBOL(rps_may_expire_flow); 3751 3752 #endif /* CONFIG_RFS_ACCEL */ 3753 3754 /* Called from hardirq (IPI) context */ 3755 static void rps_trigger_softirq(void *data) 3756 { 3757 struct softnet_data *sd = data; 3758 3759 ____napi_schedule(sd, &sd->backlog); 3760 sd->received_rps++; 3761 } 3762 3763 #endif /* CONFIG_RPS */ 3764 3765 /* 3766 * Check if this softnet_data structure is another cpu one 3767 * If yes, queue it to our IPI list and return 1 3768 * If no, return 0 3769 */ 3770 static int rps_ipi_queued(struct softnet_data *sd) 3771 { 3772 #ifdef CONFIG_RPS 3773 struct softnet_data *mysd = this_cpu_ptr(&softnet_data); 3774 3775 if (sd != mysd) { 3776 sd->rps_ipi_next = mysd->rps_ipi_list; 3777 mysd->rps_ipi_list = sd; 3778 3779 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 3780 return 1; 3781 } 3782 #endif /* CONFIG_RPS */ 3783 return 0; 3784 } 3785 3786 #ifdef CONFIG_NET_FLOW_LIMIT 3787 int netdev_flow_limit_table_len __read_mostly = (1 << 12); 3788 #endif 3789 3790 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) 3791 { 3792 #ifdef CONFIG_NET_FLOW_LIMIT 3793 struct sd_flow_limit *fl; 3794 struct softnet_data *sd; 3795 unsigned int old_flow, new_flow; 3796 3797 if (qlen < (netdev_max_backlog >> 1)) 3798 return false; 3799 3800 sd = this_cpu_ptr(&softnet_data); 3801 3802 rcu_read_lock(); 3803 fl = rcu_dereference(sd->flow_limit); 3804 if (fl) { 3805 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1); 3806 old_flow = fl->history[fl->history_head]; 3807 fl->history[fl->history_head] = new_flow; 3808 3809 fl->history_head++; 3810 fl->history_head &= FLOW_LIMIT_HISTORY - 1; 3811 3812 if (likely(fl->buckets[old_flow])) 3813 fl->buckets[old_flow]--; 3814 3815 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { 3816 fl->count++; 3817 rcu_read_unlock(); 3818 return true; 3819 } 3820 } 3821 rcu_read_unlock(); 3822 #endif 3823 return false; 3824 } 3825 3826 /* 3827 * enqueue_to_backlog is called to queue an skb to a per CPU backlog 3828 * queue (may be a remote CPU queue). 3829 */ 3830 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, 3831 unsigned int *qtail) 3832 { 3833 struct softnet_data *sd; 3834 unsigned long flags; 3835 unsigned int qlen; 3836 3837 sd = &per_cpu(softnet_data, cpu); 3838 3839 local_irq_save(flags); 3840 3841 rps_lock(sd); 3842 if (!netif_running(skb->dev)) 3843 goto drop; 3844 qlen = skb_queue_len(&sd->input_pkt_queue); 3845 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) { 3846 if (qlen) { 3847 enqueue: 3848 __skb_queue_tail(&sd->input_pkt_queue, skb); 3849 input_queue_tail_incr_save(sd, qtail); 3850 rps_unlock(sd); 3851 local_irq_restore(flags); 3852 return NET_RX_SUCCESS; 3853 } 3854 3855 /* Schedule NAPI for backlog device 3856 * We can use non atomic operation since we own the queue lock 3857 */ 3858 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) { 3859 if (!rps_ipi_queued(sd)) 3860 ____napi_schedule(sd, &sd->backlog); 3861 } 3862 goto enqueue; 3863 } 3864 3865 drop: 3866 sd->dropped++; 3867 rps_unlock(sd); 3868 3869 local_irq_restore(flags); 3870 3871 atomic_long_inc(&skb->dev->rx_dropped); 3872 kfree_skb(skb); 3873 return NET_RX_DROP; 3874 } 3875 3876 static u32 netif_receive_generic_xdp(struct sk_buff *skb, 3877 struct bpf_prog *xdp_prog) 3878 { 3879 u32 metalen, act = XDP_DROP; 3880 struct xdp_buff xdp; 3881 void *orig_data; 3882 int hlen, off; 3883 u32 mac_len; 3884 3885 /* Reinjected packets coming from act_mirred or similar should 3886 * not get XDP generic processing. 3887 */ 3888 if (skb_cloned(skb)) 3889 return XDP_PASS; 3890 3891 /* XDP packets must be linear and must have sufficient headroom 3892 * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also 3893 * native XDP provides, thus we need to do it here as well. 3894 */ 3895 if (skb_is_nonlinear(skb) || 3896 skb_headroom(skb) < XDP_PACKET_HEADROOM) { 3897 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb); 3898 int troom = skb->tail + skb->data_len - skb->end; 3899 3900 /* In case we have to go down the path and also linearize, 3901 * then lets do the pskb_expand_head() work just once here. 3902 */ 3903 if (pskb_expand_head(skb, 3904 hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0, 3905 troom > 0 ? troom + 128 : 0, GFP_ATOMIC)) 3906 goto do_drop; 3907 if (troom > 0 && __skb_linearize(skb)) 3908 goto do_drop; 3909 } 3910 3911 /* The XDP program wants to see the packet starting at the MAC 3912 * header. 3913 */ 3914 mac_len = skb->data - skb_mac_header(skb); 3915 hlen = skb_headlen(skb) + mac_len; 3916 xdp.data = skb->data - mac_len; 3917 xdp.data_meta = xdp.data; 3918 xdp.data_end = xdp.data + hlen; 3919 xdp.data_hard_start = skb->data - skb_headroom(skb); 3920 orig_data = xdp.data; 3921 3922 act = bpf_prog_run_xdp(xdp_prog, &xdp); 3923 3924 off = xdp.data - orig_data; 3925 if (off > 0) 3926 __skb_pull(skb, off); 3927 else if (off < 0) 3928 __skb_push(skb, -off); 3929 skb->mac_header += off; 3930 3931 switch (act) { 3932 case XDP_REDIRECT: 3933 case XDP_TX: 3934 __skb_push(skb, mac_len); 3935 break; 3936 case XDP_PASS: 3937 metalen = xdp.data - xdp.data_meta; 3938 if (metalen) 3939 skb_metadata_set(skb, metalen); 3940 break; 3941 default: 3942 bpf_warn_invalid_xdp_action(act); 3943 /* fall through */ 3944 case XDP_ABORTED: 3945 trace_xdp_exception(skb->dev, xdp_prog, act); 3946 /* fall through */ 3947 case XDP_DROP: 3948 do_drop: 3949 kfree_skb(skb); 3950 break; 3951 } 3952 3953 return act; 3954 } 3955 3956 /* When doing generic XDP we have to bypass the qdisc layer and the 3957 * network taps in order to match in-driver-XDP behavior. 3958 */ 3959 void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog) 3960 { 3961 struct net_device *dev = skb->dev; 3962 struct netdev_queue *txq; 3963 bool free_skb = true; 3964 int cpu, rc; 3965 3966 txq = netdev_pick_tx(dev, skb, NULL); 3967 cpu = smp_processor_id(); 3968 HARD_TX_LOCK(dev, txq, cpu); 3969 if (!netif_xmit_stopped(txq)) { 3970 rc = netdev_start_xmit(skb, dev, txq, 0); 3971 if (dev_xmit_complete(rc)) 3972 free_skb = false; 3973 } 3974 HARD_TX_UNLOCK(dev, txq); 3975 if (free_skb) { 3976 trace_xdp_exception(dev, xdp_prog, XDP_TX); 3977 kfree_skb(skb); 3978 } 3979 } 3980 EXPORT_SYMBOL_GPL(generic_xdp_tx); 3981 3982 static struct static_key generic_xdp_needed __read_mostly; 3983 3984 int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb) 3985 { 3986 if (xdp_prog) { 3987 u32 act = netif_receive_generic_xdp(skb, xdp_prog); 3988 int err; 3989 3990 if (act != XDP_PASS) { 3991 switch (act) { 3992 case XDP_REDIRECT: 3993 err = xdp_do_generic_redirect(skb->dev, skb, 3994 xdp_prog); 3995 if (err) 3996 goto out_redir; 3997 /* fallthru to submit skb */ 3998 case XDP_TX: 3999 generic_xdp_tx(skb, xdp_prog); 4000 break; 4001 } 4002 return XDP_DROP; 4003 } 4004 } 4005 return XDP_PASS; 4006 out_redir: 4007 kfree_skb(skb); 4008 return XDP_DROP; 4009 } 4010 EXPORT_SYMBOL_GPL(do_xdp_generic); 4011 4012 static int netif_rx_internal(struct sk_buff *skb) 4013 { 4014 int ret; 4015 4016 net_timestamp_check(netdev_tstamp_prequeue, skb); 4017 4018 trace_netif_rx(skb); 4019 4020 if (static_key_false(&generic_xdp_needed)) { 4021 int ret; 4022 4023 preempt_disable(); 4024 rcu_read_lock(); 4025 ret = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb); 4026 rcu_read_unlock(); 4027 preempt_enable(); 4028 4029 /* Consider XDP consuming the packet a success from 4030 * the netdev point of view we do not want to count 4031 * this as an error. 4032 */ 4033 if (ret != XDP_PASS) 4034 return NET_RX_SUCCESS; 4035 } 4036 4037 #ifdef CONFIG_RPS 4038 if (static_key_false(&rps_needed)) { 4039 struct rps_dev_flow voidflow, *rflow = &voidflow; 4040 int cpu; 4041 4042 preempt_disable(); 4043 rcu_read_lock(); 4044 4045 cpu = get_rps_cpu(skb->dev, skb, &rflow); 4046 if (cpu < 0) 4047 cpu = smp_processor_id(); 4048 4049 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 4050 4051 rcu_read_unlock(); 4052 preempt_enable(); 4053 } else 4054 #endif 4055 { 4056 unsigned int qtail; 4057 4058 ret = enqueue_to_backlog(skb, get_cpu(), &qtail); 4059 put_cpu(); 4060 } 4061 return ret; 4062 } 4063 4064 /** 4065 * netif_rx - post buffer to the network code 4066 * @skb: buffer to post 4067 * 4068 * This function receives a packet from a device driver and queues it for 4069 * the upper (protocol) levels to process. It always succeeds. The buffer 4070 * may be dropped during processing for congestion control or by the 4071 * protocol layers. 4072 * 4073 * return values: 4074 * NET_RX_SUCCESS (no congestion) 4075 * NET_RX_DROP (packet was dropped) 4076 * 4077 */ 4078 4079 int netif_rx(struct sk_buff *skb) 4080 { 4081 trace_netif_rx_entry(skb); 4082 4083 return netif_rx_internal(skb); 4084 } 4085 EXPORT_SYMBOL(netif_rx); 4086 4087 int netif_rx_ni(struct sk_buff *skb) 4088 { 4089 int err; 4090 4091 trace_netif_rx_ni_entry(skb); 4092 4093 preempt_disable(); 4094 err = netif_rx_internal(skb); 4095 if (local_softirq_pending()) 4096 do_softirq(); 4097 preempt_enable(); 4098 4099 return err; 4100 } 4101 EXPORT_SYMBOL(netif_rx_ni); 4102 4103 static __latent_entropy void net_tx_action(struct softirq_action *h) 4104 { 4105 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 4106 4107 if (sd->completion_queue) { 4108 struct sk_buff *clist; 4109 4110 local_irq_disable(); 4111 clist = sd->completion_queue; 4112 sd->completion_queue = NULL; 4113 local_irq_enable(); 4114 4115 while (clist) { 4116 struct sk_buff *skb = clist; 4117 4118 clist = clist->next; 4119 4120 WARN_ON(refcount_read(&skb->users)); 4121 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED)) 4122 trace_consume_skb(skb); 4123 else 4124 trace_kfree_skb(skb, net_tx_action); 4125 4126 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) 4127 __kfree_skb(skb); 4128 else 4129 __kfree_skb_defer(skb); 4130 } 4131 4132 __kfree_skb_flush(); 4133 } 4134 4135 if (sd->output_queue) { 4136 struct Qdisc *head; 4137 4138 local_irq_disable(); 4139 head = sd->output_queue; 4140 sd->output_queue = NULL; 4141 sd->output_queue_tailp = &sd->output_queue; 4142 local_irq_enable(); 4143 4144 while (head) { 4145 struct Qdisc *q = head; 4146 spinlock_t *root_lock; 4147 4148 head = head->next_sched; 4149 4150 root_lock = qdisc_lock(q); 4151 spin_lock(root_lock); 4152 /* We need to make sure head->next_sched is read 4153 * before clearing __QDISC_STATE_SCHED 4154 */ 4155 smp_mb__before_atomic(); 4156 clear_bit(__QDISC_STATE_SCHED, &q->state); 4157 qdisc_run(q); 4158 spin_unlock(root_lock); 4159 } 4160 } 4161 } 4162 4163 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE) 4164 /* This hook is defined here for ATM LANE */ 4165 int (*br_fdb_test_addr_hook)(struct net_device *dev, 4166 unsigned char *addr) __read_mostly; 4167 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); 4168 #endif 4169 4170 static inline struct sk_buff * 4171 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, 4172 struct net_device *orig_dev) 4173 { 4174 #ifdef CONFIG_NET_CLS_ACT 4175 struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress); 4176 struct tcf_result cl_res; 4177 4178 /* If there's at least one ingress present somewhere (so 4179 * we get here via enabled static key), remaining devices 4180 * that are not configured with an ingress qdisc will bail 4181 * out here. 4182 */ 4183 if (!miniq) 4184 return skb; 4185 4186 if (*pt_prev) { 4187 *ret = deliver_skb(skb, *pt_prev, orig_dev); 4188 *pt_prev = NULL; 4189 } 4190 4191 qdisc_skb_cb(skb)->pkt_len = skb->len; 4192 skb->tc_at_ingress = 1; 4193 mini_qdisc_bstats_cpu_update(miniq, skb); 4194 4195 switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) { 4196 case TC_ACT_OK: 4197 case TC_ACT_RECLASSIFY: 4198 skb->tc_index = TC_H_MIN(cl_res.classid); 4199 break; 4200 case TC_ACT_SHOT: 4201 mini_qdisc_qstats_cpu_drop(miniq); 4202 kfree_skb(skb); 4203 return NULL; 4204 case TC_ACT_STOLEN: 4205 case TC_ACT_QUEUED: 4206 case TC_ACT_TRAP: 4207 consume_skb(skb); 4208 return NULL; 4209 case TC_ACT_REDIRECT: 4210 /* skb_mac_header check was done by cls/act_bpf, so 4211 * we can safely push the L2 header back before 4212 * redirecting to another netdev 4213 */ 4214 __skb_push(skb, skb->mac_len); 4215 skb_do_redirect(skb); 4216 return NULL; 4217 default: 4218 break; 4219 } 4220 #endif /* CONFIG_NET_CLS_ACT */ 4221 return skb; 4222 } 4223 4224 /** 4225 * netdev_is_rx_handler_busy - check if receive handler is registered 4226 * @dev: device to check 4227 * 4228 * Check if a receive handler is already registered for a given device. 4229 * Return true if there one. 4230 * 4231 * The caller must hold the rtnl_mutex. 4232 */ 4233 bool netdev_is_rx_handler_busy(struct net_device *dev) 4234 { 4235 ASSERT_RTNL(); 4236 return dev && rtnl_dereference(dev->rx_handler); 4237 } 4238 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy); 4239 4240 /** 4241 * netdev_rx_handler_register - register receive handler 4242 * @dev: device to register a handler for 4243 * @rx_handler: receive handler to register 4244 * @rx_handler_data: data pointer that is used by rx handler 4245 * 4246 * Register a receive handler for a device. This handler will then be 4247 * called from __netif_receive_skb. A negative errno code is returned 4248 * on a failure. 4249 * 4250 * The caller must hold the rtnl_mutex. 4251 * 4252 * For a general description of rx_handler, see enum rx_handler_result. 4253 */ 4254 int netdev_rx_handler_register(struct net_device *dev, 4255 rx_handler_func_t *rx_handler, 4256 void *rx_handler_data) 4257 { 4258 if (netdev_is_rx_handler_busy(dev)) 4259 return -EBUSY; 4260 4261 /* Note: rx_handler_data must be set before rx_handler */ 4262 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); 4263 rcu_assign_pointer(dev->rx_handler, rx_handler); 4264 4265 return 0; 4266 } 4267 EXPORT_SYMBOL_GPL(netdev_rx_handler_register); 4268 4269 /** 4270 * netdev_rx_handler_unregister - unregister receive handler 4271 * @dev: device to unregister a handler from 4272 * 4273 * Unregister a receive handler from a device. 4274 * 4275 * The caller must hold the rtnl_mutex. 4276 */ 4277 void netdev_rx_handler_unregister(struct net_device *dev) 4278 { 4279 4280 ASSERT_RTNL(); 4281 RCU_INIT_POINTER(dev->rx_handler, NULL); 4282 /* a reader seeing a non NULL rx_handler in a rcu_read_lock() 4283 * section has a guarantee to see a non NULL rx_handler_data 4284 * as well. 4285 */ 4286 synchronize_net(); 4287 RCU_INIT_POINTER(dev->rx_handler_data, NULL); 4288 } 4289 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); 4290 4291 /* 4292 * Limit the use of PFMEMALLOC reserves to those protocols that implement 4293 * the special handling of PFMEMALLOC skbs. 4294 */ 4295 static bool skb_pfmemalloc_protocol(struct sk_buff *skb) 4296 { 4297 switch (skb->protocol) { 4298 case htons(ETH_P_ARP): 4299 case htons(ETH_P_IP): 4300 case htons(ETH_P_IPV6): 4301 case htons(ETH_P_8021Q): 4302 case htons(ETH_P_8021AD): 4303 return true; 4304 default: 4305 return false; 4306 } 4307 } 4308 4309 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev, 4310 int *ret, struct net_device *orig_dev) 4311 { 4312 #ifdef CONFIG_NETFILTER_INGRESS 4313 if (nf_hook_ingress_active(skb)) { 4314 int ingress_retval; 4315 4316 if (*pt_prev) { 4317 *ret = deliver_skb(skb, *pt_prev, orig_dev); 4318 *pt_prev = NULL; 4319 } 4320 4321 rcu_read_lock(); 4322 ingress_retval = nf_hook_ingress(skb); 4323 rcu_read_unlock(); 4324 return ingress_retval; 4325 } 4326 #endif /* CONFIG_NETFILTER_INGRESS */ 4327 return 0; 4328 } 4329 4330 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc) 4331 { 4332 struct packet_type *ptype, *pt_prev; 4333 rx_handler_func_t *rx_handler; 4334 struct net_device *orig_dev; 4335 bool deliver_exact = false; 4336 int ret = NET_RX_DROP; 4337 __be16 type; 4338 4339 net_timestamp_check(!netdev_tstamp_prequeue, skb); 4340 4341 trace_netif_receive_skb(skb); 4342 4343 orig_dev = skb->dev; 4344 4345 skb_reset_network_header(skb); 4346 if (!skb_transport_header_was_set(skb)) 4347 skb_reset_transport_header(skb); 4348 skb_reset_mac_len(skb); 4349 4350 pt_prev = NULL; 4351 4352 another_round: 4353 skb->skb_iif = skb->dev->ifindex; 4354 4355 __this_cpu_inc(softnet_data.processed); 4356 4357 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 4358 skb->protocol == cpu_to_be16(ETH_P_8021AD)) { 4359 skb = skb_vlan_untag(skb); 4360 if (unlikely(!skb)) 4361 goto out; 4362 } 4363 4364 if (skb_skip_tc_classify(skb)) 4365 goto skip_classify; 4366 4367 if (pfmemalloc) 4368 goto skip_taps; 4369 4370 list_for_each_entry_rcu(ptype, &ptype_all, list) { 4371 if (pt_prev) 4372 ret = deliver_skb(skb, pt_prev, orig_dev); 4373 pt_prev = ptype; 4374 } 4375 4376 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { 4377 if (pt_prev) 4378 ret = deliver_skb(skb, pt_prev, orig_dev); 4379 pt_prev = ptype; 4380 } 4381 4382 skip_taps: 4383 #ifdef CONFIG_NET_INGRESS 4384 if (static_key_false(&ingress_needed)) { 4385 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev); 4386 if (!skb) 4387 goto out; 4388 4389 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0) 4390 goto out; 4391 } 4392 #endif 4393 skb_reset_tc(skb); 4394 skip_classify: 4395 if (pfmemalloc && !skb_pfmemalloc_protocol(skb)) 4396 goto drop; 4397 4398 if (skb_vlan_tag_present(skb)) { 4399 if (pt_prev) { 4400 ret = deliver_skb(skb, pt_prev, orig_dev); 4401 pt_prev = NULL; 4402 } 4403 if (vlan_do_receive(&skb)) 4404 goto another_round; 4405 else if (unlikely(!skb)) 4406 goto out; 4407 } 4408 4409 rx_handler = rcu_dereference(skb->dev->rx_handler); 4410 if (rx_handler) { 4411 if (pt_prev) { 4412 ret = deliver_skb(skb, pt_prev, orig_dev); 4413 pt_prev = NULL; 4414 } 4415 switch (rx_handler(&skb)) { 4416 case RX_HANDLER_CONSUMED: 4417 ret = NET_RX_SUCCESS; 4418 goto out; 4419 case RX_HANDLER_ANOTHER: 4420 goto another_round; 4421 case RX_HANDLER_EXACT: 4422 deliver_exact = true; 4423 case RX_HANDLER_PASS: 4424 break; 4425 default: 4426 BUG(); 4427 } 4428 } 4429 4430 if (unlikely(skb_vlan_tag_present(skb))) { 4431 if (skb_vlan_tag_get_id(skb)) 4432 skb->pkt_type = PACKET_OTHERHOST; 4433 /* Note: we might in the future use prio bits 4434 * and set skb->priority like in vlan_do_receive() 4435 * For the time being, just ignore Priority Code Point 4436 */ 4437 skb->vlan_tci = 0; 4438 } 4439 4440 type = skb->protocol; 4441 4442 /* deliver only exact match when indicated */ 4443 if (likely(!deliver_exact)) { 4444 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4445 &ptype_base[ntohs(type) & 4446 PTYPE_HASH_MASK]); 4447 } 4448 4449 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4450 &orig_dev->ptype_specific); 4451 4452 if (unlikely(skb->dev != orig_dev)) { 4453 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 4454 &skb->dev->ptype_specific); 4455 } 4456 4457 if (pt_prev) { 4458 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) 4459 goto drop; 4460 else 4461 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 4462 } else { 4463 drop: 4464 if (!deliver_exact) 4465 atomic_long_inc(&skb->dev->rx_dropped); 4466 else 4467 atomic_long_inc(&skb->dev->rx_nohandler); 4468 kfree_skb(skb); 4469 /* Jamal, now you will not able to escape explaining 4470 * me how you were going to use this. :-) 4471 */ 4472 ret = NET_RX_DROP; 4473 } 4474 4475 out: 4476 return ret; 4477 } 4478 4479 /** 4480 * netif_receive_skb_core - special purpose version of netif_receive_skb 4481 * @skb: buffer to process 4482 * 4483 * More direct receive version of netif_receive_skb(). It should 4484 * only be used by callers that have a need to skip RPS and Generic XDP. 4485 * Caller must also take care of handling if (page_is_)pfmemalloc. 4486 * 4487 * This function may only be called from softirq context and interrupts 4488 * should be enabled. 4489 * 4490 * Return values (usually ignored): 4491 * NET_RX_SUCCESS: no congestion 4492 * NET_RX_DROP: packet was dropped 4493 */ 4494 int netif_receive_skb_core(struct sk_buff *skb) 4495 { 4496 int ret; 4497 4498 rcu_read_lock(); 4499 ret = __netif_receive_skb_core(skb, false); 4500 rcu_read_unlock(); 4501 4502 return ret; 4503 } 4504 EXPORT_SYMBOL(netif_receive_skb_core); 4505 4506 static int __netif_receive_skb(struct sk_buff *skb) 4507 { 4508 int ret; 4509 4510 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) { 4511 unsigned int noreclaim_flag; 4512 4513 /* 4514 * PFMEMALLOC skbs are special, they should 4515 * - be delivered to SOCK_MEMALLOC sockets only 4516 * - stay away from userspace 4517 * - have bounded memory usage 4518 * 4519 * Use PF_MEMALLOC as this saves us from propagating the allocation 4520 * context down to all allocation sites. 4521 */ 4522 noreclaim_flag = memalloc_noreclaim_save(); 4523 ret = __netif_receive_skb_core(skb, true); 4524 memalloc_noreclaim_restore(noreclaim_flag); 4525 } else 4526 ret = __netif_receive_skb_core(skb, false); 4527 4528 return ret; 4529 } 4530 4531 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp) 4532 { 4533 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog); 4534 struct bpf_prog *new = xdp->prog; 4535 int ret = 0; 4536 4537 switch (xdp->command) { 4538 case XDP_SETUP_PROG: 4539 rcu_assign_pointer(dev->xdp_prog, new); 4540 if (old) 4541 bpf_prog_put(old); 4542 4543 if (old && !new) { 4544 static_key_slow_dec(&generic_xdp_needed); 4545 } else if (new && !old) { 4546 static_key_slow_inc(&generic_xdp_needed); 4547 dev_disable_lro(dev); 4548 } 4549 break; 4550 4551 case XDP_QUERY_PROG: 4552 xdp->prog_attached = !!old; 4553 xdp->prog_id = old ? old->aux->id : 0; 4554 break; 4555 4556 default: 4557 ret = -EINVAL; 4558 break; 4559 } 4560 4561 return ret; 4562 } 4563 4564 static int netif_receive_skb_internal(struct sk_buff *skb) 4565 { 4566 int ret; 4567 4568 net_timestamp_check(netdev_tstamp_prequeue, skb); 4569 4570 if (skb_defer_rx_timestamp(skb)) 4571 return NET_RX_SUCCESS; 4572 4573 if (static_key_false(&generic_xdp_needed)) { 4574 int ret; 4575 4576 preempt_disable(); 4577 rcu_read_lock(); 4578 ret = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb); 4579 rcu_read_unlock(); 4580 preempt_enable(); 4581 4582 if (ret != XDP_PASS) 4583 return NET_RX_DROP; 4584 } 4585 4586 rcu_read_lock(); 4587 #ifdef CONFIG_RPS 4588 if (static_key_false(&rps_needed)) { 4589 struct rps_dev_flow voidflow, *rflow = &voidflow; 4590 int cpu = get_rps_cpu(skb->dev, skb, &rflow); 4591 4592 if (cpu >= 0) { 4593 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 4594 rcu_read_unlock(); 4595 return ret; 4596 } 4597 } 4598 #endif 4599 ret = __netif_receive_skb(skb); 4600 rcu_read_unlock(); 4601 return ret; 4602 } 4603 4604 /** 4605 * netif_receive_skb - process receive buffer from network 4606 * @skb: buffer to process 4607 * 4608 * netif_receive_skb() is the main receive data processing function. 4609 * It always succeeds. The buffer may be dropped during processing 4610 * for congestion control or by the protocol layers. 4611 * 4612 * This function may only be called from softirq context and interrupts 4613 * should be enabled. 4614 * 4615 * Return values (usually ignored): 4616 * NET_RX_SUCCESS: no congestion 4617 * NET_RX_DROP: packet was dropped 4618 */ 4619 int netif_receive_skb(struct sk_buff *skb) 4620 { 4621 trace_netif_receive_skb_entry(skb); 4622 4623 return netif_receive_skb_internal(skb); 4624 } 4625 EXPORT_SYMBOL(netif_receive_skb); 4626 4627 DEFINE_PER_CPU(struct work_struct, flush_works); 4628 4629 /* Network device is going away, flush any packets still pending */ 4630 static void flush_backlog(struct work_struct *work) 4631 { 4632 struct sk_buff *skb, *tmp; 4633 struct softnet_data *sd; 4634 4635 local_bh_disable(); 4636 sd = this_cpu_ptr(&softnet_data); 4637 4638 local_irq_disable(); 4639 rps_lock(sd); 4640 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { 4641 if (skb->dev->reg_state == NETREG_UNREGISTERING) { 4642 __skb_unlink(skb, &sd->input_pkt_queue); 4643 kfree_skb(skb); 4644 input_queue_head_incr(sd); 4645 } 4646 } 4647 rps_unlock(sd); 4648 local_irq_enable(); 4649 4650 skb_queue_walk_safe(&sd->process_queue, skb, tmp) { 4651 if (skb->dev->reg_state == NETREG_UNREGISTERING) { 4652 __skb_unlink(skb, &sd->process_queue); 4653 kfree_skb(skb); 4654 input_queue_head_incr(sd); 4655 } 4656 } 4657 local_bh_enable(); 4658 } 4659 4660 static void flush_all_backlogs(void) 4661 { 4662 unsigned int cpu; 4663 4664 get_online_cpus(); 4665 4666 for_each_online_cpu(cpu) 4667 queue_work_on(cpu, system_highpri_wq, 4668 per_cpu_ptr(&flush_works, cpu)); 4669 4670 for_each_online_cpu(cpu) 4671 flush_work(per_cpu_ptr(&flush_works, cpu)); 4672 4673 put_online_cpus(); 4674 } 4675 4676 static int napi_gro_complete(struct sk_buff *skb) 4677 { 4678 struct packet_offload *ptype; 4679 __be16 type = skb->protocol; 4680 struct list_head *head = &offload_base; 4681 int err = -ENOENT; 4682 4683 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); 4684 4685 if (NAPI_GRO_CB(skb)->count == 1) { 4686 skb_shinfo(skb)->gso_size = 0; 4687 goto out; 4688 } 4689 4690 rcu_read_lock(); 4691 list_for_each_entry_rcu(ptype, head, list) { 4692 if (ptype->type != type || !ptype->callbacks.gro_complete) 4693 continue; 4694 4695 err = ptype->callbacks.gro_complete(skb, 0); 4696 break; 4697 } 4698 rcu_read_unlock(); 4699 4700 if (err) { 4701 WARN_ON(&ptype->list == head); 4702 kfree_skb(skb); 4703 return NET_RX_SUCCESS; 4704 } 4705 4706 out: 4707 return netif_receive_skb_internal(skb); 4708 } 4709 4710 /* napi->gro_list contains packets ordered by age. 4711 * youngest packets at the head of it. 4712 * Complete skbs in reverse order to reduce latencies. 4713 */ 4714 void napi_gro_flush(struct napi_struct *napi, bool flush_old) 4715 { 4716 struct sk_buff *skb, *prev = NULL; 4717 4718 /* scan list and build reverse chain */ 4719 for (skb = napi->gro_list; skb != NULL; skb = skb->next) { 4720 skb->prev = prev; 4721 prev = skb; 4722 } 4723 4724 for (skb = prev; skb; skb = prev) { 4725 skb->next = NULL; 4726 4727 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) 4728 return; 4729 4730 prev = skb->prev; 4731 napi_gro_complete(skb); 4732 napi->gro_count--; 4733 } 4734 4735 napi->gro_list = NULL; 4736 } 4737 EXPORT_SYMBOL(napi_gro_flush); 4738 4739 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb) 4740 { 4741 struct sk_buff *p; 4742 unsigned int maclen = skb->dev->hard_header_len; 4743 u32 hash = skb_get_hash_raw(skb); 4744 4745 for (p = napi->gro_list; p; p = p->next) { 4746 unsigned long diffs; 4747 4748 NAPI_GRO_CB(p)->flush = 0; 4749 4750 if (hash != skb_get_hash_raw(p)) { 4751 NAPI_GRO_CB(p)->same_flow = 0; 4752 continue; 4753 } 4754 4755 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; 4756 diffs |= p->vlan_tci ^ skb->vlan_tci; 4757 diffs |= skb_metadata_dst_cmp(p, skb); 4758 diffs |= skb_metadata_differs(p, skb); 4759 if (maclen == ETH_HLEN) 4760 diffs |= compare_ether_header(skb_mac_header(p), 4761 skb_mac_header(skb)); 4762 else if (!diffs) 4763 diffs = memcmp(skb_mac_header(p), 4764 skb_mac_header(skb), 4765 maclen); 4766 NAPI_GRO_CB(p)->same_flow = !diffs; 4767 } 4768 } 4769 4770 static void skb_gro_reset_offset(struct sk_buff *skb) 4771 { 4772 const struct skb_shared_info *pinfo = skb_shinfo(skb); 4773 const skb_frag_t *frag0 = &pinfo->frags[0]; 4774 4775 NAPI_GRO_CB(skb)->data_offset = 0; 4776 NAPI_GRO_CB(skb)->frag0 = NULL; 4777 NAPI_GRO_CB(skb)->frag0_len = 0; 4778 4779 if (skb_mac_header(skb) == skb_tail_pointer(skb) && 4780 pinfo->nr_frags && 4781 !PageHighMem(skb_frag_page(frag0))) { 4782 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); 4783 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int, 4784 skb_frag_size(frag0), 4785 skb->end - skb->tail); 4786 } 4787 } 4788 4789 static void gro_pull_from_frag0(struct sk_buff *skb, int grow) 4790 { 4791 struct skb_shared_info *pinfo = skb_shinfo(skb); 4792 4793 BUG_ON(skb->end - skb->tail < grow); 4794 4795 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); 4796 4797 skb->data_len -= grow; 4798 skb->tail += grow; 4799 4800 pinfo->frags[0].page_offset += grow; 4801 skb_frag_size_sub(&pinfo->frags[0], grow); 4802 4803 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { 4804 skb_frag_unref(skb, 0); 4805 memmove(pinfo->frags, pinfo->frags + 1, 4806 --pinfo->nr_frags * sizeof(pinfo->frags[0])); 4807 } 4808 } 4809 4810 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 4811 { 4812 struct sk_buff **pp = NULL; 4813 struct packet_offload *ptype; 4814 __be16 type = skb->protocol; 4815 struct list_head *head = &offload_base; 4816 int same_flow; 4817 enum gro_result ret; 4818 int grow; 4819 4820 if (netif_elide_gro(skb->dev)) 4821 goto normal; 4822 4823 gro_list_prepare(napi, skb); 4824 4825 rcu_read_lock(); 4826 list_for_each_entry_rcu(ptype, head, list) { 4827 if (ptype->type != type || !ptype->callbacks.gro_receive) 4828 continue; 4829 4830 skb_set_network_header(skb, skb_gro_offset(skb)); 4831 skb_reset_mac_len(skb); 4832 NAPI_GRO_CB(skb)->same_flow = 0; 4833 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb); 4834 NAPI_GRO_CB(skb)->free = 0; 4835 NAPI_GRO_CB(skb)->encap_mark = 0; 4836 NAPI_GRO_CB(skb)->recursion_counter = 0; 4837 NAPI_GRO_CB(skb)->is_fou = 0; 4838 NAPI_GRO_CB(skb)->is_atomic = 1; 4839 NAPI_GRO_CB(skb)->gro_remcsum_start = 0; 4840 4841 /* Setup for GRO checksum validation */ 4842 switch (skb->ip_summed) { 4843 case CHECKSUM_COMPLETE: 4844 NAPI_GRO_CB(skb)->csum = skb->csum; 4845 NAPI_GRO_CB(skb)->csum_valid = 1; 4846 NAPI_GRO_CB(skb)->csum_cnt = 0; 4847 break; 4848 case CHECKSUM_UNNECESSARY: 4849 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; 4850 NAPI_GRO_CB(skb)->csum_valid = 0; 4851 break; 4852 default: 4853 NAPI_GRO_CB(skb)->csum_cnt = 0; 4854 NAPI_GRO_CB(skb)->csum_valid = 0; 4855 } 4856 4857 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb); 4858 break; 4859 } 4860 rcu_read_unlock(); 4861 4862 if (&ptype->list == head) 4863 goto normal; 4864 4865 if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) { 4866 ret = GRO_CONSUMED; 4867 goto ok; 4868 } 4869 4870 same_flow = NAPI_GRO_CB(skb)->same_flow; 4871 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; 4872 4873 if (pp) { 4874 struct sk_buff *nskb = *pp; 4875 4876 *pp = nskb->next; 4877 nskb->next = NULL; 4878 napi_gro_complete(nskb); 4879 napi->gro_count--; 4880 } 4881 4882 if (same_flow) 4883 goto ok; 4884 4885 if (NAPI_GRO_CB(skb)->flush) 4886 goto normal; 4887 4888 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) { 4889 struct sk_buff *nskb = napi->gro_list; 4890 4891 /* locate the end of the list to select the 'oldest' flow */ 4892 while (nskb->next) { 4893 pp = &nskb->next; 4894 nskb = *pp; 4895 } 4896 *pp = NULL; 4897 nskb->next = NULL; 4898 napi_gro_complete(nskb); 4899 } else { 4900 napi->gro_count++; 4901 } 4902 NAPI_GRO_CB(skb)->count = 1; 4903 NAPI_GRO_CB(skb)->age = jiffies; 4904 NAPI_GRO_CB(skb)->last = skb; 4905 skb_shinfo(skb)->gso_size = skb_gro_len(skb); 4906 skb->next = napi->gro_list; 4907 napi->gro_list = skb; 4908 ret = GRO_HELD; 4909 4910 pull: 4911 grow = skb_gro_offset(skb) - skb_headlen(skb); 4912 if (grow > 0) 4913 gro_pull_from_frag0(skb, grow); 4914 ok: 4915 return ret; 4916 4917 normal: 4918 ret = GRO_NORMAL; 4919 goto pull; 4920 } 4921 4922 struct packet_offload *gro_find_receive_by_type(__be16 type) 4923 { 4924 struct list_head *offload_head = &offload_base; 4925 struct packet_offload *ptype; 4926 4927 list_for_each_entry_rcu(ptype, offload_head, list) { 4928 if (ptype->type != type || !ptype->callbacks.gro_receive) 4929 continue; 4930 return ptype; 4931 } 4932 return NULL; 4933 } 4934 EXPORT_SYMBOL(gro_find_receive_by_type); 4935 4936 struct packet_offload *gro_find_complete_by_type(__be16 type) 4937 { 4938 struct list_head *offload_head = &offload_base; 4939 struct packet_offload *ptype; 4940 4941 list_for_each_entry_rcu(ptype, offload_head, list) { 4942 if (ptype->type != type || !ptype->callbacks.gro_complete) 4943 continue; 4944 return ptype; 4945 } 4946 return NULL; 4947 } 4948 EXPORT_SYMBOL(gro_find_complete_by_type); 4949 4950 static void napi_skb_free_stolen_head(struct sk_buff *skb) 4951 { 4952 skb_dst_drop(skb); 4953 secpath_reset(skb); 4954 kmem_cache_free(skbuff_head_cache, skb); 4955 } 4956 4957 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) 4958 { 4959 switch (ret) { 4960 case GRO_NORMAL: 4961 if (netif_receive_skb_internal(skb)) 4962 ret = GRO_DROP; 4963 break; 4964 4965 case GRO_DROP: 4966 kfree_skb(skb); 4967 break; 4968 4969 case GRO_MERGED_FREE: 4970 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) 4971 napi_skb_free_stolen_head(skb); 4972 else 4973 __kfree_skb(skb); 4974 break; 4975 4976 case GRO_HELD: 4977 case GRO_MERGED: 4978 case GRO_CONSUMED: 4979 break; 4980 } 4981 4982 return ret; 4983 } 4984 4985 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 4986 { 4987 skb_mark_napi_id(skb, napi); 4988 trace_napi_gro_receive_entry(skb); 4989 4990 skb_gro_reset_offset(skb); 4991 4992 return napi_skb_finish(dev_gro_receive(napi, skb), skb); 4993 } 4994 EXPORT_SYMBOL(napi_gro_receive); 4995 4996 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) 4997 { 4998 if (unlikely(skb->pfmemalloc)) { 4999 consume_skb(skb); 5000 return; 5001 } 5002 __skb_pull(skb, skb_headlen(skb)); 5003 /* restore the reserve we had after netdev_alloc_skb_ip_align() */ 5004 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); 5005 skb->vlan_tci = 0; 5006 skb->dev = napi->dev; 5007 skb->skb_iif = 0; 5008 skb->encapsulation = 0; 5009 skb_shinfo(skb)->gso_type = 0; 5010 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 5011 secpath_reset(skb); 5012 5013 napi->skb = skb; 5014 } 5015 5016 struct sk_buff *napi_get_frags(struct napi_struct *napi) 5017 { 5018 struct sk_buff *skb = napi->skb; 5019 5020 if (!skb) { 5021 skb = napi_alloc_skb(napi, GRO_MAX_HEAD); 5022 if (skb) { 5023 napi->skb = skb; 5024 skb_mark_napi_id(skb, napi); 5025 } 5026 } 5027 return skb; 5028 } 5029 EXPORT_SYMBOL(napi_get_frags); 5030 5031 static gro_result_t napi_frags_finish(struct napi_struct *napi, 5032 struct sk_buff *skb, 5033 gro_result_t ret) 5034 { 5035 switch (ret) { 5036 case GRO_NORMAL: 5037 case GRO_HELD: 5038 __skb_push(skb, ETH_HLEN); 5039 skb->protocol = eth_type_trans(skb, skb->dev); 5040 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb)) 5041 ret = GRO_DROP; 5042 break; 5043 5044 case GRO_DROP: 5045 napi_reuse_skb(napi, skb); 5046 break; 5047 5048 case GRO_MERGED_FREE: 5049 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) 5050 napi_skb_free_stolen_head(skb); 5051 else 5052 napi_reuse_skb(napi, skb); 5053 break; 5054 5055 case GRO_MERGED: 5056 case GRO_CONSUMED: 5057 break; 5058 } 5059 5060 return ret; 5061 } 5062 5063 /* Upper GRO stack assumes network header starts at gro_offset=0 5064 * Drivers could call both napi_gro_frags() and napi_gro_receive() 5065 * We copy ethernet header into skb->data to have a common layout. 5066 */ 5067 static struct sk_buff *napi_frags_skb(struct napi_struct *napi) 5068 { 5069 struct sk_buff *skb = napi->skb; 5070 const struct ethhdr *eth; 5071 unsigned int hlen = sizeof(*eth); 5072 5073 napi->skb = NULL; 5074 5075 skb_reset_mac_header(skb); 5076 skb_gro_reset_offset(skb); 5077 5078 eth = skb_gro_header_fast(skb, 0); 5079 if (unlikely(skb_gro_header_hard(skb, hlen))) { 5080 eth = skb_gro_header_slow(skb, hlen, 0); 5081 if (unlikely(!eth)) { 5082 net_warn_ratelimited("%s: dropping impossible skb from %s\n", 5083 __func__, napi->dev->name); 5084 napi_reuse_skb(napi, skb); 5085 return NULL; 5086 } 5087 } else { 5088 gro_pull_from_frag0(skb, hlen); 5089 NAPI_GRO_CB(skb)->frag0 += hlen; 5090 NAPI_GRO_CB(skb)->frag0_len -= hlen; 5091 } 5092 __skb_pull(skb, hlen); 5093 5094 /* 5095 * This works because the only protocols we care about don't require 5096 * special handling. 5097 * We'll fix it up properly in napi_frags_finish() 5098 */ 5099 skb->protocol = eth->h_proto; 5100 5101 return skb; 5102 } 5103 5104 gro_result_t napi_gro_frags(struct napi_struct *napi) 5105 { 5106 struct sk_buff *skb = napi_frags_skb(napi); 5107 5108 if (!skb) 5109 return GRO_DROP; 5110 5111 trace_napi_gro_frags_entry(skb); 5112 5113 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); 5114 } 5115 EXPORT_SYMBOL(napi_gro_frags); 5116 5117 /* Compute the checksum from gro_offset and return the folded value 5118 * after adding in any pseudo checksum. 5119 */ 5120 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb) 5121 { 5122 __wsum wsum; 5123 __sum16 sum; 5124 5125 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); 5126 5127 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ 5128 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); 5129 if (likely(!sum)) { 5130 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 5131 !skb->csum_complete_sw) 5132 netdev_rx_csum_fault(skb->dev); 5133 } 5134 5135 NAPI_GRO_CB(skb)->csum = wsum; 5136 NAPI_GRO_CB(skb)->csum_valid = 1; 5137 5138 return sum; 5139 } 5140 EXPORT_SYMBOL(__skb_gro_checksum_complete); 5141 5142 static void net_rps_send_ipi(struct softnet_data *remsd) 5143 { 5144 #ifdef CONFIG_RPS 5145 while (remsd) { 5146 struct softnet_data *next = remsd->rps_ipi_next; 5147 5148 if (cpu_online(remsd->cpu)) 5149 smp_call_function_single_async(remsd->cpu, &remsd->csd); 5150 remsd = next; 5151 } 5152 #endif 5153 } 5154 5155 /* 5156 * net_rps_action_and_irq_enable sends any pending IPI's for rps. 5157 * Note: called with local irq disabled, but exits with local irq enabled. 5158 */ 5159 static void net_rps_action_and_irq_enable(struct softnet_data *sd) 5160 { 5161 #ifdef CONFIG_RPS 5162 struct softnet_data *remsd = sd->rps_ipi_list; 5163 5164 if (remsd) { 5165 sd->rps_ipi_list = NULL; 5166 5167 local_irq_enable(); 5168 5169 /* Send pending IPI's to kick RPS processing on remote cpus. */ 5170 net_rps_send_ipi(remsd); 5171 } else 5172 #endif 5173 local_irq_enable(); 5174 } 5175 5176 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) 5177 { 5178 #ifdef CONFIG_RPS 5179 return sd->rps_ipi_list != NULL; 5180 #else 5181 return false; 5182 #endif 5183 } 5184 5185 static int process_backlog(struct napi_struct *napi, int quota) 5186 { 5187 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); 5188 bool again = true; 5189 int work = 0; 5190 5191 /* Check if we have pending ipi, its better to send them now, 5192 * not waiting net_rx_action() end. 5193 */ 5194 if (sd_has_rps_ipi_waiting(sd)) { 5195 local_irq_disable(); 5196 net_rps_action_and_irq_enable(sd); 5197 } 5198 5199 napi->weight = dev_rx_weight; 5200 while (again) { 5201 struct sk_buff *skb; 5202 5203 while ((skb = __skb_dequeue(&sd->process_queue))) { 5204 rcu_read_lock(); 5205 __netif_receive_skb(skb); 5206 rcu_read_unlock(); 5207 input_queue_head_incr(sd); 5208 if (++work >= quota) 5209 return work; 5210 5211 } 5212 5213 local_irq_disable(); 5214 rps_lock(sd); 5215 if (skb_queue_empty(&sd->input_pkt_queue)) { 5216 /* 5217 * Inline a custom version of __napi_complete(). 5218 * only current cpu owns and manipulates this napi, 5219 * and NAPI_STATE_SCHED is the only possible flag set 5220 * on backlog. 5221 * We can use a plain write instead of clear_bit(), 5222 * and we dont need an smp_mb() memory barrier. 5223 */ 5224 napi->state = 0; 5225 again = false; 5226 } else { 5227 skb_queue_splice_tail_init(&sd->input_pkt_queue, 5228 &sd->process_queue); 5229 } 5230 rps_unlock(sd); 5231 local_irq_enable(); 5232 } 5233 5234 return work; 5235 } 5236 5237 /** 5238 * __napi_schedule - schedule for receive 5239 * @n: entry to schedule 5240 * 5241 * The entry's receive function will be scheduled to run. 5242 * Consider using __napi_schedule_irqoff() if hard irqs are masked. 5243 */ 5244 void __napi_schedule(struct napi_struct *n) 5245 { 5246 unsigned long flags; 5247 5248 local_irq_save(flags); 5249 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 5250 local_irq_restore(flags); 5251 } 5252 EXPORT_SYMBOL(__napi_schedule); 5253 5254 /** 5255 * napi_schedule_prep - check if napi can be scheduled 5256 * @n: napi context 5257 * 5258 * Test if NAPI routine is already running, and if not mark 5259 * it as running. This is used as a condition variable 5260 * insure only one NAPI poll instance runs. We also make 5261 * sure there is no pending NAPI disable. 5262 */ 5263 bool napi_schedule_prep(struct napi_struct *n) 5264 { 5265 unsigned long val, new; 5266 5267 do { 5268 val = READ_ONCE(n->state); 5269 if (unlikely(val & NAPIF_STATE_DISABLE)) 5270 return false; 5271 new = val | NAPIF_STATE_SCHED; 5272 5273 /* Sets STATE_MISSED bit if STATE_SCHED was already set 5274 * This was suggested by Alexander Duyck, as compiler 5275 * emits better code than : 5276 * if (val & NAPIF_STATE_SCHED) 5277 * new |= NAPIF_STATE_MISSED; 5278 */ 5279 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED * 5280 NAPIF_STATE_MISSED; 5281 } while (cmpxchg(&n->state, val, new) != val); 5282 5283 return !(val & NAPIF_STATE_SCHED); 5284 } 5285 EXPORT_SYMBOL(napi_schedule_prep); 5286 5287 /** 5288 * __napi_schedule_irqoff - schedule for receive 5289 * @n: entry to schedule 5290 * 5291 * Variant of __napi_schedule() assuming hard irqs are masked 5292 */ 5293 void __napi_schedule_irqoff(struct napi_struct *n) 5294 { 5295 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 5296 } 5297 EXPORT_SYMBOL(__napi_schedule_irqoff); 5298 5299 bool napi_complete_done(struct napi_struct *n, int work_done) 5300 { 5301 unsigned long flags, val, new; 5302 5303 /* 5304 * 1) Don't let napi dequeue from the cpu poll list 5305 * just in case its running on a different cpu. 5306 * 2) If we are busy polling, do nothing here, we have 5307 * the guarantee we will be called later. 5308 */ 5309 if (unlikely(n->state & (NAPIF_STATE_NPSVC | 5310 NAPIF_STATE_IN_BUSY_POLL))) 5311 return false; 5312 5313 if (n->gro_list) { 5314 unsigned long timeout = 0; 5315 5316 if (work_done) 5317 timeout = n->dev->gro_flush_timeout; 5318 5319 if (timeout) 5320 hrtimer_start(&n->timer, ns_to_ktime(timeout), 5321 HRTIMER_MODE_REL_PINNED); 5322 else 5323 napi_gro_flush(n, false); 5324 } 5325 if (unlikely(!list_empty(&n->poll_list))) { 5326 /* If n->poll_list is not empty, we need to mask irqs */ 5327 local_irq_save(flags); 5328 list_del_init(&n->poll_list); 5329 local_irq_restore(flags); 5330 } 5331 5332 do { 5333 val = READ_ONCE(n->state); 5334 5335 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED)); 5336 5337 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED); 5338 5339 /* If STATE_MISSED was set, leave STATE_SCHED set, 5340 * because we will call napi->poll() one more time. 5341 * This C code was suggested by Alexander Duyck to help gcc. 5342 */ 5343 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED * 5344 NAPIF_STATE_SCHED; 5345 } while (cmpxchg(&n->state, val, new) != val); 5346 5347 if (unlikely(val & NAPIF_STATE_MISSED)) { 5348 __napi_schedule(n); 5349 return false; 5350 } 5351 5352 return true; 5353 } 5354 EXPORT_SYMBOL(napi_complete_done); 5355 5356 /* must be called under rcu_read_lock(), as we dont take a reference */ 5357 static struct napi_struct *napi_by_id(unsigned int napi_id) 5358 { 5359 unsigned int hash = napi_id % HASH_SIZE(napi_hash); 5360 struct napi_struct *napi; 5361 5362 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node) 5363 if (napi->napi_id == napi_id) 5364 return napi; 5365 5366 return NULL; 5367 } 5368 5369 #if defined(CONFIG_NET_RX_BUSY_POLL) 5370 5371 #define BUSY_POLL_BUDGET 8 5372 5373 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock) 5374 { 5375 int rc; 5376 5377 /* Busy polling means there is a high chance device driver hard irq 5378 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was 5379 * set in napi_schedule_prep(). 5380 * Since we are about to call napi->poll() once more, we can safely 5381 * clear NAPI_STATE_MISSED. 5382 * 5383 * Note: x86 could use a single "lock and ..." instruction 5384 * to perform these two clear_bit() 5385 */ 5386 clear_bit(NAPI_STATE_MISSED, &napi->state); 5387 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state); 5388 5389 local_bh_disable(); 5390 5391 /* All we really want here is to re-enable device interrupts. 5392 * Ideally, a new ndo_busy_poll_stop() could avoid another round. 5393 */ 5394 rc = napi->poll(napi, BUSY_POLL_BUDGET); 5395 trace_napi_poll(napi, rc, BUSY_POLL_BUDGET); 5396 netpoll_poll_unlock(have_poll_lock); 5397 if (rc == BUSY_POLL_BUDGET) 5398 __napi_schedule(napi); 5399 local_bh_enable(); 5400 } 5401 5402 void napi_busy_loop(unsigned int napi_id, 5403 bool (*loop_end)(void *, unsigned long), 5404 void *loop_end_arg) 5405 { 5406 unsigned long start_time = loop_end ? busy_loop_current_time() : 0; 5407 int (*napi_poll)(struct napi_struct *napi, int budget); 5408 void *have_poll_lock = NULL; 5409 struct napi_struct *napi; 5410 5411 restart: 5412 napi_poll = NULL; 5413 5414 rcu_read_lock(); 5415 5416 napi = napi_by_id(napi_id); 5417 if (!napi) 5418 goto out; 5419 5420 preempt_disable(); 5421 for (;;) { 5422 int work = 0; 5423 5424 local_bh_disable(); 5425 if (!napi_poll) { 5426 unsigned long val = READ_ONCE(napi->state); 5427 5428 /* If multiple threads are competing for this napi, 5429 * we avoid dirtying napi->state as much as we can. 5430 */ 5431 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED | 5432 NAPIF_STATE_IN_BUSY_POLL)) 5433 goto count; 5434 if (cmpxchg(&napi->state, val, 5435 val | NAPIF_STATE_IN_BUSY_POLL | 5436 NAPIF_STATE_SCHED) != val) 5437 goto count; 5438 have_poll_lock = netpoll_poll_lock(napi); 5439 napi_poll = napi->poll; 5440 } 5441 work = napi_poll(napi, BUSY_POLL_BUDGET); 5442 trace_napi_poll(napi, work, BUSY_POLL_BUDGET); 5443 count: 5444 if (work > 0) 5445 __NET_ADD_STATS(dev_net(napi->dev), 5446 LINUX_MIB_BUSYPOLLRXPACKETS, work); 5447 local_bh_enable(); 5448 5449 if (!loop_end || loop_end(loop_end_arg, start_time)) 5450 break; 5451 5452 if (unlikely(need_resched())) { 5453 if (napi_poll) 5454 busy_poll_stop(napi, have_poll_lock); 5455 preempt_enable(); 5456 rcu_read_unlock(); 5457 cond_resched(); 5458 if (loop_end(loop_end_arg, start_time)) 5459 return; 5460 goto restart; 5461 } 5462 cpu_relax(); 5463 } 5464 if (napi_poll) 5465 busy_poll_stop(napi, have_poll_lock); 5466 preempt_enable(); 5467 out: 5468 rcu_read_unlock(); 5469 } 5470 EXPORT_SYMBOL(napi_busy_loop); 5471 5472 #endif /* CONFIG_NET_RX_BUSY_POLL */ 5473 5474 static void napi_hash_add(struct napi_struct *napi) 5475 { 5476 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) || 5477 test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) 5478 return; 5479 5480 spin_lock(&napi_hash_lock); 5481 5482 /* 0..NR_CPUS range is reserved for sender_cpu use */ 5483 do { 5484 if (unlikely(++napi_gen_id < MIN_NAPI_ID)) 5485 napi_gen_id = MIN_NAPI_ID; 5486 } while (napi_by_id(napi_gen_id)); 5487 napi->napi_id = napi_gen_id; 5488 5489 hlist_add_head_rcu(&napi->napi_hash_node, 5490 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); 5491 5492 spin_unlock(&napi_hash_lock); 5493 } 5494 5495 /* Warning : caller is responsible to make sure rcu grace period 5496 * is respected before freeing memory containing @napi 5497 */ 5498 bool napi_hash_del(struct napi_struct *napi) 5499 { 5500 bool rcu_sync_needed = false; 5501 5502 spin_lock(&napi_hash_lock); 5503 5504 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) { 5505 rcu_sync_needed = true; 5506 hlist_del_rcu(&napi->napi_hash_node); 5507 } 5508 spin_unlock(&napi_hash_lock); 5509 return rcu_sync_needed; 5510 } 5511 EXPORT_SYMBOL_GPL(napi_hash_del); 5512 5513 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer) 5514 { 5515 struct napi_struct *napi; 5516 5517 napi = container_of(timer, struct napi_struct, timer); 5518 5519 /* Note : we use a relaxed variant of napi_schedule_prep() not setting 5520 * NAPI_STATE_MISSED, since we do not react to a device IRQ. 5521 */ 5522 if (napi->gro_list && !napi_disable_pending(napi) && 5523 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) 5524 __napi_schedule_irqoff(napi); 5525 5526 return HRTIMER_NORESTART; 5527 } 5528 5529 void netif_napi_add(struct net_device *dev, struct napi_struct *napi, 5530 int (*poll)(struct napi_struct *, int), int weight) 5531 { 5532 INIT_LIST_HEAD(&napi->poll_list); 5533 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); 5534 napi->timer.function = napi_watchdog; 5535 napi->gro_count = 0; 5536 napi->gro_list = NULL; 5537 napi->skb = NULL; 5538 napi->poll = poll; 5539 if (weight > NAPI_POLL_WEIGHT) 5540 pr_err_once("netif_napi_add() called with weight %d on device %s\n", 5541 weight, dev->name); 5542 napi->weight = weight; 5543 list_add(&napi->dev_list, &dev->napi_list); 5544 napi->dev = dev; 5545 #ifdef CONFIG_NETPOLL 5546 napi->poll_owner = -1; 5547 #endif 5548 set_bit(NAPI_STATE_SCHED, &napi->state); 5549 napi_hash_add(napi); 5550 } 5551 EXPORT_SYMBOL(netif_napi_add); 5552 5553 void napi_disable(struct napi_struct *n) 5554 { 5555 might_sleep(); 5556 set_bit(NAPI_STATE_DISABLE, &n->state); 5557 5558 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state)) 5559 msleep(1); 5560 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state)) 5561 msleep(1); 5562 5563 hrtimer_cancel(&n->timer); 5564 5565 clear_bit(NAPI_STATE_DISABLE, &n->state); 5566 } 5567 EXPORT_SYMBOL(napi_disable); 5568 5569 /* Must be called in process context */ 5570 void netif_napi_del(struct napi_struct *napi) 5571 { 5572 might_sleep(); 5573 if (napi_hash_del(napi)) 5574 synchronize_net(); 5575 list_del_init(&napi->dev_list); 5576 napi_free_frags(napi); 5577 5578 kfree_skb_list(napi->gro_list); 5579 napi->gro_list = NULL; 5580 napi->gro_count = 0; 5581 } 5582 EXPORT_SYMBOL(netif_napi_del); 5583 5584 static int napi_poll(struct napi_struct *n, struct list_head *repoll) 5585 { 5586 void *have; 5587 int work, weight; 5588 5589 list_del_init(&n->poll_list); 5590 5591 have = netpoll_poll_lock(n); 5592 5593 weight = n->weight; 5594 5595 /* This NAPI_STATE_SCHED test is for avoiding a race 5596 * with netpoll's poll_napi(). Only the entity which 5597 * obtains the lock and sees NAPI_STATE_SCHED set will 5598 * actually make the ->poll() call. Therefore we avoid 5599 * accidentally calling ->poll() when NAPI is not scheduled. 5600 */ 5601 work = 0; 5602 if (test_bit(NAPI_STATE_SCHED, &n->state)) { 5603 work = n->poll(n, weight); 5604 trace_napi_poll(n, work, weight); 5605 } 5606 5607 WARN_ON_ONCE(work > weight); 5608 5609 if (likely(work < weight)) 5610 goto out_unlock; 5611 5612 /* Drivers must not modify the NAPI state if they 5613 * consume the entire weight. In such cases this code 5614 * still "owns" the NAPI instance and therefore can 5615 * move the instance around on the list at-will. 5616 */ 5617 if (unlikely(napi_disable_pending(n))) { 5618 napi_complete(n); 5619 goto out_unlock; 5620 } 5621 5622 if (n->gro_list) { 5623 /* flush too old packets 5624 * If HZ < 1000, flush all packets. 5625 */ 5626 napi_gro_flush(n, HZ >= 1000); 5627 } 5628 5629 /* Some drivers may have called napi_schedule 5630 * prior to exhausting their budget. 5631 */ 5632 if (unlikely(!list_empty(&n->poll_list))) { 5633 pr_warn_once("%s: Budget exhausted after napi rescheduled\n", 5634 n->dev ? n->dev->name : "backlog"); 5635 goto out_unlock; 5636 } 5637 5638 list_add_tail(&n->poll_list, repoll); 5639 5640 out_unlock: 5641 netpoll_poll_unlock(have); 5642 5643 return work; 5644 } 5645 5646 static __latent_entropy void net_rx_action(struct softirq_action *h) 5647 { 5648 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 5649 unsigned long time_limit = jiffies + 5650 usecs_to_jiffies(netdev_budget_usecs); 5651 int budget = netdev_budget; 5652 LIST_HEAD(list); 5653 LIST_HEAD(repoll); 5654 5655 local_irq_disable(); 5656 list_splice_init(&sd->poll_list, &list); 5657 local_irq_enable(); 5658 5659 for (;;) { 5660 struct napi_struct *n; 5661 5662 if (list_empty(&list)) { 5663 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll)) 5664 goto out; 5665 break; 5666 } 5667 5668 n = list_first_entry(&list, struct napi_struct, poll_list); 5669 budget -= napi_poll(n, &repoll); 5670 5671 /* If softirq window is exhausted then punt. 5672 * Allow this to run for 2 jiffies since which will allow 5673 * an average latency of 1.5/HZ. 5674 */ 5675 if (unlikely(budget <= 0 || 5676 time_after_eq(jiffies, time_limit))) { 5677 sd->time_squeeze++; 5678 break; 5679 } 5680 } 5681 5682 local_irq_disable(); 5683 5684 list_splice_tail_init(&sd->poll_list, &list); 5685 list_splice_tail(&repoll, &list); 5686 list_splice(&list, &sd->poll_list); 5687 if (!list_empty(&sd->poll_list)) 5688 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 5689 5690 net_rps_action_and_irq_enable(sd); 5691 out: 5692 __kfree_skb_flush(); 5693 } 5694 5695 struct netdev_adjacent { 5696 struct net_device *dev; 5697 5698 /* upper master flag, there can only be one master device per list */ 5699 bool master; 5700 5701 /* counter for the number of times this device was added to us */ 5702 u16 ref_nr; 5703 5704 /* private field for the users */ 5705 void *private; 5706 5707 struct list_head list; 5708 struct rcu_head rcu; 5709 }; 5710 5711 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev, 5712 struct list_head *adj_list) 5713 { 5714 struct netdev_adjacent *adj; 5715 5716 list_for_each_entry(adj, adj_list, list) { 5717 if (adj->dev == adj_dev) 5718 return adj; 5719 } 5720 return NULL; 5721 } 5722 5723 static int __netdev_has_upper_dev(struct net_device *upper_dev, void *data) 5724 { 5725 struct net_device *dev = data; 5726 5727 return upper_dev == dev; 5728 } 5729 5730 /** 5731 * netdev_has_upper_dev - Check if device is linked to an upper device 5732 * @dev: device 5733 * @upper_dev: upper device to check 5734 * 5735 * Find out if a device is linked to specified upper device and return true 5736 * in case it is. Note that this checks only immediate upper device, 5737 * not through a complete stack of devices. The caller must hold the RTNL lock. 5738 */ 5739 bool netdev_has_upper_dev(struct net_device *dev, 5740 struct net_device *upper_dev) 5741 { 5742 ASSERT_RTNL(); 5743 5744 return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev, 5745 upper_dev); 5746 } 5747 EXPORT_SYMBOL(netdev_has_upper_dev); 5748 5749 /** 5750 * netdev_has_upper_dev_all - Check if device is linked to an upper device 5751 * @dev: device 5752 * @upper_dev: upper device to check 5753 * 5754 * Find out if a device is linked to specified upper device and return true 5755 * in case it is. Note that this checks the entire upper device chain. 5756 * The caller must hold rcu lock. 5757 */ 5758 5759 bool netdev_has_upper_dev_all_rcu(struct net_device *dev, 5760 struct net_device *upper_dev) 5761 { 5762 return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev, 5763 upper_dev); 5764 } 5765 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu); 5766 5767 /** 5768 * netdev_has_any_upper_dev - Check if device is linked to some device 5769 * @dev: device 5770 * 5771 * Find out if a device is linked to an upper device and return true in case 5772 * it is. The caller must hold the RTNL lock. 5773 */ 5774 bool netdev_has_any_upper_dev(struct net_device *dev) 5775 { 5776 ASSERT_RTNL(); 5777 5778 return !list_empty(&dev->adj_list.upper); 5779 } 5780 EXPORT_SYMBOL(netdev_has_any_upper_dev); 5781 5782 /** 5783 * netdev_master_upper_dev_get - Get master upper device 5784 * @dev: device 5785 * 5786 * Find a master upper device and return pointer to it or NULL in case 5787 * it's not there. The caller must hold the RTNL lock. 5788 */ 5789 struct net_device *netdev_master_upper_dev_get(struct net_device *dev) 5790 { 5791 struct netdev_adjacent *upper; 5792 5793 ASSERT_RTNL(); 5794 5795 if (list_empty(&dev->adj_list.upper)) 5796 return NULL; 5797 5798 upper = list_first_entry(&dev->adj_list.upper, 5799 struct netdev_adjacent, list); 5800 if (likely(upper->master)) 5801 return upper->dev; 5802 return NULL; 5803 } 5804 EXPORT_SYMBOL(netdev_master_upper_dev_get); 5805 5806 /** 5807 * netdev_has_any_lower_dev - Check if device is linked to some device 5808 * @dev: device 5809 * 5810 * Find out if a device is linked to a lower device and return true in case 5811 * it is. The caller must hold the RTNL lock. 5812 */ 5813 static bool netdev_has_any_lower_dev(struct net_device *dev) 5814 { 5815 ASSERT_RTNL(); 5816 5817 return !list_empty(&dev->adj_list.lower); 5818 } 5819 5820 void *netdev_adjacent_get_private(struct list_head *adj_list) 5821 { 5822 struct netdev_adjacent *adj; 5823 5824 adj = list_entry(adj_list, struct netdev_adjacent, list); 5825 5826 return adj->private; 5827 } 5828 EXPORT_SYMBOL(netdev_adjacent_get_private); 5829 5830 /** 5831 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list 5832 * @dev: device 5833 * @iter: list_head ** of the current position 5834 * 5835 * Gets the next device from the dev's upper list, starting from iter 5836 * position. The caller must hold RCU read lock. 5837 */ 5838 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev, 5839 struct list_head **iter) 5840 { 5841 struct netdev_adjacent *upper; 5842 5843 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 5844 5845 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5846 5847 if (&upper->list == &dev->adj_list.upper) 5848 return NULL; 5849 5850 *iter = &upper->list; 5851 5852 return upper->dev; 5853 } 5854 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu); 5855 5856 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev, 5857 struct list_head **iter) 5858 { 5859 struct netdev_adjacent *upper; 5860 5861 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 5862 5863 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5864 5865 if (&upper->list == &dev->adj_list.upper) 5866 return NULL; 5867 5868 *iter = &upper->list; 5869 5870 return upper->dev; 5871 } 5872 5873 int netdev_walk_all_upper_dev_rcu(struct net_device *dev, 5874 int (*fn)(struct net_device *dev, 5875 void *data), 5876 void *data) 5877 { 5878 struct net_device *udev; 5879 struct list_head *iter; 5880 int ret; 5881 5882 for (iter = &dev->adj_list.upper, 5883 udev = netdev_next_upper_dev_rcu(dev, &iter); 5884 udev; 5885 udev = netdev_next_upper_dev_rcu(dev, &iter)) { 5886 /* first is the upper device itself */ 5887 ret = fn(udev, data); 5888 if (ret) 5889 return ret; 5890 5891 /* then look at all of its upper devices */ 5892 ret = netdev_walk_all_upper_dev_rcu(udev, fn, data); 5893 if (ret) 5894 return ret; 5895 } 5896 5897 return 0; 5898 } 5899 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu); 5900 5901 /** 5902 * netdev_lower_get_next_private - Get the next ->private from the 5903 * lower neighbour list 5904 * @dev: device 5905 * @iter: list_head ** of the current position 5906 * 5907 * Gets the next netdev_adjacent->private from the dev's lower neighbour 5908 * list, starting from iter position. The caller must hold either hold the 5909 * RTNL lock or its own locking that guarantees that the neighbour lower 5910 * list will remain unchanged. 5911 */ 5912 void *netdev_lower_get_next_private(struct net_device *dev, 5913 struct list_head **iter) 5914 { 5915 struct netdev_adjacent *lower; 5916 5917 lower = list_entry(*iter, struct netdev_adjacent, list); 5918 5919 if (&lower->list == &dev->adj_list.lower) 5920 return NULL; 5921 5922 *iter = lower->list.next; 5923 5924 return lower->private; 5925 } 5926 EXPORT_SYMBOL(netdev_lower_get_next_private); 5927 5928 /** 5929 * netdev_lower_get_next_private_rcu - Get the next ->private from the 5930 * lower neighbour list, RCU 5931 * variant 5932 * @dev: device 5933 * @iter: list_head ** of the current position 5934 * 5935 * Gets the next netdev_adjacent->private from the dev's lower neighbour 5936 * list, starting from iter position. The caller must hold RCU read lock. 5937 */ 5938 void *netdev_lower_get_next_private_rcu(struct net_device *dev, 5939 struct list_head **iter) 5940 { 5941 struct netdev_adjacent *lower; 5942 5943 WARN_ON_ONCE(!rcu_read_lock_held()); 5944 5945 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 5946 5947 if (&lower->list == &dev->adj_list.lower) 5948 return NULL; 5949 5950 *iter = &lower->list; 5951 5952 return lower->private; 5953 } 5954 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu); 5955 5956 /** 5957 * netdev_lower_get_next - Get the next device from the lower neighbour 5958 * list 5959 * @dev: device 5960 * @iter: list_head ** of the current position 5961 * 5962 * Gets the next netdev_adjacent from the dev's lower neighbour 5963 * list, starting from iter position. The caller must hold RTNL lock or 5964 * its own locking that guarantees that the neighbour lower 5965 * list will remain unchanged. 5966 */ 5967 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter) 5968 { 5969 struct netdev_adjacent *lower; 5970 5971 lower = list_entry(*iter, struct netdev_adjacent, list); 5972 5973 if (&lower->list == &dev->adj_list.lower) 5974 return NULL; 5975 5976 *iter = lower->list.next; 5977 5978 return lower->dev; 5979 } 5980 EXPORT_SYMBOL(netdev_lower_get_next); 5981 5982 static struct net_device *netdev_next_lower_dev(struct net_device *dev, 5983 struct list_head **iter) 5984 { 5985 struct netdev_adjacent *lower; 5986 5987 lower = list_entry((*iter)->next, struct netdev_adjacent, list); 5988 5989 if (&lower->list == &dev->adj_list.lower) 5990 return NULL; 5991 5992 *iter = &lower->list; 5993 5994 return lower->dev; 5995 } 5996 5997 int netdev_walk_all_lower_dev(struct net_device *dev, 5998 int (*fn)(struct net_device *dev, 5999 void *data), 6000 void *data) 6001 { 6002 struct net_device *ldev; 6003 struct list_head *iter; 6004 int ret; 6005 6006 for (iter = &dev->adj_list.lower, 6007 ldev = netdev_next_lower_dev(dev, &iter); 6008 ldev; 6009 ldev = netdev_next_lower_dev(dev, &iter)) { 6010 /* first is the lower device itself */ 6011 ret = fn(ldev, data); 6012 if (ret) 6013 return ret; 6014 6015 /* then look at all of its lower devices */ 6016 ret = netdev_walk_all_lower_dev(ldev, fn, data); 6017 if (ret) 6018 return ret; 6019 } 6020 6021 return 0; 6022 } 6023 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev); 6024 6025 static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev, 6026 struct list_head **iter) 6027 { 6028 struct netdev_adjacent *lower; 6029 6030 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 6031 if (&lower->list == &dev->adj_list.lower) 6032 return NULL; 6033 6034 *iter = &lower->list; 6035 6036 return lower->dev; 6037 } 6038 6039 int netdev_walk_all_lower_dev_rcu(struct net_device *dev, 6040 int (*fn)(struct net_device *dev, 6041 void *data), 6042 void *data) 6043 { 6044 struct net_device *ldev; 6045 struct list_head *iter; 6046 int ret; 6047 6048 for (iter = &dev->adj_list.lower, 6049 ldev = netdev_next_lower_dev_rcu(dev, &iter); 6050 ldev; 6051 ldev = netdev_next_lower_dev_rcu(dev, &iter)) { 6052 /* first is the lower device itself */ 6053 ret = fn(ldev, data); 6054 if (ret) 6055 return ret; 6056 6057 /* then look at all of its lower devices */ 6058 ret = netdev_walk_all_lower_dev_rcu(ldev, fn, data); 6059 if (ret) 6060 return ret; 6061 } 6062 6063 return 0; 6064 } 6065 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu); 6066 6067 /** 6068 * netdev_lower_get_first_private_rcu - Get the first ->private from the 6069 * lower neighbour list, RCU 6070 * variant 6071 * @dev: device 6072 * 6073 * Gets the first netdev_adjacent->private from the dev's lower neighbour 6074 * list. The caller must hold RCU read lock. 6075 */ 6076 void *netdev_lower_get_first_private_rcu(struct net_device *dev) 6077 { 6078 struct netdev_adjacent *lower; 6079 6080 lower = list_first_or_null_rcu(&dev->adj_list.lower, 6081 struct netdev_adjacent, list); 6082 if (lower) 6083 return lower->private; 6084 return NULL; 6085 } 6086 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu); 6087 6088 /** 6089 * netdev_master_upper_dev_get_rcu - Get master upper device 6090 * @dev: device 6091 * 6092 * Find a master upper device and return pointer to it or NULL in case 6093 * it's not there. The caller must hold the RCU read lock. 6094 */ 6095 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev) 6096 { 6097 struct netdev_adjacent *upper; 6098 6099 upper = list_first_or_null_rcu(&dev->adj_list.upper, 6100 struct netdev_adjacent, list); 6101 if (upper && likely(upper->master)) 6102 return upper->dev; 6103 return NULL; 6104 } 6105 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu); 6106 6107 static int netdev_adjacent_sysfs_add(struct net_device *dev, 6108 struct net_device *adj_dev, 6109 struct list_head *dev_list) 6110 { 6111 char linkname[IFNAMSIZ+7]; 6112 6113 sprintf(linkname, dev_list == &dev->adj_list.upper ? 6114 "upper_%s" : "lower_%s", adj_dev->name); 6115 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), 6116 linkname); 6117 } 6118 static void netdev_adjacent_sysfs_del(struct net_device *dev, 6119 char *name, 6120 struct list_head *dev_list) 6121 { 6122 char linkname[IFNAMSIZ+7]; 6123 6124 sprintf(linkname, dev_list == &dev->adj_list.upper ? 6125 "upper_%s" : "lower_%s", name); 6126 sysfs_remove_link(&(dev->dev.kobj), linkname); 6127 } 6128 6129 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev, 6130 struct net_device *adj_dev, 6131 struct list_head *dev_list) 6132 { 6133 return (dev_list == &dev->adj_list.upper || 6134 dev_list == &dev->adj_list.lower) && 6135 net_eq(dev_net(dev), dev_net(adj_dev)); 6136 } 6137 6138 static int __netdev_adjacent_dev_insert(struct net_device *dev, 6139 struct net_device *adj_dev, 6140 struct list_head *dev_list, 6141 void *private, bool master) 6142 { 6143 struct netdev_adjacent *adj; 6144 int ret; 6145 6146 adj = __netdev_find_adj(adj_dev, dev_list); 6147 6148 if (adj) { 6149 adj->ref_nr += 1; 6150 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n", 6151 dev->name, adj_dev->name, adj->ref_nr); 6152 6153 return 0; 6154 } 6155 6156 adj = kmalloc(sizeof(*adj), GFP_KERNEL); 6157 if (!adj) 6158 return -ENOMEM; 6159 6160 adj->dev = adj_dev; 6161 adj->master = master; 6162 adj->ref_nr = 1; 6163 adj->private = private; 6164 dev_hold(adj_dev); 6165 6166 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n", 6167 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name); 6168 6169 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) { 6170 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list); 6171 if (ret) 6172 goto free_adj; 6173 } 6174 6175 /* Ensure that master link is always the first item in list. */ 6176 if (master) { 6177 ret = sysfs_create_link(&(dev->dev.kobj), 6178 &(adj_dev->dev.kobj), "master"); 6179 if (ret) 6180 goto remove_symlinks; 6181 6182 list_add_rcu(&adj->list, dev_list); 6183 } else { 6184 list_add_tail_rcu(&adj->list, dev_list); 6185 } 6186 6187 return 0; 6188 6189 remove_symlinks: 6190 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 6191 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 6192 free_adj: 6193 kfree(adj); 6194 dev_put(adj_dev); 6195 6196 return ret; 6197 } 6198 6199 static void __netdev_adjacent_dev_remove(struct net_device *dev, 6200 struct net_device *adj_dev, 6201 u16 ref_nr, 6202 struct list_head *dev_list) 6203 { 6204 struct netdev_adjacent *adj; 6205 6206 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n", 6207 dev->name, adj_dev->name, ref_nr); 6208 6209 adj = __netdev_find_adj(adj_dev, dev_list); 6210 6211 if (!adj) { 6212 pr_err("Adjacency does not exist for device %s from %s\n", 6213 dev->name, adj_dev->name); 6214 WARN_ON(1); 6215 return; 6216 } 6217 6218 if (adj->ref_nr > ref_nr) { 6219 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n", 6220 dev->name, adj_dev->name, ref_nr, 6221 adj->ref_nr - ref_nr); 6222 adj->ref_nr -= ref_nr; 6223 return; 6224 } 6225 6226 if (adj->master) 6227 sysfs_remove_link(&(dev->dev.kobj), "master"); 6228 6229 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 6230 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 6231 6232 list_del_rcu(&adj->list); 6233 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n", 6234 adj_dev->name, dev->name, adj_dev->name); 6235 dev_put(adj_dev); 6236 kfree_rcu(adj, rcu); 6237 } 6238 6239 static int __netdev_adjacent_dev_link_lists(struct net_device *dev, 6240 struct net_device *upper_dev, 6241 struct list_head *up_list, 6242 struct list_head *down_list, 6243 void *private, bool master) 6244 { 6245 int ret; 6246 6247 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, 6248 private, master); 6249 if (ret) 6250 return ret; 6251 6252 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, 6253 private, false); 6254 if (ret) { 6255 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list); 6256 return ret; 6257 } 6258 6259 return 0; 6260 } 6261 6262 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev, 6263 struct net_device *upper_dev, 6264 u16 ref_nr, 6265 struct list_head *up_list, 6266 struct list_head *down_list) 6267 { 6268 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list); 6269 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list); 6270 } 6271 6272 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev, 6273 struct net_device *upper_dev, 6274 void *private, bool master) 6275 { 6276 return __netdev_adjacent_dev_link_lists(dev, upper_dev, 6277 &dev->adj_list.upper, 6278 &upper_dev->adj_list.lower, 6279 private, master); 6280 } 6281 6282 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev, 6283 struct net_device *upper_dev) 6284 { 6285 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1, 6286 &dev->adj_list.upper, 6287 &upper_dev->adj_list.lower); 6288 } 6289 6290 static int __netdev_upper_dev_link(struct net_device *dev, 6291 struct net_device *upper_dev, bool master, 6292 void *upper_priv, void *upper_info, 6293 struct netlink_ext_ack *extack) 6294 { 6295 struct netdev_notifier_changeupper_info changeupper_info = { 6296 .info = { 6297 .dev = dev, 6298 .extack = extack, 6299 }, 6300 .upper_dev = upper_dev, 6301 .master = master, 6302 .linking = true, 6303 .upper_info = upper_info, 6304 }; 6305 int ret = 0; 6306 6307 ASSERT_RTNL(); 6308 6309 if (dev == upper_dev) 6310 return -EBUSY; 6311 6312 /* To prevent loops, check if dev is not upper device to upper_dev. */ 6313 if (netdev_has_upper_dev(upper_dev, dev)) 6314 return -EBUSY; 6315 6316 if (netdev_has_upper_dev(dev, upper_dev)) 6317 return -EEXIST; 6318 6319 if (master && netdev_master_upper_dev_get(dev)) 6320 return -EBUSY; 6321 6322 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, 6323 &changeupper_info.info); 6324 ret = notifier_to_errno(ret); 6325 if (ret) 6326 return ret; 6327 6328 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv, 6329 master); 6330 if (ret) 6331 return ret; 6332 6333 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, 6334 &changeupper_info.info); 6335 ret = notifier_to_errno(ret); 6336 if (ret) 6337 goto rollback; 6338 6339 return 0; 6340 6341 rollback: 6342 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 6343 6344 return ret; 6345 } 6346 6347 /** 6348 * netdev_upper_dev_link - Add a link to the upper device 6349 * @dev: device 6350 * @upper_dev: new upper device 6351 * 6352 * Adds a link to device which is upper to this one. The caller must hold 6353 * the RTNL lock. On a failure a negative errno code is returned. 6354 * On success the reference counts are adjusted and the function 6355 * returns zero. 6356 */ 6357 int netdev_upper_dev_link(struct net_device *dev, 6358 struct net_device *upper_dev, 6359 struct netlink_ext_ack *extack) 6360 { 6361 return __netdev_upper_dev_link(dev, upper_dev, false, 6362 NULL, NULL, extack); 6363 } 6364 EXPORT_SYMBOL(netdev_upper_dev_link); 6365 6366 /** 6367 * netdev_master_upper_dev_link - Add a master link to the upper device 6368 * @dev: device 6369 * @upper_dev: new upper device 6370 * @upper_priv: upper device private 6371 * @upper_info: upper info to be passed down via notifier 6372 * 6373 * Adds a link to device which is upper to this one. In this case, only 6374 * one master upper device can be linked, although other non-master devices 6375 * might be linked as well. The caller must hold the RTNL lock. 6376 * On a failure a negative errno code is returned. On success the reference 6377 * counts are adjusted and the function returns zero. 6378 */ 6379 int netdev_master_upper_dev_link(struct net_device *dev, 6380 struct net_device *upper_dev, 6381 void *upper_priv, void *upper_info, 6382 struct netlink_ext_ack *extack) 6383 { 6384 return __netdev_upper_dev_link(dev, upper_dev, true, 6385 upper_priv, upper_info, extack); 6386 } 6387 EXPORT_SYMBOL(netdev_master_upper_dev_link); 6388 6389 /** 6390 * netdev_upper_dev_unlink - Removes a link to upper device 6391 * @dev: device 6392 * @upper_dev: new upper device 6393 * 6394 * Removes a link to device which is upper to this one. The caller must hold 6395 * the RTNL lock. 6396 */ 6397 void netdev_upper_dev_unlink(struct net_device *dev, 6398 struct net_device *upper_dev) 6399 { 6400 struct netdev_notifier_changeupper_info changeupper_info = { 6401 .info = { 6402 .dev = dev, 6403 }, 6404 .upper_dev = upper_dev, 6405 .linking = false, 6406 }; 6407 6408 ASSERT_RTNL(); 6409 6410 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev; 6411 6412 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, 6413 &changeupper_info.info); 6414 6415 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 6416 6417 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, 6418 &changeupper_info.info); 6419 } 6420 EXPORT_SYMBOL(netdev_upper_dev_unlink); 6421 6422 /** 6423 * netdev_bonding_info_change - Dispatch event about slave change 6424 * @dev: device 6425 * @bonding_info: info to dispatch 6426 * 6427 * Send NETDEV_BONDING_INFO to netdev notifiers with info. 6428 * The caller must hold the RTNL lock. 6429 */ 6430 void netdev_bonding_info_change(struct net_device *dev, 6431 struct netdev_bonding_info *bonding_info) 6432 { 6433 struct netdev_notifier_bonding_info info = { 6434 .info.dev = dev, 6435 }; 6436 6437 memcpy(&info.bonding_info, bonding_info, 6438 sizeof(struct netdev_bonding_info)); 6439 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, 6440 &info.info); 6441 } 6442 EXPORT_SYMBOL(netdev_bonding_info_change); 6443 6444 static void netdev_adjacent_add_links(struct net_device *dev) 6445 { 6446 struct netdev_adjacent *iter; 6447 6448 struct net *net = dev_net(dev); 6449 6450 list_for_each_entry(iter, &dev->adj_list.upper, list) { 6451 if (!net_eq(net, dev_net(iter->dev))) 6452 continue; 6453 netdev_adjacent_sysfs_add(iter->dev, dev, 6454 &iter->dev->adj_list.lower); 6455 netdev_adjacent_sysfs_add(dev, iter->dev, 6456 &dev->adj_list.upper); 6457 } 6458 6459 list_for_each_entry(iter, &dev->adj_list.lower, list) { 6460 if (!net_eq(net, dev_net(iter->dev))) 6461 continue; 6462 netdev_adjacent_sysfs_add(iter->dev, dev, 6463 &iter->dev->adj_list.upper); 6464 netdev_adjacent_sysfs_add(dev, iter->dev, 6465 &dev->adj_list.lower); 6466 } 6467 } 6468 6469 static void netdev_adjacent_del_links(struct net_device *dev) 6470 { 6471 struct netdev_adjacent *iter; 6472 6473 struct net *net = dev_net(dev); 6474 6475 list_for_each_entry(iter, &dev->adj_list.upper, list) { 6476 if (!net_eq(net, dev_net(iter->dev))) 6477 continue; 6478 netdev_adjacent_sysfs_del(iter->dev, dev->name, 6479 &iter->dev->adj_list.lower); 6480 netdev_adjacent_sysfs_del(dev, iter->dev->name, 6481 &dev->adj_list.upper); 6482 } 6483 6484 list_for_each_entry(iter, &dev->adj_list.lower, list) { 6485 if (!net_eq(net, dev_net(iter->dev))) 6486 continue; 6487 netdev_adjacent_sysfs_del(iter->dev, dev->name, 6488 &iter->dev->adj_list.upper); 6489 netdev_adjacent_sysfs_del(dev, iter->dev->name, 6490 &dev->adj_list.lower); 6491 } 6492 } 6493 6494 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname) 6495 { 6496 struct netdev_adjacent *iter; 6497 6498 struct net *net = dev_net(dev); 6499 6500 list_for_each_entry(iter, &dev->adj_list.upper, list) { 6501 if (!net_eq(net, dev_net(iter->dev))) 6502 continue; 6503 netdev_adjacent_sysfs_del(iter->dev, oldname, 6504 &iter->dev->adj_list.lower); 6505 netdev_adjacent_sysfs_add(iter->dev, dev, 6506 &iter->dev->adj_list.lower); 6507 } 6508 6509 list_for_each_entry(iter, &dev->adj_list.lower, list) { 6510 if (!net_eq(net, dev_net(iter->dev))) 6511 continue; 6512 netdev_adjacent_sysfs_del(iter->dev, oldname, 6513 &iter->dev->adj_list.upper); 6514 netdev_adjacent_sysfs_add(iter->dev, dev, 6515 &iter->dev->adj_list.upper); 6516 } 6517 } 6518 6519 void *netdev_lower_dev_get_private(struct net_device *dev, 6520 struct net_device *lower_dev) 6521 { 6522 struct netdev_adjacent *lower; 6523 6524 if (!lower_dev) 6525 return NULL; 6526 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower); 6527 if (!lower) 6528 return NULL; 6529 6530 return lower->private; 6531 } 6532 EXPORT_SYMBOL(netdev_lower_dev_get_private); 6533 6534 6535 int dev_get_nest_level(struct net_device *dev) 6536 { 6537 struct net_device *lower = NULL; 6538 struct list_head *iter; 6539 int max_nest = -1; 6540 int nest; 6541 6542 ASSERT_RTNL(); 6543 6544 netdev_for_each_lower_dev(dev, lower, iter) { 6545 nest = dev_get_nest_level(lower); 6546 if (max_nest < nest) 6547 max_nest = nest; 6548 } 6549 6550 return max_nest + 1; 6551 } 6552 EXPORT_SYMBOL(dev_get_nest_level); 6553 6554 /** 6555 * netdev_lower_change - Dispatch event about lower device state change 6556 * @lower_dev: device 6557 * @lower_state_info: state to dispatch 6558 * 6559 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info. 6560 * The caller must hold the RTNL lock. 6561 */ 6562 void netdev_lower_state_changed(struct net_device *lower_dev, 6563 void *lower_state_info) 6564 { 6565 struct netdev_notifier_changelowerstate_info changelowerstate_info = { 6566 .info.dev = lower_dev, 6567 }; 6568 6569 ASSERT_RTNL(); 6570 changelowerstate_info.lower_state_info = lower_state_info; 6571 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, 6572 &changelowerstate_info.info); 6573 } 6574 EXPORT_SYMBOL(netdev_lower_state_changed); 6575 6576 static void dev_change_rx_flags(struct net_device *dev, int flags) 6577 { 6578 const struct net_device_ops *ops = dev->netdev_ops; 6579 6580 if (ops->ndo_change_rx_flags) 6581 ops->ndo_change_rx_flags(dev, flags); 6582 } 6583 6584 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify) 6585 { 6586 unsigned int old_flags = dev->flags; 6587 kuid_t uid; 6588 kgid_t gid; 6589 6590 ASSERT_RTNL(); 6591 6592 dev->flags |= IFF_PROMISC; 6593 dev->promiscuity += inc; 6594 if (dev->promiscuity == 0) { 6595 /* 6596 * Avoid overflow. 6597 * If inc causes overflow, untouch promisc and return error. 6598 */ 6599 if (inc < 0) 6600 dev->flags &= ~IFF_PROMISC; 6601 else { 6602 dev->promiscuity -= inc; 6603 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n", 6604 dev->name); 6605 return -EOVERFLOW; 6606 } 6607 } 6608 if (dev->flags != old_flags) { 6609 pr_info("device %s %s promiscuous mode\n", 6610 dev->name, 6611 dev->flags & IFF_PROMISC ? "entered" : "left"); 6612 if (audit_enabled) { 6613 current_uid_gid(&uid, &gid); 6614 audit_log(current->audit_context, GFP_ATOMIC, 6615 AUDIT_ANOM_PROMISCUOUS, 6616 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u", 6617 dev->name, (dev->flags & IFF_PROMISC), 6618 (old_flags & IFF_PROMISC), 6619 from_kuid(&init_user_ns, audit_get_loginuid(current)), 6620 from_kuid(&init_user_ns, uid), 6621 from_kgid(&init_user_ns, gid), 6622 audit_get_sessionid(current)); 6623 } 6624 6625 dev_change_rx_flags(dev, IFF_PROMISC); 6626 } 6627 if (notify) 6628 __dev_notify_flags(dev, old_flags, IFF_PROMISC); 6629 return 0; 6630 } 6631 6632 /** 6633 * dev_set_promiscuity - update promiscuity count on a device 6634 * @dev: device 6635 * @inc: modifier 6636 * 6637 * Add or remove promiscuity from a device. While the count in the device 6638 * remains above zero the interface remains promiscuous. Once it hits zero 6639 * the device reverts back to normal filtering operation. A negative inc 6640 * value is used to drop promiscuity on the device. 6641 * Return 0 if successful or a negative errno code on error. 6642 */ 6643 int dev_set_promiscuity(struct net_device *dev, int inc) 6644 { 6645 unsigned int old_flags = dev->flags; 6646 int err; 6647 6648 err = __dev_set_promiscuity(dev, inc, true); 6649 if (err < 0) 6650 return err; 6651 if (dev->flags != old_flags) 6652 dev_set_rx_mode(dev); 6653 return err; 6654 } 6655 EXPORT_SYMBOL(dev_set_promiscuity); 6656 6657 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify) 6658 { 6659 unsigned int old_flags = dev->flags, old_gflags = dev->gflags; 6660 6661 ASSERT_RTNL(); 6662 6663 dev->flags |= IFF_ALLMULTI; 6664 dev->allmulti += inc; 6665 if (dev->allmulti == 0) { 6666 /* 6667 * Avoid overflow. 6668 * If inc causes overflow, untouch allmulti and return error. 6669 */ 6670 if (inc < 0) 6671 dev->flags &= ~IFF_ALLMULTI; 6672 else { 6673 dev->allmulti -= inc; 6674 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n", 6675 dev->name); 6676 return -EOVERFLOW; 6677 } 6678 } 6679 if (dev->flags ^ old_flags) { 6680 dev_change_rx_flags(dev, IFF_ALLMULTI); 6681 dev_set_rx_mode(dev); 6682 if (notify) 6683 __dev_notify_flags(dev, old_flags, 6684 dev->gflags ^ old_gflags); 6685 } 6686 return 0; 6687 } 6688 6689 /** 6690 * dev_set_allmulti - update allmulti count on a device 6691 * @dev: device 6692 * @inc: modifier 6693 * 6694 * Add or remove reception of all multicast frames to a device. While the 6695 * count in the device remains above zero the interface remains listening 6696 * to all interfaces. Once it hits zero the device reverts back to normal 6697 * filtering operation. A negative @inc value is used to drop the counter 6698 * when releasing a resource needing all multicasts. 6699 * Return 0 if successful or a negative errno code on error. 6700 */ 6701 6702 int dev_set_allmulti(struct net_device *dev, int inc) 6703 { 6704 return __dev_set_allmulti(dev, inc, true); 6705 } 6706 EXPORT_SYMBOL(dev_set_allmulti); 6707 6708 /* 6709 * Upload unicast and multicast address lists to device and 6710 * configure RX filtering. When the device doesn't support unicast 6711 * filtering it is put in promiscuous mode while unicast addresses 6712 * are present. 6713 */ 6714 void __dev_set_rx_mode(struct net_device *dev) 6715 { 6716 const struct net_device_ops *ops = dev->netdev_ops; 6717 6718 /* dev_open will call this function so the list will stay sane. */ 6719 if (!(dev->flags&IFF_UP)) 6720 return; 6721 6722 if (!netif_device_present(dev)) 6723 return; 6724 6725 if (!(dev->priv_flags & IFF_UNICAST_FLT)) { 6726 /* Unicast addresses changes may only happen under the rtnl, 6727 * therefore calling __dev_set_promiscuity here is safe. 6728 */ 6729 if (!netdev_uc_empty(dev) && !dev->uc_promisc) { 6730 __dev_set_promiscuity(dev, 1, false); 6731 dev->uc_promisc = true; 6732 } else if (netdev_uc_empty(dev) && dev->uc_promisc) { 6733 __dev_set_promiscuity(dev, -1, false); 6734 dev->uc_promisc = false; 6735 } 6736 } 6737 6738 if (ops->ndo_set_rx_mode) 6739 ops->ndo_set_rx_mode(dev); 6740 } 6741 6742 void dev_set_rx_mode(struct net_device *dev) 6743 { 6744 netif_addr_lock_bh(dev); 6745 __dev_set_rx_mode(dev); 6746 netif_addr_unlock_bh(dev); 6747 } 6748 6749 /** 6750 * dev_get_flags - get flags reported to userspace 6751 * @dev: device 6752 * 6753 * Get the combination of flag bits exported through APIs to userspace. 6754 */ 6755 unsigned int dev_get_flags(const struct net_device *dev) 6756 { 6757 unsigned int flags; 6758 6759 flags = (dev->flags & ~(IFF_PROMISC | 6760 IFF_ALLMULTI | 6761 IFF_RUNNING | 6762 IFF_LOWER_UP | 6763 IFF_DORMANT)) | 6764 (dev->gflags & (IFF_PROMISC | 6765 IFF_ALLMULTI)); 6766 6767 if (netif_running(dev)) { 6768 if (netif_oper_up(dev)) 6769 flags |= IFF_RUNNING; 6770 if (netif_carrier_ok(dev)) 6771 flags |= IFF_LOWER_UP; 6772 if (netif_dormant(dev)) 6773 flags |= IFF_DORMANT; 6774 } 6775 6776 return flags; 6777 } 6778 EXPORT_SYMBOL(dev_get_flags); 6779 6780 int __dev_change_flags(struct net_device *dev, unsigned int flags) 6781 { 6782 unsigned int old_flags = dev->flags; 6783 int ret; 6784 6785 ASSERT_RTNL(); 6786 6787 /* 6788 * Set the flags on our device. 6789 */ 6790 6791 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | 6792 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL | 6793 IFF_AUTOMEDIA)) | 6794 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | 6795 IFF_ALLMULTI)); 6796 6797 /* 6798 * Load in the correct multicast list now the flags have changed. 6799 */ 6800 6801 if ((old_flags ^ flags) & IFF_MULTICAST) 6802 dev_change_rx_flags(dev, IFF_MULTICAST); 6803 6804 dev_set_rx_mode(dev); 6805 6806 /* 6807 * Have we downed the interface. We handle IFF_UP ourselves 6808 * according to user attempts to set it, rather than blindly 6809 * setting it. 6810 */ 6811 6812 ret = 0; 6813 if ((old_flags ^ flags) & IFF_UP) { 6814 if (old_flags & IFF_UP) 6815 __dev_close(dev); 6816 else 6817 ret = __dev_open(dev); 6818 } 6819 6820 if ((flags ^ dev->gflags) & IFF_PROMISC) { 6821 int inc = (flags & IFF_PROMISC) ? 1 : -1; 6822 unsigned int old_flags = dev->flags; 6823 6824 dev->gflags ^= IFF_PROMISC; 6825 6826 if (__dev_set_promiscuity(dev, inc, false) >= 0) 6827 if (dev->flags != old_flags) 6828 dev_set_rx_mode(dev); 6829 } 6830 6831 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI 6832 * is important. Some (broken) drivers set IFF_PROMISC, when 6833 * IFF_ALLMULTI is requested not asking us and not reporting. 6834 */ 6835 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { 6836 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; 6837 6838 dev->gflags ^= IFF_ALLMULTI; 6839 __dev_set_allmulti(dev, inc, false); 6840 } 6841 6842 return ret; 6843 } 6844 6845 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, 6846 unsigned int gchanges) 6847 { 6848 unsigned int changes = dev->flags ^ old_flags; 6849 6850 if (gchanges) 6851 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC); 6852 6853 if (changes & IFF_UP) { 6854 if (dev->flags & IFF_UP) 6855 call_netdevice_notifiers(NETDEV_UP, dev); 6856 else 6857 call_netdevice_notifiers(NETDEV_DOWN, dev); 6858 } 6859 6860 if (dev->flags & IFF_UP && 6861 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) { 6862 struct netdev_notifier_change_info change_info = { 6863 .info = { 6864 .dev = dev, 6865 }, 6866 .flags_changed = changes, 6867 }; 6868 6869 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info); 6870 } 6871 } 6872 6873 /** 6874 * dev_change_flags - change device settings 6875 * @dev: device 6876 * @flags: device state flags 6877 * 6878 * Change settings on device based state flags. The flags are 6879 * in the userspace exported format. 6880 */ 6881 int dev_change_flags(struct net_device *dev, unsigned int flags) 6882 { 6883 int ret; 6884 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; 6885 6886 ret = __dev_change_flags(dev, flags); 6887 if (ret < 0) 6888 return ret; 6889 6890 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); 6891 __dev_notify_flags(dev, old_flags, changes); 6892 return ret; 6893 } 6894 EXPORT_SYMBOL(dev_change_flags); 6895 6896 int __dev_set_mtu(struct net_device *dev, int new_mtu) 6897 { 6898 const struct net_device_ops *ops = dev->netdev_ops; 6899 6900 if (ops->ndo_change_mtu) 6901 return ops->ndo_change_mtu(dev, new_mtu); 6902 6903 dev->mtu = new_mtu; 6904 return 0; 6905 } 6906 EXPORT_SYMBOL(__dev_set_mtu); 6907 6908 /** 6909 * dev_set_mtu - Change maximum transfer unit 6910 * @dev: device 6911 * @new_mtu: new transfer unit 6912 * 6913 * Change the maximum transfer size of the network device. 6914 */ 6915 int dev_set_mtu(struct net_device *dev, int new_mtu) 6916 { 6917 int err, orig_mtu; 6918 6919 if (new_mtu == dev->mtu) 6920 return 0; 6921 6922 /* MTU must be positive, and in range */ 6923 if (new_mtu < 0 || new_mtu < dev->min_mtu) { 6924 net_err_ratelimited("%s: Invalid MTU %d requested, hw min %d\n", 6925 dev->name, new_mtu, dev->min_mtu); 6926 return -EINVAL; 6927 } 6928 6929 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) { 6930 net_err_ratelimited("%s: Invalid MTU %d requested, hw max %d\n", 6931 dev->name, new_mtu, dev->max_mtu); 6932 return -EINVAL; 6933 } 6934 6935 if (!netif_device_present(dev)) 6936 return -ENODEV; 6937 6938 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev); 6939 err = notifier_to_errno(err); 6940 if (err) 6941 return err; 6942 6943 orig_mtu = dev->mtu; 6944 err = __dev_set_mtu(dev, new_mtu); 6945 6946 if (!err) { 6947 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); 6948 err = notifier_to_errno(err); 6949 if (err) { 6950 /* setting mtu back and notifying everyone again, 6951 * so that they have a chance to revert changes. 6952 */ 6953 __dev_set_mtu(dev, orig_mtu); 6954 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); 6955 } 6956 } 6957 return err; 6958 } 6959 EXPORT_SYMBOL(dev_set_mtu); 6960 6961 /** 6962 * dev_set_group - Change group this device belongs to 6963 * @dev: device 6964 * @new_group: group this device should belong to 6965 */ 6966 void dev_set_group(struct net_device *dev, int new_group) 6967 { 6968 dev->group = new_group; 6969 } 6970 EXPORT_SYMBOL(dev_set_group); 6971 6972 /** 6973 * dev_set_mac_address - Change Media Access Control Address 6974 * @dev: device 6975 * @sa: new address 6976 * 6977 * Change the hardware (MAC) address of the device 6978 */ 6979 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa) 6980 { 6981 const struct net_device_ops *ops = dev->netdev_ops; 6982 int err; 6983 6984 if (!ops->ndo_set_mac_address) 6985 return -EOPNOTSUPP; 6986 if (sa->sa_family != dev->type) 6987 return -EINVAL; 6988 if (!netif_device_present(dev)) 6989 return -ENODEV; 6990 err = ops->ndo_set_mac_address(dev, sa); 6991 if (err) 6992 return err; 6993 dev->addr_assign_type = NET_ADDR_SET; 6994 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 6995 add_device_randomness(dev->dev_addr, dev->addr_len); 6996 return 0; 6997 } 6998 EXPORT_SYMBOL(dev_set_mac_address); 6999 7000 /** 7001 * dev_change_carrier - Change device carrier 7002 * @dev: device 7003 * @new_carrier: new value 7004 * 7005 * Change device carrier 7006 */ 7007 int dev_change_carrier(struct net_device *dev, bool new_carrier) 7008 { 7009 const struct net_device_ops *ops = dev->netdev_ops; 7010 7011 if (!ops->ndo_change_carrier) 7012 return -EOPNOTSUPP; 7013 if (!netif_device_present(dev)) 7014 return -ENODEV; 7015 return ops->ndo_change_carrier(dev, new_carrier); 7016 } 7017 EXPORT_SYMBOL(dev_change_carrier); 7018 7019 /** 7020 * dev_get_phys_port_id - Get device physical port ID 7021 * @dev: device 7022 * @ppid: port ID 7023 * 7024 * Get device physical port ID 7025 */ 7026 int dev_get_phys_port_id(struct net_device *dev, 7027 struct netdev_phys_item_id *ppid) 7028 { 7029 const struct net_device_ops *ops = dev->netdev_ops; 7030 7031 if (!ops->ndo_get_phys_port_id) 7032 return -EOPNOTSUPP; 7033 return ops->ndo_get_phys_port_id(dev, ppid); 7034 } 7035 EXPORT_SYMBOL(dev_get_phys_port_id); 7036 7037 /** 7038 * dev_get_phys_port_name - Get device physical port name 7039 * @dev: device 7040 * @name: port name 7041 * @len: limit of bytes to copy to name 7042 * 7043 * Get device physical port name 7044 */ 7045 int dev_get_phys_port_name(struct net_device *dev, 7046 char *name, size_t len) 7047 { 7048 const struct net_device_ops *ops = dev->netdev_ops; 7049 7050 if (!ops->ndo_get_phys_port_name) 7051 return -EOPNOTSUPP; 7052 return ops->ndo_get_phys_port_name(dev, name, len); 7053 } 7054 EXPORT_SYMBOL(dev_get_phys_port_name); 7055 7056 /** 7057 * dev_change_proto_down - update protocol port state information 7058 * @dev: device 7059 * @proto_down: new value 7060 * 7061 * This info can be used by switch drivers to set the phys state of the 7062 * port. 7063 */ 7064 int dev_change_proto_down(struct net_device *dev, bool proto_down) 7065 { 7066 const struct net_device_ops *ops = dev->netdev_ops; 7067 7068 if (!ops->ndo_change_proto_down) 7069 return -EOPNOTSUPP; 7070 if (!netif_device_present(dev)) 7071 return -ENODEV; 7072 return ops->ndo_change_proto_down(dev, proto_down); 7073 } 7074 EXPORT_SYMBOL(dev_change_proto_down); 7075 7076 u8 __dev_xdp_attached(struct net_device *dev, bpf_op_t bpf_op, u32 *prog_id) 7077 { 7078 struct netdev_bpf xdp; 7079 7080 memset(&xdp, 0, sizeof(xdp)); 7081 xdp.command = XDP_QUERY_PROG; 7082 7083 /* Query must always succeed. */ 7084 WARN_ON(bpf_op(dev, &xdp) < 0); 7085 if (prog_id) 7086 *prog_id = xdp.prog_id; 7087 7088 return xdp.prog_attached; 7089 } 7090 7091 static int dev_xdp_install(struct net_device *dev, bpf_op_t bpf_op, 7092 struct netlink_ext_ack *extack, u32 flags, 7093 struct bpf_prog *prog) 7094 { 7095 struct netdev_bpf xdp; 7096 7097 memset(&xdp, 0, sizeof(xdp)); 7098 if (flags & XDP_FLAGS_HW_MODE) 7099 xdp.command = XDP_SETUP_PROG_HW; 7100 else 7101 xdp.command = XDP_SETUP_PROG; 7102 xdp.extack = extack; 7103 xdp.flags = flags; 7104 xdp.prog = prog; 7105 7106 return bpf_op(dev, &xdp); 7107 } 7108 7109 /** 7110 * dev_change_xdp_fd - set or clear a bpf program for a device rx path 7111 * @dev: device 7112 * @extack: netlink extended ack 7113 * @fd: new program fd or negative value to clear 7114 * @flags: xdp-related flags 7115 * 7116 * Set or clear a bpf program for a device 7117 */ 7118 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, 7119 int fd, u32 flags) 7120 { 7121 const struct net_device_ops *ops = dev->netdev_ops; 7122 struct bpf_prog *prog = NULL; 7123 bpf_op_t bpf_op, bpf_chk; 7124 int err; 7125 7126 ASSERT_RTNL(); 7127 7128 bpf_op = bpf_chk = ops->ndo_bpf; 7129 if (!bpf_op && (flags & (XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE))) 7130 return -EOPNOTSUPP; 7131 if (!bpf_op || (flags & XDP_FLAGS_SKB_MODE)) 7132 bpf_op = generic_xdp_install; 7133 if (bpf_op == bpf_chk) 7134 bpf_chk = generic_xdp_install; 7135 7136 if (fd >= 0) { 7137 if (bpf_chk && __dev_xdp_attached(dev, bpf_chk, NULL)) 7138 return -EEXIST; 7139 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && 7140 __dev_xdp_attached(dev, bpf_op, NULL)) 7141 return -EBUSY; 7142 7143 prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP, 7144 bpf_op == ops->ndo_bpf); 7145 if (IS_ERR(prog)) 7146 return PTR_ERR(prog); 7147 7148 if (!(flags & XDP_FLAGS_HW_MODE) && 7149 bpf_prog_is_dev_bound(prog->aux)) { 7150 NL_SET_ERR_MSG(extack, "using device-bound program without HW_MODE flag is not supported"); 7151 bpf_prog_put(prog); 7152 return -EINVAL; 7153 } 7154 } 7155 7156 err = dev_xdp_install(dev, bpf_op, extack, flags, prog); 7157 if (err < 0 && prog) 7158 bpf_prog_put(prog); 7159 7160 return err; 7161 } 7162 7163 /** 7164 * dev_new_index - allocate an ifindex 7165 * @net: the applicable net namespace 7166 * 7167 * Returns a suitable unique value for a new device interface 7168 * number. The caller must hold the rtnl semaphore or the 7169 * dev_base_lock to be sure it remains unique. 7170 */ 7171 static int dev_new_index(struct net *net) 7172 { 7173 int ifindex = net->ifindex; 7174 7175 for (;;) { 7176 if (++ifindex <= 0) 7177 ifindex = 1; 7178 if (!__dev_get_by_index(net, ifindex)) 7179 return net->ifindex = ifindex; 7180 } 7181 } 7182 7183 /* Delayed registration/unregisteration */ 7184 static LIST_HEAD(net_todo_list); 7185 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq); 7186 7187 static void net_set_todo(struct net_device *dev) 7188 { 7189 list_add_tail(&dev->todo_list, &net_todo_list); 7190 dev_net(dev)->dev_unreg_count++; 7191 } 7192 7193 static void rollback_registered_many(struct list_head *head) 7194 { 7195 struct net_device *dev, *tmp; 7196 LIST_HEAD(close_head); 7197 7198 BUG_ON(dev_boot_phase); 7199 ASSERT_RTNL(); 7200 7201 list_for_each_entry_safe(dev, tmp, head, unreg_list) { 7202 /* Some devices call without registering 7203 * for initialization unwind. Remove those 7204 * devices and proceed with the remaining. 7205 */ 7206 if (dev->reg_state == NETREG_UNINITIALIZED) { 7207 pr_debug("unregister_netdevice: device %s/%p never was registered\n", 7208 dev->name, dev); 7209 7210 WARN_ON(1); 7211 list_del(&dev->unreg_list); 7212 continue; 7213 } 7214 dev->dismantle = true; 7215 BUG_ON(dev->reg_state != NETREG_REGISTERED); 7216 } 7217 7218 /* If device is running, close it first. */ 7219 list_for_each_entry(dev, head, unreg_list) 7220 list_add_tail(&dev->close_list, &close_head); 7221 dev_close_many(&close_head, true); 7222 7223 list_for_each_entry(dev, head, unreg_list) { 7224 /* And unlink it from device chain. */ 7225 unlist_netdevice(dev); 7226 7227 dev->reg_state = NETREG_UNREGISTERING; 7228 } 7229 flush_all_backlogs(); 7230 7231 synchronize_net(); 7232 7233 list_for_each_entry(dev, head, unreg_list) { 7234 struct sk_buff *skb = NULL; 7235 7236 /* Shutdown queueing discipline. */ 7237 dev_shutdown(dev); 7238 7239 7240 /* Notify protocols, that we are about to destroy 7241 * this device. They should clean all the things. 7242 */ 7243 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 7244 7245 if (!dev->rtnl_link_ops || 7246 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 7247 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0, 7248 GFP_KERNEL, NULL); 7249 7250 /* 7251 * Flush the unicast and multicast chains 7252 */ 7253 dev_uc_flush(dev); 7254 dev_mc_flush(dev); 7255 7256 if (dev->netdev_ops->ndo_uninit) 7257 dev->netdev_ops->ndo_uninit(dev); 7258 7259 if (skb) 7260 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL); 7261 7262 /* Notifier chain MUST detach us all upper devices. */ 7263 WARN_ON(netdev_has_any_upper_dev(dev)); 7264 WARN_ON(netdev_has_any_lower_dev(dev)); 7265 7266 /* Remove entries from kobject tree */ 7267 netdev_unregister_kobject(dev); 7268 #ifdef CONFIG_XPS 7269 /* Remove XPS queueing entries */ 7270 netif_reset_xps_queues_gt(dev, 0); 7271 #endif 7272 } 7273 7274 synchronize_net(); 7275 7276 list_for_each_entry(dev, head, unreg_list) 7277 dev_put(dev); 7278 } 7279 7280 static void rollback_registered(struct net_device *dev) 7281 { 7282 LIST_HEAD(single); 7283 7284 list_add(&dev->unreg_list, &single); 7285 rollback_registered_many(&single); 7286 list_del(&single); 7287 } 7288 7289 static netdev_features_t netdev_sync_upper_features(struct net_device *lower, 7290 struct net_device *upper, netdev_features_t features) 7291 { 7292 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 7293 netdev_features_t feature; 7294 int feature_bit; 7295 7296 for_each_netdev_feature(&upper_disables, feature_bit) { 7297 feature = __NETIF_F_BIT(feature_bit); 7298 if (!(upper->wanted_features & feature) 7299 && (features & feature)) { 7300 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n", 7301 &feature, upper->name); 7302 features &= ~feature; 7303 } 7304 } 7305 7306 return features; 7307 } 7308 7309 static void netdev_sync_lower_features(struct net_device *upper, 7310 struct net_device *lower, netdev_features_t features) 7311 { 7312 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 7313 netdev_features_t feature; 7314 int feature_bit; 7315 7316 for_each_netdev_feature(&upper_disables, feature_bit) { 7317 feature = __NETIF_F_BIT(feature_bit); 7318 if (!(features & feature) && (lower->features & feature)) { 7319 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n", 7320 &feature, lower->name); 7321 lower->wanted_features &= ~feature; 7322 netdev_update_features(lower); 7323 7324 if (unlikely(lower->features & feature)) 7325 netdev_WARN(upper, "failed to disable %pNF on %s!\n", 7326 &feature, lower->name); 7327 } 7328 } 7329 } 7330 7331 static netdev_features_t netdev_fix_features(struct net_device *dev, 7332 netdev_features_t features) 7333 { 7334 /* Fix illegal checksum combinations */ 7335 if ((features & NETIF_F_HW_CSUM) && 7336 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { 7337 netdev_warn(dev, "mixed HW and IP checksum settings.\n"); 7338 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); 7339 } 7340 7341 /* TSO requires that SG is present as well. */ 7342 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) { 7343 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n"); 7344 features &= ~NETIF_F_ALL_TSO; 7345 } 7346 7347 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) && 7348 !(features & NETIF_F_IP_CSUM)) { 7349 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n"); 7350 features &= ~NETIF_F_TSO; 7351 features &= ~NETIF_F_TSO_ECN; 7352 } 7353 7354 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) && 7355 !(features & NETIF_F_IPV6_CSUM)) { 7356 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n"); 7357 features &= ~NETIF_F_TSO6; 7358 } 7359 7360 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */ 7361 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO)) 7362 features &= ~NETIF_F_TSO_MANGLEID; 7363 7364 /* TSO ECN requires that TSO is present as well. */ 7365 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN) 7366 features &= ~NETIF_F_TSO_ECN; 7367 7368 /* Software GSO depends on SG. */ 7369 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) { 7370 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n"); 7371 features &= ~NETIF_F_GSO; 7372 } 7373 7374 /* GSO partial features require GSO partial be set */ 7375 if ((features & dev->gso_partial_features) && 7376 !(features & NETIF_F_GSO_PARTIAL)) { 7377 netdev_dbg(dev, 7378 "Dropping partially supported GSO features since no GSO partial.\n"); 7379 features &= ~dev->gso_partial_features; 7380 } 7381 7382 return features; 7383 } 7384 7385 int __netdev_update_features(struct net_device *dev) 7386 { 7387 struct net_device *upper, *lower; 7388 netdev_features_t features; 7389 struct list_head *iter; 7390 int err = -1; 7391 7392 ASSERT_RTNL(); 7393 7394 features = netdev_get_wanted_features(dev); 7395 7396 if (dev->netdev_ops->ndo_fix_features) 7397 features = dev->netdev_ops->ndo_fix_features(dev, features); 7398 7399 /* driver might be less strict about feature dependencies */ 7400 features = netdev_fix_features(dev, features); 7401 7402 /* some features can't be enabled if they're off an an upper device */ 7403 netdev_for_each_upper_dev_rcu(dev, upper, iter) 7404 features = netdev_sync_upper_features(dev, upper, features); 7405 7406 if (dev->features == features) 7407 goto sync_lower; 7408 7409 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", 7410 &dev->features, &features); 7411 7412 if (dev->netdev_ops->ndo_set_features) 7413 err = dev->netdev_ops->ndo_set_features(dev, features); 7414 else 7415 err = 0; 7416 7417 if (unlikely(err < 0)) { 7418 netdev_err(dev, 7419 "set_features() failed (%d); wanted %pNF, left %pNF\n", 7420 err, &features, &dev->features); 7421 /* return non-0 since some features might have changed and 7422 * it's better to fire a spurious notification than miss it 7423 */ 7424 return -1; 7425 } 7426 7427 sync_lower: 7428 /* some features must be disabled on lower devices when disabled 7429 * on an upper device (think: bonding master or bridge) 7430 */ 7431 netdev_for_each_lower_dev(dev, lower, iter) 7432 netdev_sync_lower_features(dev, lower, features); 7433 7434 if (!err) { 7435 netdev_features_t diff = features ^ dev->features; 7436 7437 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) { 7438 /* udp_tunnel_{get,drop}_rx_info both need 7439 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the 7440 * device, or they won't do anything. 7441 * Thus we need to update dev->features 7442 * *before* calling udp_tunnel_get_rx_info, 7443 * but *after* calling udp_tunnel_drop_rx_info. 7444 */ 7445 if (features & NETIF_F_RX_UDP_TUNNEL_PORT) { 7446 dev->features = features; 7447 udp_tunnel_get_rx_info(dev); 7448 } else { 7449 udp_tunnel_drop_rx_info(dev); 7450 } 7451 } 7452 7453 dev->features = features; 7454 } 7455 7456 return err < 0 ? 0 : 1; 7457 } 7458 7459 /** 7460 * netdev_update_features - recalculate device features 7461 * @dev: the device to check 7462 * 7463 * Recalculate dev->features set and send notifications if it 7464 * has changed. Should be called after driver or hardware dependent 7465 * conditions might have changed that influence the features. 7466 */ 7467 void netdev_update_features(struct net_device *dev) 7468 { 7469 if (__netdev_update_features(dev)) 7470 netdev_features_change(dev); 7471 } 7472 EXPORT_SYMBOL(netdev_update_features); 7473 7474 /** 7475 * netdev_change_features - recalculate device features 7476 * @dev: the device to check 7477 * 7478 * Recalculate dev->features set and send notifications even 7479 * if they have not changed. Should be called instead of 7480 * netdev_update_features() if also dev->vlan_features might 7481 * have changed to allow the changes to be propagated to stacked 7482 * VLAN devices. 7483 */ 7484 void netdev_change_features(struct net_device *dev) 7485 { 7486 __netdev_update_features(dev); 7487 netdev_features_change(dev); 7488 } 7489 EXPORT_SYMBOL(netdev_change_features); 7490 7491 /** 7492 * netif_stacked_transfer_operstate - transfer operstate 7493 * @rootdev: the root or lower level device to transfer state from 7494 * @dev: the device to transfer operstate to 7495 * 7496 * Transfer operational state from root to device. This is normally 7497 * called when a stacking relationship exists between the root 7498 * device and the device(a leaf device). 7499 */ 7500 void netif_stacked_transfer_operstate(const struct net_device *rootdev, 7501 struct net_device *dev) 7502 { 7503 if (rootdev->operstate == IF_OPER_DORMANT) 7504 netif_dormant_on(dev); 7505 else 7506 netif_dormant_off(dev); 7507 7508 if (netif_carrier_ok(rootdev)) 7509 netif_carrier_on(dev); 7510 else 7511 netif_carrier_off(dev); 7512 } 7513 EXPORT_SYMBOL(netif_stacked_transfer_operstate); 7514 7515 #ifdef CONFIG_SYSFS 7516 static int netif_alloc_rx_queues(struct net_device *dev) 7517 { 7518 unsigned int i, count = dev->num_rx_queues; 7519 struct netdev_rx_queue *rx; 7520 size_t sz = count * sizeof(*rx); 7521 7522 BUG_ON(count < 1); 7523 7524 rx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL); 7525 if (!rx) 7526 return -ENOMEM; 7527 7528 dev->_rx = rx; 7529 7530 for (i = 0; i < count; i++) 7531 rx[i].dev = dev; 7532 return 0; 7533 } 7534 #endif 7535 7536 static void netdev_init_one_queue(struct net_device *dev, 7537 struct netdev_queue *queue, void *_unused) 7538 { 7539 /* Initialize queue lock */ 7540 spin_lock_init(&queue->_xmit_lock); 7541 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); 7542 queue->xmit_lock_owner = -1; 7543 netdev_queue_numa_node_write(queue, NUMA_NO_NODE); 7544 queue->dev = dev; 7545 #ifdef CONFIG_BQL 7546 dql_init(&queue->dql, HZ); 7547 #endif 7548 } 7549 7550 static void netif_free_tx_queues(struct net_device *dev) 7551 { 7552 kvfree(dev->_tx); 7553 } 7554 7555 static int netif_alloc_netdev_queues(struct net_device *dev) 7556 { 7557 unsigned int count = dev->num_tx_queues; 7558 struct netdev_queue *tx; 7559 size_t sz = count * sizeof(*tx); 7560 7561 if (count < 1 || count > 0xffff) 7562 return -EINVAL; 7563 7564 tx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL); 7565 if (!tx) 7566 return -ENOMEM; 7567 7568 dev->_tx = tx; 7569 7570 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); 7571 spin_lock_init(&dev->tx_global_lock); 7572 7573 return 0; 7574 } 7575 7576 void netif_tx_stop_all_queues(struct net_device *dev) 7577 { 7578 unsigned int i; 7579 7580 for (i = 0; i < dev->num_tx_queues; i++) { 7581 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 7582 7583 netif_tx_stop_queue(txq); 7584 } 7585 } 7586 EXPORT_SYMBOL(netif_tx_stop_all_queues); 7587 7588 /** 7589 * register_netdevice - register a network device 7590 * @dev: device to register 7591 * 7592 * Take a completed network device structure and add it to the kernel 7593 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 7594 * chain. 0 is returned on success. A negative errno code is returned 7595 * on a failure to set up the device, or if the name is a duplicate. 7596 * 7597 * Callers must hold the rtnl semaphore. You may want 7598 * register_netdev() instead of this. 7599 * 7600 * BUGS: 7601 * The locking appears insufficient to guarantee two parallel registers 7602 * will not get the same name. 7603 */ 7604 7605 int register_netdevice(struct net_device *dev) 7606 { 7607 int ret; 7608 struct net *net = dev_net(dev); 7609 7610 BUG_ON(dev_boot_phase); 7611 ASSERT_RTNL(); 7612 7613 might_sleep(); 7614 7615 /* When net_device's are persistent, this will be fatal. */ 7616 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); 7617 BUG_ON(!net); 7618 7619 spin_lock_init(&dev->addr_list_lock); 7620 netdev_set_addr_lockdep_class(dev); 7621 7622 ret = dev_get_valid_name(net, dev, dev->name); 7623 if (ret < 0) 7624 goto out; 7625 7626 /* Init, if this function is available */ 7627 if (dev->netdev_ops->ndo_init) { 7628 ret = dev->netdev_ops->ndo_init(dev); 7629 if (ret) { 7630 if (ret > 0) 7631 ret = -EIO; 7632 goto out; 7633 } 7634 } 7635 7636 if (((dev->hw_features | dev->features) & 7637 NETIF_F_HW_VLAN_CTAG_FILTER) && 7638 (!dev->netdev_ops->ndo_vlan_rx_add_vid || 7639 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { 7640 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n"); 7641 ret = -EINVAL; 7642 goto err_uninit; 7643 } 7644 7645 ret = -EBUSY; 7646 if (!dev->ifindex) 7647 dev->ifindex = dev_new_index(net); 7648 else if (__dev_get_by_index(net, dev->ifindex)) 7649 goto err_uninit; 7650 7651 /* Transfer changeable features to wanted_features and enable 7652 * software offloads (GSO and GRO). 7653 */ 7654 dev->hw_features |= NETIF_F_SOFT_FEATURES; 7655 dev->features |= NETIF_F_SOFT_FEATURES; 7656 7657 if (dev->netdev_ops->ndo_udp_tunnel_add) { 7658 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT; 7659 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT; 7660 } 7661 7662 dev->wanted_features = dev->features & dev->hw_features; 7663 7664 if (!(dev->flags & IFF_LOOPBACK)) 7665 dev->hw_features |= NETIF_F_NOCACHE_COPY; 7666 7667 /* If IPv4 TCP segmentation offload is supported we should also 7668 * allow the device to enable segmenting the frame with the option 7669 * of ignoring a static IP ID value. This doesn't enable the 7670 * feature itself but allows the user to enable it later. 7671 */ 7672 if (dev->hw_features & NETIF_F_TSO) 7673 dev->hw_features |= NETIF_F_TSO_MANGLEID; 7674 if (dev->vlan_features & NETIF_F_TSO) 7675 dev->vlan_features |= NETIF_F_TSO_MANGLEID; 7676 if (dev->mpls_features & NETIF_F_TSO) 7677 dev->mpls_features |= NETIF_F_TSO_MANGLEID; 7678 if (dev->hw_enc_features & NETIF_F_TSO) 7679 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 7680 7681 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. 7682 */ 7683 dev->vlan_features |= NETIF_F_HIGHDMA; 7684 7685 /* Make NETIF_F_SG inheritable to tunnel devices. 7686 */ 7687 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL; 7688 7689 /* Make NETIF_F_SG inheritable to MPLS. 7690 */ 7691 dev->mpls_features |= NETIF_F_SG; 7692 7693 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); 7694 ret = notifier_to_errno(ret); 7695 if (ret) 7696 goto err_uninit; 7697 7698 ret = netdev_register_kobject(dev); 7699 if (ret) 7700 goto err_uninit; 7701 dev->reg_state = NETREG_REGISTERED; 7702 7703 __netdev_update_features(dev); 7704 7705 /* 7706 * Default initial state at registry is that the 7707 * device is present. 7708 */ 7709 7710 set_bit(__LINK_STATE_PRESENT, &dev->state); 7711 7712 linkwatch_init_dev(dev); 7713 7714 dev_init_scheduler(dev); 7715 dev_hold(dev); 7716 list_netdevice(dev); 7717 add_device_randomness(dev->dev_addr, dev->addr_len); 7718 7719 /* If the device has permanent device address, driver should 7720 * set dev_addr and also addr_assign_type should be set to 7721 * NET_ADDR_PERM (default value). 7722 */ 7723 if (dev->addr_assign_type == NET_ADDR_PERM) 7724 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); 7725 7726 /* Notify protocols, that a new device appeared. */ 7727 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); 7728 ret = notifier_to_errno(ret); 7729 if (ret) { 7730 rollback_registered(dev); 7731 dev->reg_state = NETREG_UNREGISTERED; 7732 } 7733 /* 7734 * Prevent userspace races by waiting until the network 7735 * device is fully setup before sending notifications. 7736 */ 7737 if (!dev->rtnl_link_ops || 7738 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 7739 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); 7740 7741 out: 7742 return ret; 7743 7744 err_uninit: 7745 if (dev->netdev_ops->ndo_uninit) 7746 dev->netdev_ops->ndo_uninit(dev); 7747 if (dev->priv_destructor) 7748 dev->priv_destructor(dev); 7749 goto out; 7750 } 7751 EXPORT_SYMBOL(register_netdevice); 7752 7753 /** 7754 * init_dummy_netdev - init a dummy network device for NAPI 7755 * @dev: device to init 7756 * 7757 * This takes a network device structure and initialize the minimum 7758 * amount of fields so it can be used to schedule NAPI polls without 7759 * registering a full blown interface. This is to be used by drivers 7760 * that need to tie several hardware interfaces to a single NAPI 7761 * poll scheduler due to HW limitations. 7762 */ 7763 int init_dummy_netdev(struct net_device *dev) 7764 { 7765 /* Clear everything. Note we don't initialize spinlocks 7766 * are they aren't supposed to be taken by any of the 7767 * NAPI code and this dummy netdev is supposed to be 7768 * only ever used for NAPI polls 7769 */ 7770 memset(dev, 0, sizeof(struct net_device)); 7771 7772 /* make sure we BUG if trying to hit standard 7773 * register/unregister code path 7774 */ 7775 dev->reg_state = NETREG_DUMMY; 7776 7777 /* NAPI wants this */ 7778 INIT_LIST_HEAD(&dev->napi_list); 7779 7780 /* a dummy interface is started by default */ 7781 set_bit(__LINK_STATE_PRESENT, &dev->state); 7782 set_bit(__LINK_STATE_START, &dev->state); 7783 7784 /* Note : We dont allocate pcpu_refcnt for dummy devices, 7785 * because users of this 'device' dont need to change 7786 * its refcount. 7787 */ 7788 7789 return 0; 7790 } 7791 EXPORT_SYMBOL_GPL(init_dummy_netdev); 7792 7793 7794 /** 7795 * register_netdev - register a network device 7796 * @dev: device to register 7797 * 7798 * Take a completed network device structure and add it to the kernel 7799 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 7800 * chain. 0 is returned on success. A negative errno code is returned 7801 * on a failure to set up the device, or if the name is a duplicate. 7802 * 7803 * This is a wrapper around register_netdevice that takes the rtnl semaphore 7804 * and expands the device name if you passed a format string to 7805 * alloc_netdev. 7806 */ 7807 int register_netdev(struct net_device *dev) 7808 { 7809 int err; 7810 7811 rtnl_lock(); 7812 err = register_netdevice(dev); 7813 rtnl_unlock(); 7814 return err; 7815 } 7816 EXPORT_SYMBOL(register_netdev); 7817 7818 int netdev_refcnt_read(const struct net_device *dev) 7819 { 7820 int i, refcnt = 0; 7821 7822 for_each_possible_cpu(i) 7823 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); 7824 return refcnt; 7825 } 7826 EXPORT_SYMBOL(netdev_refcnt_read); 7827 7828 /** 7829 * netdev_wait_allrefs - wait until all references are gone. 7830 * @dev: target net_device 7831 * 7832 * This is called when unregistering network devices. 7833 * 7834 * Any protocol or device that holds a reference should register 7835 * for netdevice notification, and cleanup and put back the 7836 * reference if they receive an UNREGISTER event. 7837 * We can get stuck here if buggy protocols don't correctly 7838 * call dev_put. 7839 */ 7840 static void netdev_wait_allrefs(struct net_device *dev) 7841 { 7842 unsigned long rebroadcast_time, warning_time; 7843 int refcnt; 7844 7845 linkwatch_forget_dev(dev); 7846 7847 rebroadcast_time = warning_time = jiffies; 7848 refcnt = netdev_refcnt_read(dev); 7849 7850 while (refcnt != 0) { 7851 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { 7852 rtnl_lock(); 7853 7854 /* Rebroadcast unregister notification */ 7855 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 7856 7857 __rtnl_unlock(); 7858 rcu_barrier(); 7859 rtnl_lock(); 7860 7861 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 7862 if (test_bit(__LINK_STATE_LINKWATCH_PENDING, 7863 &dev->state)) { 7864 /* We must not have linkwatch events 7865 * pending on unregister. If this 7866 * happens, we simply run the queue 7867 * unscheduled, resulting in a noop 7868 * for this device. 7869 */ 7870 linkwatch_run_queue(); 7871 } 7872 7873 __rtnl_unlock(); 7874 7875 rebroadcast_time = jiffies; 7876 } 7877 7878 msleep(250); 7879 7880 refcnt = netdev_refcnt_read(dev); 7881 7882 if (time_after(jiffies, warning_time + 10 * HZ)) { 7883 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", 7884 dev->name, refcnt); 7885 warning_time = jiffies; 7886 } 7887 } 7888 } 7889 7890 /* The sequence is: 7891 * 7892 * rtnl_lock(); 7893 * ... 7894 * register_netdevice(x1); 7895 * register_netdevice(x2); 7896 * ... 7897 * unregister_netdevice(y1); 7898 * unregister_netdevice(y2); 7899 * ... 7900 * rtnl_unlock(); 7901 * free_netdev(y1); 7902 * free_netdev(y2); 7903 * 7904 * We are invoked by rtnl_unlock(). 7905 * This allows us to deal with problems: 7906 * 1) We can delete sysfs objects which invoke hotplug 7907 * without deadlocking with linkwatch via keventd. 7908 * 2) Since we run with the RTNL semaphore not held, we can sleep 7909 * safely in order to wait for the netdev refcnt to drop to zero. 7910 * 7911 * We must not return until all unregister events added during 7912 * the interval the lock was held have been completed. 7913 */ 7914 void netdev_run_todo(void) 7915 { 7916 struct list_head list; 7917 7918 /* Snapshot list, allow later requests */ 7919 list_replace_init(&net_todo_list, &list); 7920 7921 __rtnl_unlock(); 7922 7923 7924 /* Wait for rcu callbacks to finish before next phase */ 7925 if (!list_empty(&list)) 7926 rcu_barrier(); 7927 7928 while (!list_empty(&list)) { 7929 struct net_device *dev 7930 = list_first_entry(&list, struct net_device, todo_list); 7931 list_del(&dev->todo_list); 7932 7933 rtnl_lock(); 7934 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 7935 __rtnl_unlock(); 7936 7937 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { 7938 pr_err("network todo '%s' but state %d\n", 7939 dev->name, dev->reg_state); 7940 dump_stack(); 7941 continue; 7942 } 7943 7944 dev->reg_state = NETREG_UNREGISTERED; 7945 7946 netdev_wait_allrefs(dev); 7947 7948 /* paranoia */ 7949 BUG_ON(netdev_refcnt_read(dev)); 7950 BUG_ON(!list_empty(&dev->ptype_all)); 7951 BUG_ON(!list_empty(&dev->ptype_specific)); 7952 WARN_ON(rcu_access_pointer(dev->ip_ptr)); 7953 WARN_ON(rcu_access_pointer(dev->ip6_ptr)); 7954 WARN_ON(dev->dn_ptr); 7955 7956 if (dev->priv_destructor) 7957 dev->priv_destructor(dev); 7958 if (dev->needs_free_netdev) 7959 free_netdev(dev); 7960 7961 /* Report a network device has been unregistered */ 7962 rtnl_lock(); 7963 dev_net(dev)->dev_unreg_count--; 7964 __rtnl_unlock(); 7965 wake_up(&netdev_unregistering_wq); 7966 7967 /* Free network device */ 7968 kobject_put(&dev->dev.kobj); 7969 } 7970 } 7971 7972 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has 7973 * all the same fields in the same order as net_device_stats, with only 7974 * the type differing, but rtnl_link_stats64 may have additional fields 7975 * at the end for newer counters. 7976 */ 7977 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64, 7978 const struct net_device_stats *netdev_stats) 7979 { 7980 #if BITS_PER_LONG == 64 7981 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats)); 7982 memcpy(stats64, netdev_stats, sizeof(*netdev_stats)); 7983 /* zero out counters that only exist in rtnl_link_stats64 */ 7984 memset((char *)stats64 + sizeof(*netdev_stats), 0, 7985 sizeof(*stats64) - sizeof(*netdev_stats)); 7986 #else 7987 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long); 7988 const unsigned long *src = (const unsigned long *)netdev_stats; 7989 u64 *dst = (u64 *)stats64; 7990 7991 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64)); 7992 for (i = 0; i < n; i++) 7993 dst[i] = src[i]; 7994 /* zero out counters that only exist in rtnl_link_stats64 */ 7995 memset((char *)stats64 + n * sizeof(u64), 0, 7996 sizeof(*stats64) - n * sizeof(u64)); 7997 #endif 7998 } 7999 EXPORT_SYMBOL(netdev_stats_to_stats64); 8000 8001 /** 8002 * dev_get_stats - get network device statistics 8003 * @dev: device to get statistics from 8004 * @storage: place to store stats 8005 * 8006 * Get network statistics from device. Return @storage. 8007 * The device driver may provide its own method by setting 8008 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats; 8009 * otherwise the internal statistics structure is used. 8010 */ 8011 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, 8012 struct rtnl_link_stats64 *storage) 8013 { 8014 const struct net_device_ops *ops = dev->netdev_ops; 8015 8016 if (ops->ndo_get_stats64) { 8017 memset(storage, 0, sizeof(*storage)); 8018 ops->ndo_get_stats64(dev, storage); 8019 } else if (ops->ndo_get_stats) { 8020 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); 8021 } else { 8022 netdev_stats_to_stats64(storage, &dev->stats); 8023 } 8024 storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped); 8025 storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped); 8026 storage->rx_nohandler += (unsigned long)atomic_long_read(&dev->rx_nohandler); 8027 return storage; 8028 } 8029 EXPORT_SYMBOL(dev_get_stats); 8030 8031 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) 8032 { 8033 struct netdev_queue *queue = dev_ingress_queue(dev); 8034 8035 #ifdef CONFIG_NET_CLS_ACT 8036 if (queue) 8037 return queue; 8038 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 8039 if (!queue) 8040 return NULL; 8041 netdev_init_one_queue(dev, queue, NULL); 8042 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); 8043 queue->qdisc_sleeping = &noop_qdisc; 8044 rcu_assign_pointer(dev->ingress_queue, queue); 8045 #endif 8046 return queue; 8047 } 8048 8049 static const struct ethtool_ops default_ethtool_ops; 8050 8051 void netdev_set_default_ethtool_ops(struct net_device *dev, 8052 const struct ethtool_ops *ops) 8053 { 8054 if (dev->ethtool_ops == &default_ethtool_ops) 8055 dev->ethtool_ops = ops; 8056 } 8057 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops); 8058 8059 void netdev_freemem(struct net_device *dev) 8060 { 8061 char *addr = (char *)dev - dev->padded; 8062 8063 kvfree(addr); 8064 } 8065 8066 /** 8067 * alloc_netdev_mqs - allocate network device 8068 * @sizeof_priv: size of private data to allocate space for 8069 * @name: device name format string 8070 * @name_assign_type: origin of device name 8071 * @setup: callback to initialize device 8072 * @txqs: the number of TX subqueues to allocate 8073 * @rxqs: the number of RX subqueues to allocate 8074 * 8075 * Allocates a struct net_device with private data area for driver use 8076 * and performs basic initialization. Also allocates subqueue structs 8077 * for each queue on the device. 8078 */ 8079 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, 8080 unsigned char name_assign_type, 8081 void (*setup)(struct net_device *), 8082 unsigned int txqs, unsigned int rxqs) 8083 { 8084 struct net_device *dev; 8085 unsigned int alloc_size; 8086 struct net_device *p; 8087 8088 BUG_ON(strlen(name) >= sizeof(dev->name)); 8089 8090 if (txqs < 1) { 8091 pr_err("alloc_netdev: Unable to allocate device with zero queues\n"); 8092 return NULL; 8093 } 8094 8095 #ifdef CONFIG_SYSFS 8096 if (rxqs < 1) { 8097 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n"); 8098 return NULL; 8099 } 8100 #endif 8101 8102 alloc_size = sizeof(struct net_device); 8103 if (sizeof_priv) { 8104 /* ensure 32-byte alignment of private area */ 8105 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN); 8106 alloc_size += sizeof_priv; 8107 } 8108 /* ensure 32-byte alignment of whole construct */ 8109 alloc_size += NETDEV_ALIGN - 1; 8110 8111 p = kvzalloc(alloc_size, GFP_KERNEL | __GFP_RETRY_MAYFAIL); 8112 if (!p) 8113 return NULL; 8114 8115 dev = PTR_ALIGN(p, NETDEV_ALIGN); 8116 dev->padded = (char *)dev - (char *)p; 8117 8118 dev->pcpu_refcnt = alloc_percpu(int); 8119 if (!dev->pcpu_refcnt) 8120 goto free_dev; 8121 8122 if (dev_addr_init(dev)) 8123 goto free_pcpu; 8124 8125 dev_mc_init(dev); 8126 dev_uc_init(dev); 8127 8128 dev_net_set(dev, &init_net); 8129 8130 dev->gso_max_size = GSO_MAX_SIZE; 8131 dev->gso_max_segs = GSO_MAX_SEGS; 8132 8133 INIT_LIST_HEAD(&dev->napi_list); 8134 INIT_LIST_HEAD(&dev->unreg_list); 8135 INIT_LIST_HEAD(&dev->close_list); 8136 INIT_LIST_HEAD(&dev->link_watch_list); 8137 INIT_LIST_HEAD(&dev->adj_list.upper); 8138 INIT_LIST_HEAD(&dev->adj_list.lower); 8139 INIT_LIST_HEAD(&dev->ptype_all); 8140 INIT_LIST_HEAD(&dev->ptype_specific); 8141 #ifdef CONFIG_NET_SCHED 8142 hash_init(dev->qdisc_hash); 8143 #endif 8144 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; 8145 setup(dev); 8146 8147 if (!dev->tx_queue_len) { 8148 dev->priv_flags |= IFF_NO_QUEUE; 8149 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; 8150 } 8151 8152 dev->num_tx_queues = txqs; 8153 dev->real_num_tx_queues = txqs; 8154 if (netif_alloc_netdev_queues(dev)) 8155 goto free_all; 8156 8157 #ifdef CONFIG_SYSFS 8158 dev->num_rx_queues = rxqs; 8159 dev->real_num_rx_queues = rxqs; 8160 if (netif_alloc_rx_queues(dev)) 8161 goto free_all; 8162 #endif 8163 8164 strcpy(dev->name, name); 8165 dev->name_assign_type = name_assign_type; 8166 dev->group = INIT_NETDEV_GROUP; 8167 if (!dev->ethtool_ops) 8168 dev->ethtool_ops = &default_ethtool_ops; 8169 8170 nf_hook_ingress_init(dev); 8171 8172 return dev; 8173 8174 free_all: 8175 free_netdev(dev); 8176 return NULL; 8177 8178 free_pcpu: 8179 free_percpu(dev->pcpu_refcnt); 8180 free_dev: 8181 netdev_freemem(dev); 8182 return NULL; 8183 } 8184 EXPORT_SYMBOL(alloc_netdev_mqs); 8185 8186 /** 8187 * free_netdev - free network device 8188 * @dev: device 8189 * 8190 * This function does the last stage of destroying an allocated device 8191 * interface. The reference to the device object is released. If this 8192 * is the last reference then it will be freed.Must be called in process 8193 * context. 8194 */ 8195 void free_netdev(struct net_device *dev) 8196 { 8197 struct napi_struct *p, *n; 8198 struct bpf_prog *prog; 8199 8200 might_sleep(); 8201 netif_free_tx_queues(dev); 8202 #ifdef CONFIG_SYSFS 8203 kvfree(dev->_rx); 8204 #endif 8205 8206 kfree(rcu_dereference_protected(dev->ingress_queue, 1)); 8207 8208 /* Flush device addresses */ 8209 dev_addr_flush(dev); 8210 8211 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) 8212 netif_napi_del(p); 8213 8214 free_percpu(dev->pcpu_refcnt); 8215 dev->pcpu_refcnt = NULL; 8216 8217 prog = rcu_dereference_protected(dev->xdp_prog, 1); 8218 if (prog) { 8219 bpf_prog_put(prog); 8220 static_key_slow_dec(&generic_xdp_needed); 8221 } 8222 8223 /* Compatibility with error handling in drivers */ 8224 if (dev->reg_state == NETREG_UNINITIALIZED) { 8225 netdev_freemem(dev); 8226 return; 8227 } 8228 8229 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); 8230 dev->reg_state = NETREG_RELEASED; 8231 8232 /* will free via device release */ 8233 put_device(&dev->dev); 8234 } 8235 EXPORT_SYMBOL(free_netdev); 8236 8237 /** 8238 * synchronize_net - Synchronize with packet receive processing 8239 * 8240 * Wait for packets currently being received to be done. 8241 * Does not block later packets from starting. 8242 */ 8243 void synchronize_net(void) 8244 { 8245 might_sleep(); 8246 if (rtnl_is_locked()) 8247 synchronize_rcu_expedited(); 8248 else 8249 synchronize_rcu(); 8250 } 8251 EXPORT_SYMBOL(synchronize_net); 8252 8253 /** 8254 * unregister_netdevice_queue - remove device from the kernel 8255 * @dev: device 8256 * @head: list 8257 * 8258 * This function shuts down a device interface and removes it 8259 * from the kernel tables. 8260 * If head not NULL, device is queued to be unregistered later. 8261 * 8262 * Callers must hold the rtnl semaphore. You may want 8263 * unregister_netdev() instead of this. 8264 */ 8265 8266 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) 8267 { 8268 ASSERT_RTNL(); 8269 8270 if (head) { 8271 list_move_tail(&dev->unreg_list, head); 8272 } else { 8273 rollback_registered(dev); 8274 /* Finish processing unregister after unlock */ 8275 net_set_todo(dev); 8276 } 8277 } 8278 EXPORT_SYMBOL(unregister_netdevice_queue); 8279 8280 /** 8281 * unregister_netdevice_many - unregister many devices 8282 * @head: list of devices 8283 * 8284 * Note: As most callers use a stack allocated list_head, 8285 * we force a list_del() to make sure stack wont be corrupted later. 8286 */ 8287 void unregister_netdevice_many(struct list_head *head) 8288 { 8289 struct net_device *dev; 8290 8291 if (!list_empty(head)) { 8292 rollback_registered_many(head); 8293 list_for_each_entry(dev, head, unreg_list) 8294 net_set_todo(dev); 8295 list_del(head); 8296 } 8297 } 8298 EXPORT_SYMBOL(unregister_netdevice_many); 8299 8300 /** 8301 * unregister_netdev - remove device from the kernel 8302 * @dev: device 8303 * 8304 * This function shuts down a device interface and removes it 8305 * from the kernel tables. 8306 * 8307 * This is just a wrapper for unregister_netdevice that takes 8308 * the rtnl semaphore. In general you want to use this and not 8309 * unregister_netdevice. 8310 */ 8311 void unregister_netdev(struct net_device *dev) 8312 { 8313 rtnl_lock(); 8314 unregister_netdevice(dev); 8315 rtnl_unlock(); 8316 } 8317 EXPORT_SYMBOL(unregister_netdev); 8318 8319 /** 8320 * dev_change_net_namespace - move device to different nethost namespace 8321 * @dev: device 8322 * @net: network namespace 8323 * @pat: If not NULL name pattern to try if the current device name 8324 * is already taken in the destination network namespace. 8325 * 8326 * This function shuts down a device interface and moves it 8327 * to a new network namespace. On success 0 is returned, on 8328 * a failure a netagive errno code is returned. 8329 * 8330 * Callers must hold the rtnl semaphore. 8331 */ 8332 8333 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat) 8334 { 8335 int err, new_nsid; 8336 8337 ASSERT_RTNL(); 8338 8339 /* Don't allow namespace local devices to be moved. */ 8340 err = -EINVAL; 8341 if (dev->features & NETIF_F_NETNS_LOCAL) 8342 goto out; 8343 8344 /* Ensure the device has been registrered */ 8345 if (dev->reg_state != NETREG_REGISTERED) 8346 goto out; 8347 8348 /* Get out if there is nothing todo */ 8349 err = 0; 8350 if (net_eq(dev_net(dev), net)) 8351 goto out; 8352 8353 /* Pick the destination device name, and ensure 8354 * we can use it in the destination network namespace. 8355 */ 8356 err = -EEXIST; 8357 if (__dev_get_by_name(net, dev->name)) { 8358 /* We get here if we can't use the current device name */ 8359 if (!pat) 8360 goto out; 8361 if (dev_get_valid_name(net, dev, pat) < 0) 8362 goto out; 8363 } 8364 8365 /* 8366 * And now a mini version of register_netdevice unregister_netdevice. 8367 */ 8368 8369 /* If device is running close it first. */ 8370 dev_close(dev); 8371 8372 /* And unlink it from device chain */ 8373 err = -ENODEV; 8374 unlist_netdevice(dev); 8375 8376 synchronize_net(); 8377 8378 /* Shutdown queueing discipline. */ 8379 dev_shutdown(dev); 8380 8381 /* Notify protocols, that we are about to destroy 8382 * this device. They should clean all the things. 8383 * 8384 * Note that dev->reg_state stays at NETREG_REGISTERED. 8385 * This is wanted because this way 8021q and macvlan know 8386 * the device is just moving and can keep their slaves up. 8387 */ 8388 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 8389 rcu_barrier(); 8390 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); 8391 if (dev->rtnl_link_ops && dev->rtnl_link_ops->get_link_net) 8392 new_nsid = peernet2id_alloc(dev_net(dev), net); 8393 else 8394 new_nsid = peernet2id(dev_net(dev), net); 8395 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid); 8396 8397 /* 8398 * Flush the unicast and multicast chains 8399 */ 8400 dev_uc_flush(dev); 8401 dev_mc_flush(dev); 8402 8403 /* Send a netdev-removed uevent to the old namespace */ 8404 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); 8405 netdev_adjacent_del_links(dev); 8406 8407 /* Actually switch the network namespace */ 8408 dev_net_set(dev, net); 8409 8410 /* If there is an ifindex conflict assign a new one */ 8411 if (__dev_get_by_index(net, dev->ifindex)) 8412 dev->ifindex = dev_new_index(net); 8413 8414 /* Send a netdev-add uevent to the new namespace */ 8415 kobject_uevent(&dev->dev.kobj, KOBJ_ADD); 8416 netdev_adjacent_add_links(dev); 8417 8418 /* Fixup kobjects */ 8419 err = device_rename(&dev->dev, dev->name); 8420 WARN_ON(err); 8421 8422 /* Add the device back in the hashes */ 8423 list_netdevice(dev); 8424 8425 /* Notify protocols, that a new device appeared. */ 8426 call_netdevice_notifiers(NETDEV_REGISTER, dev); 8427 8428 /* 8429 * Prevent userspace races by waiting until the network 8430 * device is fully setup before sending notifications. 8431 */ 8432 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); 8433 8434 synchronize_net(); 8435 err = 0; 8436 out: 8437 return err; 8438 } 8439 EXPORT_SYMBOL_GPL(dev_change_net_namespace); 8440 8441 static int dev_cpu_dead(unsigned int oldcpu) 8442 { 8443 struct sk_buff **list_skb; 8444 struct sk_buff *skb; 8445 unsigned int cpu; 8446 struct softnet_data *sd, *oldsd, *remsd = NULL; 8447 8448 local_irq_disable(); 8449 cpu = smp_processor_id(); 8450 sd = &per_cpu(softnet_data, cpu); 8451 oldsd = &per_cpu(softnet_data, oldcpu); 8452 8453 /* Find end of our completion_queue. */ 8454 list_skb = &sd->completion_queue; 8455 while (*list_skb) 8456 list_skb = &(*list_skb)->next; 8457 /* Append completion queue from offline CPU. */ 8458 *list_skb = oldsd->completion_queue; 8459 oldsd->completion_queue = NULL; 8460 8461 /* Append output queue from offline CPU. */ 8462 if (oldsd->output_queue) { 8463 *sd->output_queue_tailp = oldsd->output_queue; 8464 sd->output_queue_tailp = oldsd->output_queue_tailp; 8465 oldsd->output_queue = NULL; 8466 oldsd->output_queue_tailp = &oldsd->output_queue; 8467 } 8468 /* Append NAPI poll list from offline CPU, with one exception : 8469 * process_backlog() must be called by cpu owning percpu backlog. 8470 * We properly handle process_queue & input_pkt_queue later. 8471 */ 8472 while (!list_empty(&oldsd->poll_list)) { 8473 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, 8474 struct napi_struct, 8475 poll_list); 8476 8477 list_del_init(&napi->poll_list); 8478 if (napi->poll == process_backlog) 8479 napi->state = 0; 8480 else 8481 ____napi_schedule(sd, napi); 8482 } 8483 8484 raise_softirq_irqoff(NET_TX_SOFTIRQ); 8485 local_irq_enable(); 8486 8487 #ifdef CONFIG_RPS 8488 remsd = oldsd->rps_ipi_list; 8489 oldsd->rps_ipi_list = NULL; 8490 #endif 8491 /* send out pending IPI's on offline CPU */ 8492 net_rps_send_ipi(remsd); 8493 8494 /* Process offline CPU's input_pkt_queue */ 8495 while ((skb = __skb_dequeue(&oldsd->process_queue))) { 8496 netif_rx_ni(skb); 8497 input_queue_head_incr(oldsd); 8498 } 8499 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { 8500 netif_rx_ni(skb); 8501 input_queue_head_incr(oldsd); 8502 } 8503 8504 return 0; 8505 } 8506 8507 /** 8508 * netdev_increment_features - increment feature set by one 8509 * @all: current feature set 8510 * @one: new feature set 8511 * @mask: mask feature set 8512 * 8513 * Computes a new feature set after adding a device with feature set 8514 * @one to the master device with current feature set @all. Will not 8515 * enable anything that is off in @mask. Returns the new feature set. 8516 */ 8517 netdev_features_t netdev_increment_features(netdev_features_t all, 8518 netdev_features_t one, netdev_features_t mask) 8519 { 8520 if (mask & NETIF_F_HW_CSUM) 8521 mask |= NETIF_F_CSUM_MASK; 8522 mask |= NETIF_F_VLAN_CHALLENGED; 8523 8524 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask; 8525 all &= one | ~NETIF_F_ALL_FOR_ALL; 8526 8527 /* If one device supports hw checksumming, set for all. */ 8528 if (all & NETIF_F_HW_CSUM) 8529 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM); 8530 8531 return all; 8532 } 8533 EXPORT_SYMBOL(netdev_increment_features); 8534 8535 static struct hlist_head * __net_init netdev_create_hash(void) 8536 { 8537 int i; 8538 struct hlist_head *hash; 8539 8540 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL); 8541 if (hash != NULL) 8542 for (i = 0; i < NETDEV_HASHENTRIES; i++) 8543 INIT_HLIST_HEAD(&hash[i]); 8544 8545 return hash; 8546 } 8547 8548 /* Initialize per network namespace state */ 8549 static int __net_init netdev_init(struct net *net) 8550 { 8551 if (net != &init_net) 8552 INIT_LIST_HEAD(&net->dev_base_head); 8553 8554 net->dev_name_head = netdev_create_hash(); 8555 if (net->dev_name_head == NULL) 8556 goto err_name; 8557 8558 net->dev_index_head = netdev_create_hash(); 8559 if (net->dev_index_head == NULL) 8560 goto err_idx; 8561 8562 return 0; 8563 8564 err_idx: 8565 kfree(net->dev_name_head); 8566 err_name: 8567 return -ENOMEM; 8568 } 8569 8570 /** 8571 * netdev_drivername - network driver for the device 8572 * @dev: network device 8573 * 8574 * Determine network driver for device. 8575 */ 8576 const char *netdev_drivername(const struct net_device *dev) 8577 { 8578 const struct device_driver *driver; 8579 const struct device *parent; 8580 const char *empty = ""; 8581 8582 parent = dev->dev.parent; 8583 if (!parent) 8584 return empty; 8585 8586 driver = parent->driver; 8587 if (driver && driver->name) 8588 return driver->name; 8589 return empty; 8590 } 8591 8592 static void __netdev_printk(const char *level, const struct net_device *dev, 8593 struct va_format *vaf) 8594 { 8595 if (dev && dev->dev.parent) { 8596 dev_printk_emit(level[1] - '0', 8597 dev->dev.parent, 8598 "%s %s %s%s: %pV", 8599 dev_driver_string(dev->dev.parent), 8600 dev_name(dev->dev.parent), 8601 netdev_name(dev), netdev_reg_state(dev), 8602 vaf); 8603 } else if (dev) { 8604 printk("%s%s%s: %pV", 8605 level, netdev_name(dev), netdev_reg_state(dev), vaf); 8606 } else { 8607 printk("%s(NULL net_device): %pV", level, vaf); 8608 } 8609 } 8610 8611 void netdev_printk(const char *level, const struct net_device *dev, 8612 const char *format, ...) 8613 { 8614 struct va_format vaf; 8615 va_list args; 8616 8617 va_start(args, format); 8618 8619 vaf.fmt = format; 8620 vaf.va = &args; 8621 8622 __netdev_printk(level, dev, &vaf); 8623 8624 va_end(args); 8625 } 8626 EXPORT_SYMBOL(netdev_printk); 8627 8628 #define define_netdev_printk_level(func, level) \ 8629 void func(const struct net_device *dev, const char *fmt, ...) \ 8630 { \ 8631 struct va_format vaf; \ 8632 va_list args; \ 8633 \ 8634 va_start(args, fmt); \ 8635 \ 8636 vaf.fmt = fmt; \ 8637 vaf.va = &args; \ 8638 \ 8639 __netdev_printk(level, dev, &vaf); \ 8640 \ 8641 va_end(args); \ 8642 } \ 8643 EXPORT_SYMBOL(func); 8644 8645 define_netdev_printk_level(netdev_emerg, KERN_EMERG); 8646 define_netdev_printk_level(netdev_alert, KERN_ALERT); 8647 define_netdev_printk_level(netdev_crit, KERN_CRIT); 8648 define_netdev_printk_level(netdev_err, KERN_ERR); 8649 define_netdev_printk_level(netdev_warn, KERN_WARNING); 8650 define_netdev_printk_level(netdev_notice, KERN_NOTICE); 8651 define_netdev_printk_level(netdev_info, KERN_INFO); 8652 8653 static void __net_exit netdev_exit(struct net *net) 8654 { 8655 kfree(net->dev_name_head); 8656 kfree(net->dev_index_head); 8657 if (net != &init_net) 8658 WARN_ON_ONCE(!list_empty(&net->dev_base_head)); 8659 } 8660 8661 static struct pernet_operations __net_initdata netdev_net_ops = { 8662 .init = netdev_init, 8663 .exit = netdev_exit, 8664 }; 8665 8666 static void __net_exit default_device_exit(struct net *net) 8667 { 8668 struct net_device *dev, *aux; 8669 /* 8670 * Push all migratable network devices back to the 8671 * initial network namespace 8672 */ 8673 rtnl_lock(); 8674 for_each_netdev_safe(net, dev, aux) { 8675 int err; 8676 char fb_name[IFNAMSIZ]; 8677 8678 /* Ignore unmoveable devices (i.e. loopback) */ 8679 if (dev->features & NETIF_F_NETNS_LOCAL) 8680 continue; 8681 8682 /* Leave virtual devices for the generic cleanup */ 8683 if (dev->rtnl_link_ops) 8684 continue; 8685 8686 /* Push remaining network devices to init_net */ 8687 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); 8688 err = dev_change_net_namespace(dev, &init_net, fb_name); 8689 if (err) { 8690 pr_emerg("%s: failed to move %s to init_net: %d\n", 8691 __func__, dev->name, err); 8692 BUG(); 8693 } 8694 } 8695 rtnl_unlock(); 8696 } 8697 8698 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list) 8699 { 8700 /* Return with the rtnl_lock held when there are no network 8701 * devices unregistering in any network namespace in net_list. 8702 */ 8703 struct net *net; 8704 bool unregistering; 8705 DEFINE_WAIT_FUNC(wait, woken_wake_function); 8706 8707 add_wait_queue(&netdev_unregistering_wq, &wait); 8708 for (;;) { 8709 unregistering = false; 8710 rtnl_lock(); 8711 list_for_each_entry(net, net_list, exit_list) { 8712 if (net->dev_unreg_count > 0) { 8713 unregistering = true; 8714 break; 8715 } 8716 } 8717 if (!unregistering) 8718 break; 8719 __rtnl_unlock(); 8720 8721 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); 8722 } 8723 remove_wait_queue(&netdev_unregistering_wq, &wait); 8724 } 8725 8726 static void __net_exit default_device_exit_batch(struct list_head *net_list) 8727 { 8728 /* At exit all network devices most be removed from a network 8729 * namespace. Do this in the reverse order of registration. 8730 * Do this across as many network namespaces as possible to 8731 * improve batching efficiency. 8732 */ 8733 struct net_device *dev; 8734 struct net *net; 8735 LIST_HEAD(dev_kill_list); 8736 8737 /* To prevent network device cleanup code from dereferencing 8738 * loopback devices or network devices that have been freed 8739 * wait here for all pending unregistrations to complete, 8740 * before unregistring the loopback device and allowing the 8741 * network namespace be freed. 8742 * 8743 * The netdev todo list containing all network devices 8744 * unregistrations that happen in default_device_exit_batch 8745 * will run in the rtnl_unlock() at the end of 8746 * default_device_exit_batch. 8747 */ 8748 rtnl_lock_unregistering(net_list); 8749 list_for_each_entry(net, net_list, exit_list) { 8750 for_each_netdev_reverse(net, dev) { 8751 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) 8752 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); 8753 else 8754 unregister_netdevice_queue(dev, &dev_kill_list); 8755 } 8756 } 8757 unregister_netdevice_many(&dev_kill_list); 8758 rtnl_unlock(); 8759 } 8760 8761 static struct pernet_operations __net_initdata default_device_ops = { 8762 .exit = default_device_exit, 8763 .exit_batch = default_device_exit_batch, 8764 }; 8765 8766 /* 8767 * Initialize the DEV module. At boot time this walks the device list and 8768 * unhooks any devices that fail to initialise (normally hardware not 8769 * present) and leaves us with a valid list of present and active devices. 8770 * 8771 */ 8772 8773 /* 8774 * This is called single threaded during boot, so no need 8775 * to take the rtnl semaphore. 8776 */ 8777 static int __init net_dev_init(void) 8778 { 8779 int i, rc = -ENOMEM; 8780 8781 BUG_ON(!dev_boot_phase); 8782 8783 if (dev_proc_init()) 8784 goto out; 8785 8786 if (netdev_kobject_init()) 8787 goto out; 8788 8789 INIT_LIST_HEAD(&ptype_all); 8790 for (i = 0; i < PTYPE_HASH_SIZE; i++) 8791 INIT_LIST_HEAD(&ptype_base[i]); 8792 8793 INIT_LIST_HEAD(&offload_base); 8794 8795 if (register_pernet_subsys(&netdev_net_ops)) 8796 goto out; 8797 8798 /* 8799 * Initialise the packet receive queues. 8800 */ 8801 8802 for_each_possible_cpu(i) { 8803 struct work_struct *flush = per_cpu_ptr(&flush_works, i); 8804 struct softnet_data *sd = &per_cpu(softnet_data, i); 8805 8806 INIT_WORK(flush, flush_backlog); 8807 8808 skb_queue_head_init(&sd->input_pkt_queue); 8809 skb_queue_head_init(&sd->process_queue); 8810 INIT_LIST_HEAD(&sd->poll_list); 8811 sd->output_queue_tailp = &sd->output_queue; 8812 #ifdef CONFIG_RPS 8813 sd->csd.func = rps_trigger_softirq; 8814 sd->csd.info = sd; 8815 sd->cpu = i; 8816 #endif 8817 8818 sd->backlog.poll = process_backlog; 8819 sd->backlog.weight = weight_p; 8820 } 8821 8822 dev_boot_phase = 0; 8823 8824 /* The loopback device is special if any other network devices 8825 * is present in a network namespace the loopback device must 8826 * be present. Since we now dynamically allocate and free the 8827 * loopback device ensure this invariant is maintained by 8828 * keeping the loopback device as the first device on the 8829 * list of network devices. Ensuring the loopback devices 8830 * is the first device that appears and the last network device 8831 * that disappears. 8832 */ 8833 if (register_pernet_device(&loopback_net_ops)) 8834 goto out; 8835 8836 if (register_pernet_device(&default_device_ops)) 8837 goto out; 8838 8839 open_softirq(NET_TX_SOFTIRQ, net_tx_action); 8840 open_softirq(NET_RX_SOFTIRQ, net_rx_action); 8841 8842 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead", 8843 NULL, dev_cpu_dead); 8844 WARN_ON(rc < 0); 8845 rc = 0; 8846 out: 8847 return rc; 8848 } 8849 8850 subsys_initcall(net_dev_init); 8851