1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * NET3 Protocol independent device support routines. 4 * 5 * Derived from the non IP parts of dev.c 1.0.19 6 * Authors: Ross Biro 7 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 8 * Mark Evans, <evansmp@uhura.aston.ac.uk> 9 * 10 * Additional Authors: 11 * Florian la Roche <rzsfl@rz.uni-sb.de> 12 * Alan Cox <gw4pts@gw4pts.ampr.org> 13 * David Hinds <dahinds@users.sourceforge.net> 14 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 15 * Adam Sulmicki <adam@cfar.umd.edu> 16 * Pekka Riikonen <priikone@poesidon.pspt.fi> 17 * 18 * Changes: 19 * D.J. Barrow : Fixed bug where dev->refcnt gets set 20 * to 2 if register_netdev gets called 21 * before net_dev_init & also removed a 22 * few lines of code in the process. 23 * Alan Cox : device private ioctl copies fields back. 24 * Alan Cox : Transmit queue code does relevant 25 * stunts to keep the queue safe. 26 * Alan Cox : Fixed double lock. 27 * Alan Cox : Fixed promisc NULL pointer trap 28 * ???????? : Support the full private ioctl range 29 * Alan Cox : Moved ioctl permission check into 30 * drivers 31 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI 32 * Alan Cox : 100 backlog just doesn't cut it when 33 * you start doing multicast video 8) 34 * Alan Cox : Rewrote net_bh and list manager. 35 * Alan Cox : Fix ETH_P_ALL echoback lengths. 36 * Alan Cox : Took out transmit every packet pass 37 * Saved a few bytes in the ioctl handler 38 * Alan Cox : Network driver sets packet type before 39 * calling netif_rx. Saves a function 40 * call a packet. 41 * Alan Cox : Hashed net_bh() 42 * Richard Kooijman: Timestamp fixes. 43 * Alan Cox : Wrong field in SIOCGIFDSTADDR 44 * Alan Cox : Device lock protection. 45 * Alan Cox : Fixed nasty side effect of device close 46 * changes. 47 * Rudi Cilibrasi : Pass the right thing to 48 * set_mac_address() 49 * Dave Miller : 32bit quantity for the device lock to 50 * make it work out on a Sparc. 51 * Bjorn Ekwall : Added KERNELD hack. 52 * Alan Cox : Cleaned up the backlog initialise. 53 * Craig Metz : SIOCGIFCONF fix if space for under 54 * 1 device. 55 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there 56 * is no device open function. 57 * Andi Kleen : Fix error reporting for SIOCGIFCONF 58 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF 59 * Cyrus Durgin : Cleaned for KMOD 60 * Adam Sulmicki : Bug Fix : Network Device Unload 61 * A network device unload needs to purge 62 * the backlog queue. 63 * Paul Rusty Russell : SIOCSIFNAME 64 * Pekka Riikonen : Netdev boot-time settings code 65 * Andrew Morton : Make unregister_netdevice wait 66 * indefinitely on dev->refcnt 67 * J Hadi Salim : - Backlog queue sampling 68 * - netif_rx() feedback 69 */ 70 71 #include <linux/uaccess.h> 72 #include <linux/bitops.h> 73 #include <linux/capability.h> 74 #include <linux/cpu.h> 75 #include <linux/types.h> 76 #include <linux/kernel.h> 77 #include <linux/hash.h> 78 #include <linux/slab.h> 79 #include <linux/sched.h> 80 #include <linux/sched/mm.h> 81 #include <linux/mutex.h> 82 #include <linux/rwsem.h> 83 #include <linux/string.h> 84 #include <linux/mm.h> 85 #include <linux/socket.h> 86 #include <linux/sockios.h> 87 #include <linux/errno.h> 88 #include <linux/interrupt.h> 89 #include <linux/if_ether.h> 90 #include <linux/netdevice.h> 91 #include <linux/etherdevice.h> 92 #include <linux/ethtool.h> 93 #include <linux/skbuff.h> 94 #include <linux/bpf.h> 95 #include <linux/bpf_trace.h> 96 #include <net/net_namespace.h> 97 #include <net/sock.h> 98 #include <net/busy_poll.h> 99 #include <linux/rtnetlink.h> 100 #include <linux/stat.h> 101 #include <net/dst.h> 102 #include <net/dst_metadata.h> 103 #include <net/pkt_sched.h> 104 #include <net/pkt_cls.h> 105 #include <net/checksum.h> 106 #include <net/xfrm.h> 107 #include <linux/highmem.h> 108 #include <linux/init.h> 109 #include <linux/module.h> 110 #include <linux/netpoll.h> 111 #include <linux/rcupdate.h> 112 #include <linux/delay.h> 113 #include <net/iw_handler.h> 114 #include <asm/current.h> 115 #include <linux/audit.h> 116 #include <linux/dmaengine.h> 117 #include <linux/err.h> 118 #include <linux/ctype.h> 119 #include <linux/if_arp.h> 120 #include <linux/if_vlan.h> 121 #include <linux/ip.h> 122 #include <net/ip.h> 123 #include <net/mpls.h> 124 #include <linux/ipv6.h> 125 #include <linux/in.h> 126 #include <linux/jhash.h> 127 #include <linux/random.h> 128 #include <trace/events/napi.h> 129 #include <trace/events/net.h> 130 #include <trace/events/skb.h> 131 #include <linux/inetdevice.h> 132 #include <linux/cpu_rmap.h> 133 #include <linux/static_key.h> 134 #include <linux/hashtable.h> 135 #include <linux/vmalloc.h> 136 #include <linux/if_macvlan.h> 137 #include <linux/errqueue.h> 138 #include <linux/hrtimer.h> 139 #include <linux/netfilter_ingress.h> 140 #include <linux/crash_dump.h> 141 #include <linux/sctp.h> 142 #include <net/udp_tunnel.h> 143 #include <linux/net_namespace.h> 144 #include <linux/indirect_call_wrapper.h> 145 #include <net/devlink.h> 146 #include <linux/pm_runtime.h> 147 148 #include "net-sysfs.h" 149 150 #define MAX_GRO_SKBS 8 151 152 /* This should be increased if a protocol with a bigger head is added. */ 153 #define GRO_MAX_HEAD (MAX_HEADER + 128) 154 155 static DEFINE_SPINLOCK(ptype_lock); 156 static DEFINE_SPINLOCK(offload_lock); 157 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; 158 struct list_head ptype_all __read_mostly; /* Taps */ 159 static struct list_head offload_base __read_mostly; 160 161 static int netif_rx_internal(struct sk_buff *skb); 162 static int call_netdevice_notifiers_info(unsigned long val, 163 struct netdev_notifier_info *info); 164 static int call_netdevice_notifiers_extack(unsigned long val, 165 struct net_device *dev, 166 struct netlink_ext_ack *extack); 167 static struct napi_struct *napi_by_id(unsigned int napi_id); 168 169 /* 170 * The @dev_base_head list is protected by @dev_base_lock and the rtnl 171 * semaphore. 172 * 173 * Pure readers hold dev_base_lock for reading, or rcu_read_lock() 174 * 175 * Writers must hold the rtnl semaphore while they loop through the 176 * dev_base_head list, and hold dev_base_lock for writing when they do the 177 * actual updates. This allows pure readers to access the list even 178 * while a writer is preparing to update it. 179 * 180 * To put it another way, dev_base_lock is held for writing only to 181 * protect against pure readers; the rtnl semaphore provides the 182 * protection against other writers. 183 * 184 * See, for example usages, register_netdevice() and 185 * unregister_netdevice(), which must be called with the rtnl 186 * semaphore held. 187 */ 188 DEFINE_RWLOCK(dev_base_lock); 189 EXPORT_SYMBOL(dev_base_lock); 190 191 static DEFINE_MUTEX(ifalias_mutex); 192 193 /* protects napi_hash addition/deletion and napi_gen_id */ 194 static DEFINE_SPINLOCK(napi_hash_lock); 195 196 static unsigned int napi_gen_id = NR_CPUS; 197 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8); 198 199 static DECLARE_RWSEM(devnet_rename_sem); 200 201 static inline void dev_base_seq_inc(struct net *net) 202 { 203 while (++net->dev_base_seq == 0) 204 ; 205 } 206 207 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name) 208 { 209 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ)); 210 211 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; 212 } 213 214 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) 215 { 216 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; 217 } 218 219 static inline void rps_lock(struct softnet_data *sd) 220 { 221 #ifdef CONFIG_RPS 222 spin_lock(&sd->input_pkt_queue.lock); 223 #endif 224 } 225 226 static inline void rps_unlock(struct softnet_data *sd) 227 { 228 #ifdef CONFIG_RPS 229 spin_unlock(&sd->input_pkt_queue.lock); 230 #endif 231 } 232 233 static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev, 234 const char *name) 235 { 236 struct netdev_name_node *name_node; 237 238 name_node = kmalloc(sizeof(*name_node), GFP_KERNEL); 239 if (!name_node) 240 return NULL; 241 INIT_HLIST_NODE(&name_node->hlist); 242 name_node->dev = dev; 243 name_node->name = name; 244 return name_node; 245 } 246 247 static struct netdev_name_node * 248 netdev_name_node_head_alloc(struct net_device *dev) 249 { 250 struct netdev_name_node *name_node; 251 252 name_node = netdev_name_node_alloc(dev, dev->name); 253 if (!name_node) 254 return NULL; 255 INIT_LIST_HEAD(&name_node->list); 256 return name_node; 257 } 258 259 static void netdev_name_node_free(struct netdev_name_node *name_node) 260 { 261 kfree(name_node); 262 } 263 264 static void netdev_name_node_add(struct net *net, 265 struct netdev_name_node *name_node) 266 { 267 hlist_add_head_rcu(&name_node->hlist, 268 dev_name_hash(net, name_node->name)); 269 } 270 271 static void netdev_name_node_del(struct netdev_name_node *name_node) 272 { 273 hlist_del_rcu(&name_node->hlist); 274 } 275 276 static struct netdev_name_node *netdev_name_node_lookup(struct net *net, 277 const char *name) 278 { 279 struct hlist_head *head = dev_name_hash(net, name); 280 struct netdev_name_node *name_node; 281 282 hlist_for_each_entry(name_node, head, hlist) 283 if (!strcmp(name_node->name, name)) 284 return name_node; 285 return NULL; 286 } 287 288 static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net, 289 const char *name) 290 { 291 struct hlist_head *head = dev_name_hash(net, name); 292 struct netdev_name_node *name_node; 293 294 hlist_for_each_entry_rcu(name_node, head, hlist) 295 if (!strcmp(name_node->name, name)) 296 return name_node; 297 return NULL; 298 } 299 300 int netdev_name_node_alt_create(struct net_device *dev, const char *name) 301 { 302 struct netdev_name_node *name_node; 303 struct net *net = dev_net(dev); 304 305 name_node = netdev_name_node_lookup(net, name); 306 if (name_node) 307 return -EEXIST; 308 name_node = netdev_name_node_alloc(dev, name); 309 if (!name_node) 310 return -ENOMEM; 311 netdev_name_node_add(net, name_node); 312 /* The node that holds dev->name acts as a head of per-device list. */ 313 list_add_tail(&name_node->list, &dev->name_node->list); 314 315 return 0; 316 } 317 EXPORT_SYMBOL(netdev_name_node_alt_create); 318 319 static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node) 320 { 321 list_del(&name_node->list); 322 netdev_name_node_del(name_node); 323 kfree(name_node->name); 324 netdev_name_node_free(name_node); 325 } 326 327 int netdev_name_node_alt_destroy(struct net_device *dev, const char *name) 328 { 329 struct netdev_name_node *name_node; 330 struct net *net = dev_net(dev); 331 332 name_node = netdev_name_node_lookup(net, name); 333 if (!name_node) 334 return -ENOENT; 335 /* lookup might have found our primary name or a name belonging 336 * to another device. 337 */ 338 if (name_node == dev->name_node || name_node->dev != dev) 339 return -EINVAL; 340 341 __netdev_name_node_alt_destroy(name_node); 342 343 return 0; 344 } 345 EXPORT_SYMBOL(netdev_name_node_alt_destroy); 346 347 static void netdev_name_node_alt_flush(struct net_device *dev) 348 { 349 struct netdev_name_node *name_node, *tmp; 350 351 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list) 352 __netdev_name_node_alt_destroy(name_node); 353 } 354 355 /* Device list insertion */ 356 static void list_netdevice(struct net_device *dev) 357 { 358 struct net *net = dev_net(dev); 359 360 ASSERT_RTNL(); 361 362 write_lock_bh(&dev_base_lock); 363 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); 364 netdev_name_node_add(net, dev->name_node); 365 hlist_add_head_rcu(&dev->index_hlist, 366 dev_index_hash(net, dev->ifindex)); 367 write_unlock_bh(&dev_base_lock); 368 369 dev_base_seq_inc(net); 370 } 371 372 /* Device list removal 373 * caller must respect a RCU grace period before freeing/reusing dev 374 */ 375 static void unlist_netdevice(struct net_device *dev) 376 { 377 ASSERT_RTNL(); 378 379 /* Unlink dev from the device chain */ 380 write_lock_bh(&dev_base_lock); 381 list_del_rcu(&dev->dev_list); 382 netdev_name_node_del(dev->name_node); 383 hlist_del_rcu(&dev->index_hlist); 384 write_unlock_bh(&dev_base_lock); 385 386 dev_base_seq_inc(dev_net(dev)); 387 } 388 389 /* 390 * Our notifier list 391 */ 392 393 static RAW_NOTIFIER_HEAD(netdev_chain); 394 395 /* 396 * Device drivers call our routines to queue packets here. We empty the 397 * queue in the local softnet handler. 398 */ 399 400 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data); 401 EXPORT_PER_CPU_SYMBOL(softnet_data); 402 403 #ifdef CONFIG_LOCKDEP 404 /* 405 * register_netdevice() inits txq->_xmit_lock and sets lockdep class 406 * according to dev->type 407 */ 408 static const unsigned short netdev_lock_type[] = { 409 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25, 410 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET, 411 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM, 412 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, 413 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, 414 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, 415 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, 416 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, 417 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, 418 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, 419 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, 420 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, 421 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM, 422 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE, 423 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE}; 424 425 static const char *const netdev_lock_name[] = { 426 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", 427 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET", 428 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM", 429 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", 430 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", 431 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", 432 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", 433 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", 434 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", 435 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", 436 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", 437 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", 438 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM", 439 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE", 440 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"}; 441 442 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; 443 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; 444 445 static inline unsigned short netdev_lock_pos(unsigned short dev_type) 446 { 447 int i; 448 449 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++) 450 if (netdev_lock_type[i] == dev_type) 451 return i; 452 /* the last key is used by default */ 453 return ARRAY_SIZE(netdev_lock_type) - 1; 454 } 455 456 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 457 unsigned short dev_type) 458 { 459 int i; 460 461 i = netdev_lock_pos(dev_type); 462 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i], 463 netdev_lock_name[i]); 464 } 465 466 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 467 { 468 int i; 469 470 i = netdev_lock_pos(dev->type); 471 lockdep_set_class_and_name(&dev->addr_list_lock, 472 &netdev_addr_lock_key[i], 473 netdev_lock_name[i]); 474 } 475 #else 476 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 477 unsigned short dev_type) 478 { 479 } 480 481 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 482 { 483 } 484 #endif 485 486 /******************************************************************************* 487 * 488 * Protocol management and registration routines 489 * 490 *******************************************************************************/ 491 492 493 /* 494 * Add a protocol ID to the list. Now that the input handler is 495 * smarter we can dispense with all the messy stuff that used to be 496 * here. 497 * 498 * BEWARE!!! Protocol handlers, mangling input packets, 499 * MUST BE last in hash buckets and checking protocol handlers 500 * MUST start from promiscuous ptype_all chain in net_bh. 501 * It is true now, do not change it. 502 * Explanation follows: if protocol handler, mangling packet, will 503 * be the first on list, it is not able to sense, that packet 504 * is cloned and should be copied-on-write, so that it will 505 * change it and subsequent readers will get broken packet. 506 * --ANK (980803) 507 */ 508 509 static inline struct list_head *ptype_head(const struct packet_type *pt) 510 { 511 if (pt->type == htons(ETH_P_ALL)) 512 return pt->dev ? &pt->dev->ptype_all : &ptype_all; 513 else 514 return pt->dev ? &pt->dev->ptype_specific : 515 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; 516 } 517 518 /** 519 * dev_add_pack - add packet handler 520 * @pt: packet type declaration 521 * 522 * Add a protocol handler to the networking stack. The passed &packet_type 523 * is linked into kernel lists and may not be freed until it has been 524 * removed from the kernel lists. 525 * 526 * This call does not sleep therefore it can not 527 * guarantee all CPU's that are in middle of receiving packets 528 * will see the new packet type (until the next received packet). 529 */ 530 531 void dev_add_pack(struct packet_type *pt) 532 { 533 struct list_head *head = ptype_head(pt); 534 535 spin_lock(&ptype_lock); 536 list_add_rcu(&pt->list, head); 537 spin_unlock(&ptype_lock); 538 } 539 EXPORT_SYMBOL(dev_add_pack); 540 541 /** 542 * __dev_remove_pack - remove packet handler 543 * @pt: packet type declaration 544 * 545 * Remove a protocol handler that was previously added to the kernel 546 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 547 * from the kernel lists and can be freed or reused once this function 548 * returns. 549 * 550 * The packet type might still be in use by receivers 551 * and must not be freed until after all the CPU's have gone 552 * through a quiescent state. 553 */ 554 void __dev_remove_pack(struct packet_type *pt) 555 { 556 struct list_head *head = ptype_head(pt); 557 struct packet_type *pt1; 558 559 spin_lock(&ptype_lock); 560 561 list_for_each_entry(pt1, head, list) { 562 if (pt == pt1) { 563 list_del_rcu(&pt->list); 564 goto out; 565 } 566 } 567 568 pr_warn("dev_remove_pack: %p not found\n", pt); 569 out: 570 spin_unlock(&ptype_lock); 571 } 572 EXPORT_SYMBOL(__dev_remove_pack); 573 574 /** 575 * dev_remove_pack - remove packet handler 576 * @pt: packet type declaration 577 * 578 * Remove a protocol handler that was previously added to the kernel 579 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 580 * from the kernel lists and can be freed or reused once this function 581 * returns. 582 * 583 * This call sleeps to guarantee that no CPU is looking at the packet 584 * type after return. 585 */ 586 void dev_remove_pack(struct packet_type *pt) 587 { 588 __dev_remove_pack(pt); 589 590 synchronize_net(); 591 } 592 EXPORT_SYMBOL(dev_remove_pack); 593 594 595 /** 596 * dev_add_offload - register offload handlers 597 * @po: protocol offload declaration 598 * 599 * Add protocol offload handlers to the networking stack. The passed 600 * &proto_offload is linked into kernel lists and may not be freed until 601 * it has been removed from the kernel lists. 602 * 603 * This call does not sleep therefore it can not 604 * guarantee all CPU's that are in middle of receiving packets 605 * will see the new offload handlers (until the next received packet). 606 */ 607 void dev_add_offload(struct packet_offload *po) 608 { 609 struct packet_offload *elem; 610 611 spin_lock(&offload_lock); 612 list_for_each_entry(elem, &offload_base, list) { 613 if (po->priority < elem->priority) 614 break; 615 } 616 list_add_rcu(&po->list, elem->list.prev); 617 spin_unlock(&offload_lock); 618 } 619 EXPORT_SYMBOL(dev_add_offload); 620 621 /** 622 * __dev_remove_offload - remove offload handler 623 * @po: packet offload declaration 624 * 625 * Remove a protocol offload handler that was previously added to the 626 * kernel offload handlers by dev_add_offload(). The passed &offload_type 627 * is removed from the kernel lists and can be freed or reused once this 628 * function returns. 629 * 630 * The packet type might still be in use by receivers 631 * and must not be freed until after all the CPU's have gone 632 * through a quiescent state. 633 */ 634 static void __dev_remove_offload(struct packet_offload *po) 635 { 636 struct list_head *head = &offload_base; 637 struct packet_offload *po1; 638 639 spin_lock(&offload_lock); 640 641 list_for_each_entry(po1, head, list) { 642 if (po == po1) { 643 list_del_rcu(&po->list); 644 goto out; 645 } 646 } 647 648 pr_warn("dev_remove_offload: %p not found\n", po); 649 out: 650 spin_unlock(&offload_lock); 651 } 652 653 /** 654 * dev_remove_offload - remove packet offload handler 655 * @po: packet offload declaration 656 * 657 * Remove a packet offload handler that was previously added to the kernel 658 * offload handlers by dev_add_offload(). The passed &offload_type is 659 * removed from the kernel lists and can be freed or reused once this 660 * function returns. 661 * 662 * This call sleeps to guarantee that no CPU is looking at the packet 663 * type after return. 664 */ 665 void dev_remove_offload(struct packet_offload *po) 666 { 667 __dev_remove_offload(po); 668 669 synchronize_net(); 670 } 671 EXPORT_SYMBOL(dev_remove_offload); 672 673 /****************************************************************************** 674 * 675 * Device Boot-time Settings Routines 676 * 677 ******************************************************************************/ 678 679 /* Boot time configuration table */ 680 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX]; 681 682 /** 683 * netdev_boot_setup_add - add new setup entry 684 * @name: name of the device 685 * @map: configured settings for the device 686 * 687 * Adds new setup entry to the dev_boot_setup list. The function 688 * returns 0 on error and 1 on success. This is a generic routine to 689 * all netdevices. 690 */ 691 static int netdev_boot_setup_add(char *name, struct ifmap *map) 692 { 693 struct netdev_boot_setup *s; 694 int i; 695 696 s = dev_boot_setup; 697 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 698 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') { 699 memset(s[i].name, 0, sizeof(s[i].name)); 700 strlcpy(s[i].name, name, IFNAMSIZ); 701 memcpy(&s[i].map, map, sizeof(s[i].map)); 702 break; 703 } 704 } 705 706 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1; 707 } 708 709 /** 710 * netdev_boot_setup_check - check boot time settings 711 * @dev: the netdevice 712 * 713 * Check boot time settings for the device. 714 * The found settings are set for the device to be used 715 * later in the device probing. 716 * Returns 0 if no settings found, 1 if they are. 717 */ 718 int netdev_boot_setup_check(struct net_device *dev) 719 { 720 struct netdev_boot_setup *s = dev_boot_setup; 721 int i; 722 723 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 724 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' && 725 !strcmp(dev->name, s[i].name)) { 726 dev->irq = s[i].map.irq; 727 dev->base_addr = s[i].map.base_addr; 728 dev->mem_start = s[i].map.mem_start; 729 dev->mem_end = s[i].map.mem_end; 730 return 1; 731 } 732 } 733 return 0; 734 } 735 EXPORT_SYMBOL(netdev_boot_setup_check); 736 737 738 /** 739 * netdev_boot_base - get address from boot time settings 740 * @prefix: prefix for network device 741 * @unit: id for network device 742 * 743 * Check boot time settings for the base address of device. 744 * The found settings are set for the device to be used 745 * later in the device probing. 746 * Returns 0 if no settings found. 747 */ 748 unsigned long netdev_boot_base(const char *prefix, int unit) 749 { 750 const struct netdev_boot_setup *s = dev_boot_setup; 751 char name[IFNAMSIZ]; 752 int i; 753 754 sprintf(name, "%s%d", prefix, unit); 755 756 /* 757 * If device already registered then return base of 1 758 * to indicate not to probe for this interface 759 */ 760 if (__dev_get_by_name(&init_net, name)) 761 return 1; 762 763 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) 764 if (!strcmp(name, s[i].name)) 765 return s[i].map.base_addr; 766 return 0; 767 } 768 769 /* 770 * Saves at boot time configured settings for any netdevice. 771 */ 772 int __init netdev_boot_setup(char *str) 773 { 774 int ints[5]; 775 struct ifmap map; 776 777 str = get_options(str, ARRAY_SIZE(ints), ints); 778 if (!str || !*str) 779 return 0; 780 781 /* Save settings */ 782 memset(&map, 0, sizeof(map)); 783 if (ints[0] > 0) 784 map.irq = ints[1]; 785 if (ints[0] > 1) 786 map.base_addr = ints[2]; 787 if (ints[0] > 2) 788 map.mem_start = ints[3]; 789 if (ints[0] > 3) 790 map.mem_end = ints[4]; 791 792 /* Add new entry to the list */ 793 return netdev_boot_setup_add(str, &map); 794 } 795 796 __setup("netdev=", netdev_boot_setup); 797 798 /******************************************************************************* 799 * 800 * Device Interface Subroutines 801 * 802 *******************************************************************************/ 803 804 /** 805 * dev_get_iflink - get 'iflink' value of a interface 806 * @dev: targeted interface 807 * 808 * Indicates the ifindex the interface is linked to. 809 * Physical interfaces have the same 'ifindex' and 'iflink' values. 810 */ 811 812 int dev_get_iflink(const struct net_device *dev) 813 { 814 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) 815 return dev->netdev_ops->ndo_get_iflink(dev); 816 817 return dev->ifindex; 818 } 819 EXPORT_SYMBOL(dev_get_iflink); 820 821 /** 822 * dev_fill_metadata_dst - Retrieve tunnel egress information. 823 * @dev: targeted interface 824 * @skb: The packet. 825 * 826 * For better visibility of tunnel traffic OVS needs to retrieve 827 * egress tunnel information for a packet. Following API allows 828 * user to get this info. 829 */ 830 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb) 831 { 832 struct ip_tunnel_info *info; 833 834 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst) 835 return -EINVAL; 836 837 info = skb_tunnel_info_unclone(skb); 838 if (!info) 839 return -ENOMEM; 840 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX))) 841 return -EINVAL; 842 843 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb); 844 } 845 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst); 846 847 /** 848 * __dev_get_by_name - find a device by its name 849 * @net: the applicable net namespace 850 * @name: name to find 851 * 852 * Find an interface by name. Must be called under RTNL semaphore 853 * or @dev_base_lock. If the name is found a pointer to the device 854 * is returned. If the name is not found then %NULL is returned. The 855 * reference counters are not incremented so the caller must be 856 * careful with locks. 857 */ 858 859 struct net_device *__dev_get_by_name(struct net *net, const char *name) 860 { 861 struct netdev_name_node *node_name; 862 863 node_name = netdev_name_node_lookup(net, name); 864 return node_name ? node_name->dev : NULL; 865 } 866 EXPORT_SYMBOL(__dev_get_by_name); 867 868 /** 869 * dev_get_by_name_rcu - find a device by its name 870 * @net: the applicable net namespace 871 * @name: name to find 872 * 873 * Find an interface by name. 874 * If the name is found a pointer to the device is returned. 875 * If the name is not found then %NULL is returned. 876 * The reference counters are not incremented so the caller must be 877 * careful with locks. The caller must hold RCU lock. 878 */ 879 880 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name) 881 { 882 struct netdev_name_node *node_name; 883 884 node_name = netdev_name_node_lookup_rcu(net, name); 885 return node_name ? node_name->dev : NULL; 886 } 887 EXPORT_SYMBOL(dev_get_by_name_rcu); 888 889 /** 890 * dev_get_by_name - find a device by its name 891 * @net: the applicable net namespace 892 * @name: name to find 893 * 894 * Find an interface by name. This can be called from any 895 * context and does its own locking. The returned handle has 896 * the usage count incremented and the caller must use dev_put() to 897 * release it when it is no longer needed. %NULL is returned if no 898 * matching device is found. 899 */ 900 901 struct net_device *dev_get_by_name(struct net *net, const char *name) 902 { 903 struct net_device *dev; 904 905 rcu_read_lock(); 906 dev = dev_get_by_name_rcu(net, name); 907 if (dev) 908 dev_hold(dev); 909 rcu_read_unlock(); 910 return dev; 911 } 912 EXPORT_SYMBOL(dev_get_by_name); 913 914 /** 915 * __dev_get_by_index - find a device by its ifindex 916 * @net: the applicable net namespace 917 * @ifindex: index of device 918 * 919 * Search for an interface by index. Returns %NULL if the device 920 * is not found or a pointer to the device. The device has not 921 * had its reference counter increased so the caller must be careful 922 * about locking. The caller must hold either the RTNL semaphore 923 * or @dev_base_lock. 924 */ 925 926 struct net_device *__dev_get_by_index(struct net *net, int ifindex) 927 { 928 struct net_device *dev; 929 struct hlist_head *head = dev_index_hash(net, ifindex); 930 931 hlist_for_each_entry(dev, head, index_hlist) 932 if (dev->ifindex == ifindex) 933 return dev; 934 935 return NULL; 936 } 937 EXPORT_SYMBOL(__dev_get_by_index); 938 939 /** 940 * dev_get_by_index_rcu - find a device by its ifindex 941 * @net: the applicable net namespace 942 * @ifindex: index of device 943 * 944 * Search for an interface by index. Returns %NULL if the device 945 * is not found or a pointer to the device. The device has not 946 * had its reference counter increased so the caller must be careful 947 * about locking. The caller must hold RCU lock. 948 */ 949 950 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex) 951 { 952 struct net_device *dev; 953 struct hlist_head *head = dev_index_hash(net, ifindex); 954 955 hlist_for_each_entry_rcu(dev, head, index_hlist) 956 if (dev->ifindex == ifindex) 957 return dev; 958 959 return NULL; 960 } 961 EXPORT_SYMBOL(dev_get_by_index_rcu); 962 963 964 /** 965 * dev_get_by_index - find a device by its ifindex 966 * @net: the applicable net namespace 967 * @ifindex: index of device 968 * 969 * Search for an interface by index. Returns NULL if the device 970 * is not found or a pointer to the device. The device returned has 971 * had a reference added and the pointer is safe until the user calls 972 * dev_put to indicate they have finished with it. 973 */ 974 975 struct net_device *dev_get_by_index(struct net *net, int ifindex) 976 { 977 struct net_device *dev; 978 979 rcu_read_lock(); 980 dev = dev_get_by_index_rcu(net, ifindex); 981 if (dev) 982 dev_hold(dev); 983 rcu_read_unlock(); 984 return dev; 985 } 986 EXPORT_SYMBOL(dev_get_by_index); 987 988 /** 989 * dev_get_by_napi_id - find a device by napi_id 990 * @napi_id: ID of the NAPI struct 991 * 992 * Search for an interface by NAPI ID. Returns %NULL if the device 993 * is not found or a pointer to the device. The device has not had 994 * its reference counter increased so the caller must be careful 995 * about locking. The caller must hold RCU lock. 996 */ 997 998 struct net_device *dev_get_by_napi_id(unsigned int napi_id) 999 { 1000 struct napi_struct *napi; 1001 1002 WARN_ON_ONCE(!rcu_read_lock_held()); 1003 1004 if (napi_id < MIN_NAPI_ID) 1005 return NULL; 1006 1007 napi = napi_by_id(napi_id); 1008 1009 return napi ? napi->dev : NULL; 1010 } 1011 EXPORT_SYMBOL(dev_get_by_napi_id); 1012 1013 /** 1014 * netdev_get_name - get a netdevice name, knowing its ifindex. 1015 * @net: network namespace 1016 * @name: a pointer to the buffer where the name will be stored. 1017 * @ifindex: the ifindex of the interface to get the name from. 1018 */ 1019 int netdev_get_name(struct net *net, char *name, int ifindex) 1020 { 1021 struct net_device *dev; 1022 int ret; 1023 1024 down_read(&devnet_rename_sem); 1025 rcu_read_lock(); 1026 1027 dev = dev_get_by_index_rcu(net, ifindex); 1028 if (!dev) { 1029 ret = -ENODEV; 1030 goto out; 1031 } 1032 1033 strcpy(name, dev->name); 1034 1035 ret = 0; 1036 out: 1037 rcu_read_unlock(); 1038 up_read(&devnet_rename_sem); 1039 return ret; 1040 } 1041 1042 /** 1043 * dev_getbyhwaddr_rcu - find a device by its hardware address 1044 * @net: the applicable net namespace 1045 * @type: media type of device 1046 * @ha: hardware address 1047 * 1048 * Search for an interface by MAC address. Returns NULL if the device 1049 * is not found or a pointer to the device. 1050 * The caller must hold RCU or RTNL. 1051 * The returned device has not had its ref count increased 1052 * and the caller must therefore be careful about locking 1053 * 1054 */ 1055 1056 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type, 1057 const char *ha) 1058 { 1059 struct net_device *dev; 1060 1061 for_each_netdev_rcu(net, dev) 1062 if (dev->type == type && 1063 !memcmp(dev->dev_addr, ha, dev->addr_len)) 1064 return dev; 1065 1066 return NULL; 1067 } 1068 EXPORT_SYMBOL(dev_getbyhwaddr_rcu); 1069 1070 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type) 1071 { 1072 struct net_device *dev; 1073 1074 ASSERT_RTNL(); 1075 for_each_netdev(net, dev) 1076 if (dev->type == type) 1077 return dev; 1078 1079 return NULL; 1080 } 1081 EXPORT_SYMBOL(__dev_getfirstbyhwtype); 1082 1083 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) 1084 { 1085 struct net_device *dev, *ret = NULL; 1086 1087 rcu_read_lock(); 1088 for_each_netdev_rcu(net, dev) 1089 if (dev->type == type) { 1090 dev_hold(dev); 1091 ret = dev; 1092 break; 1093 } 1094 rcu_read_unlock(); 1095 return ret; 1096 } 1097 EXPORT_SYMBOL(dev_getfirstbyhwtype); 1098 1099 /** 1100 * __dev_get_by_flags - find any device with given flags 1101 * @net: the applicable net namespace 1102 * @if_flags: IFF_* values 1103 * @mask: bitmask of bits in if_flags to check 1104 * 1105 * Search for any interface with the given flags. Returns NULL if a device 1106 * is not found or a pointer to the device. Must be called inside 1107 * rtnl_lock(), and result refcount is unchanged. 1108 */ 1109 1110 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags, 1111 unsigned short mask) 1112 { 1113 struct net_device *dev, *ret; 1114 1115 ASSERT_RTNL(); 1116 1117 ret = NULL; 1118 for_each_netdev(net, dev) { 1119 if (((dev->flags ^ if_flags) & mask) == 0) { 1120 ret = dev; 1121 break; 1122 } 1123 } 1124 return ret; 1125 } 1126 EXPORT_SYMBOL(__dev_get_by_flags); 1127 1128 /** 1129 * dev_valid_name - check if name is okay for network device 1130 * @name: name string 1131 * 1132 * Network device names need to be valid file names to 1133 * to allow sysfs to work. We also disallow any kind of 1134 * whitespace. 1135 */ 1136 bool dev_valid_name(const char *name) 1137 { 1138 if (*name == '\0') 1139 return false; 1140 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ) 1141 return false; 1142 if (!strcmp(name, ".") || !strcmp(name, "..")) 1143 return false; 1144 1145 while (*name) { 1146 if (*name == '/' || *name == ':' || isspace(*name)) 1147 return false; 1148 name++; 1149 } 1150 return true; 1151 } 1152 EXPORT_SYMBOL(dev_valid_name); 1153 1154 /** 1155 * __dev_alloc_name - allocate a name for a device 1156 * @net: network namespace to allocate the device name in 1157 * @name: name format string 1158 * @buf: scratch buffer and result name string 1159 * 1160 * Passed a format string - eg "lt%d" it will try and find a suitable 1161 * id. It scans list of devices to build up a free map, then chooses 1162 * the first empty slot. The caller must hold the dev_base or rtnl lock 1163 * while allocating the name and adding the device in order to avoid 1164 * duplicates. 1165 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1166 * Returns the number of the unit assigned or a negative errno code. 1167 */ 1168 1169 static int __dev_alloc_name(struct net *net, const char *name, char *buf) 1170 { 1171 int i = 0; 1172 const char *p; 1173 const int max_netdevices = 8*PAGE_SIZE; 1174 unsigned long *inuse; 1175 struct net_device *d; 1176 1177 if (!dev_valid_name(name)) 1178 return -EINVAL; 1179 1180 p = strchr(name, '%'); 1181 if (p) { 1182 /* 1183 * Verify the string as this thing may have come from 1184 * the user. There must be either one "%d" and no other "%" 1185 * characters. 1186 */ 1187 if (p[1] != 'd' || strchr(p + 2, '%')) 1188 return -EINVAL; 1189 1190 /* Use one page as a bit array of possible slots */ 1191 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC); 1192 if (!inuse) 1193 return -ENOMEM; 1194 1195 for_each_netdev(net, d) { 1196 if (!sscanf(d->name, name, &i)) 1197 continue; 1198 if (i < 0 || i >= max_netdevices) 1199 continue; 1200 1201 /* avoid cases where sscanf is not exact inverse of printf */ 1202 snprintf(buf, IFNAMSIZ, name, i); 1203 if (!strncmp(buf, d->name, IFNAMSIZ)) 1204 set_bit(i, inuse); 1205 } 1206 1207 i = find_first_zero_bit(inuse, max_netdevices); 1208 free_page((unsigned long) inuse); 1209 } 1210 1211 snprintf(buf, IFNAMSIZ, name, i); 1212 if (!__dev_get_by_name(net, buf)) 1213 return i; 1214 1215 /* It is possible to run out of possible slots 1216 * when the name is long and there isn't enough space left 1217 * for the digits, or if all bits are used. 1218 */ 1219 return -ENFILE; 1220 } 1221 1222 static int dev_alloc_name_ns(struct net *net, 1223 struct net_device *dev, 1224 const char *name) 1225 { 1226 char buf[IFNAMSIZ]; 1227 int ret; 1228 1229 BUG_ON(!net); 1230 ret = __dev_alloc_name(net, name, buf); 1231 if (ret >= 0) 1232 strlcpy(dev->name, buf, IFNAMSIZ); 1233 return ret; 1234 } 1235 1236 /** 1237 * dev_alloc_name - allocate a name for a device 1238 * @dev: device 1239 * @name: name format string 1240 * 1241 * Passed a format string - eg "lt%d" it will try and find a suitable 1242 * id. It scans list of devices to build up a free map, then chooses 1243 * the first empty slot. The caller must hold the dev_base or rtnl lock 1244 * while allocating the name and adding the device in order to avoid 1245 * duplicates. 1246 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1247 * Returns the number of the unit assigned or a negative errno code. 1248 */ 1249 1250 int dev_alloc_name(struct net_device *dev, const char *name) 1251 { 1252 return dev_alloc_name_ns(dev_net(dev), dev, name); 1253 } 1254 EXPORT_SYMBOL(dev_alloc_name); 1255 1256 static int dev_get_valid_name(struct net *net, struct net_device *dev, 1257 const char *name) 1258 { 1259 BUG_ON(!net); 1260 1261 if (!dev_valid_name(name)) 1262 return -EINVAL; 1263 1264 if (strchr(name, '%')) 1265 return dev_alloc_name_ns(net, dev, name); 1266 else if (__dev_get_by_name(net, name)) 1267 return -EEXIST; 1268 else if (dev->name != name) 1269 strlcpy(dev->name, name, IFNAMSIZ); 1270 1271 return 0; 1272 } 1273 1274 /** 1275 * dev_change_name - change name of a device 1276 * @dev: device 1277 * @newname: name (or format string) must be at least IFNAMSIZ 1278 * 1279 * Change name of a device, can pass format strings "eth%d". 1280 * for wildcarding. 1281 */ 1282 int dev_change_name(struct net_device *dev, const char *newname) 1283 { 1284 unsigned char old_assign_type; 1285 char oldname[IFNAMSIZ]; 1286 int err = 0; 1287 int ret; 1288 struct net *net; 1289 1290 ASSERT_RTNL(); 1291 BUG_ON(!dev_net(dev)); 1292 1293 net = dev_net(dev); 1294 1295 /* Some auto-enslaved devices e.g. failover slaves are 1296 * special, as userspace might rename the device after 1297 * the interface had been brought up and running since 1298 * the point kernel initiated auto-enslavement. Allow 1299 * live name change even when these slave devices are 1300 * up and running. 1301 * 1302 * Typically, users of these auto-enslaving devices 1303 * don't actually care about slave name change, as 1304 * they are supposed to operate on master interface 1305 * directly. 1306 */ 1307 if (dev->flags & IFF_UP && 1308 likely(!(dev->priv_flags & IFF_LIVE_RENAME_OK))) 1309 return -EBUSY; 1310 1311 down_write(&devnet_rename_sem); 1312 1313 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) { 1314 up_write(&devnet_rename_sem); 1315 return 0; 1316 } 1317 1318 memcpy(oldname, dev->name, IFNAMSIZ); 1319 1320 err = dev_get_valid_name(net, dev, newname); 1321 if (err < 0) { 1322 up_write(&devnet_rename_sem); 1323 return err; 1324 } 1325 1326 if (oldname[0] && !strchr(oldname, '%')) 1327 netdev_info(dev, "renamed from %s\n", oldname); 1328 1329 old_assign_type = dev->name_assign_type; 1330 dev->name_assign_type = NET_NAME_RENAMED; 1331 1332 rollback: 1333 ret = device_rename(&dev->dev, dev->name); 1334 if (ret) { 1335 memcpy(dev->name, oldname, IFNAMSIZ); 1336 dev->name_assign_type = old_assign_type; 1337 up_write(&devnet_rename_sem); 1338 return ret; 1339 } 1340 1341 up_write(&devnet_rename_sem); 1342 1343 netdev_adjacent_rename_links(dev, oldname); 1344 1345 write_lock_bh(&dev_base_lock); 1346 netdev_name_node_del(dev->name_node); 1347 write_unlock_bh(&dev_base_lock); 1348 1349 synchronize_rcu(); 1350 1351 write_lock_bh(&dev_base_lock); 1352 netdev_name_node_add(net, dev->name_node); 1353 write_unlock_bh(&dev_base_lock); 1354 1355 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); 1356 ret = notifier_to_errno(ret); 1357 1358 if (ret) { 1359 /* err >= 0 after dev_alloc_name() or stores the first errno */ 1360 if (err >= 0) { 1361 err = ret; 1362 down_write(&devnet_rename_sem); 1363 memcpy(dev->name, oldname, IFNAMSIZ); 1364 memcpy(oldname, newname, IFNAMSIZ); 1365 dev->name_assign_type = old_assign_type; 1366 old_assign_type = NET_NAME_RENAMED; 1367 goto rollback; 1368 } else { 1369 pr_err("%s: name change rollback failed: %d\n", 1370 dev->name, ret); 1371 } 1372 } 1373 1374 return err; 1375 } 1376 1377 /** 1378 * dev_set_alias - change ifalias of a device 1379 * @dev: device 1380 * @alias: name up to IFALIASZ 1381 * @len: limit of bytes to copy from info 1382 * 1383 * Set ifalias for a device, 1384 */ 1385 int dev_set_alias(struct net_device *dev, const char *alias, size_t len) 1386 { 1387 struct dev_ifalias *new_alias = NULL; 1388 1389 if (len >= IFALIASZ) 1390 return -EINVAL; 1391 1392 if (len) { 1393 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL); 1394 if (!new_alias) 1395 return -ENOMEM; 1396 1397 memcpy(new_alias->ifalias, alias, len); 1398 new_alias->ifalias[len] = 0; 1399 } 1400 1401 mutex_lock(&ifalias_mutex); 1402 new_alias = rcu_replace_pointer(dev->ifalias, new_alias, 1403 mutex_is_locked(&ifalias_mutex)); 1404 mutex_unlock(&ifalias_mutex); 1405 1406 if (new_alias) 1407 kfree_rcu(new_alias, rcuhead); 1408 1409 return len; 1410 } 1411 EXPORT_SYMBOL(dev_set_alias); 1412 1413 /** 1414 * dev_get_alias - get ifalias of a device 1415 * @dev: device 1416 * @name: buffer to store name of ifalias 1417 * @len: size of buffer 1418 * 1419 * get ifalias for a device. Caller must make sure dev cannot go 1420 * away, e.g. rcu read lock or own a reference count to device. 1421 */ 1422 int dev_get_alias(const struct net_device *dev, char *name, size_t len) 1423 { 1424 const struct dev_ifalias *alias; 1425 int ret = 0; 1426 1427 rcu_read_lock(); 1428 alias = rcu_dereference(dev->ifalias); 1429 if (alias) 1430 ret = snprintf(name, len, "%s", alias->ifalias); 1431 rcu_read_unlock(); 1432 1433 return ret; 1434 } 1435 1436 /** 1437 * netdev_features_change - device changes features 1438 * @dev: device to cause notification 1439 * 1440 * Called to indicate a device has changed features. 1441 */ 1442 void netdev_features_change(struct net_device *dev) 1443 { 1444 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); 1445 } 1446 EXPORT_SYMBOL(netdev_features_change); 1447 1448 /** 1449 * netdev_state_change - device changes state 1450 * @dev: device to cause notification 1451 * 1452 * Called to indicate a device has changed state. This function calls 1453 * the notifier chains for netdev_chain and sends a NEWLINK message 1454 * to the routing socket. 1455 */ 1456 void netdev_state_change(struct net_device *dev) 1457 { 1458 if (dev->flags & IFF_UP) { 1459 struct netdev_notifier_change_info change_info = { 1460 .info.dev = dev, 1461 }; 1462 1463 call_netdevice_notifiers_info(NETDEV_CHANGE, 1464 &change_info.info); 1465 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL); 1466 } 1467 } 1468 EXPORT_SYMBOL(netdev_state_change); 1469 1470 /** 1471 * netdev_notify_peers - notify network peers about existence of @dev 1472 * @dev: network device 1473 * 1474 * Generate traffic such that interested network peers are aware of 1475 * @dev, such as by generating a gratuitous ARP. This may be used when 1476 * a device wants to inform the rest of the network about some sort of 1477 * reconfiguration such as a failover event or virtual machine 1478 * migration. 1479 */ 1480 void netdev_notify_peers(struct net_device *dev) 1481 { 1482 rtnl_lock(); 1483 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev); 1484 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev); 1485 rtnl_unlock(); 1486 } 1487 EXPORT_SYMBOL(netdev_notify_peers); 1488 1489 static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack) 1490 { 1491 const struct net_device_ops *ops = dev->netdev_ops; 1492 int ret; 1493 1494 ASSERT_RTNL(); 1495 1496 if (!netif_device_present(dev)) { 1497 /* may be detached because parent is runtime-suspended */ 1498 if (dev->dev.parent) 1499 pm_runtime_resume(dev->dev.parent); 1500 if (!netif_device_present(dev)) 1501 return -ENODEV; 1502 } 1503 1504 /* Block netpoll from trying to do any rx path servicing. 1505 * If we don't do this there is a chance ndo_poll_controller 1506 * or ndo_poll may be running while we open the device 1507 */ 1508 netpoll_poll_disable(dev); 1509 1510 ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack); 1511 ret = notifier_to_errno(ret); 1512 if (ret) 1513 return ret; 1514 1515 set_bit(__LINK_STATE_START, &dev->state); 1516 1517 if (ops->ndo_validate_addr) 1518 ret = ops->ndo_validate_addr(dev); 1519 1520 if (!ret && ops->ndo_open) 1521 ret = ops->ndo_open(dev); 1522 1523 netpoll_poll_enable(dev); 1524 1525 if (ret) 1526 clear_bit(__LINK_STATE_START, &dev->state); 1527 else { 1528 dev->flags |= IFF_UP; 1529 dev_set_rx_mode(dev); 1530 dev_activate(dev); 1531 add_device_randomness(dev->dev_addr, dev->addr_len); 1532 } 1533 1534 return ret; 1535 } 1536 1537 /** 1538 * dev_open - prepare an interface for use. 1539 * @dev: device to open 1540 * @extack: netlink extended ack 1541 * 1542 * Takes a device from down to up state. The device's private open 1543 * function is invoked and then the multicast lists are loaded. Finally 1544 * the device is moved into the up state and a %NETDEV_UP message is 1545 * sent to the netdev notifier chain. 1546 * 1547 * Calling this function on an active interface is a nop. On a failure 1548 * a negative errno code is returned. 1549 */ 1550 int dev_open(struct net_device *dev, struct netlink_ext_ack *extack) 1551 { 1552 int ret; 1553 1554 if (dev->flags & IFF_UP) 1555 return 0; 1556 1557 ret = __dev_open(dev, extack); 1558 if (ret < 0) 1559 return ret; 1560 1561 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); 1562 call_netdevice_notifiers(NETDEV_UP, dev); 1563 1564 return ret; 1565 } 1566 EXPORT_SYMBOL(dev_open); 1567 1568 static void __dev_close_many(struct list_head *head) 1569 { 1570 struct net_device *dev; 1571 1572 ASSERT_RTNL(); 1573 might_sleep(); 1574 1575 list_for_each_entry(dev, head, close_list) { 1576 /* Temporarily disable netpoll until the interface is down */ 1577 netpoll_poll_disable(dev); 1578 1579 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); 1580 1581 clear_bit(__LINK_STATE_START, &dev->state); 1582 1583 /* Synchronize to scheduled poll. We cannot touch poll list, it 1584 * can be even on different cpu. So just clear netif_running(). 1585 * 1586 * dev->stop() will invoke napi_disable() on all of it's 1587 * napi_struct instances on this device. 1588 */ 1589 smp_mb__after_atomic(); /* Commit netif_running(). */ 1590 } 1591 1592 dev_deactivate_many(head); 1593 1594 list_for_each_entry(dev, head, close_list) { 1595 const struct net_device_ops *ops = dev->netdev_ops; 1596 1597 /* 1598 * Call the device specific close. This cannot fail. 1599 * Only if device is UP 1600 * 1601 * We allow it to be called even after a DETACH hot-plug 1602 * event. 1603 */ 1604 if (ops->ndo_stop) 1605 ops->ndo_stop(dev); 1606 1607 dev->flags &= ~IFF_UP; 1608 netpoll_poll_enable(dev); 1609 } 1610 } 1611 1612 static void __dev_close(struct net_device *dev) 1613 { 1614 LIST_HEAD(single); 1615 1616 list_add(&dev->close_list, &single); 1617 __dev_close_many(&single); 1618 list_del(&single); 1619 } 1620 1621 void dev_close_many(struct list_head *head, bool unlink) 1622 { 1623 struct net_device *dev, *tmp; 1624 1625 /* Remove the devices that don't need to be closed */ 1626 list_for_each_entry_safe(dev, tmp, head, close_list) 1627 if (!(dev->flags & IFF_UP)) 1628 list_del_init(&dev->close_list); 1629 1630 __dev_close_many(head); 1631 1632 list_for_each_entry_safe(dev, tmp, head, close_list) { 1633 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); 1634 call_netdevice_notifiers(NETDEV_DOWN, dev); 1635 if (unlink) 1636 list_del_init(&dev->close_list); 1637 } 1638 } 1639 EXPORT_SYMBOL(dev_close_many); 1640 1641 /** 1642 * dev_close - shutdown an interface. 1643 * @dev: device to shutdown 1644 * 1645 * This function moves an active device into down state. A 1646 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device 1647 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier 1648 * chain. 1649 */ 1650 void dev_close(struct net_device *dev) 1651 { 1652 if (dev->flags & IFF_UP) { 1653 LIST_HEAD(single); 1654 1655 list_add(&dev->close_list, &single); 1656 dev_close_many(&single, true); 1657 list_del(&single); 1658 } 1659 } 1660 EXPORT_SYMBOL(dev_close); 1661 1662 1663 /** 1664 * dev_disable_lro - disable Large Receive Offload on a device 1665 * @dev: device 1666 * 1667 * Disable Large Receive Offload (LRO) on a net device. Must be 1668 * called under RTNL. This is needed if received packets may be 1669 * forwarded to another interface. 1670 */ 1671 void dev_disable_lro(struct net_device *dev) 1672 { 1673 struct net_device *lower_dev; 1674 struct list_head *iter; 1675 1676 dev->wanted_features &= ~NETIF_F_LRO; 1677 netdev_update_features(dev); 1678 1679 if (unlikely(dev->features & NETIF_F_LRO)) 1680 netdev_WARN(dev, "failed to disable LRO!\n"); 1681 1682 netdev_for_each_lower_dev(dev, lower_dev, iter) 1683 dev_disable_lro(lower_dev); 1684 } 1685 EXPORT_SYMBOL(dev_disable_lro); 1686 1687 /** 1688 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device 1689 * @dev: device 1690 * 1691 * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be 1692 * called under RTNL. This is needed if Generic XDP is installed on 1693 * the device. 1694 */ 1695 static void dev_disable_gro_hw(struct net_device *dev) 1696 { 1697 dev->wanted_features &= ~NETIF_F_GRO_HW; 1698 netdev_update_features(dev); 1699 1700 if (unlikely(dev->features & NETIF_F_GRO_HW)) 1701 netdev_WARN(dev, "failed to disable GRO_HW!\n"); 1702 } 1703 1704 const char *netdev_cmd_to_name(enum netdev_cmd cmd) 1705 { 1706 #define N(val) \ 1707 case NETDEV_##val: \ 1708 return "NETDEV_" __stringify(val); 1709 switch (cmd) { 1710 N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER) 1711 N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE) 1712 N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE) 1713 N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER) 1714 N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO) 1715 N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO) 1716 N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN) 1717 N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO) 1718 N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO) 1719 N(PRE_CHANGEADDR) 1720 } 1721 #undef N 1722 return "UNKNOWN_NETDEV_EVENT"; 1723 } 1724 EXPORT_SYMBOL_GPL(netdev_cmd_to_name); 1725 1726 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val, 1727 struct net_device *dev) 1728 { 1729 struct netdev_notifier_info info = { 1730 .dev = dev, 1731 }; 1732 1733 return nb->notifier_call(nb, val, &info); 1734 } 1735 1736 static int call_netdevice_register_notifiers(struct notifier_block *nb, 1737 struct net_device *dev) 1738 { 1739 int err; 1740 1741 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev); 1742 err = notifier_to_errno(err); 1743 if (err) 1744 return err; 1745 1746 if (!(dev->flags & IFF_UP)) 1747 return 0; 1748 1749 call_netdevice_notifier(nb, NETDEV_UP, dev); 1750 return 0; 1751 } 1752 1753 static void call_netdevice_unregister_notifiers(struct notifier_block *nb, 1754 struct net_device *dev) 1755 { 1756 if (dev->flags & IFF_UP) { 1757 call_netdevice_notifier(nb, NETDEV_GOING_DOWN, 1758 dev); 1759 call_netdevice_notifier(nb, NETDEV_DOWN, dev); 1760 } 1761 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); 1762 } 1763 1764 static int call_netdevice_register_net_notifiers(struct notifier_block *nb, 1765 struct net *net) 1766 { 1767 struct net_device *dev; 1768 int err; 1769 1770 for_each_netdev(net, dev) { 1771 err = call_netdevice_register_notifiers(nb, dev); 1772 if (err) 1773 goto rollback; 1774 } 1775 return 0; 1776 1777 rollback: 1778 for_each_netdev_continue_reverse(net, dev) 1779 call_netdevice_unregister_notifiers(nb, dev); 1780 return err; 1781 } 1782 1783 static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb, 1784 struct net *net) 1785 { 1786 struct net_device *dev; 1787 1788 for_each_netdev(net, dev) 1789 call_netdevice_unregister_notifiers(nb, dev); 1790 } 1791 1792 static int dev_boot_phase = 1; 1793 1794 /** 1795 * register_netdevice_notifier - register a network notifier block 1796 * @nb: notifier 1797 * 1798 * Register a notifier to be called when network device events occur. 1799 * The notifier passed is linked into the kernel structures and must 1800 * not be reused until it has been unregistered. A negative errno code 1801 * is returned on a failure. 1802 * 1803 * When registered all registration and up events are replayed 1804 * to the new notifier to allow device to have a race free 1805 * view of the network device list. 1806 */ 1807 1808 int register_netdevice_notifier(struct notifier_block *nb) 1809 { 1810 struct net *net; 1811 int err; 1812 1813 /* Close race with setup_net() and cleanup_net() */ 1814 down_write(&pernet_ops_rwsem); 1815 rtnl_lock(); 1816 err = raw_notifier_chain_register(&netdev_chain, nb); 1817 if (err) 1818 goto unlock; 1819 if (dev_boot_phase) 1820 goto unlock; 1821 for_each_net(net) { 1822 err = call_netdevice_register_net_notifiers(nb, net); 1823 if (err) 1824 goto rollback; 1825 } 1826 1827 unlock: 1828 rtnl_unlock(); 1829 up_write(&pernet_ops_rwsem); 1830 return err; 1831 1832 rollback: 1833 for_each_net_continue_reverse(net) 1834 call_netdevice_unregister_net_notifiers(nb, net); 1835 1836 raw_notifier_chain_unregister(&netdev_chain, nb); 1837 goto unlock; 1838 } 1839 EXPORT_SYMBOL(register_netdevice_notifier); 1840 1841 /** 1842 * unregister_netdevice_notifier - unregister a network notifier block 1843 * @nb: notifier 1844 * 1845 * Unregister a notifier previously registered by 1846 * register_netdevice_notifier(). The notifier is unlinked into the 1847 * kernel structures and may then be reused. A negative errno code 1848 * is returned on a failure. 1849 * 1850 * After unregistering unregister and down device events are synthesized 1851 * for all devices on the device list to the removed notifier to remove 1852 * the need for special case cleanup code. 1853 */ 1854 1855 int unregister_netdevice_notifier(struct notifier_block *nb) 1856 { 1857 struct net *net; 1858 int err; 1859 1860 /* Close race with setup_net() and cleanup_net() */ 1861 down_write(&pernet_ops_rwsem); 1862 rtnl_lock(); 1863 err = raw_notifier_chain_unregister(&netdev_chain, nb); 1864 if (err) 1865 goto unlock; 1866 1867 for_each_net(net) 1868 call_netdevice_unregister_net_notifiers(nb, net); 1869 1870 unlock: 1871 rtnl_unlock(); 1872 up_write(&pernet_ops_rwsem); 1873 return err; 1874 } 1875 EXPORT_SYMBOL(unregister_netdevice_notifier); 1876 1877 static int __register_netdevice_notifier_net(struct net *net, 1878 struct notifier_block *nb, 1879 bool ignore_call_fail) 1880 { 1881 int err; 1882 1883 err = raw_notifier_chain_register(&net->netdev_chain, nb); 1884 if (err) 1885 return err; 1886 if (dev_boot_phase) 1887 return 0; 1888 1889 err = call_netdevice_register_net_notifiers(nb, net); 1890 if (err && !ignore_call_fail) 1891 goto chain_unregister; 1892 1893 return 0; 1894 1895 chain_unregister: 1896 raw_notifier_chain_unregister(&net->netdev_chain, nb); 1897 return err; 1898 } 1899 1900 static int __unregister_netdevice_notifier_net(struct net *net, 1901 struct notifier_block *nb) 1902 { 1903 int err; 1904 1905 err = raw_notifier_chain_unregister(&net->netdev_chain, nb); 1906 if (err) 1907 return err; 1908 1909 call_netdevice_unregister_net_notifiers(nb, net); 1910 return 0; 1911 } 1912 1913 /** 1914 * register_netdevice_notifier_net - register a per-netns network notifier block 1915 * @net: network namespace 1916 * @nb: notifier 1917 * 1918 * Register a notifier to be called when network device events occur. 1919 * The notifier passed is linked into the kernel structures and must 1920 * not be reused until it has been unregistered. A negative errno code 1921 * is returned on a failure. 1922 * 1923 * When registered all registration and up events are replayed 1924 * to the new notifier to allow device to have a race free 1925 * view of the network device list. 1926 */ 1927 1928 int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb) 1929 { 1930 int err; 1931 1932 rtnl_lock(); 1933 err = __register_netdevice_notifier_net(net, nb, false); 1934 rtnl_unlock(); 1935 return err; 1936 } 1937 EXPORT_SYMBOL(register_netdevice_notifier_net); 1938 1939 /** 1940 * unregister_netdevice_notifier_net - unregister a per-netns 1941 * network notifier block 1942 * @net: network namespace 1943 * @nb: notifier 1944 * 1945 * Unregister a notifier previously registered by 1946 * register_netdevice_notifier(). The notifier is unlinked into the 1947 * kernel structures and may then be reused. A negative errno code 1948 * is returned on a failure. 1949 * 1950 * After unregistering unregister and down device events are synthesized 1951 * for all devices on the device list to the removed notifier to remove 1952 * the need for special case cleanup code. 1953 */ 1954 1955 int unregister_netdevice_notifier_net(struct net *net, 1956 struct notifier_block *nb) 1957 { 1958 int err; 1959 1960 rtnl_lock(); 1961 err = __unregister_netdevice_notifier_net(net, nb); 1962 rtnl_unlock(); 1963 return err; 1964 } 1965 EXPORT_SYMBOL(unregister_netdevice_notifier_net); 1966 1967 int register_netdevice_notifier_dev_net(struct net_device *dev, 1968 struct notifier_block *nb, 1969 struct netdev_net_notifier *nn) 1970 { 1971 int err; 1972 1973 rtnl_lock(); 1974 err = __register_netdevice_notifier_net(dev_net(dev), nb, false); 1975 if (!err) { 1976 nn->nb = nb; 1977 list_add(&nn->list, &dev->net_notifier_list); 1978 } 1979 rtnl_unlock(); 1980 return err; 1981 } 1982 EXPORT_SYMBOL(register_netdevice_notifier_dev_net); 1983 1984 int unregister_netdevice_notifier_dev_net(struct net_device *dev, 1985 struct notifier_block *nb, 1986 struct netdev_net_notifier *nn) 1987 { 1988 int err; 1989 1990 rtnl_lock(); 1991 list_del(&nn->list); 1992 err = __unregister_netdevice_notifier_net(dev_net(dev), nb); 1993 rtnl_unlock(); 1994 return err; 1995 } 1996 EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net); 1997 1998 static void move_netdevice_notifiers_dev_net(struct net_device *dev, 1999 struct net *net) 2000 { 2001 struct netdev_net_notifier *nn; 2002 2003 list_for_each_entry(nn, &dev->net_notifier_list, list) { 2004 __unregister_netdevice_notifier_net(dev_net(dev), nn->nb); 2005 __register_netdevice_notifier_net(net, nn->nb, true); 2006 } 2007 } 2008 2009 /** 2010 * call_netdevice_notifiers_info - call all network notifier blocks 2011 * @val: value passed unmodified to notifier function 2012 * @info: notifier information data 2013 * 2014 * Call all network notifier blocks. Parameters and return value 2015 * are as for raw_notifier_call_chain(). 2016 */ 2017 2018 static int call_netdevice_notifiers_info(unsigned long val, 2019 struct netdev_notifier_info *info) 2020 { 2021 struct net *net = dev_net(info->dev); 2022 int ret; 2023 2024 ASSERT_RTNL(); 2025 2026 /* Run per-netns notifier block chain first, then run the global one. 2027 * Hopefully, one day, the global one is going to be removed after 2028 * all notifier block registrators get converted to be per-netns. 2029 */ 2030 ret = raw_notifier_call_chain(&net->netdev_chain, val, info); 2031 if (ret & NOTIFY_STOP_MASK) 2032 return ret; 2033 return raw_notifier_call_chain(&netdev_chain, val, info); 2034 } 2035 2036 static int call_netdevice_notifiers_extack(unsigned long val, 2037 struct net_device *dev, 2038 struct netlink_ext_ack *extack) 2039 { 2040 struct netdev_notifier_info info = { 2041 .dev = dev, 2042 .extack = extack, 2043 }; 2044 2045 return call_netdevice_notifiers_info(val, &info); 2046 } 2047 2048 /** 2049 * call_netdevice_notifiers - call all network notifier blocks 2050 * @val: value passed unmodified to notifier function 2051 * @dev: net_device pointer passed unmodified to notifier function 2052 * 2053 * Call all network notifier blocks. Parameters and return value 2054 * are as for raw_notifier_call_chain(). 2055 */ 2056 2057 int call_netdevice_notifiers(unsigned long val, struct net_device *dev) 2058 { 2059 return call_netdevice_notifiers_extack(val, dev, NULL); 2060 } 2061 EXPORT_SYMBOL(call_netdevice_notifiers); 2062 2063 /** 2064 * call_netdevice_notifiers_mtu - call all network notifier blocks 2065 * @val: value passed unmodified to notifier function 2066 * @dev: net_device pointer passed unmodified to notifier function 2067 * @arg: additional u32 argument passed to the notifier function 2068 * 2069 * Call all network notifier blocks. Parameters and return value 2070 * are as for raw_notifier_call_chain(). 2071 */ 2072 static int call_netdevice_notifiers_mtu(unsigned long val, 2073 struct net_device *dev, u32 arg) 2074 { 2075 struct netdev_notifier_info_ext info = { 2076 .info.dev = dev, 2077 .ext.mtu = arg, 2078 }; 2079 2080 BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0); 2081 2082 return call_netdevice_notifiers_info(val, &info.info); 2083 } 2084 2085 #ifdef CONFIG_NET_INGRESS 2086 static DEFINE_STATIC_KEY_FALSE(ingress_needed_key); 2087 2088 void net_inc_ingress_queue(void) 2089 { 2090 static_branch_inc(&ingress_needed_key); 2091 } 2092 EXPORT_SYMBOL_GPL(net_inc_ingress_queue); 2093 2094 void net_dec_ingress_queue(void) 2095 { 2096 static_branch_dec(&ingress_needed_key); 2097 } 2098 EXPORT_SYMBOL_GPL(net_dec_ingress_queue); 2099 #endif 2100 2101 #ifdef CONFIG_NET_EGRESS 2102 static DEFINE_STATIC_KEY_FALSE(egress_needed_key); 2103 2104 void net_inc_egress_queue(void) 2105 { 2106 static_branch_inc(&egress_needed_key); 2107 } 2108 EXPORT_SYMBOL_GPL(net_inc_egress_queue); 2109 2110 void net_dec_egress_queue(void) 2111 { 2112 static_branch_dec(&egress_needed_key); 2113 } 2114 EXPORT_SYMBOL_GPL(net_dec_egress_queue); 2115 #endif 2116 2117 static DEFINE_STATIC_KEY_FALSE(netstamp_needed_key); 2118 #ifdef CONFIG_JUMP_LABEL 2119 static atomic_t netstamp_needed_deferred; 2120 static atomic_t netstamp_wanted; 2121 static void netstamp_clear(struct work_struct *work) 2122 { 2123 int deferred = atomic_xchg(&netstamp_needed_deferred, 0); 2124 int wanted; 2125 2126 wanted = atomic_add_return(deferred, &netstamp_wanted); 2127 if (wanted > 0) 2128 static_branch_enable(&netstamp_needed_key); 2129 else 2130 static_branch_disable(&netstamp_needed_key); 2131 } 2132 static DECLARE_WORK(netstamp_work, netstamp_clear); 2133 #endif 2134 2135 void net_enable_timestamp(void) 2136 { 2137 #ifdef CONFIG_JUMP_LABEL 2138 int wanted; 2139 2140 while (1) { 2141 wanted = atomic_read(&netstamp_wanted); 2142 if (wanted <= 0) 2143 break; 2144 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted) 2145 return; 2146 } 2147 atomic_inc(&netstamp_needed_deferred); 2148 schedule_work(&netstamp_work); 2149 #else 2150 static_branch_inc(&netstamp_needed_key); 2151 #endif 2152 } 2153 EXPORT_SYMBOL(net_enable_timestamp); 2154 2155 void net_disable_timestamp(void) 2156 { 2157 #ifdef CONFIG_JUMP_LABEL 2158 int wanted; 2159 2160 while (1) { 2161 wanted = atomic_read(&netstamp_wanted); 2162 if (wanted <= 1) 2163 break; 2164 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted) 2165 return; 2166 } 2167 atomic_dec(&netstamp_needed_deferred); 2168 schedule_work(&netstamp_work); 2169 #else 2170 static_branch_dec(&netstamp_needed_key); 2171 #endif 2172 } 2173 EXPORT_SYMBOL(net_disable_timestamp); 2174 2175 static inline void net_timestamp_set(struct sk_buff *skb) 2176 { 2177 skb->tstamp = 0; 2178 if (static_branch_unlikely(&netstamp_needed_key)) 2179 __net_timestamp(skb); 2180 } 2181 2182 #define net_timestamp_check(COND, SKB) \ 2183 if (static_branch_unlikely(&netstamp_needed_key)) { \ 2184 if ((COND) && !(SKB)->tstamp) \ 2185 __net_timestamp(SKB); \ 2186 } \ 2187 2188 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb) 2189 { 2190 unsigned int len; 2191 2192 if (!(dev->flags & IFF_UP)) 2193 return false; 2194 2195 len = dev->mtu + dev->hard_header_len + VLAN_HLEN; 2196 if (skb->len <= len) 2197 return true; 2198 2199 /* if TSO is enabled, we don't care about the length as the packet 2200 * could be forwarded without being segmented before 2201 */ 2202 if (skb_is_gso(skb)) 2203 return true; 2204 2205 return false; 2206 } 2207 EXPORT_SYMBOL_GPL(is_skb_forwardable); 2208 2209 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 2210 { 2211 int ret = ____dev_forward_skb(dev, skb); 2212 2213 if (likely(!ret)) { 2214 skb->protocol = eth_type_trans(skb, dev); 2215 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); 2216 } 2217 2218 return ret; 2219 } 2220 EXPORT_SYMBOL_GPL(__dev_forward_skb); 2221 2222 /** 2223 * dev_forward_skb - loopback an skb to another netif 2224 * 2225 * @dev: destination network device 2226 * @skb: buffer to forward 2227 * 2228 * return values: 2229 * NET_RX_SUCCESS (no congestion) 2230 * NET_RX_DROP (packet was dropped, but freed) 2231 * 2232 * dev_forward_skb can be used for injecting an skb from the 2233 * start_xmit function of one device into the receive queue 2234 * of another device. 2235 * 2236 * The receiving device may be in another namespace, so 2237 * we have to clear all information in the skb that could 2238 * impact namespace isolation. 2239 */ 2240 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 2241 { 2242 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb); 2243 } 2244 EXPORT_SYMBOL_GPL(dev_forward_skb); 2245 2246 static inline int deliver_skb(struct sk_buff *skb, 2247 struct packet_type *pt_prev, 2248 struct net_device *orig_dev) 2249 { 2250 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) 2251 return -ENOMEM; 2252 refcount_inc(&skb->users); 2253 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 2254 } 2255 2256 static inline void deliver_ptype_list_skb(struct sk_buff *skb, 2257 struct packet_type **pt, 2258 struct net_device *orig_dev, 2259 __be16 type, 2260 struct list_head *ptype_list) 2261 { 2262 struct packet_type *ptype, *pt_prev = *pt; 2263 2264 list_for_each_entry_rcu(ptype, ptype_list, list) { 2265 if (ptype->type != type) 2266 continue; 2267 if (pt_prev) 2268 deliver_skb(skb, pt_prev, orig_dev); 2269 pt_prev = ptype; 2270 } 2271 *pt = pt_prev; 2272 } 2273 2274 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) 2275 { 2276 if (!ptype->af_packet_priv || !skb->sk) 2277 return false; 2278 2279 if (ptype->id_match) 2280 return ptype->id_match(ptype, skb->sk); 2281 else if ((struct sock *)ptype->af_packet_priv == skb->sk) 2282 return true; 2283 2284 return false; 2285 } 2286 2287 /** 2288 * dev_nit_active - return true if any network interface taps are in use 2289 * 2290 * @dev: network device to check for the presence of taps 2291 */ 2292 bool dev_nit_active(struct net_device *dev) 2293 { 2294 return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all); 2295 } 2296 EXPORT_SYMBOL_GPL(dev_nit_active); 2297 2298 /* 2299 * Support routine. Sends outgoing frames to any network 2300 * taps currently in use. 2301 */ 2302 2303 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) 2304 { 2305 struct packet_type *ptype; 2306 struct sk_buff *skb2 = NULL; 2307 struct packet_type *pt_prev = NULL; 2308 struct list_head *ptype_list = &ptype_all; 2309 2310 rcu_read_lock(); 2311 again: 2312 list_for_each_entry_rcu(ptype, ptype_list, list) { 2313 if (ptype->ignore_outgoing) 2314 continue; 2315 2316 /* Never send packets back to the socket 2317 * they originated from - MvS (miquels@drinkel.ow.org) 2318 */ 2319 if (skb_loop_sk(ptype, skb)) 2320 continue; 2321 2322 if (pt_prev) { 2323 deliver_skb(skb2, pt_prev, skb->dev); 2324 pt_prev = ptype; 2325 continue; 2326 } 2327 2328 /* need to clone skb, done only once */ 2329 skb2 = skb_clone(skb, GFP_ATOMIC); 2330 if (!skb2) 2331 goto out_unlock; 2332 2333 net_timestamp_set(skb2); 2334 2335 /* skb->nh should be correctly 2336 * set by sender, so that the second statement is 2337 * just protection against buggy protocols. 2338 */ 2339 skb_reset_mac_header(skb2); 2340 2341 if (skb_network_header(skb2) < skb2->data || 2342 skb_network_header(skb2) > skb_tail_pointer(skb2)) { 2343 net_crit_ratelimited("protocol %04x is buggy, dev %s\n", 2344 ntohs(skb2->protocol), 2345 dev->name); 2346 skb_reset_network_header(skb2); 2347 } 2348 2349 skb2->transport_header = skb2->network_header; 2350 skb2->pkt_type = PACKET_OUTGOING; 2351 pt_prev = ptype; 2352 } 2353 2354 if (ptype_list == &ptype_all) { 2355 ptype_list = &dev->ptype_all; 2356 goto again; 2357 } 2358 out_unlock: 2359 if (pt_prev) { 2360 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC)) 2361 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); 2362 else 2363 kfree_skb(skb2); 2364 } 2365 rcu_read_unlock(); 2366 } 2367 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit); 2368 2369 /** 2370 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change 2371 * @dev: Network device 2372 * @txq: number of queues available 2373 * 2374 * If real_num_tx_queues is changed the tc mappings may no longer be 2375 * valid. To resolve this verify the tc mapping remains valid and if 2376 * not NULL the mapping. With no priorities mapping to this 2377 * offset/count pair it will no longer be used. In the worst case TC0 2378 * is invalid nothing can be done so disable priority mappings. If is 2379 * expected that drivers will fix this mapping if they can before 2380 * calling netif_set_real_num_tx_queues. 2381 */ 2382 static void netif_setup_tc(struct net_device *dev, unsigned int txq) 2383 { 2384 int i; 2385 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 2386 2387 /* If TC0 is invalidated disable TC mapping */ 2388 if (tc->offset + tc->count > txq) { 2389 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n"); 2390 dev->num_tc = 0; 2391 return; 2392 } 2393 2394 /* Invalidated prio to tc mappings set to TC0 */ 2395 for (i = 1; i < TC_BITMASK + 1; i++) { 2396 int q = netdev_get_prio_tc_map(dev, i); 2397 2398 tc = &dev->tc_to_txq[q]; 2399 if (tc->offset + tc->count > txq) { 2400 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n", 2401 i, q); 2402 netdev_set_prio_tc_map(dev, i, 0); 2403 } 2404 } 2405 } 2406 2407 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq) 2408 { 2409 if (dev->num_tc) { 2410 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 2411 int i; 2412 2413 /* walk through the TCs and see if it falls into any of them */ 2414 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) { 2415 if ((txq - tc->offset) < tc->count) 2416 return i; 2417 } 2418 2419 /* didn't find it, just return -1 to indicate no match */ 2420 return -1; 2421 } 2422 2423 return 0; 2424 } 2425 EXPORT_SYMBOL(netdev_txq_to_tc); 2426 2427 #ifdef CONFIG_XPS 2428 struct static_key xps_needed __read_mostly; 2429 EXPORT_SYMBOL(xps_needed); 2430 struct static_key xps_rxqs_needed __read_mostly; 2431 EXPORT_SYMBOL(xps_rxqs_needed); 2432 static DEFINE_MUTEX(xps_map_mutex); 2433 #define xmap_dereference(P) \ 2434 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex)) 2435 2436 static bool remove_xps_queue(struct xps_dev_maps *dev_maps, 2437 int tci, u16 index) 2438 { 2439 struct xps_map *map = NULL; 2440 int pos; 2441 2442 if (dev_maps) 2443 map = xmap_dereference(dev_maps->attr_map[tci]); 2444 if (!map) 2445 return false; 2446 2447 for (pos = map->len; pos--;) { 2448 if (map->queues[pos] != index) 2449 continue; 2450 2451 if (map->len > 1) { 2452 map->queues[pos] = map->queues[--map->len]; 2453 break; 2454 } 2455 2456 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL); 2457 kfree_rcu(map, rcu); 2458 return false; 2459 } 2460 2461 return true; 2462 } 2463 2464 static bool remove_xps_queue_cpu(struct net_device *dev, 2465 struct xps_dev_maps *dev_maps, 2466 int cpu, u16 offset, u16 count) 2467 { 2468 int num_tc = dev->num_tc ? : 1; 2469 bool active = false; 2470 int tci; 2471 2472 for (tci = cpu * num_tc; num_tc--; tci++) { 2473 int i, j; 2474 2475 for (i = count, j = offset; i--; j++) { 2476 if (!remove_xps_queue(dev_maps, tci, j)) 2477 break; 2478 } 2479 2480 active |= i < 0; 2481 } 2482 2483 return active; 2484 } 2485 2486 static void reset_xps_maps(struct net_device *dev, 2487 struct xps_dev_maps *dev_maps, 2488 bool is_rxqs_map) 2489 { 2490 if (is_rxqs_map) { 2491 static_key_slow_dec_cpuslocked(&xps_rxqs_needed); 2492 RCU_INIT_POINTER(dev->xps_rxqs_map, NULL); 2493 } else { 2494 RCU_INIT_POINTER(dev->xps_cpus_map, NULL); 2495 } 2496 static_key_slow_dec_cpuslocked(&xps_needed); 2497 kfree_rcu(dev_maps, rcu); 2498 } 2499 2500 static void clean_xps_maps(struct net_device *dev, const unsigned long *mask, 2501 struct xps_dev_maps *dev_maps, unsigned int nr_ids, 2502 u16 offset, u16 count, bool is_rxqs_map) 2503 { 2504 bool active = false; 2505 int i, j; 2506 2507 for (j = -1; j = netif_attrmask_next(j, mask, nr_ids), 2508 j < nr_ids;) 2509 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, 2510 count); 2511 if (!active) 2512 reset_xps_maps(dev, dev_maps, is_rxqs_map); 2513 2514 if (!is_rxqs_map) { 2515 for (i = offset + (count - 1); count--; i--) { 2516 netdev_queue_numa_node_write( 2517 netdev_get_tx_queue(dev, i), 2518 NUMA_NO_NODE); 2519 } 2520 } 2521 } 2522 2523 static void netif_reset_xps_queues(struct net_device *dev, u16 offset, 2524 u16 count) 2525 { 2526 const unsigned long *possible_mask = NULL; 2527 struct xps_dev_maps *dev_maps; 2528 unsigned int nr_ids; 2529 2530 if (!static_key_false(&xps_needed)) 2531 return; 2532 2533 cpus_read_lock(); 2534 mutex_lock(&xps_map_mutex); 2535 2536 if (static_key_false(&xps_rxqs_needed)) { 2537 dev_maps = xmap_dereference(dev->xps_rxqs_map); 2538 if (dev_maps) { 2539 nr_ids = dev->num_rx_queues; 2540 clean_xps_maps(dev, possible_mask, dev_maps, nr_ids, 2541 offset, count, true); 2542 } 2543 } 2544 2545 dev_maps = xmap_dereference(dev->xps_cpus_map); 2546 if (!dev_maps) 2547 goto out_no_maps; 2548 2549 if (num_possible_cpus() > 1) 2550 possible_mask = cpumask_bits(cpu_possible_mask); 2551 nr_ids = nr_cpu_ids; 2552 clean_xps_maps(dev, possible_mask, dev_maps, nr_ids, offset, count, 2553 false); 2554 2555 out_no_maps: 2556 mutex_unlock(&xps_map_mutex); 2557 cpus_read_unlock(); 2558 } 2559 2560 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index) 2561 { 2562 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index); 2563 } 2564 2565 static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index, 2566 u16 index, bool is_rxqs_map) 2567 { 2568 struct xps_map *new_map; 2569 int alloc_len = XPS_MIN_MAP_ALLOC; 2570 int i, pos; 2571 2572 for (pos = 0; map && pos < map->len; pos++) { 2573 if (map->queues[pos] != index) 2574 continue; 2575 return map; 2576 } 2577 2578 /* Need to add tx-queue to this CPU's/rx-queue's existing map */ 2579 if (map) { 2580 if (pos < map->alloc_len) 2581 return map; 2582 2583 alloc_len = map->alloc_len * 2; 2584 } 2585 2586 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's 2587 * map 2588 */ 2589 if (is_rxqs_map) 2590 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL); 2591 else 2592 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL, 2593 cpu_to_node(attr_index)); 2594 if (!new_map) 2595 return NULL; 2596 2597 for (i = 0; i < pos; i++) 2598 new_map->queues[i] = map->queues[i]; 2599 new_map->alloc_len = alloc_len; 2600 new_map->len = pos; 2601 2602 return new_map; 2603 } 2604 2605 /* Must be called under cpus_read_lock */ 2606 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask, 2607 u16 index, bool is_rxqs_map) 2608 { 2609 const unsigned long *online_mask = NULL, *possible_mask = NULL; 2610 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL; 2611 int i, j, tci, numa_node_id = -2; 2612 int maps_sz, num_tc = 1, tc = 0; 2613 struct xps_map *map, *new_map; 2614 bool active = false; 2615 unsigned int nr_ids; 2616 2617 if (dev->num_tc) { 2618 /* Do not allow XPS on subordinate device directly */ 2619 num_tc = dev->num_tc; 2620 if (num_tc < 0) 2621 return -EINVAL; 2622 2623 /* If queue belongs to subordinate dev use its map */ 2624 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev; 2625 2626 tc = netdev_txq_to_tc(dev, index); 2627 if (tc < 0) 2628 return -EINVAL; 2629 } 2630 2631 mutex_lock(&xps_map_mutex); 2632 if (is_rxqs_map) { 2633 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues); 2634 dev_maps = xmap_dereference(dev->xps_rxqs_map); 2635 nr_ids = dev->num_rx_queues; 2636 } else { 2637 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc); 2638 if (num_possible_cpus() > 1) { 2639 online_mask = cpumask_bits(cpu_online_mask); 2640 possible_mask = cpumask_bits(cpu_possible_mask); 2641 } 2642 dev_maps = xmap_dereference(dev->xps_cpus_map); 2643 nr_ids = nr_cpu_ids; 2644 } 2645 2646 if (maps_sz < L1_CACHE_BYTES) 2647 maps_sz = L1_CACHE_BYTES; 2648 2649 /* allocate memory for queue storage */ 2650 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids), 2651 j < nr_ids;) { 2652 if (!new_dev_maps) 2653 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL); 2654 if (!new_dev_maps) { 2655 mutex_unlock(&xps_map_mutex); 2656 return -ENOMEM; 2657 } 2658 2659 tci = j * num_tc + tc; 2660 map = dev_maps ? xmap_dereference(dev_maps->attr_map[tci]) : 2661 NULL; 2662 2663 map = expand_xps_map(map, j, index, is_rxqs_map); 2664 if (!map) 2665 goto error; 2666 2667 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); 2668 } 2669 2670 if (!new_dev_maps) 2671 goto out_no_new_maps; 2672 2673 if (!dev_maps) { 2674 /* Increment static keys at most once per type */ 2675 static_key_slow_inc_cpuslocked(&xps_needed); 2676 if (is_rxqs_map) 2677 static_key_slow_inc_cpuslocked(&xps_rxqs_needed); 2678 } 2679 2680 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids), 2681 j < nr_ids;) { 2682 /* copy maps belonging to foreign traffic classes */ 2683 for (i = tc, tci = j * num_tc; dev_maps && i--; tci++) { 2684 /* fill in the new device map from the old device map */ 2685 map = xmap_dereference(dev_maps->attr_map[tci]); 2686 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); 2687 } 2688 2689 /* We need to explicitly update tci as prevous loop 2690 * could break out early if dev_maps is NULL. 2691 */ 2692 tci = j * num_tc + tc; 2693 2694 if (netif_attr_test_mask(j, mask, nr_ids) && 2695 netif_attr_test_online(j, online_mask, nr_ids)) { 2696 /* add tx-queue to CPU/rx-queue maps */ 2697 int pos = 0; 2698 2699 map = xmap_dereference(new_dev_maps->attr_map[tci]); 2700 while ((pos < map->len) && (map->queues[pos] != index)) 2701 pos++; 2702 2703 if (pos == map->len) 2704 map->queues[map->len++] = index; 2705 #ifdef CONFIG_NUMA 2706 if (!is_rxqs_map) { 2707 if (numa_node_id == -2) 2708 numa_node_id = cpu_to_node(j); 2709 else if (numa_node_id != cpu_to_node(j)) 2710 numa_node_id = -1; 2711 } 2712 #endif 2713 } else if (dev_maps) { 2714 /* fill in the new device map from the old device map */ 2715 map = xmap_dereference(dev_maps->attr_map[tci]); 2716 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); 2717 } 2718 2719 /* copy maps belonging to foreign traffic classes */ 2720 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) { 2721 /* fill in the new device map from the old device map */ 2722 map = xmap_dereference(dev_maps->attr_map[tci]); 2723 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); 2724 } 2725 } 2726 2727 if (is_rxqs_map) 2728 rcu_assign_pointer(dev->xps_rxqs_map, new_dev_maps); 2729 else 2730 rcu_assign_pointer(dev->xps_cpus_map, new_dev_maps); 2731 2732 /* Cleanup old maps */ 2733 if (!dev_maps) 2734 goto out_no_old_maps; 2735 2736 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids), 2737 j < nr_ids;) { 2738 for (i = num_tc, tci = j * num_tc; i--; tci++) { 2739 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); 2740 map = xmap_dereference(dev_maps->attr_map[tci]); 2741 if (map && map != new_map) 2742 kfree_rcu(map, rcu); 2743 } 2744 } 2745 2746 kfree_rcu(dev_maps, rcu); 2747 2748 out_no_old_maps: 2749 dev_maps = new_dev_maps; 2750 active = true; 2751 2752 out_no_new_maps: 2753 if (!is_rxqs_map) { 2754 /* update Tx queue numa node */ 2755 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index), 2756 (numa_node_id >= 0) ? 2757 numa_node_id : NUMA_NO_NODE); 2758 } 2759 2760 if (!dev_maps) 2761 goto out_no_maps; 2762 2763 /* removes tx-queue from unused CPUs/rx-queues */ 2764 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids), 2765 j < nr_ids;) { 2766 for (i = tc, tci = j * num_tc; i--; tci++) 2767 active |= remove_xps_queue(dev_maps, tci, index); 2768 if (!netif_attr_test_mask(j, mask, nr_ids) || 2769 !netif_attr_test_online(j, online_mask, nr_ids)) 2770 active |= remove_xps_queue(dev_maps, tci, index); 2771 for (i = num_tc - tc, tci++; --i; tci++) 2772 active |= remove_xps_queue(dev_maps, tci, index); 2773 } 2774 2775 /* free map if not active */ 2776 if (!active) 2777 reset_xps_maps(dev, dev_maps, is_rxqs_map); 2778 2779 out_no_maps: 2780 mutex_unlock(&xps_map_mutex); 2781 2782 return 0; 2783 error: 2784 /* remove any maps that we added */ 2785 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids), 2786 j < nr_ids;) { 2787 for (i = num_tc, tci = j * num_tc; i--; tci++) { 2788 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); 2789 map = dev_maps ? 2790 xmap_dereference(dev_maps->attr_map[tci]) : 2791 NULL; 2792 if (new_map && new_map != map) 2793 kfree(new_map); 2794 } 2795 } 2796 2797 mutex_unlock(&xps_map_mutex); 2798 2799 kfree(new_dev_maps); 2800 return -ENOMEM; 2801 } 2802 EXPORT_SYMBOL_GPL(__netif_set_xps_queue); 2803 2804 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask, 2805 u16 index) 2806 { 2807 int ret; 2808 2809 cpus_read_lock(); 2810 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, false); 2811 cpus_read_unlock(); 2812 2813 return ret; 2814 } 2815 EXPORT_SYMBOL(netif_set_xps_queue); 2816 2817 #endif 2818 static void netdev_unbind_all_sb_channels(struct net_device *dev) 2819 { 2820 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; 2821 2822 /* Unbind any subordinate channels */ 2823 while (txq-- != &dev->_tx[0]) { 2824 if (txq->sb_dev) 2825 netdev_unbind_sb_channel(dev, txq->sb_dev); 2826 } 2827 } 2828 2829 void netdev_reset_tc(struct net_device *dev) 2830 { 2831 #ifdef CONFIG_XPS 2832 netif_reset_xps_queues_gt(dev, 0); 2833 #endif 2834 netdev_unbind_all_sb_channels(dev); 2835 2836 /* Reset TC configuration of device */ 2837 dev->num_tc = 0; 2838 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq)); 2839 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map)); 2840 } 2841 EXPORT_SYMBOL(netdev_reset_tc); 2842 2843 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset) 2844 { 2845 if (tc >= dev->num_tc) 2846 return -EINVAL; 2847 2848 #ifdef CONFIG_XPS 2849 netif_reset_xps_queues(dev, offset, count); 2850 #endif 2851 dev->tc_to_txq[tc].count = count; 2852 dev->tc_to_txq[tc].offset = offset; 2853 return 0; 2854 } 2855 EXPORT_SYMBOL(netdev_set_tc_queue); 2856 2857 int netdev_set_num_tc(struct net_device *dev, u8 num_tc) 2858 { 2859 if (num_tc > TC_MAX_QUEUE) 2860 return -EINVAL; 2861 2862 #ifdef CONFIG_XPS 2863 netif_reset_xps_queues_gt(dev, 0); 2864 #endif 2865 netdev_unbind_all_sb_channels(dev); 2866 2867 dev->num_tc = num_tc; 2868 return 0; 2869 } 2870 EXPORT_SYMBOL(netdev_set_num_tc); 2871 2872 void netdev_unbind_sb_channel(struct net_device *dev, 2873 struct net_device *sb_dev) 2874 { 2875 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; 2876 2877 #ifdef CONFIG_XPS 2878 netif_reset_xps_queues_gt(sb_dev, 0); 2879 #endif 2880 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq)); 2881 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map)); 2882 2883 while (txq-- != &dev->_tx[0]) { 2884 if (txq->sb_dev == sb_dev) 2885 txq->sb_dev = NULL; 2886 } 2887 } 2888 EXPORT_SYMBOL(netdev_unbind_sb_channel); 2889 2890 int netdev_bind_sb_channel_queue(struct net_device *dev, 2891 struct net_device *sb_dev, 2892 u8 tc, u16 count, u16 offset) 2893 { 2894 /* Make certain the sb_dev and dev are already configured */ 2895 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc) 2896 return -EINVAL; 2897 2898 /* We cannot hand out queues we don't have */ 2899 if ((offset + count) > dev->real_num_tx_queues) 2900 return -EINVAL; 2901 2902 /* Record the mapping */ 2903 sb_dev->tc_to_txq[tc].count = count; 2904 sb_dev->tc_to_txq[tc].offset = offset; 2905 2906 /* Provide a way for Tx queue to find the tc_to_txq map or 2907 * XPS map for itself. 2908 */ 2909 while (count--) 2910 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev; 2911 2912 return 0; 2913 } 2914 EXPORT_SYMBOL(netdev_bind_sb_channel_queue); 2915 2916 int netdev_set_sb_channel(struct net_device *dev, u16 channel) 2917 { 2918 /* Do not use a multiqueue device to represent a subordinate channel */ 2919 if (netif_is_multiqueue(dev)) 2920 return -ENODEV; 2921 2922 /* We allow channels 1 - 32767 to be used for subordinate channels. 2923 * Channel 0 is meant to be "native" mode and used only to represent 2924 * the main root device. We allow writing 0 to reset the device back 2925 * to normal mode after being used as a subordinate channel. 2926 */ 2927 if (channel > S16_MAX) 2928 return -EINVAL; 2929 2930 dev->num_tc = -channel; 2931 2932 return 0; 2933 } 2934 EXPORT_SYMBOL(netdev_set_sb_channel); 2935 2936 /* 2937 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues 2938 * greater than real_num_tx_queues stale skbs on the qdisc must be flushed. 2939 */ 2940 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) 2941 { 2942 bool disabling; 2943 int rc; 2944 2945 disabling = txq < dev->real_num_tx_queues; 2946 2947 if (txq < 1 || txq > dev->num_tx_queues) 2948 return -EINVAL; 2949 2950 if (dev->reg_state == NETREG_REGISTERED || 2951 dev->reg_state == NETREG_UNREGISTERING) { 2952 ASSERT_RTNL(); 2953 2954 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, 2955 txq); 2956 if (rc) 2957 return rc; 2958 2959 if (dev->num_tc) 2960 netif_setup_tc(dev, txq); 2961 2962 dev->real_num_tx_queues = txq; 2963 2964 if (disabling) { 2965 synchronize_net(); 2966 qdisc_reset_all_tx_gt(dev, txq); 2967 #ifdef CONFIG_XPS 2968 netif_reset_xps_queues_gt(dev, txq); 2969 #endif 2970 } 2971 } else { 2972 dev->real_num_tx_queues = txq; 2973 } 2974 2975 return 0; 2976 } 2977 EXPORT_SYMBOL(netif_set_real_num_tx_queues); 2978 2979 #ifdef CONFIG_SYSFS 2980 /** 2981 * netif_set_real_num_rx_queues - set actual number of RX queues used 2982 * @dev: Network device 2983 * @rxq: Actual number of RX queues 2984 * 2985 * This must be called either with the rtnl_lock held or before 2986 * registration of the net device. Returns 0 on success, or a 2987 * negative error code. If called before registration, it always 2988 * succeeds. 2989 */ 2990 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) 2991 { 2992 int rc; 2993 2994 if (rxq < 1 || rxq > dev->num_rx_queues) 2995 return -EINVAL; 2996 2997 if (dev->reg_state == NETREG_REGISTERED) { 2998 ASSERT_RTNL(); 2999 3000 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, 3001 rxq); 3002 if (rc) 3003 return rc; 3004 } 3005 3006 dev->real_num_rx_queues = rxq; 3007 return 0; 3008 } 3009 EXPORT_SYMBOL(netif_set_real_num_rx_queues); 3010 #endif 3011 3012 /** 3013 * netif_get_num_default_rss_queues - default number of RSS queues 3014 * 3015 * This routine should set an upper limit on the number of RSS queues 3016 * used by default by multiqueue devices. 3017 */ 3018 int netif_get_num_default_rss_queues(void) 3019 { 3020 return is_kdump_kernel() ? 3021 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus()); 3022 } 3023 EXPORT_SYMBOL(netif_get_num_default_rss_queues); 3024 3025 static void __netif_reschedule(struct Qdisc *q) 3026 { 3027 struct softnet_data *sd; 3028 unsigned long flags; 3029 3030 local_irq_save(flags); 3031 sd = this_cpu_ptr(&softnet_data); 3032 q->next_sched = NULL; 3033 *sd->output_queue_tailp = q; 3034 sd->output_queue_tailp = &q->next_sched; 3035 raise_softirq_irqoff(NET_TX_SOFTIRQ); 3036 local_irq_restore(flags); 3037 } 3038 3039 void __netif_schedule(struct Qdisc *q) 3040 { 3041 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) 3042 __netif_reschedule(q); 3043 } 3044 EXPORT_SYMBOL(__netif_schedule); 3045 3046 struct dev_kfree_skb_cb { 3047 enum skb_free_reason reason; 3048 }; 3049 3050 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb) 3051 { 3052 return (struct dev_kfree_skb_cb *)skb->cb; 3053 } 3054 3055 void netif_schedule_queue(struct netdev_queue *txq) 3056 { 3057 rcu_read_lock(); 3058 if (!netif_xmit_stopped(txq)) { 3059 struct Qdisc *q = rcu_dereference(txq->qdisc); 3060 3061 __netif_schedule(q); 3062 } 3063 rcu_read_unlock(); 3064 } 3065 EXPORT_SYMBOL(netif_schedule_queue); 3066 3067 void netif_tx_wake_queue(struct netdev_queue *dev_queue) 3068 { 3069 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { 3070 struct Qdisc *q; 3071 3072 rcu_read_lock(); 3073 q = rcu_dereference(dev_queue->qdisc); 3074 __netif_schedule(q); 3075 rcu_read_unlock(); 3076 } 3077 } 3078 EXPORT_SYMBOL(netif_tx_wake_queue); 3079 3080 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason) 3081 { 3082 unsigned long flags; 3083 3084 if (unlikely(!skb)) 3085 return; 3086 3087 if (likely(refcount_read(&skb->users) == 1)) { 3088 smp_rmb(); 3089 refcount_set(&skb->users, 0); 3090 } else if (likely(!refcount_dec_and_test(&skb->users))) { 3091 return; 3092 } 3093 get_kfree_skb_cb(skb)->reason = reason; 3094 local_irq_save(flags); 3095 skb->next = __this_cpu_read(softnet_data.completion_queue); 3096 __this_cpu_write(softnet_data.completion_queue, skb); 3097 raise_softirq_irqoff(NET_TX_SOFTIRQ); 3098 local_irq_restore(flags); 3099 } 3100 EXPORT_SYMBOL(__dev_kfree_skb_irq); 3101 3102 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason) 3103 { 3104 if (in_irq() || irqs_disabled()) 3105 __dev_kfree_skb_irq(skb, reason); 3106 else 3107 dev_kfree_skb(skb); 3108 } 3109 EXPORT_SYMBOL(__dev_kfree_skb_any); 3110 3111 3112 /** 3113 * netif_device_detach - mark device as removed 3114 * @dev: network device 3115 * 3116 * Mark device as removed from system and therefore no longer available. 3117 */ 3118 void netif_device_detach(struct net_device *dev) 3119 { 3120 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && 3121 netif_running(dev)) { 3122 netif_tx_stop_all_queues(dev); 3123 } 3124 } 3125 EXPORT_SYMBOL(netif_device_detach); 3126 3127 /** 3128 * netif_device_attach - mark device as attached 3129 * @dev: network device 3130 * 3131 * Mark device as attached from system and restart if needed. 3132 */ 3133 void netif_device_attach(struct net_device *dev) 3134 { 3135 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && 3136 netif_running(dev)) { 3137 netif_tx_wake_all_queues(dev); 3138 __netdev_watchdog_up(dev); 3139 } 3140 } 3141 EXPORT_SYMBOL(netif_device_attach); 3142 3143 /* 3144 * Returns a Tx hash based on the given packet descriptor a Tx queues' number 3145 * to be used as a distribution range. 3146 */ 3147 static u16 skb_tx_hash(const struct net_device *dev, 3148 const struct net_device *sb_dev, 3149 struct sk_buff *skb) 3150 { 3151 u32 hash; 3152 u16 qoffset = 0; 3153 u16 qcount = dev->real_num_tx_queues; 3154 3155 if (dev->num_tc) { 3156 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); 3157 3158 qoffset = sb_dev->tc_to_txq[tc].offset; 3159 qcount = sb_dev->tc_to_txq[tc].count; 3160 } 3161 3162 if (skb_rx_queue_recorded(skb)) { 3163 hash = skb_get_rx_queue(skb); 3164 if (hash >= qoffset) 3165 hash -= qoffset; 3166 while (unlikely(hash >= qcount)) 3167 hash -= qcount; 3168 return hash + qoffset; 3169 } 3170 3171 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset; 3172 } 3173 3174 static void skb_warn_bad_offload(const struct sk_buff *skb) 3175 { 3176 static const netdev_features_t null_features; 3177 struct net_device *dev = skb->dev; 3178 const char *name = ""; 3179 3180 if (!net_ratelimit()) 3181 return; 3182 3183 if (dev) { 3184 if (dev->dev.parent) 3185 name = dev_driver_string(dev->dev.parent); 3186 else 3187 name = netdev_name(dev); 3188 } 3189 skb_dump(KERN_WARNING, skb, false); 3190 WARN(1, "%s: caps=(%pNF, %pNF)\n", 3191 name, dev ? &dev->features : &null_features, 3192 skb->sk ? &skb->sk->sk_route_caps : &null_features); 3193 } 3194 3195 /* 3196 * Invalidate hardware checksum when packet is to be mangled, and 3197 * complete checksum manually on outgoing path. 3198 */ 3199 int skb_checksum_help(struct sk_buff *skb) 3200 { 3201 __wsum csum; 3202 int ret = 0, offset; 3203 3204 if (skb->ip_summed == CHECKSUM_COMPLETE) 3205 goto out_set_summed; 3206 3207 if (unlikely(skb_shinfo(skb)->gso_size)) { 3208 skb_warn_bad_offload(skb); 3209 return -EINVAL; 3210 } 3211 3212 /* Before computing a checksum, we should make sure no frag could 3213 * be modified by an external entity : checksum could be wrong. 3214 */ 3215 if (skb_has_shared_frag(skb)) { 3216 ret = __skb_linearize(skb); 3217 if (ret) 3218 goto out; 3219 } 3220 3221 offset = skb_checksum_start_offset(skb); 3222 BUG_ON(offset >= skb_headlen(skb)); 3223 csum = skb_checksum(skb, offset, skb->len - offset, 0); 3224 3225 offset += skb->csum_offset; 3226 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb)); 3227 3228 ret = skb_ensure_writable(skb, offset + sizeof(__sum16)); 3229 if (ret) 3230 goto out; 3231 3232 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0; 3233 out_set_summed: 3234 skb->ip_summed = CHECKSUM_NONE; 3235 out: 3236 return ret; 3237 } 3238 EXPORT_SYMBOL(skb_checksum_help); 3239 3240 int skb_crc32c_csum_help(struct sk_buff *skb) 3241 { 3242 __le32 crc32c_csum; 3243 int ret = 0, offset, start; 3244 3245 if (skb->ip_summed != CHECKSUM_PARTIAL) 3246 goto out; 3247 3248 if (unlikely(skb_is_gso(skb))) 3249 goto out; 3250 3251 /* Before computing a checksum, we should make sure no frag could 3252 * be modified by an external entity : checksum could be wrong. 3253 */ 3254 if (unlikely(skb_has_shared_frag(skb))) { 3255 ret = __skb_linearize(skb); 3256 if (ret) 3257 goto out; 3258 } 3259 start = skb_checksum_start_offset(skb); 3260 offset = start + offsetof(struct sctphdr, checksum); 3261 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) { 3262 ret = -EINVAL; 3263 goto out; 3264 } 3265 3266 ret = skb_ensure_writable(skb, offset + sizeof(__le32)); 3267 if (ret) 3268 goto out; 3269 3270 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start, 3271 skb->len - start, ~(__u32)0, 3272 crc32c_csum_stub)); 3273 *(__le32 *)(skb->data + offset) = crc32c_csum; 3274 skb->ip_summed = CHECKSUM_NONE; 3275 skb->csum_not_inet = 0; 3276 out: 3277 return ret; 3278 } 3279 3280 __be16 skb_network_protocol(struct sk_buff *skb, int *depth) 3281 { 3282 __be16 type = skb->protocol; 3283 3284 /* Tunnel gso handlers can set protocol to ethernet. */ 3285 if (type == htons(ETH_P_TEB)) { 3286 struct ethhdr *eth; 3287 3288 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr)))) 3289 return 0; 3290 3291 eth = (struct ethhdr *)skb->data; 3292 type = eth->h_proto; 3293 } 3294 3295 return __vlan_get_protocol(skb, type, depth); 3296 } 3297 3298 /** 3299 * skb_mac_gso_segment - mac layer segmentation handler. 3300 * @skb: buffer to segment 3301 * @features: features for the output path (see dev->features) 3302 */ 3303 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, 3304 netdev_features_t features) 3305 { 3306 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); 3307 struct packet_offload *ptype; 3308 int vlan_depth = skb->mac_len; 3309 __be16 type = skb_network_protocol(skb, &vlan_depth); 3310 3311 if (unlikely(!type)) 3312 return ERR_PTR(-EINVAL); 3313 3314 __skb_pull(skb, vlan_depth); 3315 3316 rcu_read_lock(); 3317 list_for_each_entry_rcu(ptype, &offload_base, list) { 3318 if (ptype->type == type && ptype->callbacks.gso_segment) { 3319 segs = ptype->callbacks.gso_segment(skb, features); 3320 break; 3321 } 3322 } 3323 rcu_read_unlock(); 3324 3325 __skb_push(skb, skb->data - skb_mac_header(skb)); 3326 3327 return segs; 3328 } 3329 EXPORT_SYMBOL(skb_mac_gso_segment); 3330 3331 3332 /* openvswitch calls this on rx path, so we need a different check. 3333 */ 3334 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) 3335 { 3336 if (tx_path) 3337 return skb->ip_summed != CHECKSUM_PARTIAL && 3338 skb->ip_summed != CHECKSUM_UNNECESSARY; 3339 3340 return skb->ip_summed == CHECKSUM_NONE; 3341 } 3342 3343 /** 3344 * __skb_gso_segment - Perform segmentation on skb. 3345 * @skb: buffer to segment 3346 * @features: features for the output path (see dev->features) 3347 * @tx_path: whether it is called in TX path 3348 * 3349 * This function segments the given skb and returns a list of segments. 3350 * 3351 * It may return NULL if the skb requires no segmentation. This is 3352 * only possible when GSO is used for verifying header integrity. 3353 * 3354 * Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb. 3355 */ 3356 struct sk_buff *__skb_gso_segment(struct sk_buff *skb, 3357 netdev_features_t features, bool tx_path) 3358 { 3359 struct sk_buff *segs; 3360 3361 if (unlikely(skb_needs_check(skb, tx_path))) { 3362 int err; 3363 3364 /* We're going to init ->check field in TCP or UDP header */ 3365 err = skb_cow_head(skb, 0); 3366 if (err < 0) 3367 return ERR_PTR(err); 3368 } 3369 3370 /* Only report GSO partial support if it will enable us to 3371 * support segmentation on this frame without needing additional 3372 * work. 3373 */ 3374 if (features & NETIF_F_GSO_PARTIAL) { 3375 netdev_features_t partial_features = NETIF_F_GSO_ROBUST; 3376 struct net_device *dev = skb->dev; 3377 3378 partial_features |= dev->features & dev->gso_partial_features; 3379 if (!skb_gso_ok(skb, features | partial_features)) 3380 features &= ~NETIF_F_GSO_PARTIAL; 3381 } 3382 3383 BUILD_BUG_ON(SKB_GSO_CB_OFFSET + 3384 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb)); 3385 3386 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb); 3387 SKB_GSO_CB(skb)->encap_level = 0; 3388 3389 skb_reset_mac_header(skb); 3390 skb_reset_mac_len(skb); 3391 3392 segs = skb_mac_gso_segment(skb, features); 3393 3394 if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs))) 3395 skb_warn_bad_offload(skb); 3396 3397 return segs; 3398 } 3399 EXPORT_SYMBOL(__skb_gso_segment); 3400 3401 /* Take action when hardware reception checksum errors are detected. */ 3402 #ifdef CONFIG_BUG 3403 void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb) 3404 { 3405 if (net_ratelimit()) { 3406 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>"); 3407 skb_dump(KERN_ERR, skb, true); 3408 dump_stack(); 3409 } 3410 } 3411 EXPORT_SYMBOL(netdev_rx_csum_fault); 3412 #endif 3413 3414 /* XXX: check that highmem exists at all on the given machine. */ 3415 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) 3416 { 3417 #ifdef CONFIG_HIGHMEM 3418 int i; 3419 3420 if (!(dev->features & NETIF_F_HIGHDMA)) { 3421 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3422 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3423 3424 if (PageHighMem(skb_frag_page(frag))) 3425 return 1; 3426 } 3427 } 3428 #endif 3429 return 0; 3430 } 3431 3432 /* If MPLS offload request, verify we are testing hardware MPLS features 3433 * instead of standard features for the netdev. 3434 */ 3435 #if IS_ENABLED(CONFIG_NET_MPLS_GSO) 3436 static netdev_features_t net_mpls_features(struct sk_buff *skb, 3437 netdev_features_t features, 3438 __be16 type) 3439 { 3440 if (eth_p_mpls(type)) 3441 features &= skb->dev->mpls_features; 3442 3443 return features; 3444 } 3445 #else 3446 static netdev_features_t net_mpls_features(struct sk_buff *skb, 3447 netdev_features_t features, 3448 __be16 type) 3449 { 3450 return features; 3451 } 3452 #endif 3453 3454 static netdev_features_t harmonize_features(struct sk_buff *skb, 3455 netdev_features_t features) 3456 { 3457 __be16 type; 3458 3459 type = skb_network_protocol(skb, NULL); 3460 features = net_mpls_features(skb, features, type); 3461 3462 if (skb->ip_summed != CHECKSUM_NONE && 3463 !can_checksum_protocol(features, type)) { 3464 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 3465 } 3466 if (illegal_highdma(skb->dev, skb)) 3467 features &= ~NETIF_F_SG; 3468 3469 return features; 3470 } 3471 3472 netdev_features_t passthru_features_check(struct sk_buff *skb, 3473 struct net_device *dev, 3474 netdev_features_t features) 3475 { 3476 return features; 3477 } 3478 EXPORT_SYMBOL(passthru_features_check); 3479 3480 static netdev_features_t dflt_features_check(struct sk_buff *skb, 3481 struct net_device *dev, 3482 netdev_features_t features) 3483 { 3484 return vlan_features_check(skb, features); 3485 } 3486 3487 static netdev_features_t gso_features_check(const struct sk_buff *skb, 3488 struct net_device *dev, 3489 netdev_features_t features) 3490 { 3491 u16 gso_segs = skb_shinfo(skb)->gso_segs; 3492 3493 if (gso_segs > dev->gso_max_segs) 3494 return features & ~NETIF_F_GSO_MASK; 3495 3496 /* Support for GSO partial features requires software 3497 * intervention before we can actually process the packets 3498 * so we need to strip support for any partial features now 3499 * and we can pull them back in after we have partially 3500 * segmented the frame. 3501 */ 3502 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)) 3503 features &= ~dev->gso_partial_features; 3504 3505 /* Make sure to clear the IPv4 ID mangling feature if the 3506 * IPv4 header has the potential to be fragmented. 3507 */ 3508 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { 3509 struct iphdr *iph = skb->encapsulation ? 3510 inner_ip_hdr(skb) : ip_hdr(skb); 3511 3512 if (!(iph->frag_off & htons(IP_DF))) 3513 features &= ~NETIF_F_TSO_MANGLEID; 3514 } 3515 3516 return features; 3517 } 3518 3519 netdev_features_t netif_skb_features(struct sk_buff *skb) 3520 { 3521 struct net_device *dev = skb->dev; 3522 netdev_features_t features = dev->features; 3523 3524 if (skb_is_gso(skb)) 3525 features = gso_features_check(skb, dev, features); 3526 3527 /* If encapsulation offload request, verify we are testing 3528 * hardware encapsulation features instead of standard 3529 * features for the netdev 3530 */ 3531 if (skb->encapsulation) 3532 features &= dev->hw_enc_features; 3533 3534 if (skb_vlan_tagged(skb)) 3535 features = netdev_intersect_features(features, 3536 dev->vlan_features | 3537 NETIF_F_HW_VLAN_CTAG_TX | 3538 NETIF_F_HW_VLAN_STAG_TX); 3539 3540 if (dev->netdev_ops->ndo_features_check) 3541 features &= dev->netdev_ops->ndo_features_check(skb, dev, 3542 features); 3543 else 3544 features &= dflt_features_check(skb, dev, features); 3545 3546 return harmonize_features(skb, features); 3547 } 3548 EXPORT_SYMBOL(netif_skb_features); 3549 3550 static int xmit_one(struct sk_buff *skb, struct net_device *dev, 3551 struct netdev_queue *txq, bool more) 3552 { 3553 unsigned int len; 3554 int rc; 3555 3556 if (dev_nit_active(dev)) 3557 dev_queue_xmit_nit(skb, dev); 3558 3559 len = skb->len; 3560 trace_net_dev_start_xmit(skb, dev); 3561 rc = netdev_start_xmit(skb, dev, txq, more); 3562 trace_net_dev_xmit(skb, rc, dev, len); 3563 3564 return rc; 3565 } 3566 3567 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev, 3568 struct netdev_queue *txq, int *ret) 3569 { 3570 struct sk_buff *skb = first; 3571 int rc = NETDEV_TX_OK; 3572 3573 while (skb) { 3574 struct sk_buff *next = skb->next; 3575 3576 skb_mark_not_on_list(skb); 3577 rc = xmit_one(skb, dev, txq, next != NULL); 3578 if (unlikely(!dev_xmit_complete(rc))) { 3579 skb->next = next; 3580 goto out; 3581 } 3582 3583 skb = next; 3584 if (netif_tx_queue_stopped(txq) && skb) { 3585 rc = NETDEV_TX_BUSY; 3586 break; 3587 } 3588 } 3589 3590 out: 3591 *ret = rc; 3592 return skb; 3593 } 3594 3595 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, 3596 netdev_features_t features) 3597 { 3598 if (skb_vlan_tag_present(skb) && 3599 !vlan_hw_offload_capable(features, skb->vlan_proto)) 3600 skb = __vlan_hwaccel_push_inside(skb); 3601 return skb; 3602 } 3603 3604 int skb_csum_hwoffload_help(struct sk_buff *skb, 3605 const netdev_features_t features) 3606 { 3607 if (unlikely(skb->csum_not_inet)) 3608 return !!(features & NETIF_F_SCTP_CRC) ? 0 : 3609 skb_crc32c_csum_help(skb); 3610 3611 return !!(features & NETIF_F_CSUM_MASK) ? 0 : skb_checksum_help(skb); 3612 } 3613 EXPORT_SYMBOL(skb_csum_hwoffload_help); 3614 3615 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again) 3616 { 3617 netdev_features_t features; 3618 3619 features = netif_skb_features(skb); 3620 skb = validate_xmit_vlan(skb, features); 3621 if (unlikely(!skb)) 3622 goto out_null; 3623 3624 skb = sk_validate_xmit_skb(skb, dev); 3625 if (unlikely(!skb)) 3626 goto out_null; 3627 3628 if (netif_needs_gso(skb, features)) { 3629 struct sk_buff *segs; 3630 3631 segs = skb_gso_segment(skb, features); 3632 if (IS_ERR(segs)) { 3633 goto out_kfree_skb; 3634 } else if (segs) { 3635 consume_skb(skb); 3636 skb = segs; 3637 } 3638 } else { 3639 if (skb_needs_linearize(skb, features) && 3640 __skb_linearize(skb)) 3641 goto out_kfree_skb; 3642 3643 /* If packet is not checksummed and device does not 3644 * support checksumming for this protocol, complete 3645 * checksumming here. 3646 */ 3647 if (skb->ip_summed == CHECKSUM_PARTIAL) { 3648 if (skb->encapsulation) 3649 skb_set_inner_transport_header(skb, 3650 skb_checksum_start_offset(skb)); 3651 else 3652 skb_set_transport_header(skb, 3653 skb_checksum_start_offset(skb)); 3654 if (skb_csum_hwoffload_help(skb, features)) 3655 goto out_kfree_skb; 3656 } 3657 } 3658 3659 skb = validate_xmit_xfrm(skb, features, again); 3660 3661 return skb; 3662 3663 out_kfree_skb: 3664 kfree_skb(skb); 3665 out_null: 3666 atomic_long_inc(&dev->tx_dropped); 3667 return NULL; 3668 } 3669 3670 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again) 3671 { 3672 struct sk_buff *next, *head = NULL, *tail; 3673 3674 for (; skb != NULL; skb = next) { 3675 next = skb->next; 3676 skb_mark_not_on_list(skb); 3677 3678 /* in case skb wont be segmented, point to itself */ 3679 skb->prev = skb; 3680 3681 skb = validate_xmit_skb(skb, dev, again); 3682 if (!skb) 3683 continue; 3684 3685 if (!head) 3686 head = skb; 3687 else 3688 tail->next = skb; 3689 /* If skb was segmented, skb->prev points to 3690 * the last segment. If not, it still contains skb. 3691 */ 3692 tail = skb->prev; 3693 } 3694 return head; 3695 } 3696 EXPORT_SYMBOL_GPL(validate_xmit_skb_list); 3697 3698 static void qdisc_pkt_len_init(struct sk_buff *skb) 3699 { 3700 const struct skb_shared_info *shinfo = skb_shinfo(skb); 3701 3702 qdisc_skb_cb(skb)->pkt_len = skb->len; 3703 3704 /* To get more precise estimation of bytes sent on wire, 3705 * we add to pkt_len the headers size of all segments 3706 */ 3707 if (shinfo->gso_size && skb_transport_header_was_set(skb)) { 3708 unsigned int hdr_len; 3709 u16 gso_segs = shinfo->gso_segs; 3710 3711 /* mac layer + network layer */ 3712 hdr_len = skb_transport_header(skb) - skb_mac_header(skb); 3713 3714 /* + transport layer */ 3715 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { 3716 const struct tcphdr *th; 3717 struct tcphdr _tcphdr; 3718 3719 th = skb_header_pointer(skb, skb_transport_offset(skb), 3720 sizeof(_tcphdr), &_tcphdr); 3721 if (likely(th)) 3722 hdr_len += __tcp_hdrlen(th); 3723 } else { 3724 struct udphdr _udphdr; 3725 3726 if (skb_header_pointer(skb, skb_transport_offset(skb), 3727 sizeof(_udphdr), &_udphdr)) 3728 hdr_len += sizeof(struct udphdr); 3729 } 3730 3731 if (shinfo->gso_type & SKB_GSO_DODGY) 3732 gso_segs = DIV_ROUND_UP(skb->len - hdr_len, 3733 shinfo->gso_size); 3734 3735 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; 3736 } 3737 } 3738 3739 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 3740 struct net_device *dev, 3741 struct netdev_queue *txq) 3742 { 3743 spinlock_t *root_lock = qdisc_lock(q); 3744 struct sk_buff *to_free = NULL; 3745 bool contended; 3746 int rc; 3747 3748 qdisc_calculate_pkt_len(skb, q); 3749 3750 if (q->flags & TCQ_F_NOLOCK) { 3751 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK; 3752 qdisc_run(q); 3753 3754 if (unlikely(to_free)) 3755 kfree_skb_list(to_free); 3756 return rc; 3757 } 3758 3759 /* 3760 * Heuristic to force contended enqueues to serialize on a 3761 * separate lock before trying to get qdisc main lock. 3762 * This permits qdisc->running owner to get the lock more 3763 * often and dequeue packets faster. 3764 */ 3765 contended = qdisc_is_running(q); 3766 if (unlikely(contended)) 3767 spin_lock(&q->busylock); 3768 3769 spin_lock(root_lock); 3770 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { 3771 __qdisc_drop(skb, &to_free); 3772 rc = NET_XMIT_DROP; 3773 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && 3774 qdisc_run_begin(q)) { 3775 /* 3776 * This is a work-conserving queue; there are no old skbs 3777 * waiting to be sent out; and the qdisc is not running - 3778 * xmit the skb directly. 3779 */ 3780 3781 qdisc_bstats_update(q, skb); 3782 3783 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) { 3784 if (unlikely(contended)) { 3785 spin_unlock(&q->busylock); 3786 contended = false; 3787 } 3788 __qdisc_run(q); 3789 } 3790 3791 qdisc_run_end(q); 3792 rc = NET_XMIT_SUCCESS; 3793 } else { 3794 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK; 3795 if (qdisc_run_begin(q)) { 3796 if (unlikely(contended)) { 3797 spin_unlock(&q->busylock); 3798 contended = false; 3799 } 3800 __qdisc_run(q); 3801 qdisc_run_end(q); 3802 } 3803 } 3804 spin_unlock(root_lock); 3805 if (unlikely(to_free)) 3806 kfree_skb_list(to_free); 3807 if (unlikely(contended)) 3808 spin_unlock(&q->busylock); 3809 return rc; 3810 } 3811 3812 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO) 3813 static void skb_update_prio(struct sk_buff *skb) 3814 { 3815 const struct netprio_map *map; 3816 const struct sock *sk; 3817 unsigned int prioidx; 3818 3819 if (skb->priority) 3820 return; 3821 map = rcu_dereference_bh(skb->dev->priomap); 3822 if (!map) 3823 return; 3824 sk = skb_to_full_sk(skb); 3825 if (!sk) 3826 return; 3827 3828 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data); 3829 3830 if (prioidx < map->priomap_len) 3831 skb->priority = map->priomap[prioidx]; 3832 } 3833 #else 3834 #define skb_update_prio(skb) 3835 #endif 3836 3837 /** 3838 * dev_loopback_xmit - loop back @skb 3839 * @net: network namespace this loopback is happening in 3840 * @sk: sk needed to be a netfilter okfn 3841 * @skb: buffer to transmit 3842 */ 3843 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb) 3844 { 3845 skb_reset_mac_header(skb); 3846 __skb_pull(skb, skb_network_offset(skb)); 3847 skb->pkt_type = PACKET_LOOPBACK; 3848 skb->ip_summed = CHECKSUM_UNNECESSARY; 3849 WARN_ON(!skb_dst(skb)); 3850 skb_dst_force(skb); 3851 netif_rx_ni(skb); 3852 return 0; 3853 } 3854 EXPORT_SYMBOL(dev_loopback_xmit); 3855 3856 #ifdef CONFIG_NET_EGRESS 3857 static struct sk_buff * 3858 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) 3859 { 3860 struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress); 3861 struct tcf_result cl_res; 3862 3863 if (!miniq) 3864 return skb; 3865 3866 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */ 3867 mini_qdisc_bstats_cpu_update(miniq, skb); 3868 3869 switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) { 3870 case TC_ACT_OK: 3871 case TC_ACT_RECLASSIFY: 3872 skb->tc_index = TC_H_MIN(cl_res.classid); 3873 break; 3874 case TC_ACT_SHOT: 3875 mini_qdisc_qstats_cpu_drop(miniq); 3876 *ret = NET_XMIT_DROP; 3877 kfree_skb(skb); 3878 return NULL; 3879 case TC_ACT_STOLEN: 3880 case TC_ACT_QUEUED: 3881 case TC_ACT_TRAP: 3882 *ret = NET_XMIT_SUCCESS; 3883 consume_skb(skb); 3884 return NULL; 3885 case TC_ACT_REDIRECT: 3886 /* No need to push/pop skb's mac_header here on egress! */ 3887 skb_do_redirect(skb); 3888 *ret = NET_XMIT_SUCCESS; 3889 return NULL; 3890 default: 3891 break; 3892 } 3893 3894 return skb; 3895 } 3896 #endif /* CONFIG_NET_EGRESS */ 3897 3898 #ifdef CONFIG_XPS 3899 static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb, 3900 struct xps_dev_maps *dev_maps, unsigned int tci) 3901 { 3902 struct xps_map *map; 3903 int queue_index = -1; 3904 3905 if (dev->num_tc) { 3906 tci *= dev->num_tc; 3907 tci += netdev_get_prio_tc_map(dev, skb->priority); 3908 } 3909 3910 map = rcu_dereference(dev_maps->attr_map[tci]); 3911 if (map) { 3912 if (map->len == 1) 3913 queue_index = map->queues[0]; 3914 else 3915 queue_index = map->queues[reciprocal_scale( 3916 skb_get_hash(skb), map->len)]; 3917 if (unlikely(queue_index >= dev->real_num_tx_queues)) 3918 queue_index = -1; 3919 } 3920 return queue_index; 3921 } 3922 #endif 3923 3924 static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev, 3925 struct sk_buff *skb) 3926 { 3927 #ifdef CONFIG_XPS 3928 struct xps_dev_maps *dev_maps; 3929 struct sock *sk = skb->sk; 3930 int queue_index = -1; 3931 3932 if (!static_key_false(&xps_needed)) 3933 return -1; 3934 3935 rcu_read_lock(); 3936 if (!static_key_false(&xps_rxqs_needed)) 3937 goto get_cpus_map; 3938 3939 dev_maps = rcu_dereference(sb_dev->xps_rxqs_map); 3940 if (dev_maps) { 3941 int tci = sk_rx_queue_get(sk); 3942 3943 if (tci >= 0 && tci < dev->num_rx_queues) 3944 queue_index = __get_xps_queue_idx(dev, skb, dev_maps, 3945 tci); 3946 } 3947 3948 get_cpus_map: 3949 if (queue_index < 0) { 3950 dev_maps = rcu_dereference(sb_dev->xps_cpus_map); 3951 if (dev_maps) { 3952 unsigned int tci = skb->sender_cpu - 1; 3953 3954 queue_index = __get_xps_queue_idx(dev, skb, dev_maps, 3955 tci); 3956 } 3957 } 3958 rcu_read_unlock(); 3959 3960 return queue_index; 3961 #else 3962 return -1; 3963 #endif 3964 } 3965 3966 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb, 3967 struct net_device *sb_dev) 3968 { 3969 return 0; 3970 } 3971 EXPORT_SYMBOL(dev_pick_tx_zero); 3972 3973 u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb, 3974 struct net_device *sb_dev) 3975 { 3976 return (u16)raw_smp_processor_id() % dev->real_num_tx_queues; 3977 } 3978 EXPORT_SYMBOL(dev_pick_tx_cpu_id); 3979 3980 u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb, 3981 struct net_device *sb_dev) 3982 { 3983 struct sock *sk = skb->sk; 3984 int queue_index = sk_tx_queue_get(sk); 3985 3986 sb_dev = sb_dev ? : dev; 3987 3988 if (queue_index < 0 || skb->ooo_okay || 3989 queue_index >= dev->real_num_tx_queues) { 3990 int new_index = get_xps_queue(dev, sb_dev, skb); 3991 3992 if (new_index < 0) 3993 new_index = skb_tx_hash(dev, sb_dev, skb); 3994 3995 if (queue_index != new_index && sk && 3996 sk_fullsock(sk) && 3997 rcu_access_pointer(sk->sk_dst_cache)) 3998 sk_tx_queue_set(sk, new_index); 3999 4000 queue_index = new_index; 4001 } 4002 4003 return queue_index; 4004 } 4005 EXPORT_SYMBOL(netdev_pick_tx); 4006 4007 struct netdev_queue *netdev_core_pick_tx(struct net_device *dev, 4008 struct sk_buff *skb, 4009 struct net_device *sb_dev) 4010 { 4011 int queue_index = 0; 4012 4013 #ifdef CONFIG_XPS 4014 u32 sender_cpu = skb->sender_cpu - 1; 4015 4016 if (sender_cpu >= (u32)NR_CPUS) 4017 skb->sender_cpu = raw_smp_processor_id() + 1; 4018 #endif 4019 4020 if (dev->real_num_tx_queues != 1) { 4021 const struct net_device_ops *ops = dev->netdev_ops; 4022 4023 if (ops->ndo_select_queue) 4024 queue_index = ops->ndo_select_queue(dev, skb, sb_dev); 4025 else 4026 queue_index = netdev_pick_tx(dev, skb, sb_dev); 4027 4028 queue_index = netdev_cap_txqueue(dev, queue_index); 4029 } 4030 4031 skb_set_queue_mapping(skb, queue_index); 4032 return netdev_get_tx_queue(dev, queue_index); 4033 } 4034 4035 /** 4036 * __dev_queue_xmit - transmit a buffer 4037 * @skb: buffer to transmit 4038 * @sb_dev: suboordinate device used for L2 forwarding offload 4039 * 4040 * Queue a buffer for transmission to a network device. The caller must 4041 * have set the device and priority and built the buffer before calling 4042 * this function. The function can be called from an interrupt. 4043 * 4044 * A negative errno code is returned on a failure. A success does not 4045 * guarantee the frame will be transmitted as it may be dropped due 4046 * to congestion or traffic shaping. 4047 * 4048 * ----------------------------------------------------------------------------------- 4049 * I notice this method can also return errors from the queue disciplines, 4050 * including NET_XMIT_DROP, which is a positive value. So, errors can also 4051 * be positive. 4052 * 4053 * Regardless of the return value, the skb is consumed, so it is currently 4054 * difficult to retry a send to this method. (You can bump the ref count 4055 * before sending to hold a reference for retry if you are careful.) 4056 * 4057 * When calling this method, interrupts MUST be enabled. This is because 4058 * the BH enable code must have IRQs enabled so that it will not deadlock. 4059 * --BLG 4060 */ 4061 static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) 4062 { 4063 struct net_device *dev = skb->dev; 4064 struct netdev_queue *txq; 4065 struct Qdisc *q; 4066 int rc = -ENOMEM; 4067 bool again = false; 4068 4069 skb_reset_mac_header(skb); 4070 4071 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP)) 4072 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED); 4073 4074 /* Disable soft irqs for various locks below. Also 4075 * stops preemption for RCU. 4076 */ 4077 rcu_read_lock_bh(); 4078 4079 skb_update_prio(skb); 4080 4081 qdisc_pkt_len_init(skb); 4082 #ifdef CONFIG_NET_CLS_ACT 4083 skb->tc_at_ingress = 0; 4084 # ifdef CONFIG_NET_EGRESS 4085 if (static_branch_unlikely(&egress_needed_key)) { 4086 skb = sch_handle_egress(skb, &rc, dev); 4087 if (!skb) 4088 goto out; 4089 } 4090 # endif 4091 #endif 4092 /* If device/qdisc don't need skb->dst, release it right now while 4093 * its hot in this cpu cache. 4094 */ 4095 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) 4096 skb_dst_drop(skb); 4097 else 4098 skb_dst_force(skb); 4099 4100 txq = netdev_core_pick_tx(dev, skb, sb_dev); 4101 q = rcu_dereference_bh(txq->qdisc); 4102 4103 trace_net_dev_queue(skb); 4104 if (q->enqueue) { 4105 rc = __dev_xmit_skb(skb, q, dev, txq); 4106 goto out; 4107 } 4108 4109 /* The device has no queue. Common case for software devices: 4110 * loopback, all the sorts of tunnels... 4111 4112 * Really, it is unlikely that netif_tx_lock protection is necessary 4113 * here. (f.e. loopback and IP tunnels are clean ignoring statistics 4114 * counters.) 4115 * However, it is possible, that they rely on protection 4116 * made by us here. 4117 4118 * Check this and shot the lock. It is not prone from deadlocks. 4119 *Either shot noqueue qdisc, it is even simpler 8) 4120 */ 4121 if (dev->flags & IFF_UP) { 4122 int cpu = smp_processor_id(); /* ok because BHs are off */ 4123 4124 if (txq->xmit_lock_owner != cpu) { 4125 if (dev_xmit_recursion()) 4126 goto recursion_alert; 4127 4128 skb = validate_xmit_skb(skb, dev, &again); 4129 if (!skb) 4130 goto out; 4131 4132 HARD_TX_LOCK(dev, txq, cpu); 4133 4134 if (!netif_xmit_stopped(txq)) { 4135 dev_xmit_recursion_inc(); 4136 skb = dev_hard_start_xmit(skb, dev, txq, &rc); 4137 dev_xmit_recursion_dec(); 4138 if (dev_xmit_complete(rc)) { 4139 HARD_TX_UNLOCK(dev, txq); 4140 goto out; 4141 } 4142 } 4143 HARD_TX_UNLOCK(dev, txq); 4144 net_crit_ratelimited("Virtual device %s asks to queue packet!\n", 4145 dev->name); 4146 } else { 4147 /* Recursion is detected! It is possible, 4148 * unfortunately 4149 */ 4150 recursion_alert: 4151 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", 4152 dev->name); 4153 } 4154 } 4155 4156 rc = -ENETDOWN; 4157 rcu_read_unlock_bh(); 4158 4159 atomic_long_inc(&dev->tx_dropped); 4160 kfree_skb_list(skb); 4161 return rc; 4162 out: 4163 rcu_read_unlock_bh(); 4164 return rc; 4165 } 4166 4167 int dev_queue_xmit(struct sk_buff *skb) 4168 { 4169 return __dev_queue_xmit(skb, NULL); 4170 } 4171 EXPORT_SYMBOL(dev_queue_xmit); 4172 4173 int dev_queue_xmit_accel(struct sk_buff *skb, struct net_device *sb_dev) 4174 { 4175 return __dev_queue_xmit(skb, sb_dev); 4176 } 4177 EXPORT_SYMBOL(dev_queue_xmit_accel); 4178 4179 int dev_direct_xmit(struct sk_buff *skb, u16 queue_id) 4180 { 4181 struct net_device *dev = skb->dev; 4182 struct sk_buff *orig_skb = skb; 4183 struct netdev_queue *txq; 4184 int ret = NETDEV_TX_BUSY; 4185 bool again = false; 4186 4187 if (unlikely(!netif_running(dev) || 4188 !netif_carrier_ok(dev))) 4189 goto drop; 4190 4191 skb = validate_xmit_skb_list(skb, dev, &again); 4192 if (skb != orig_skb) 4193 goto drop; 4194 4195 skb_set_queue_mapping(skb, queue_id); 4196 txq = skb_get_tx_queue(dev, skb); 4197 4198 local_bh_disable(); 4199 4200 dev_xmit_recursion_inc(); 4201 HARD_TX_LOCK(dev, txq, smp_processor_id()); 4202 if (!netif_xmit_frozen_or_drv_stopped(txq)) 4203 ret = netdev_start_xmit(skb, dev, txq, false); 4204 HARD_TX_UNLOCK(dev, txq); 4205 dev_xmit_recursion_dec(); 4206 4207 local_bh_enable(); 4208 4209 if (!dev_xmit_complete(ret)) 4210 kfree_skb(skb); 4211 4212 return ret; 4213 drop: 4214 atomic_long_inc(&dev->tx_dropped); 4215 kfree_skb_list(skb); 4216 return NET_XMIT_DROP; 4217 } 4218 EXPORT_SYMBOL(dev_direct_xmit); 4219 4220 /************************************************************************* 4221 * Receiver routines 4222 *************************************************************************/ 4223 4224 int netdev_max_backlog __read_mostly = 1000; 4225 EXPORT_SYMBOL(netdev_max_backlog); 4226 4227 int netdev_tstamp_prequeue __read_mostly = 1; 4228 int netdev_budget __read_mostly = 300; 4229 /* Must be at least 2 jiffes to guarantee 1 jiffy timeout */ 4230 unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ; 4231 int weight_p __read_mostly = 64; /* old backlog weight */ 4232 int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */ 4233 int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */ 4234 int dev_rx_weight __read_mostly = 64; 4235 int dev_tx_weight __read_mostly = 64; 4236 /* Maximum number of GRO_NORMAL skbs to batch up for list-RX */ 4237 int gro_normal_batch __read_mostly = 8; 4238 4239 /* Called with irq disabled */ 4240 static inline void ____napi_schedule(struct softnet_data *sd, 4241 struct napi_struct *napi) 4242 { 4243 list_add_tail(&napi->poll_list, &sd->poll_list); 4244 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 4245 } 4246 4247 #ifdef CONFIG_RPS 4248 4249 /* One global table that all flow-based protocols share. */ 4250 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; 4251 EXPORT_SYMBOL(rps_sock_flow_table); 4252 u32 rps_cpu_mask __read_mostly; 4253 EXPORT_SYMBOL(rps_cpu_mask); 4254 4255 struct static_key_false rps_needed __read_mostly; 4256 EXPORT_SYMBOL(rps_needed); 4257 struct static_key_false rfs_needed __read_mostly; 4258 EXPORT_SYMBOL(rfs_needed); 4259 4260 static struct rps_dev_flow * 4261 set_rps_cpu(struct net_device *dev, struct sk_buff *skb, 4262 struct rps_dev_flow *rflow, u16 next_cpu) 4263 { 4264 if (next_cpu < nr_cpu_ids) { 4265 #ifdef CONFIG_RFS_ACCEL 4266 struct netdev_rx_queue *rxqueue; 4267 struct rps_dev_flow_table *flow_table; 4268 struct rps_dev_flow *old_rflow; 4269 u32 flow_id; 4270 u16 rxq_index; 4271 int rc; 4272 4273 /* Should we steer this flow to a different hardware queue? */ 4274 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || 4275 !(dev->features & NETIF_F_NTUPLE)) 4276 goto out; 4277 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); 4278 if (rxq_index == skb_get_rx_queue(skb)) 4279 goto out; 4280 4281 rxqueue = dev->_rx + rxq_index; 4282 flow_table = rcu_dereference(rxqueue->rps_flow_table); 4283 if (!flow_table) 4284 goto out; 4285 flow_id = skb_get_hash(skb) & flow_table->mask; 4286 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, 4287 rxq_index, flow_id); 4288 if (rc < 0) 4289 goto out; 4290 old_rflow = rflow; 4291 rflow = &flow_table->flows[flow_id]; 4292 rflow->filter = rc; 4293 if (old_rflow->filter == rflow->filter) 4294 old_rflow->filter = RPS_NO_FILTER; 4295 out: 4296 #endif 4297 rflow->last_qtail = 4298 per_cpu(softnet_data, next_cpu).input_queue_head; 4299 } 4300 4301 rflow->cpu = next_cpu; 4302 return rflow; 4303 } 4304 4305 /* 4306 * get_rps_cpu is called from netif_receive_skb and returns the target 4307 * CPU from the RPS map of the receiving queue for a given skb. 4308 * rcu_read_lock must be held on entry. 4309 */ 4310 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, 4311 struct rps_dev_flow **rflowp) 4312 { 4313 const struct rps_sock_flow_table *sock_flow_table; 4314 struct netdev_rx_queue *rxqueue = dev->_rx; 4315 struct rps_dev_flow_table *flow_table; 4316 struct rps_map *map; 4317 int cpu = -1; 4318 u32 tcpu; 4319 u32 hash; 4320 4321 if (skb_rx_queue_recorded(skb)) { 4322 u16 index = skb_get_rx_queue(skb); 4323 4324 if (unlikely(index >= dev->real_num_rx_queues)) { 4325 WARN_ONCE(dev->real_num_rx_queues > 1, 4326 "%s received packet on queue %u, but number " 4327 "of RX queues is %u\n", 4328 dev->name, index, dev->real_num_rx_queues); 4329 goto done; 4330 } 4331 rxqueue += index; 4332 } 4333 4334 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */ 4335 4336 flow_table = rcu_dereference(rxqueue->rps_flow_table); 4337 map = rcu_dereference(rxqueue->rps_map); 4338 if (!flow_table && !map) 4339 goto done; 4340 4341 skb_reset_network_header(skb); 4342 hash = skb_get_hash(skb); 4343 if (!hash) 4344 goto done; 4345 4346 sock_flow_table = rcu_dereference(rps_sock_flow_table); 4347 if (flow_table && sock_flow_table) { 4348 struct rps_dev_flow *rflow; 4349 u32 next_cpu; 4350 u32 ident; 4351 4352 /* First check into global flow table if there is a match */ 4353 ident = sock_flow_table->ents[hash & sock_flow_table->mask]; 4354 if ((ident ^ hash) & ~rps_cpu_mask) 4355 goto try_rps; 4356 4357 next_cpu = ident & rps_cpu_mask; 4358 4359 /* OK, now we know there is a match, 4360 * we can look at the local (per receive queue) flow table 4361 */ 4362 rflow = &flow_table->flows[hash & flow_table->mask]; 4363 tcpu = rflow->cpu; 4364 4365 /* 4366 * If the desired CPU (where last recvmsg was done) is 4367 * different from current CPU (one in the rx-queue flow 4368 * table entry), switch if one of the following holds: 4369 * - Current CPU is unset (>= nr_cpu_ids). 4370 * - Current CPU is offline. 4371 * - The current CPU's queue tail has advanced beyond the 4372 * last packet that was enqueued using this table entry. 4373 * This guarantees that all previous packets for the flow 4374 * have been dequeued, thus preserving in order delivery. 4375 */ 4376 if (unlikely(tcpu != next_cpu) && 4377 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) || 4378 ((int)(per_cpu(softnet_data, tcpu).input_queue_head - 4379 rflow->last_qtail)) >= 0)) { 4380 tcpu = next_cpu; 4381 rflow = set_rps_cpu(dev, skb, rflow, next_cpu); 4382 } 4383 4384 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { 4385 *rflowp = rflow; 4386 cpu = tcpu; 4387 goto done; 4388 } 4389 } 4390 4391 try_rps: 4392 4393 if (map) { 4394 tcpu = map->cpus[reciprocal_scale(hash, map->len)]; 4395 if (cpu_online(tcpu)) { 4396 cpu = tcpu; 4397 goto done; 4398 } 4399 } 4400 4401 done: 4402 return cpu; 4403 } 4404 4405 #ifdef CONFIG_RFS_ACCEL 4406 4407 /** 4408 * rps_may_expire_flow - check whether an RFS hardware filter may be removed 4409 * @dev: Device on which the filter was set 4410 * @rxq_index: RX queue index 4411 * @flow_id: Flow ID passed to ndo_rx_flow_steer() 4412 * @filter_id: Filter ID returned by ndo_rx_flow_steer() 4413 * 4414 * Drivers that implement ndo_rx_flow_steer() should periodically call 4415 * this function for each installed filter and remove the filters for 4416 * which it returns %true. 4417 */ 4418 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, 4419 u32 flow_id, u16 filter_id) 4420 { 4421 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; 4422 struct rps_dev_flow_table *flow_table; 4423 struct rps_dev_flow *rflow; 4424 bool expire = true; 4425 unsigned int cpu; 4426 4427 rcu_read_lock(); 4428 flow_table = rcu_dereference(rxqueue->rps_flow_table); 4429 if (flow_table && flow_id <= flow_table->mask) { 4430 rflow = &flow_table->flows[flow_id]; 4431 cpu = READ_ONCE(rflow->cpu); 4432 if (rflow->filter == filter_id && cpu < nr_cpu_ids && 4433 ((int)(per_cpu(softnet_data, cpu).input_queue_head - 4434 rflow->last_qtail) < 4435 (int)(10 * flow_table->mask))) 4436 expire = false; 4437 } 4438 rcu_read_unlock(); 4439 return expire; 4440 } 4441 EXPORT_SYMBOL(rps_may_expire_flow); 4442 4443 #endif /* CONFIG_RFS_ACCEL */ 4444 4445 /* Called from hardirq (IPI) context */ 4446 static void rps_trigger_softirq(void *data) 4447 { 4448 struct softnet_data *sd = data; 4449 4450 ____napi_schedule(sd, &sd->backlog); 4451 sd->received_rps++; 4452 } 4453 4454 #endif /* CONFIG_RPS */ 4455 4456 /* 4457 * Check if this softnet_data structure is another cpu one 4458 * If yes, queue it to our IPI list and return 1 4459 * If no, return 0 4460 */ 4461 static int rps_ipi_queued(struct softnet_data *sd) 4462 { 4463 #ifdef CONFIG_RPS 4464 struct softnet_data *mysd = this_cpu_ptr(&softnet_data); 4465 4466 if (sd != mysd) { 4467 sd->rps_ipi_next = mysd->rps_ipi_list; 4468 mysd->rps_ipi_list = sd; 4469 4470 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 4471 return 1; 4472 } 4473 #endif /* CONFIG_RPS */ 4474 return 0; 4475 } 4476 4477 #ifdef CONFIG_NET_FLOW_LIMIT 4478 int netdev_flow_limit_table_len __read_mostly = (1 << 12); 4479 #endif 4480 4481 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) 4482 { 4483 #ifdef CONFIG_NET_FLOW_LIMIT 4484 struct sd_flow_limit *fl; 4485 struct softnet_data *sd; 4486 unsigned int old_flow, new_flow; 4487 4488 if (qlen < (netdev_max_backlog >> 1)) 4489 return false; 4490 4491 sd = this_cpu_ptr(&softnet_data); 4492 4493 rcu_read_lock(); 4494 fl = rcu_dereference(sd->flow_limit); 4495 if (fl) { 4496 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1); 4497 old_flow = fl->history[fl->history_head]; 4498 fl->history[fl->history_head] = new_flow; 4499 4500 fl->history_head++; 4501 fl->history_head &= FLOW_LIMIT_HISTORY - 1; 4502 4503 if (likely(fl->buckets[old_flow])) 4504 fl->buckets[old_flow]--; 4505 4506 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { 4507 fl->count++; 4508 rcu_read_unlock(); 4509 return true; 4510 } 4511 } 4512 rcu_read_unlock(); 4513 #endif 4514 return false; 4515 } 4516 4517 /* 4518 * enqueue_to_backlog is called to queue an skb to a per CPU backlog 4519 * queue (may be a remote CPU queue). 4520 */ 4521 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, 4522 unsigned int *qtail) 4523 { 4524 struct softnet_data *sd; 4525 unsigned long flags; 4526 unsigned int qlen; 4527 4528 sd = &per_cpu(softnet_data, cpu); 4529 4530 local_irq_save(flags); 4531 4532 rps_lock(sd); 4533 if (!netif_running(skb->dev)) 4534 goto drop; 4535 qlen = skb_queue_len(&sd->input_pkt_queue); 4536 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) { 4537 if (qlen) { 4538 enqueue: 4539 __skb_queue_tail(&sd->input_pkt_queue, skb); 4540 input_queue_tail_incr_save(sd, qtail); 4541 rps_unlock(sd); 4542 local_irq_restore(flags); 4543 return NET_RX_SUCCESS; 4544 } 4545 4546 /* Schedule NAPI for backlog device 4547 * We can use non atomic operation since we own the queue lock 4548 */ 4549 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) { 4550 if (!rps_ipi_queued(sd)) 4551 ____napi_schedule(sd, &sd->backlog); 4552 } 4553 goto enqueue; 4554 } 4555 4556 drop: 4557 sd->dropped++; 4558 rps_unlock(sd); 4559 4560 local_irq_restore(flags); 4561 4562 atomic_long_inc(&skb->dev->rx_dropped); 4563 kfree_skb(skb); 4564 return NET_RX_DROP; 4565 } 4566 4567 static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb) 4568 { 4569 struct net_device *dev = skb->dev; 4570 struct netdev_rx_queue *rxqueue; 4571 4572 rxqueue = dev->_rx; 4573 4574 if (skb_rx_queue_recorded(skb)) { 4575 u16 index = skb_get_rx_queue(skb); 4576 4577 if (unlikely(index >= dev->real_num_rx_queues)) { 4578 WARN_ONCE(dev->real_num_rx_queues > 1, 4579 "%s received packet on queue %u, but number " 4580 "of RX queues is %u\n", 4581 dev->name, index, dev->real_num_rx_queues); 4582 4583 return rxqueue; /* Return first rxqueue */ 4584 } 4585 rxqueue += index; 4586 } 4587 return rxqueue; 4588 } 4589 4590 static u32 netif_receive_generic_xdp(struct sk_buff *skb, 4591 struct xdp_buff *xdp, 4592 struct bpf_prog *xdp_prog) 4593 { 4594 struct netdev_rx_queue *rxqueue; 4595 void *orig_data, *orig_data_end; 4596 u32 metalen, act = XDP_DROP; 4597 __be16 orig_eth_type; 4598 struct ethhdr *eth; 4599 bool orig_bcast; 4600 int hlen, off; 4601 u32 mac_len; 4602 4603 /* Reinjected packets coming from act_mirred or similar should 4604 * not get XDP generic processing. 4605 */ 4606 if (skb_is_redirected(skb)) 4607 return XDP_PASS; 4608 4609 /* XDP packets must be linear and must have sufficient headroom 4610 * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also 4611 * native XDP provides, thus we need to do it here as well. 4612 */ 4613 if (skb_cloned(skb) || skb_is_nonlinear(skb) || 4614 skb_headroom(skb) < XDP_PACKET_HEADROOM) { 4615 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb); 4616 int troom = skb->tail + skb->data_len - skb->end; 4617 4618 /* In case we have to go down the path and also linearize, 4619 * then lets do the pskb_expand_head() work just once here. 4620 */ 4621 if (pskb_expand_head(skb, 4622 hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0, 4623 troom > 0 ? troom + 128 : 0, GFP_ATOMIC)) 4624 goto do_drop; 4625 if (skb_linearize(skb)) 4626 goto do_drop; 4627 } 4628 4629 /* The XDP program wants to see the packet starting at the MAC 4630 * header. 4631 */ 4632 mac_len = skb->data - skb_mac_header(skb); 4633 hlen = skb_headlen(skb) + mac_len; 4634 xdp->data = skb->data - mac_len; 4635 xdp->data_meta = xdp->data; 4636 xdp->data_end = xdp->data + hlen; 4637 xdp->data_hard_start = skb->data - skb_headroom(skb); 4638 4639 /* SKB "head" area always have tailroom for skb_shared_info */ 4640 xdp->frame_sz = (void *)skb_end_pointer(skb) - xdp->data_hard_start; 4641 xdp->frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 4642 4643 orig_data_end = xdp->data_end; 4644 orig_data = xdp->data; 4645 eth = (struct ethhdr *)xdp->data; 4646 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest); 4647 orig_eth_type = eth->h_proto; 4648 4649 rxqueue = netif_get_rxqueue(skb); 4650 xdp->rxq = &rxqueue->xdp_rxq; 4651 4652 act = bpf_prog_run_xdp(xdp_prog, xdp); 4653 4654 /* check if bpf_xdp_adjust_head was used */ 4655 off = xdp->data - orig_data; 4656 if (off) { 4657 if (off > 0) 4658 __skb_pull(skb, off); 4659 else if (off < 0) 4660 __skb_push(skb, -off); 4661 4662 skb->mac_header += off; 4663 skb_reset_network_header(skb); 4664 } 4665 4666 /* check if bpf_xdp_adjust_tail was used */ 4667 off = xdp->data_end - orig_data_end; 4668 if (off != 0) { 4669 skb_set_tail_pointer(skb, xdp->data_end - xdp->data); 4670 skb->len += off; /* positive on grow, negative on shrink */ 4671 } 4672 4673 /* check if XDP changed eth hdr such SKB needs update */ 4674 eth = (struct ethhdr *)xdp->data; 4675 if ((orig_eth_type != eth->h_proto) || 4676 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) { 4677 __skb_push(skb, ETH_HLEN); 4678 skb->protocol = eth_type_trans(skb, skb->dev); 4679 } 4680 4681 switch (act) { 4682 case XDP_REDIRECT: 4683 case XDP_TX: 4684 __skb_push(skb, mac_len); 4685 break; 4686 case XDP_PASS: 4687 metalen = xdp->data - xdp->data_meta; 4688 if (metalen) 4689 skb_metadata_set(skb, metalen); 4690 break; 4691 default: 4692 bpf_warn_invalid_xdp_action(act); 4693 fallthrough; 4694 case XDP_ABORTED: 4695 trace_xdp_exception(skb->dev, xdp_prog, act); 4696 fallthrough; 4697 case XDP_DROP: 4698 do_drop: 4699 kfree_skb(skb); 4700 break; 4701 } 4702 4703 return act; 4704 } 4705 4706 /* When doing generic XDP we have to bypass the qdisc layer and the 4707 * network taps in order to match in-driver-XDP behavior. 4708 */ 4709 void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog) 4710 { 4711 struct net_device *dev = skb->dev; 4712 struct netdev_queue *txq; 4713 bool free_skb = true; 4714 int cpu, rc; 4715 4716 txq = netdev_core_pick_tx(dev, skb, NULL); 4717 cpu = smp_processor_id(); 4718 HARD_TX_LOCK(dev, txq, cpu); 4719 if (!netif_xmit_stopped(txq)) { 4720 rc = netdev_start_xmit(skb, dev, txq, 0); 4721 if (dev_xmit_complete(rc)) 4722 free_skb = false; 4723 } 4724 HARD_TX_UNLOCK(dev, txq); 4725 if (free_skb) { 4726 trace_xdp_exception(dev, xdp_prog, XDP_TX); 4727 kfree_skb(skb); 4728 } 4729 } 4730 4731 static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key); 4732 4733 int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb) 4734 { 4735 if (xdp_prog) { 4736 struct xdp_buff xdp; 4737 u32 act; 4738 int err; 4739 4740 act = netif_receive_generic_xdp(skb, &xdp, xdp_prog); 4741 if (act != XDP_PASS) { 4742 switch (act) { 4743 case XDP_REDIRECT: 4744 err = xdp_do_generic_redirect(skb->dev, skb, 4745 &xdp, xdp_prog); 4746 if (err) 4747 goto out_redir; 4748 break; 4749 case XDP_TX: 4750 generic_xdp_tx(skb, xdp_prog); 4751 break; 4752 } 4753 return XDP_DROP; 4754 } 4755 } 4756 return XDP_PASS; 4757 out_redir: 4758 kfree_skb(skb); 4759 return XDP_DROP; 4760 } 4761 EXPORT_SYMBOL_GPL(do_xdp_generic); 4762 4763 static int netif_rx_internal(struct sk_buff *skb) 4764 { 4765 int ret; 4766 4767 net_timestamp_check(netdev_tstamp_prequeue, skb); 4768 4769 trace_netif_rx(skb); 4770 4771 #ifdef CONFIG_RPS 4772 if (static_branch_unlikely(&rps_needed)) { 4773 struct rps_dev_flow voidflow, *rflow = &voidflow; 4774 int cpu; 4775 4776 preempt_disable(); 4777 rcu_read_lock(); 4778 4779 cpu = get_rps_cpu(skb->dev, skb, &rflow); 4780 if (cpu < 0) 4781 cpu = smp_processor_id(); 4782 4783 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 4784 4785 rcu_read_unlock(); 4786 preempt_enable(); 4787 } else 4788 #endif 4789 { 4790 unsigned int qtail; 4791 4792 ret = enqueue_to_backlog(skb, get_cpu(), &qtail); 4793 put_cpu(); 4794 } 4795 return ret; 4796 } 4797 4798 /** 4799 * netif_rx - post buffer to the network code 4800 * @skb: buffer to post 4801 * 4802 * This function receives a packet from a device driver and queues it for 4803 * the upper (protocol) levels to process. It always succeeds. The buffer 4804 * may be dropped during processing for congestion control or by the 4805 * protocol layers. 4806 * 4807 * return values: 4808 * NET_RX_SUCCESS (no congestion) 4809 * NET_RX_DROP (packet was dropped) 4810 * 4811 */ 4812 4813 int netif_rx(struct sk_buff *skb) 4814 { 4815 int ret; 4816 4817 trace_netif_rx_entry(skb); 4818 4819 ret = netif_rx_internal(skb); 4820 trace_netif_rx_exit(ret); 4821 4822 return ret; 4823 } 4824 EXPORT_SYMBOL(netif_rx); 4825 4826 int netif_rx_ni(struct sk_buff *skb) 4827 { 4828 int err; 4829 4830 trace_netif_rx_ni_entry(skb); 4831 4832 preempt_disable(); 4833 err = netif_rx_internal(skb); 4834 if (local_softirq_pending()) 4835 do_softirq(); 4836 preempt_enable(); 4837 trace_netif_rx_ni_exit(err); 4838 4839 return err; 4840 } 4841 EXPORT_SYMBOL(netif_rx_ni); 4842 4843 static __latent_entropy void net_tx_action(struct softirq_action *h) 4844 { 4845 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 4846 4847 if (sd->completion_queue) { 4848 struct sk_buff *clist; 4849 4850 local_irq_disable(); 4851 clist = sd->completion_queue; 4852 sd->completion_queue = NULL; 4853 local_irq_enable(); 4854 4855 while (clist) { 4856 struct sk_buff *skb = clist; 4857 4858 clist = clist->next; 4859 4860 WARN_ON(refcount_read(&skb->users)); 4861 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED)) 4862 trace_consume_skb(skb); 4863 else 4864 trace_kfree_skb(skb, net_tx_action); 4865 4866 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) 4867 __kfree_skb(skb); 4868 else 4869 __kfree_skb_defer(skb); 4870 } 4871 4872 __kfree_skb_flush(); 4873 } 4874 4875 if (sd->output_queue) { 4876 struct Qdisc *head; 4877 4878 local_irq_disable(); 4879 head = sd->output_queue; 4880 sd->output_queue = NULL; 4881 sd->output_queue_tailp = &sd->output_queue; 4882 local_irq_enable(); 4883 4884 while (head) { 4885 struct Qdisc *q = head; 4886 spinlock_t *root_lock = NULL; 4887 4888 head = head->next_sched; 4889 4890 if (!(q->flags & TCQ_F_NOLOCK)) { 4891 root_lock = qdisc_lock(q); 4892 spin_lock(root_lock); 4893 } 4894 /* We need to make sure head->next_sched is read 4895 * before clearing __QDISC_STATE_SCHED 4896 */ 4897 smp_mb__before_atomic(); 4898 clear_bit(__QDISC_STATE_SCHED, &q->state); 4899 qdisc_run(q); 4900 if (root_lock) 4901 spin_unlock(root_lock); 4902 } 4903 } 4904 4905 xfrm_dev_backlog(sd); 4906 } 4907 4908 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE) 4909 /* This hook is defined here for ATM LANE */ 4910 int (*br_fdb_test_addr_hook)(struct net_device *dev, 4911 unsigned char *addr) __read_mostly; 4912 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); 4913 #endif 4914 4915 static inline struct sk_buff * 4916 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, 4917 struct net_device *orig_dev) 4918 { 4919 #ifdef CONFIG_NET_CLS_ACT 4920 struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress); 4921 struct tcf_result cl_res; 4922 4923 /* If there's at least one ingress present somewhere (so 4924 * we get here via enabled static key), remaining devices 4925 * that are not configured with an ingress qdisc will bail 4926 * out here. 4927 */ 4928 if (!miniq) 4929 return skb; 4930 4931 if (*pt_prev) { 4932 *ret = deliver_skb(skb, *pt_prev, orig_dev); 4933 *pt_prev = NULL; 4934 } 4935 4936 qdisc_skb_cb(skb)->pkt_len = skb->len; 4937 skb->tc_at_ingress = 1; 4938 mini_qdisc_bstats_cpu_update(miniq, skb); 4939 4940 switch (tcf_classify_ingress(skb, miniq->block, miniq->filter_list, 4941 &cl_res, false)) { 4942 case TC_ACT_OK: 4943 case TC_ACT_RECLASSIFY: 4944 skb->tc_index = TC_H_MIN(cl_res.classid); 4945 break; 4946 case TC_ACT_SHOT: 4947 mini_qdisc_qstats_cpu_drop(miniq); 4948 kfree_skb(skb); 4949 return NULL; 4950 case TC_ACT_STOLEN: 4951 case TC_ACT_QUEUED: 4952 case TC_ACT_TRAP: 4953 consume_skb(skb); 4954 return NULL; 4955 case TC_ACT_REDIRECT: 4956 /* skb_mac_header check was done by cls/act_bpf, so 4957 * we can safely push the L2 header back before 4958 * redirecting to another netdev 4959 */ 4960 __skb_push(skb, skb->mac_len); 4961 skb_do_redirect(skb); 4962 return NULL; 4963 case TC_ACT_CONSUMED: 4964 return NULL; 4965 default: 4966 break; 4967 } 4968 #endif /* CONFIG_NET_CLS_ACT */ 4969 return skb; 4970 } 4971 4972 /** 4973 * netdev_is_rx_handler_busy - check if receive handler is registered 4974 * @dev: device to check 4975 * 4976 * Check if a receive handler is already registered for a given device. 4977 * Return true if there one. 4978 * 4979 * The caller must hold the rtnl_mutex. 4980 */ 4981 bool netdev_is_rx_handler_busy(struct net_device *dev) 4982 { 4983 ASSERT_RTNL(); 4984 return dev && rtnl_dereference(dev->rx_handler); 4985 } 4986 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy); 4987 4988 /** 4989 * netdev_rx_handler_register - register receive handler 4990 * @dev: device to register a handler for 4991 * @rx_handler: receive handler to register 4992 * @rx_handler_data: data pointer that is used by rx handler 4993 * 4994 * Register a receive handler for a device. This handler will then be 4995 * called from __netif_receive_skb. A negative errno code is returned 4996 * on a failure. 4997 * 4998 * The caller must hold the rtnl_mutex. 4999 * 5000 * For a general description of rx_handler, see enum rx_handler_result. 5001 */ 5002 int netdev_rx_handler_register(struct net_device *dev, 5003 rx_handler_func_t *rx_handler, 5004 void *rx_handler_data) 5005 { 5006 if (netdev_is_rx_handler_busy(dev)) 5007 return -EBUSY; 5008 5009 if (dev->priv_flags & IFF_NO_RX_HANDLER) 5010 return -EINVAL; 5011 5012 /* Note: rx_handler_data must be set before rx_handler */ 5013 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); 5014 rcu_assign_pointer(dev->rx_handler, rx_handler); 5015 5016 return 0; 5017 } 5018 EXPORT_SYMBOL_GPL(netdev_rx_handler_register); 5019 5020 /** 5021 * netdev_rx_handler_unregister - unregister receive handler 5022 * @dev: device to unregister a handler from 5023 * 5024 * Unregister a receive handler from a device. 5025 * 5026 * The caller must hold the rtnl_mutex. 5027 */ 5028 void netdev_rx_handler_unregister(struct net_device *dev) 5029 { 5030 5031 ASSERT_RTNL(); 5032 RCU_INIT_POINTER(dev->rx_handler, NULL); 5033 /* a reader seeing a non NULL rx_handler in a rcu_read_lock() 5034 * section has a guarantee to see a non NULL rx_handler_data 5035 * as well. 5036 */ 5037 synchronize_net(); 5038 RCU_INIT_POINTER(dev->rx_handler_data, NULL); 5039 } 5040 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); 5041 5042 /* 5043 * Limit the use of PFMEMALLOC reserves to those protocols that implement 5044 * the special handling of PFMEMALLOC skbs. 5045 */ 5046 static bool skb_pfmemalloc_protocol(struct sk_buff *skb) 5047 { 5048 switch (skb->protocol) { 5049 case htons(ETH_P_ARP): 5050 case htons(ETH_P_IP): 5051 case htons(ETH_P_IPV6): 5052 case htons(ETH_P_8021Q): 5053 case htons(ETH_P_8021AD): 5054 return true; 5055 default: 5056 return false; 5057 } 5058 } 5059 5060 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev, 5061 int *ret, struct net_device *orig_dev) 5062 { 5063 if (nf_hook_ingress_active(skb)) { 5064 int ingress_retval; 5065 5066 if (*pt_prev) { 5067 *ret = deliver_skb(skb, *pt_prev, orig_dev); 5068 *pt_prev = NULL; 5069 } 5070 5071 rcu_read_lock(); 5072 ingress_retval = nf_hook_ingress(skb); 5073 rcu_read_unlock(); 5074 return ingress_retval; 5075 } 5076 return 0; 5077 } 5078 5079 static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc, 5080 struct packet_type **ppt_prev) 5081 { 5082 struct packet_type *ptype, *pt_prev; 5083 rx_handler_func_t *rx_handler; 5084 struct sk_buff *skb = *pskb; 5085 struct net_device *orig_dev; 5086 bool deliver_exact = false; 5087 int ret = NET_RX_DROP; 5088 __be16 type; 5089 5090 net_timestamp_check(!netdev_tstamp_prequeue, skb); 5091 5092 trace_netif_receive_skb(skb); 5093 5094 orig_dev = skb->dev; 5095 5096 skb_reset_network_header(skb); 5097 if (!skb_transport_header_was_set(skb)) 5098 skb_reset_transport_header(skb); 5099 skb_reset_mac_len(skb); 5100 5101 pt_prev = NULL; 5102 5103 another_round: 5104 skb->skb_iif = skb->dev->ifindex; 5105 5106 __this_cpu_inc(softnet_data.processed); 5107 5108 if (static_branch_unlikely(&generic_xdp_needed_key)) { 5109 int ret2; 5110 5111 preempt_disable(); 5112 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb); 5113 preempt_enable(); 5114 5115 if (ret2 != XDP_PASS) { 5116 ret = NET_RX_DROP; 5117 goto out; 5118 } 5119 skb_reset_mac_len(skb); 5120 } 5121 5122 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 5123 skb->protocol == cpu_to_be16(ETH_P_8021AD)) { 5124 skb = skb_vlan_untag(skb); 5125 if (unlikely(!skb)) 5126 goto out; 5127 } 5128 5129 if (skb_skip_tc_classify(skb)) 5130 goto skip_classify; 5131 5132 if (pfmemalloc) 5133 goto skip_taps; 5134 5135 list_for_each_entry_rcu(ptype, &ptype_all, list) { 5136 if (pt_prev) 5137 ret = deliver_skb(skb, pt_prev, orig_dev); 5138 pt_prev = ptype; 5139 } 5140 5141 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { 5142 if (pt_prev) 5143 ret = deliver_skb(skb, pt_prev, orig_dev); 5144 pt_prev = ptype; 5145 } 5146 5147 skip_taps: 5148 #ifdef CONFIG_NET_INGRESS 5149 if (static_branch_unlikely(&ingress_needed_key)) { 5150 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev); 5151 if (!skb) 5152 goto out; 5153 5154 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0) 5155 goto out; 5156 } 5157 #endif 5158 skb_reset_redirect(skb); 5159 skip_classify: 5160 if (pfmemalloc && !skb_pfmemalloc_protocol(skb)) 5161 goto drop; 5162 5163 if (skb_vlan_tag_present(skb)) { 5164 if (pt_prev) { 5165 ret = deliver_skb(skb, pt_prev, orig_dev); 5166 pt_prev = NULL; 5167 } 5168 if (vlan_do_receive(&skb)) 5169 goto another_round; 5170 else if (unlikely(!skb)) 5171 goto out; 5172 } 5173 5174 rx_handler = rcu_dereference(skb->dev->rx_handler); 5175 if (rx_handler) { 5176 if (pt_prev) { 5177 ret = deliver_skb(skb, pt_prev, orig_dev); 5178 pt_prev = NULL; 5179 } 5180 switch (rx_handler(&skb)) { 5181 case RX_HANDLER_CONSUMED: 5182 ret = NET_RX_SUCCESS; 5183 goto out; 5184 case RX_HANDLER_ANOTHER: 5185 goto another_round; 5186 case RX_HANDLER_EXACT: 5187 deliver_exact = true; 5188 case RX_HANDLER_PASS: 5189 break; 5190 default: 5191 BUG(); 5192 } 5193 } 5194 5195 if (unlikely(skb_vlan_tag_present(skb))) { 5196 check_vlan_id: 5197 if (skb_vlan_tag_get_id(skb)) { 5198 /* Vlan id is non 0 and vlan_do_receive() above couldn't 5199 * find vlan device. 5200 */ 5201 skb->pkt_type = PACKET_OTHERHOST; 5202 } else if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 5203 skb->protocol == cpu_to_be16(ETH_P_8021AD)) { 5204 /* Outer header is 802.1P with vlan 0, inner header is 5205 * 802.1Q or 802.1AD and vlan_do_receive() above could 5206 * not find vlan dev for vlan id 0. 5207 */ 5208 __vlan_hwaccel_clear_tag(skb); 5209 skb = skb_vlan_untag(skb); 5210 if (unlikely(!skb)) 5211 goto out; 5212 if (vlan_do_receive(&skb)) 5213 /* After stripping off 802.1P header with vlan 0 5214 * vlan dev is found for inner header. 5215 */ 5216 goto another_round; 5217 else if (unlikely(!skb)) 5218 goto out; 5219 else 5220 /* We have stripped outer 802.1P vlan 0 header. 5221 * But could not find vlan dev. 5222 * check again for vlan id to set OTHERHOST. 5223 */ 5224 goto check_vlan_id; 5225 } 5226 /* Note: we might in the future use prio bits 5227 * and set skb->priority like in vlan_do_receive() 5228 * For the time being, just ignore Priority Code Point 5229 */ 5230 __vlan_hwaccel_clear_tag(skb); 5231 } 5232 5233 type = skb->protocol; 5234 5235 /* deliver only exact match when indicated */ 5236 if (likely(!deliver_exact)) { 5237 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 5238 &ptype_base[ntohs(type) & 5239 PTYPE_HASH_MASK]); 5240 } 5241 5242 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 5243 &orig_dev->ptype_specific); 5244 5245 if (unlikely(skb->dev != orig_dev)) { 5246 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 5247 &skb->dev->ptype_specific); 5248 } 5249 5250 if (pt_prev) { 5251 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) 5252 goto drop; 5253 *ppt_prev = pt_prev; 5254 } else { 5255 drop: 5256 if (!deliver_exact) 5257 atomic_long_inc(&skb->dev->rx_dropped); 5258 else 5259 atomic_long_inc(&skb->dev->rx_nohandler); 5260 kfree_skb(skb); 5261 /* Jamal, now you will not able to escape explaining 5262 * me how you were going to use this. :-) 5263 */ 5264 ret = NET_RX_DROP; 5265 } 5266 5267 out: 5268 /* The invariant here is that if *ppt_prev is not NULL 5269 * then skb should also be non-NULL. 5270 * 5271 * Apparently *ppt_prev assignment above holds this invariant due to 5272 * skb dereferencing near it. 5273 */ 5274 *pskb = skb; 5275 return ret; 5276 } 5277 5278 static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc) 5279 { 5280 struct net_device *orig_dev = skb->dev; 5281 struct packet_type *pt_prev = NULL; 5282 int ret; 5283 5284 ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev); 5285 if (pt_prev) 5286 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb, 5287 skb->dev, pt_prev, orig_dev); 5288 return ret; 5289 } 5290 5291 /** 5292 * netif_receive_skb_core - special purpose version of netif_receive_skb 5293 * @skb: buffer to process 5294 * 5295 * More direct receive version of netif_receive_skb(). It should 5296 * only be used by callers that have a need to skip RPS and Generic XDP. 5297 * Caller must also take care of handling if ``(page_is_)pfmemalloc``. 5298 * 5299 * This function may only be called from softirq context and interrupts 5300 * should be enabled. 5301 * 5302 * Return values (usually ignored): 5303 * NET_RX_SUCCESS: no congestion 5304 * NET_RX_DROP: packet was dropped 5305 */ 5306 int netif_receive_skb_core(struct sk_buff *skb) 5307 { 5308 int ret; 5309 5310 rcu_read_lock(); 5311 ret = __netif_receive_skb_one_core(skb, false); 5312 rcu_read_unlock(); 5313 5314 return ret; 5315 } 5316 EXPORT_SYMBOL(netif_receive_skb_core); 5317 5318 static inline void __netif_receive_skb_list_ptype(struct list_head *head, 5319 struct packet_type *pt_prev, 5320 struct net_device *orig_dev) 5321 { 5322 struct sk_buff *skb, *next; 5323 5324 if (!pt_prev) 5325 return; 5326 if (list_empty(head)) 5327 return; 5328 if (pt_prev->list_func != NULL) 5329 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv, 5330 ip_list_rcv, head, pt_prev, orig_dev); 5331 else 5332 list_for_each_entry_safe(skb, next, head, list) { 5333 skb_list_del_init(skb); 5334 pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 5335 } 5336 } 5337 5338 static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc) 5339 { 5340 /* Fast-path assumptions: 5341 * - There is no RX handler. 5342 * - Only one packet_type matches. 5343 * If either of these fails, we will end up doing some per-packet 5344 * processing in-line, then handling the 'last ptype' for the whole 5345 * sublist. This can't cause out-of-order delivery to any single ptype, 5346 * because the 'last ptype' must be constant across the sublist, and all 5347 * other ptypes are handled per-packet. 5348 */ 5349 /* Current (common) ptype of sublist */ 5350 struct packet_type *pt_curr = NULL; 5351 /* Current (common) orig_dev of sublist */ 5352 struct net_device *od_curr = NULL; 5353 struct list_head sublist; 5354 struct sk_buff *skb, *next; 5355 5356 INIT_LIST_HEAD(&sublist); 5357 list_for_each_entry_safe(skb, next, head, list) { 5358 struct net_device *orig_dev = skb->dev; 5359 struct packet_type *pt_prev = NULL; 5360 5361 skb_list_del_init(skb); 5362 __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev); 5363 if (!pt_prev) 5364 continue; 5365 if (pt_curr != pt_prev || od_curr != orig_dev) { 5366 /* dispatch old sublist */ 5367 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr); 5368 /* start new sublist */ 5369 INIT_LIST_HEAD(&sublist); 5370 pt_curr = pt_prev; 5371 od_curr = orig_dev; 5372 } 5373 list_add_tail(&skb->list, &sublist); 5374 } 5375 5376 /* dispatch final sublist */ 5377 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr); 5378 } 5379 5380 static int __netif_receive_skb(struct sk_buff *skb) 5381 { 5382 int ret; 5383 5384 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) { 5385 unsigned int noreclaim_flag; 5386 5387 /* 5388 * PFMEMALLOC skbs are special, they should 5389 * - be delivered to SOCK_MEMALLOC sockets only 5390 * - stay away from userspace 5391 * - have bounded memory usage 5392 * 5393 * Use PF_MEMALLOC as this saves us from propagating the allocation 5394 * context down to all allocation sites. 5395 */ 5396 noreclaim_flag = memalloc_noreclaim_save(); 5397 ret = __netif_receive_skb_one_core(skb, true); 5398 memalloc_noreclaim_restore(noreclaim_flag); 5399 } else 5400 ret = __netif_receive_skb_one_core(skb, false); 5401 5402 return ret; 5403 } 5404 5405 static void __netif_receive_skb_list(struct list_head *head) 5406 { 5407 unsigned long noreclaim_flag = 0; 5408 struct sk_buff *skb, *next; 5409 bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */ 5410 5411 list_for_each_entry_safe(skb, next, head, list) { 5412 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) { 5413 struct list_head sublist; 5414 5415 /* Handle the previous sublist */ 5416 list_cut_before(&sublist, head, &skb->list); 5417 if (!list_empty(&sublist)) 5418 __netif_receive_skb_list_core(&sublist, pfmemalloc); 5419 pfmemalloc = !pfmemalloc; 5420 /* See comments in __netif_receive_skb */ 5421 if (pfmemalloc) 5422 noreclaim_flag = memalloc_noreclaim_save(); 5423 else 5424 memalloc_noreclaim_restore(noreclaim_flag); 5425 } 5426 } 5427 /* Handle the remaining sublist */ 5428 if (!list_empty(head)) 5429 __netif_receive_skb_list_core(head, pfmemalloc); 5430 /* Restore pflags */ 5431 if (pfmemalloc) 5432 memalloc_noreclaim_restore(noreclaim_flag); 5433 } 5434 5435 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp) 5436 { 5437 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog); 5438 struct bpf_prog *new = xdp->prog; 5439 int ret = 0; 5440 5441 if (new) { 5442 u32 i; 5443 5444 /* generic XDP does not work with DEVMAPs that can 5445 * have a bpf_prog installed on an entry 5446 */ 5447 for (i = 0; i < new->aux->used_map_cnt; i++) { 5448 if (dev_map_can_have_prog(new->aux->used_maps[i])) 5449 return -EINVAL; 5450 if (cpu_map_prog_allowed(new->aux->used_maps[i])) 5451 return -EINVAL; 5452 } 5453 } 5454 5455 switch (xdp->command) { 5456 case XDP_SETUP_PROG: 5457 rcu_assign_pointer(dev->xdp_prog, new); 5458 if (old) 5459 bpf_prog_put(old); 5460 5461 if (old && !new) { 5462 static_branch_dec(&generic_xdp_needed_key); 5463 } else if (new && !old) { 5464 static_branch_inc(&generic_xdp_needed_key); 5465 dev_disable_lro(dev); 5466 dev_disable_gro_hw(dev); 5467 } 5468 break; 5469 5470 default: 5471 ret = -EINVAL; 5472 break; 5473 } 5474 5475 return ret; 5476 } 5477 5478 static int netif_receive_skb_internal(struct sk_buff *skb) 5479 { 5480 int ret; 5481 5482 net_timestamp_check(netdev_tstamp_prequeue, skb); 5483 5484 if (skb_defer_rx_timestamp(skb)) 5485 return NET_RX_SUCCESS; 5486 5487 rcu_read_lock(); 5488 #ifdef CONFIG_RPS 5489 if (static_branch_unlikely(&rps_needed)) { 5490 struct rps_dev_flow voidflow, *rflow = &voidflow; 5491 int cpu = get_rps_cpu(skb->dev, skb, &rflow); 5492 5493 if (cpu >= 0) { 5494 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 5495 rcu_read_unlock(); 5496 return ret; 5497 } 5498 } 5499 #endif 5500 ret = __netif_receive_skb(skb); 5501 rcu_read_unlock(); 5502 return ret; 5503 } 5504 5505 static void netif_receive_skb_list_internal(struct list_head *head) 5506 { 5507 struct sk_buff *skb, *next; 5508 struct list_head sublist; 5509 5510 INIT_LIST_HEAD(&sublist); 5511 list_for_each_entry_safe(skb, next, head, list) { 5512 net_timestamp_check(netdev_tstamp_prequeue, skb); 5513 skb_list_del_init(skb); 5514 if (!skb_defer_rx_timestamp(skb)) 5515 list_add_tail(&skb->list, &sublist); 5516 } 5517 list_splice_init(&sublist, head); 5518 5519 rcu_read_lock(); 5520 #ifdef CONFIG_RPS 5521 if (static_branch_unlikely(&rps_needed)) { 5522 list_for_each_entry_safe(skb, next, head, list) { 5523 struct rps_dev_flow voidflow, *rflow = &voidflow; 5524 int cpu = get_rps_cpu(skb->dev, skb, &rflow); 5525 5526 if (cpu >= 0) { 5527 /* Will be handled, remove from list */ 5528 skb_list_del_init(skb); 5529 enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 5530 } 5531 } 5532 } 5533 #endif 5534 __netif_receive_skb_list(head); 5535 rcu_read_unlock(); 5536 } 5537 5538 /** 5539 * netif_receive_skb - process receive buffer from network 5540 * @skb: buffer to process 5541 * 5542 * netif_receive_skb() is the main receive data processing function. 5543 * It always succeeds. The buffer may be dropped during processing 5544 * for congestion control or by the protocol layers. 5545 * 5546 * This function may only be called from softirq context and interrupts 5547 * should be enabled. 5548 * 5549 * Return values (usually ignored): 5550 * NET_RX_SUCCESS: no congestion 5551 * NET_RX_DROP: packet was dropped 5552 */ 5553 int netif_receive_skb(struct sk_buff *skb) 5554 { 5555 int ret; 5556 5557 trace_netif_receive_skb_entry(skb); 5558 5559 ret = netif_receive_skb_internal(skb); 5560 trace_netif_receive_skb_exit(ret); 5561 5562 return ret; 5563 } 5564 EXPORT_SYMBOL(netif_receive_skb); 5565 5566 /** 5567 * netif_receive_skb_list - process many receive buffers from network 5568 * @head: list of skbs to process. 5569 * 5570 * Since return value of netif_receive_skb() is normally ignored, and 5571 * wouldn't be meaningful for a list, this function returns void. 5572 * 5573 * This function may only be called from softirq context and interrupts 5574 * should be enabled. 5575 */ 5576 void netif_receive_skb_list(struct list_head *head) 5577 { 5578 struct sk_buff *skb; 5579 5580 if (list_empty(head)) 5581 return; 5582 if (trace_netif_receive_skb_list_entry_enabled()) { 5583 list_for_each_entry(skb, head, list) 5584 trace_netif_receive_skb_list_entry(skb); 5585 } 5586 netif_receive_skb_list_internal(head); 5587 trace_netif_receive_skb_list_exit(0); 5588 } 5589 EXPORT_SYMBOL(netif_receive_skb_list); 5590 5591 static DEFINE_PER_CPU(struct work_struct, flush_works); 5592 5593 /* Network device is going away, flush any packets still pending */ 5594 static void flush_backlog(struct work_struct *work) 5595 { 5596 struct sk_buff *skb, *tmp; 5597 struct softnet_data *sd; 5598 5599 local_bh_disable(); 5600 sd = this_cpu_ptr(&softnet_data); 5601 5602 local_irq_disable(); 5603 rps_lock(sd); 5604 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { 5605 if (skb->dev->reg_state == NETREG_UNREGISTERING) { 5606 __skb_unlink(skb, &sd->input_pkt_queue); 5607 dev_kfree_skb_irq(skb); 5608 input_queue_head_incr(sd); 5609 } 5610 } 5611 rps_unlock(sd); 5612 local_irq_enable(); 5613 5614 skb_queue_walk_safe(&sd->process_queue, skb, tmp) { 5615 if (skb->dev->reg_state == NETREG_UNREGISTERING) { 5616 __skb_unlink(skb, &sd->process_queue); 5617 kfree_skb(skb); 5618 input_queue_head_incr(sd); 5619 } 5620 } 5621 local_bh_enable(); 5622 } 5623 5624 static void flush_all_backlogs(void) 5625 { 5626 unsigned int cpu; 5627 5628 get_online_cpus(); 5629 5630 for_each_online_cpu(cpu) 5631 queue_work_on(cpu, system_highpri_wq, 5632 per_cpu_ptr(&flush_works, cpu)); 5633 5634 for_each_online_cpu(cpu) 5635 flush_work(per_cpu_ptr(&flush_works, cpu)); 5636 5637 put_online_cpus(); 5638 } 5639 5640 /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */ 5641 static void gro_normal_list(struct napi_struct *napi) 5642 { 5643 if (!napi->rx_count) 5644 return; 5645 netif_receive_skb_list_internal(&napi->rx_list); 5646 INIT_LIST_HEAD(&napi->rx_list); 5647 napi->rx_count = 0; 5648 } 5649 5650 /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded, 5651 * pass the whole batch up to the stack. 5652 */ 5653 static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb) 5654 { 5655 list_add_tail(&skb->list, &napi->rx_list); 5656 if (++napi->rx_count >= gro_normal_batch) 5657 gro_normal_list(napi); 5658 } 5659 5660 INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int)); 5661 INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int)); 5662 static int napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb) 5663 { 5664 struct packet_offload *ptype; 5665 __be16 type = skb->protocol; 5666 struct list_head *head = &offload_base; 5667 int err = -ENOENT; 5668 5669 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); 5670 5671 if (NAPI_GRO_CB(skb)->count == 1) { 5672 skb_shinfo(skb)->gso_size = 0; 5673 goto out; 5674 } 5675 5676 rcu_read_lock(); 5677 list_for_each_entry_rcu(ptype, head, list) { 5678 if (ptype->type != type || !ptype->callbacks.gro_complete) 5679 continue; 5680 5681 err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete, 5682 ipv6_gro_complete, inet_gro_complete, 5683 skb, 0); 5684 break; 5685 } 5686 rcu_read_unlock(); 5687 5688 if (err) { 5689 WARN_ON(&ptype->list == head); 5690 kfree_skb(skb); 5691 return NET_RX_SUCCESS; 5692 } 5693 5694 out: 5695 gro_normal_one(napi, skb); 5696 return NET_RX_SUCCESS; 5697 } 5698 5699 static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index, 5700 bool flush_old) 5701 { 5702 struct list_head *head = &napi->gro_hash[index].list; 5703 struct sk_buff *skb, *p; 5704 5705 list_for_each_entry_safe_reverse(skb, p, head, list) { 5706 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) 5707 return; 5708 skb_list_del_init(skb); 5709 napi_gro_complete(napi, skb); 5710 napi->gro_hash[index].count--; 5711 } 5712 5713 if (!napi->gro_hash[index].count) 5714 __clear_bit(index, &napi->gro_bitmask); 5715 } 5716 5717 /* napi->gro_hash[].list contains packets ordered by age. 5718 * youngest packets at the head of it. 5719 * Complete skbs in reverse order to reduce latencies. 5720 */ 5721 void napi_gro_flush(struct napi_struct *napi, bool flush_old) 5722 { 5723 unsigned long bitmask = napi->gro_bitmask; 5724 unsigned int i, base = ~0U; 5725 5726 while ((i = ffs(bitmask)) != 0) { 5727 bitmask >>= i; 5728 base += i; 5729 __napi_gro_flush_chain(napi, base, flush_old); 5730 } 5731 } 5732 EXPORT_SYMBOL(napi_gro_flush); 5733 5734 static struct list_head *gro_list_prepare(struct napi_struct *napi, 5735 struct sk_buff *skb) 5736 { 5737 unsigned int maclen = skb->dev->hard_header_len; 5738 u32 hash = skb_get_hash_raw(skb); 5739 struct list_head *head; 5740 struct sk_buff *p; 5741 5742 head = &napi->gro_hash[hash & (GRO_HASH_BUCKETS - 1)].list; 5743 list_for_each_entry(p, head, list) { 5744 unsigned long diffs; 5745 5746 NAPI_GRO_CB(p)->flush = 0; 5747 5748 if (hash != skb_get_hash_raw(p)) { 5749 NAPI_GRO_CB(p)->same_flow = 0; 5750 continue; 5751 } 5752 5753 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; 5754 diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb); 5755 if (skb_vlan_tag_present(p)) 5756 diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb); 5757 diffs |= skb_metadata_dst_cmp(p, skb); 5758 diffs |= skb_metadata_differs(p, skb); 5759 if (maclen == ETH_HLEN) 5760 diffs |= compare_ether_header(skb_mac_header(p), 5761 skb_mac_header(skb)); 5762 else if (!diffs) 5763 diffs = memcmp(skb_mac_header(p), 5764 skb_mac_header(skb), 5765 maclen); 5766 NAPI_GRO_CB(p)->same_flow = !diffs; 5767 } 5768 5769 return head; 5770 } 5771 5772 static void skb_gro_reset_offset(struct sk_buff *skb) 5773 { 5774 const struct skb_shared_info *pinfo = skb_shinfo(skb); 5775 const skb_frag_t *frag0 = &pinfo->frags[0]; 5776 5777 NAPI_GRO_CB(skb)->data_offset = 0; 5778 NAPI_GRO_CB(skb)->frag0 = NULL; 5779 NAPI_GRO_CB(skb)->frag0_len = 0; 5780 5781 if (!skb_headlen(skb) && pinfo->nr_frags && 5782 !PageHighMem(skb_frag_page(frag0))) { 5783 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); 5784 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int, 5785 skb_frag_size(frag0), 5786 skb->end - skb->tail); 5787 } 5788 } 5789 5790 static void gro_pull_from_frag0(struct sk_buff *skb, int grow) 5791 { 5792 struct skb_shared_info *pinfo = skb_shinfo(skb); 5793 5794 BUG_ON(skb->end - skb->tail < grow); 5795 5796 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); 5797 5798 skb->data_len -= grow; 5799 skb->tail += grow; 5800 5801 skb_frag_off_add(&pinfo->frags[0], grow); 5802 skb_frag_size_sub(&pinfo->frags[0], grow); 5803 5804 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { 5805 skb_frag_unref(skb, 0); 5806 memmove(pinfo->frags, pinfo->frags + 1, 5807 --pinfo->nr_frags * sizeof(pinfo->frags[0])); 5808 } 5809 } 5810 5811 static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head) 5812 { 5813 struct sk_buff *oldest; 5814 5815 oldest = list_last_entry(head, struct sk_buff, list); 5816 5817 /* We are called with head length >= MAX_GRO_SKBS, so this is 5818 * impossible. 5819 */ 5820 if (WARN_ON_ONCE(!oldest)) 5821 return; 5822 5823 /* Do not adjust napi->gro_hash[].count, caller is adding a new 5824 * SKB to the chain. 5825 */ 5826 skb_list_del_init(oldest); 5827 napi_gro_complete(napi, oldest); 5828 } 5829 5830 INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *, 5831 struct sk_buff *)); 5832 INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *, 5833 struct sk_buff *)); 5834 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 5835 { 5836 u32 hash = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1); 5837 struct list_head *head = &offload_base; 5838 struct packet_offload *ptype; 5839 __be16 type = skb->protocol; 5840 struct list_head *gro_head; 5841 struct sk_buff *pp = NULL; 5842 enum gro_result ret; 5843 int same_flow; 5844 int grow; 5845 5846 if (netif_elide_gro(skb->dev)) 5847 goto normal; 5848 5849 gro_head = gro_list_prepare(napi, skb); 5850 5851 rcu_read_lock(); 5852 list_for_each_entry_rcu(ptype, head, list) { 5853 if (ptype->type != type || !ptype->callbacks.gro_receive) 5854 continue; 5855 5856 skb_set_network_header(skb, skb_gro_offset(skb)); 5857 skb_reset_mac_len(skb); 5858 NAPI_GRO_CB(skb)->same_flow = 0; 5859 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb); 5860 NAPI_GRO_CB(skb)->free = 0; 5861 NAPI_GRO_CB(skb)->encap_mark = 0; 5862 NAPI_GRO_CB(skb)->recursion_counter = 0; 5863 NAPI_GRO_CB(skb)->is_fou = 0; 5864 NAPI_GRO_CB(skb)->is_atomic = 1; 5865 NAPI_GRO_CB(skb)->gro_remcsum_start = 0; 5866 5867 /* Setup for GRO checksum validation */ 5868 switch (skb->ip_summed) { 5869 case CHECKSUM_COMPLETE: 5870 NAPI_GRO_CB(skb)->csum = skb->csum; 5871 NAPI_GRO_CB(skb)->csum_valid = 1; 5872 NAPI_GRO_CB(skb)->csum_cnt = 0; 5873 break; 5874 case CHECKSUM_UNNECESSARY: 5875 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; 5876 NAPI_GRO_CB(skb)->csum_valid = 0; 5877 break; 5878 default: 5879 NAPI_GRO_CB(skb)->csum_cnt = 0; 5880 NAPI_GRO_CB(skb)->csum_valid = 0; 5881 } 5882 5883 pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive, 5884 ipv6_gro_receive, inet_gro_receive, 5885 gro_head, skb); 5886 break; 5887 } 5888 rcu_read_unlock(); 5889 5890 if (&ptype->list == head) 5891 goto normal; 5892 5893 if (PTR_ERR(pp) == -EINPROGRESS) { 5894 ret = GRO_CONSUMED; 5895 goto ok; 5896 } 5897 5898 same_flow = NAPI_GRO_CB(skb)->same_flow; 5899 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; 5900 5901 if (pp) { 5902 skb_list_del_init(pp); 5903 napi_gro_complete(napi, pp); 5904 napi->gro_hash[hash].count--; 5905 } 5906 5907 if (same_flow) 5908 goto ok; 5909 5910 if (NAPI_GRO_CB(skb)->flush) 5911 goto normal; 5912 5913 if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) { 5914 gro_flush_oldest(napi, gro_head); 5915 } else { 5916 napi->gro_hash[hash].count++; 5917 } 5918 NAPI_GRO_CB(skb)->count = 1; 5919 NAPI_GRO_CB(skb)->age = jiffies; 5920 NAPI_GRO_CB(skb)->last = skb; 5921 skb_shinfo(skb)->gso_size = skb_gro_len(skb); 5922 list_add(&skb->list, gro_head); 5923 ret = GRO_HELD; 5924 5925 pull: 5926 grow = skb_gro_offset(skb) - skb_headlen(skb); 5927 if (grow > 0) 5928 gro_pull_from_frag0(skb, grow); 5929 ok: 5930 if (napi->gro_hash[hash].count) { 5931 if (!test_bit(hash, &napi->gro_bitmask)) 5932 __set_bit(hash, &napi->gro_bitmask); 5933 } else if (test_bit(hash, &napi->gro_bitmask)) { 5934 __clear_bit(hash, &napi->gro_bitmask); 5935 } 5936 5937 return ret; 5938 5939 normal: 5940 ret = GRO_NORMAL; 5941 goto pull; 5942 } 5943 5944 struct packet_offload *gro_find_receive_by_type(__be16 type) 5945 { 5946 struct list_head *offload_head = &offload_base; 5947 struct packet_offload *ptype; 5948 5949 list_for_each_entry_rcu(ptype, offload_head, list) { 5950 if (ptype->type != type || !ptype->callbacks.gro_receive) 5951 continue; 5952 return ptype; 5953 } 5954 return NULL; 5955 } 5956 EXPORT_SYMBOL(gro_find_receive_by_type); 5957 5958 struct packet_offload *gro_find_complete_by_type(__be16 type) 5959 { 5960 struct list_head *offload_head = &offload_base; 5961 struct packet_offload *ptype; 5962 5963 list_for_each_entry_rcu(ptype, offload_head, list) { 5964 if (ptype->type != type || !ptype->callbacks.gro_complete) 5965 continue; 5966 return ptype; 5967 } 5968 return NULL; 5969 } 5970 EXPORT_SYMBOL(gro_find_complete_by_type); 5971 5972 static void napi_skb_free_stolen_head(struct sk_buff *skb) 5973 { 5974 skb_dst_drop(skb); 5975 skb_ext_put(skb); 5976 kmem_cache_free(skbuff_head_cache, skb); 5977 } 5978 5979 static gro_result_t napi_skb_finish(struct napi_struct *napi, 5980 struct sk_buff *skb, 5981 gro_result_t ret) 5982 { 5983 switch (ret) { 5984 case GRO_NORMAL: 5985 gro_normal_one(napi, skb); 5986 break; 5987 5988 case GRO_DROP: 5989 kfree_skb(skb); 5990 break; 5991 5992 case GRO_MERGED_FREE: 5993 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) 5994 napi_skb_free_stolen_head(skb); 5995 else 5996 __kfree_skb(skb); 5997 break; 5998 5999 case GRO_HELD: 6000 case GRO_MERGED: 6001 case GRO_CONSUMED: 6002 break; 6003 } 6004 6005 return ret; 6006 } 6007 6008 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 6009 { 6010 gro_result_t ret; 6011 6012 skb_mark_napi_id(skb, napi); 6013 trace_napi_gro_receive_entry(skb); 6014 6015 skb_gro_reset_offset(skb); 6016 6017 ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb)); 6018 trace_napi_gro_receive_exit(ret); 6019 6020 return ret; 6021 } 6022 EXPORT_SYMBOL(napi_gro_receive); 6023 6024 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) 6025 { 6026 if (unlikely(skb->pfmemalloc)) { 6027 consume_skb(skb); 6028 return; 6029 } 6030 __skb_pull(skb, skb_headlen(skb)); 6031 /* restore the reserve we had after netdev_alloc_skb_ip_align() */ 6032 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); 6033 __vlan_hwaccel_clear_tag(skb); 6034 skb->dev = napi->dev; 6035 skb->skb_iif = 0; 6036 6037 /* eth_type_trans() assumes pkt_type is PACKET_HOST */ 6038 skb->pkt_type = PACKET_HOST; 6039 6040 skb->encapsulation = 0; 6041 skb_shinfo(skb)->gso_type = 0; 6042 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 6043 skb_ext_reset(skb); 6044 6045 napi->skb = skb; 6046 } 6047 6048 struct sk_buff *napi_get_frags(struct napi_struct *napi) 6049 { 6050 struct sk_buff *skb = napi->skb; 6051 6052 if (!skb) { 6053 skb = napi_alloc_skb(napi, GRO_MAX_HEAD); 6054 if (skb) { 6055 napi->skb = skb; 6056 skb_mark_napi_id(skb, napi); 6057 } 6058 } 6059 return skb; 6060 } 6061 EXPORT_SYMBOL(napi_get_frags); 6062 6063 static gro_result_t napi_frags_finish(struct napi_struct *napi, 6064 struct sk_buff *skb, 6065 gro_result_t ret) 6066 { 6067 switch (ret) { 6068 case GRO_NORMAL: 6069 case GRO_HELD: 6070 __skb_push(skb, ETH_HLEN); 6071 skb->protocol = eth_type_trans(skb, skb->dev); 6072 if (ret == GRO_NORMAL) 6073 gro_normal_one(napi, skb); 6074 break; 6075 6076 case GRO_DROP: 6077 napi_reuse_skb(napi, skb); 6078 break; 6079 6080 case GRO_MERGED_FREE: 6081 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) 6082 napi_skb_free_stolen_head(skb); 6083 else 6084 napi_reuse_skb(napi, skb); 6085 break; 6086 6087 case GRO_MERGED: 6088 case GRO_CONSUMED: 6089 break; 6090 } 6091 6092 return ret; 6093 } 6094 6095 /* Upper GRO stack assumes network header starts at gro_offset=0 6096 * Drivers could call both napi_gro_frags() and napi_gro_receive() 6097 * We copy ethernet header into skb->data to have a common layout. 6098 */ 6099 static struct sk_buff *napi_frags_skb(struct napi_struct *napi) 6100 { 6101 struct sk_buff *skb = napi->skb; 6102 const struct ethhdr *eth; 6103 unsigned int hlen = sizeof(*eth); 6104 6105 napi->skb = NULL; 6106 6107 skb_reset_mac_header(skb); 6108 skb_gro_reset_offset(skb); 6109 6110 if (unlikely(skb_gro_header_hard(skb, hlen))) { 6111 eth = skb_gro_header_slow(skb, hlen, 0); 6112 if (unlikely(!eth)) { 6113 net_warn_ratelimited("%s: dropping impossible skb from %s\n", 6114 __func__, napi->dev->name); 6115 napi_reuse_skb(napi, skb); 6116 return NULL; 6117 } 6118 } else { 6119 eth = (const struct ethhdr *)skb->data; 6120 gro_pull_from_frag0(skb, hlen); 6121 NAPI_GRO_CB(skb)->frag0 += hlen; 6122 NAPI_GRO_CB(skb)->frag0_len -= hlen; 6123 } 6124 __skb_pull(skb, hlen); 6125 6126 /* 6127 * This works because the only protocols we care about don't require 6128 * special handling. 6129 * We'll fix it up properly in napi_frags_finish() 6130 */ 6131 skb->protocol = eth->h_proto; 6132 6133 return skb; 6134 } 6135 6136 gro_result_t napi_gro_frags(struct napi_struct *napi) 6137 { 6138 gro_result_t ret; 6139 struct sk_buff *skb = napi_frags_skb(napi); 6140 6141 if (!skb) 6142 return GRO_DROP; 6143 6144 trace_napi_gro_frags_entry(skb); 6145 6146 ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); 6147 trace_napi_gro_frags_exit(ret); 6148 6149 return ret; 6150 } 6151 EXPORT_SYMBOL(napi_gro_frags); 6152 6153 /* Compute the checksum from gro_offset and return the folded value 6154 * after adding in any pseudo checksum. 6155 */ 6156 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb) 6157 { 6158 __wsum wsum; 6159 __sum16 sum; 6160 6161 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); 6162 6163 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ 6164 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); 6165 /* See comments in __skb_checksum_complete(). */ 6166 if (likely(!sum)) { 6167 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 6168 !skb->csum_complete_sw) 6169 netdev_rx_csum_fault(skb->dev, skb); 6170 } 6171 6172 NAPI_GRO_CB(skb)->csum = wsum; 6173 NAPI_GRO_CB(skb)->csum_valid = 1; 6174 6175 return sum; 6176 } 6177 EXPORT_SYMBOL(__skb_gro_checksum_complete); 6178 6179 static void net_rps_send_ipi(struct softnet_data *remsd) 6180 { 6181 #ifdef CONFIG_RPS 6182 while (remsd) { 6183 struct softnet_data *next = remsd->rps_ipi_next; 6184 6185 if (cpu_online(remsd->cpu)) 6186 smp_call_function_single_async(remsd->cpu, &remsd->csd); 6187 remsd = next; 6188 } 6189 #endif 6190 } 6191 6192 /* 6193 * net_rps_action_and_irq_enable sends any pending IPI's for rps. 6194 * Note: called with local irq disabled, but exits with local irq enabled. 6195 */ 6196 static void net_rps_action_and_irq_enable(struct softnet_data *sd) 6197 { 6198 #ifdef CONFIG_RPS 6199 struct softnet_data *remsd = sd->rps_ipi_list; 6200 6201 if (remsd) { 6202 sd->rps_ipi_list = NULL; 6203 6204 local_irq_enable(); 6205 6206 /* Send pending IPI's to kick RPS processing on remote cpus. */ 6207 net_rps_send_ipi(remsd); 6208 } else 6209 #endif 6210 local_irq_enable(); 6211 } 6212 6213 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) 6214 { 6215 #ifdef CONFIG_RPS 6216 return sd->rps_ipi_list != NULL; 6217 #else 6218 return false; 6219 #endif 6220 } 6221 6222 static int process_backlog(struct napi_struct *napi, int quota) 6223 { 6224 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); 6225 bool again = true; 6226 int work = 0; 6227 6228 /* Check if we have pending ipi, its better to send them now, 6229 * not waiting net_rx_action() end. 6230 */ 6231 if (sd_has_rps_ipi_waiting(sd)) { 6232 local_irq_disable(); 6233 net_rps_action_and_irq_enable(sd); 6234 } 6235 6236 napi->weight = dev_rx_weight; 6237 while (again) { 6238 struct sk_buff *skb; 6239 6240 while ((skb = __skb_dequeue(&sd->process_queue))) { 6241 rcu_read_lock(); 6242 __netif_receive_skb(skb); 6243 rcu_read_unlock(); 6244 input_queue_head_incr(sd); 6245 if (++work >= quota) 6246 return work; 6247 6248 } 6249 6250 local_irq_disable(); 6251 rps_lock(sd); 6252 if (skb_queue_empty(&sd->input_pkt_queue)) { 6253 /* 6254 * Inline a custom version of __napi_complete(). 6255 * only current cpu owns and manipulates this napi, 6256 * and NAPI_STATE_SCHED is the only possible flag set 6257 * on backlog. 6258 * We can use a plain write instead of clear_bit(), 6259 * and we dont need an smp_mb() memory barrier. 6260 */ 6261 napi->state = 0; 6262 again = false; 6263 } else { 6264 skb_queue_splice_tail_init(&sd->input_pkt_queue, 6265 &sd->process_queue); 6266 } 6267 rps_unlock(sd); 6268 local_irq_enable(); 6269 } 6270 6271 return work; 6272 } 6273 6274 /** 6275 * __napi_schedule - schedule for receive 6276 * @n: entry to schedule 6277 * 6278 * The entry's receive function will be scheduled to run. 6279 * Consider using __napi_schedule_irqoff() if hard irqs are masked. 6280 */ 6281 void __napi_schedule(struct napi_struct *n) 6282 { 6283 unsigned long flags; 6284 6285 local_irq_save(flags); 6286 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 6287 local_irq_restore(flags); 6288 } 6289 EXPORT_SYMBOL(__napi_schedule); 6290 6291 /** 6292 * napi_schedule_prep - check if napi can be scheduled 6293 * @n: napi context 6294 * 6295 * Test if NAPI routine is already running, and if not mark 6296 * it as running. This is used as a condition variable 6297 * insure only one NAPI poll instance runs. We also make 6298 * sure there is no pending NAPI disable. 6299 */ 6300 bool napi_schedule_prep(struct napi_struct *n) 6301 { 6302 unsigned long val, new; 6303 6304 do { 6305 val = READ_ONCE(n->state); 6306 if (unlikely(val & NAPIF_STATE_DISABLE)) 6307 return false; 6308 new = val | NAPIF_STATE_SCHED; 6309 6310 /* Sets STATE_MISSED bit if STATE_SCHED was already set 6311 * This was suggested by Alexander Duyck, as compiler 6312 * emits better code than : 6313 * if (val & NAPIF_STATE_SCHED) 6314 * new |= NAPIF_STATE_MISSED; 6315 */ 6316 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED * 6317 NAPIF_STATE_MISSED; 6318 } while (cmpxchg(&n->state, val, new) != val); 6319 6320 return !(val & NAPIF_STATE_SCHED); 6321 } 6322 EXPORT_SYMBOL(napi_schedule_prep); 6323 6324 /** 6325 * __napi_schedule_irqoff - schedule for receive 6326 * @n: entry to schedule 6327 * 6328 * Variant of __napi_schedule() assuming hard irqs are masked 6329 */ 6330 void __napi_schedule_irqoff(struct napi_struct *n) 6331 { 6332 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 6333 } 6334 EXPORT_SYMBOL(__napi_schedule_irqoff); 6335 6336 bool napi_complete_done(struct napi_struct *n, int work_done) 6337 { 6338 unsigned long flags, val, new, timeout = 0; 6339 bool ret = true; 6340 6341 /* 6342 * 1) Don't let napi dequeue from the cpu poll list 6343 * just in case its running on a different cpu. 6344 * 2) If we are busy polling, do nothing here, we have 6345 * the guarantee we will be called later. 6346 */ 6347 if (unlikely(n->state & (NAPIF_STATE_NPSVC | 6348 NAPIF_STATE_IN_BUSY_POLL))) 6349 return false; 6350 6351 if (work_done) { 6352 if (n->gro_bitmask) 6353 timeout = READ_ONCE(n->dev->gro_flush_timeout); 6354 n->defer_hard_irqs_count = READ_ONCE(n->dev->napi_defer_hard_irqs); 6355 } 6356 if (n->defer_hard_irqs_count > 0) { 6357 n->defer_hard_irqs_count--; 6358 timeout = READ_ONCE(n->dev->gro_flush_timeout); 6359 if (timeout) 6360 ret = false; 6361 } 6362 if (n->gro_bitmask) { 6363 /* When the NAPI instance uses a timeout and keeps postponing 6364 * it, we need to bound somehow the time packets are kept in 6365 * the GRO layer 6366 */ 6367 napi_gro_flush(n, !!timeout); 6368 } 6369 6370 gro_normal_list(n); 6371 6372 if (unlikely(!list_empty(&n->poll_list))) { 6373 /* If n->poll_list is not empty, we need to mask irqs */ 6374 local_irq_save(flags); 6375 list_del_init(&n->poll_list); 6376 local_irq_restore(flags); 6377 } 6378 6379 do { 6380 val = READ_ONCE(n->state); 6381 6382 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED)); 6383 6384 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED); 6385 6386 /* If STATE_MISSED was set, leave STATE_SCHED set, 6387 * because we will call napi->poll() one more time. 6388 * This C code was suggested by Alexander Duyck to help gcc. 6389 */ 6390 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED * 6391 NAPIF_STATE_SCHED; 6392 } while (cmpxchg(&n->state, val, new) != val); 6393 6394 if (unlikely(val & NAPIF_STATE_MISSED)) { 6395 __napi_schedule(n); 6396 return false; 6397 } 6398 6399 if (timeout) 6400 hrtimer_start(&n->timer, ns_to_ktime(timeout), 6401 HRTIMER_MODE_REL_PINNED); 6402 return ret; 6403 } 6404 EXPORT_SYMBOL(napi_complete_done); 6405 6406 /* must be called under rcu_read_lock(), as we dont take a reference */ 6407 static struct napi_struct *napi_by_id(unsigned int napi_id) 6408 { 6409 unsigned int hash = napi_id % HASH_SIZE(napi_hash); 6410 struct napi_struct *napi; 6411 6412 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node) 6413 if (napi->napi_id == napi_id) 6414 return napi; 6415 6416 return NULL; 6417 } 6418 6419 #if defined(CONFIG_NET_RX_BUSY_POLL) 6420 6421 #define BUSY_POLL_BUDGET 8 6422 6423 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock) 6424 { 6425 int rc; 6426 6427 /* Busy polling means there is a high chance device driver hard irq 6428 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was 6429 * set in napi_schedule_prep(). 6430 * Since we are about to call napi->poll() once more, we can safely 6431 * clear NAPI_STATE_MISSED. 6432 * 6433 * Note: x86 could use a single "lock and ..." instruction 6434 * to perform these two clear_bit() 6435 */ 6436 clear_bit(NAPI_STATE_MISSED, &napi->state); 6437 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state); 6438 6439 local_bh_disable(); 6440 6441 /* All we really want here is to re-enable device interrupts. 6442 * Ideally, a new ndo_busy_poll_stop() could avoid another round. 6443 */ 6444 rc = napi->poll(napi, BUSY_POLL_BUDGET); 6445 /* We can't gro_normal_list() here, because napi->poll() might have 6446 * rearmed the napi (napi_complete_done()) in which case it could 6447 * already be running on another CPU. 6448 */ 6449 trace_napi_poll(napi, rc, BUSY_POLL_BUDGET); 6450 netpoll_poll_unlock(have_poll_lock); 6451 if (rc == BUSY_POLL_BUDGET) { 6452 /* As the whole budget was spent, we still own the napi so can 6453 * safely handle the rx_list. 6454 */ 6455 gro_normal_list(napi); 6456 __napi_schedule(napi); 6457 } 6458 local_bh_enable(); 6459 } 6460 6461 void napi_busy_loop(unsigned int napi_id, 6462 bool (*loop_end)(void *, unsigned long), 6463 void *loop_end_arg) 6464 { 6465 unsigned long start_time = loop_end ? busy_loop_current_time() : 0; 6466 int (*napi_poll)(struct napi_struct *napi, int budget); 6467 void *have_poll_lock = NULL; 6468 struct napi_struct *napi; 6469 6470 restart: 6471 napi_poll = NULL; 6472 6473 rcu_read_lock(); 6474 6475 napi = napi_by_id(napi_id); 6476 if (!napi) 6477 goto out; 6478 6479 preempt_disable(); 6480 for (;;) { 6481 int work = 0; 6482 6483 local_bh_disable(); 6484 if (!napi_poll) { 6485 unsigned long val = READ_ONCE(napi->state); 6486 6487 /* If multiple threads are competing for this napi, 6488 * we avoid dirtying napi->state as much as we can. 6489 */ 6490 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED | 6491 NAPIF_STATE_IN_BUSY_POLL)) 6492 goto count; 6493 if (cmpxchg(&napi->state, val, 6494 val | NAPIF_STATE_IN_BUSY_POLL | 6495 NAPIF_STATE_SCHED) != val) 6496 goto count; 6497 have_poll_lock = netpoll_poll_lock(napi); 6498 napi_poll = napi->poll; 6499 } 6500 work = napi_poll(napi, BUSY_POLL_BUDGET); 6501 trace_napi_poll(napi, work, BUSY_POLL_BUDGET); 6502 gro_normal_list(napi); 6503 count: 6504 if (work > 0) 6505 __NET_ADD_STATS(dev_net(napi->dev), 6506 LINUX_MIB_BUSYPOLLRXPACKETS, work); 6507 local_bh_enable(); 6508 6509 if (!loop_end || loop_end(loop_end_arg, start_time)) 6510 break; 6511 6512 if (unlikely(need_resched())) { 6513 if (napi_poll) 6514 busy_poll_stop(napi, have_poll_lock); 6515 preempt_enable(); 6516 rcu_read_unlock(); 6517 cond_resched(); 6518 if (loop_end(loop_end_arg, start_time)) 6519 return; 6520 goto restart; 6521 } 6522 cpu_relax(); 6523 } 6524 if (napi_poll) 6525 busy_poll_stop(napi, have_poll_lock); 6526 preempt_enable(); 6527 out: 6528 rcu_read_unlock(); 6529 } 6530 EXPORT_SYMBOL(napi_busy_loop); 6531 6532 #endif /* CONFIG_NET_RX_BUSY_POLL */ 6533 6534 static void napi_hash_add(struct napi_struct *napi) 6535 { 6536 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) || 6537 test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) 6538 return; 6539 6540 spin_lock(&napi_hash_lock); 6541 6542 /* 0..NR_CPUS range is reserved for sender_cpu use */ 6543 do { 6544 if (unlikely(++napi_gen_id < MIN_NAPI_ID)) 6545 napi_gen_id = MIN_NAPI_ID; 6546 } while (napi_by_id(napi_gen_id)); 6547 napi->napi_id = napi_gen_id; 6548 6549 hlist_add_head_rcu(&napi->napi_hash_node, 6550 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); 6551 6552 spin_unlock(&napi_hash_lock); 6553 } 6554 6555 /* Warning : caller is responsible to make sure rcu grace period 6556 * is respected before freeing memory containing @napi 6557 */ 6558 bool napi_hash_del(struct napi_struct *napi) 6559 { 6560 bool rcu_sync_needed = false; 6561 6562 spin_lock(&napi_hash_lock); 6563 6564 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) { 6565 rcu_sync_needed = true; 6566 hlist_del_rcu(&napi->napi_hash_node); 6567 } 6568 spin_unlock(&napi_hash_lock); 6569 return rcu_sync_needed; 6570 } 6571 EXPORT_SYMBOL_GPL(napi_hash_del); 6572 6573 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer) 6574 { 6575 struct napi_struct *napi; 6576 6577 napi = container_of(timer, struct napi_struct, timer); 6578 6579 /* Note : we use a relaxed variant of napi_schedule_prep() not setting 6580 * NAPI_STATE_MISSED, since we do not react to a device IRQ. 6581 */ 6582 if (!napi_disable_pending(napi) && 6583 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) 6584 __napi_schedule_irqoff(napi); 6585 6586 return HRTIMER_NORESTART; 6587 } 6588 6589 static void init_gro_hash(struct napi_struct *napi) 6590 { 6591 int i; 6592 6593 for (i = 0; i < GRO_HASH_BUCKETS; i++) { 6594 INIT_LIST_HEAD(&napi->gro_hash[i].list); 6595 napi->gro_hash[i].count = 0; 6596 } 6597 napi->gro_bitmask = 0; 6598 } 6599 6600 void netif_napi_add(struct net_device *dev, struct napi_struct *napi, 6601 int (*poll)(struct napi_struct *, int), int weight) 6602 { 6603 INIT_LIST_HEAD(&napi->poll_list); 6604 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); 6605 napi->timer.function = napi_watchdog; 6606 init_gro_hash(napi); 6607 napi->skb = NULL; 6608 INIT_LIST_HEAD(&napi->rx_list); 6609 napi->rx_count = 0; 6610 napi->poll = poll; 6611 if (weight > NAPI_POLL_WEIGHT) 6612 netdev_err_once(dev, "%s() called with weight %d\n", __func__, 6613 weight); 6614 napi->weight = weight; 6615 napi->dev = dev; 6616 #ifdef CONFIG_NETPOLL 6617 napi->poll_owner = -1; 6618 #endif 6619 set_bit(NAPI_STATE_SCHED, &napi->state); 6620 set_bit(NAPI_STATE_NPSVC, &napi->state); 6621 list_add_rcu(&napi->dev_list, &dev->napi_list); 6622 napi_hash_add(napi); 6623 } 6624 EXPORT_SYMBOL(netif_napi_add); 6625 6626 void napi_disable(struct napi_struct *n) 6627 { 6628 might_sleep(); 6629 set_bit(NAPI_STATE_DISABLE, &n->state); 6630 6631 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state)) 6632 msleep(1); 6633 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state)) 6634 msleep(1); 6635 6636 hrtimer_cancel(&n->timer); 6637 6638 clear_bit(NAPI_STATE_DISABLE, &n->state); 6639 } 6640 EXPORT_SYMBOL(napi_disable); 6641 6642 static void flush_gro_hash(struct napi_struct *napi) 6643 { 6644 int i; 6645 6646 for (i = 0; i < GRO_HASH_BUCKETS; i++) { 6647 struct sk_buff *skb, *n; 6648 6649 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list) 6650 kfree_skb(skb); 6651 napi->gro_hash[i].count = 0; 6652 } 6653 } 6654 6655 /* Must be called in process context */ 6656 void netif_napi_del(struct napi_struct *napi) 6657 { 6658 might_sleep(); 6659 if (napi_hash_del(napi)) 6660 synchronize_net(); 6661 list_del_init(&napi->dev_list); 6662 napi_free_frags(napi); 6663 6664 flush_gro_hash(napi); 6665 napi->gro_bitmask = 0; 6666 } 6667 EXPORT_SYMBOL(netif_napi_del); 6668 6669 static int napi_poll(struct napi_struct *n, struct list_head *repoll) 6670 { 6671 void *have; 6672 int work, weight; 6673 6674 list_del_init(&n->poll_list); 6675 6676 have = netpoll_poll_lock(n); 6677 6678 weight = n->weight; 6679 6680 /* This NAPI_STATE_SCHED test is for avoiding a race 6681 * with netpoll's poll_napi(). Only the entity which 6682 * obtains the lock and sees NAPI_STATE_SCHED set will 6683 * actually make the ->poll() call. Therefore we avoid 6684 * accidentally calling ->poll() when NAPI is not scheduled. 6685 */ 6686 work = 0; 6687 if (test_bit(NAPI_STATE_SCHED, &n->state)) { 6688 work = n->poll(n, weight); 6689 trace_napi_poll(n, work, weight); 6690 } 6691 6692 if (unlikely(work > weight)) 6693 pr_err_once("NAPI poll function %pS returned %d, exceeding its budget of %d.\n", 6694 n->poll, work, weight); 6695 6696 if (likely(work < weight)) 6697 goto out_unlock; 6698 6699 /* Drivers must not modify the NAPI state if they 6700 * consume the entire weight. In such cases this code 6701 * still "owns" the NAPI instance and therefore can 6702 * move the instance around on the list at-will. 6703 */ 6704 if (unlikely(napi_disable_pending(n))) { 6705 napi_complete(n); 6706 goto out_unlock; 6707 } 6708 6709 if (n->gro_bitmask) { 6710 /* flush too old packets 6711 * If HZ < 1000, flush all packets. 6712 */ 6713 napi_gro_flush(n, HZ >= 1000); 6714 } 6715 6716 gro_normal_list(n); 6717 6718 /* Some drivers may have called napi_schedule 6719 * prior to exhausting their budget. 6720 */ 6721 if (unlikely(!list_empty(&n->poll_list))) { 6722 pr_warn_once("%s: Budget exhausted after napi rescheduled\n", 6723 n->dev ? n->dev->name : "backlog"); 6724 goto out_unlock; 6725 } 6726 6727 list_add_tail(&n->poll_list, repoll); 6728 6729 out_unlock: 6730 netpoll_poll_unlock(have); 6731 6732 return work; 6733 } 6734 6735 static __latent_entropy void net_rx_action(struct softirq_action *h) 6736 { 6737 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 6738 unsigned long time_limit = jiffies + 6739 usecs_to_jiffies(netdev_budget_usecs); 6740 int budget = netdev_budget; 6741 LIST_HEAD(list); 6742 LIST_HEAD(repoll); 6743 6744 local_irq_disable(); 6745 list_splice_init(&sd->poll_list, &list); 6746 local_irq_enable(); 6747 6748 for (;;) { 6749 struct napi_struct *n; 6750 6751 if (list_empty(&list)) { 6752 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll)) 6753 goto out; 6754 break; 6755 } 6756 6757 n = list_first_entry(&list, struct napi_struct, poll_list); 6758 budget -= napi_poll(n, &repoll); 6759 6760 /* If softirq window is exhausted then punt. 6761 * Allow this to run for 2 jiffies since which will allow 6762 * an average latency of 1.5/HZ. 6763 */ 6764 if (unlikely(budget <= 0 || 6765 time_after_eq(jiffies, time_limit))) { 6766 sd->time_squeeze++; 6767 break; 6768 } 6769 } 6770 6771 local_irq_disable(); 6772 6773 list_splice_tail_init(&sd->poll_list, &list); 6774 list_splice_tail(&repoll, &list); 6775 list_splice(&list, &sd->poll_list); 6776 if (!list_empty(&sd->poll_list)) 6777 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 6778 6779 net_rps_action_and_irq_enable(sd); 6780 out: 6781 __kfree_skb_flush(); 6782 } 6783 6784 struct netdev_adjacent { 6785 struct net_device *dev; 6786 6787 /* upper master flag, there can only be one master device per list */ 6788 bool master; 6789 6790 /* lookup ignore flag */ 6791 bool ignore; 6792 6793 /* counter for the number of times this device was added to us */ 6794 u16 ref_nr; 6795 6796 /* private field for the users */ 6797 void *private; 6798 6799 struct list_head list; 6800 struct rcu_head rcu; 6801 }; 6802 6803 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev, 6804 struct list_head *adj_list) 6805 { 6806 struct netdev_adjacent *adj; 6807 6808 list_for_each_entry(adj, adj_list, list) { 6809 if (adj->dev == adj_dev) 6810 return adj; 6811 } 6812 return NULL; 6813 } 6814 6815 static int ____netdev_has_upper_dev(struct net_device *upper_dev, void *data) 6816 { 6817 struct net_device *dev = data; 6818 6819 return upper_dev == dev; 6820 } 6821 6822 /** 6823 * netdev_has_upper_dev - Check if device is linked to an upper device 6824 * @dev: device 6825 * @upper_dev: upper device to check 6826 * 6827 * Find out if a device is linked to specified upper device and return true 6828 * in case it is. Note that this checks only immediate upper device, 6829 * not through a complete stack of devices. The caller must hold the RTNL lock. 6830 */ 6831 bool netdev_has_upper_dev(struct net_device *dev, 6832 struct net_device *upper_dev) 6833 { 6834 ASSERT_RTNL(); 6835 6836 return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev, 6837 upper_dev); 6838 } 6839 EXPORT_SYMBOL(netdev_has_upper_dev); 6840 6841 /** 6842 * netdev_has_upper_dev_all - Check if device is linked to an upper device 6843 * @dev: device 6844 * @upper_dev: upper device to check 6845 * 6846 * Find out if a device is linked to specified upper device and return true 6847 * in case it is. Note that this checks the entire upper device chain. 6848 * The caller must hold rcu lock. 6849 */ 6850 6851 bool netdev_has_upper_dev_all_rcu(struct net_device *dev, 6852 struct net_device *upper_dev) 6853 { 6854 return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev, 6855 upper_dev); 6856 } 6857 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu); 6858 6859 /** 6860 * netdev_has_any_upper_dev - Check if device is linked to some device 6861 * @dev: device 6862 * 6863 * Find out if a device is linked to an upper device and return true in case 6864 * it is. The caller must hold the RTNL lock. 6865 */ 6866 bool netdev_has_any_upper_dev(struct net_device *dev) 6867 { 6868 ASSERT_RTNL(); 6869 6870 return !list_empty(&dev->adj_list.upper); 6871 } 6872 EXPORT_SYMBOL(netdev_has_any_upper_dev); 6873 6874 /** 6875 * netdev_master_upper_dev_get - Get master upper device 6876 * @dev: device 6877 * 6878 * Find a master upper device and return pointer to it or NULL in case 6879 * it's not there. The caller must hold the RTNL lock. 6880 */ 6881 struct net_device *netdev_master_upper_dev_get(struct net_device *dev) 6882 { 6883 struct netdev_adjacent *upper; 6884 6885 ASSERT_RTNL(); 6886 6887 if (list_empty(&dev->adj_list.upper)) 6888 return NULL; 6889 6890 upper = list_first_entry(&dev->adj_list.upper, 6891 struct netdev_adjacent, list); 6892 if (likely(upper->master)) 6893 return upper->dev; 6894 return NULL; 6895 } 6896 EXPORT_SYMBOL(netdev_master_upper_dev_get); 6897 6898 static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev) 6899 { 6900 struct netdev_adjacent *upper; 6901 6902 ASSERT_RTNL(); 6903 6904 if (list_empty(&dev->adj_list.upper)) 6905 return NULL; 6906 6907 upper = list_first_entry(&dev->adj_list.upper, 6908 struct netdev_adjacent, list); 6909 if (likely(upper->master) && !upper->ignore) 6910 return upper->dev; 6911 return NULL; 6912 } 6913 6914 /** 6915 * netdev_has_any_lower_dev - Check if device is linked to some device 6916 * @dev: device 6917 * 6918 * Find out if a device is linked to a lower device and return true in case 6919 * it is. The caller must hold the RTNL lock. 6920 */ 6921 static bool netdev_has_any_lower_dev(struct net_device *dev) 6922 { 6923 ASSERT_RTNL(); 6924 6925 return !list_empty(&dev->adj_list.lower); 6926 } 6927 6928 void *netdev_adjacent_get_private(struct list_head *adj_list) 6929 { 6930 struct netdev_adjacent *adj; 6931 6932 adj = list_entry(adj_list, struct netdev_adjacent, list); 6933 6934 return adj->private; 6935 } 6936 EXPORT_SYMBOL(netdev_adjacent_get_private); 6937 6938 /** 6939 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list 6940 * @dev: device 6941 * @iter: list_head ** of the current position 6942 * 6943 * Gets the next device from the dev's upper list, starting from iter 6944 * position. The caller must hold RCU read lock. 6945 */ 6946 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev, 6947 struct list_head **iter) 6948 { 6949 struct netdev_adjacent *upper; 6950 6951 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 6952 6953 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 6954 6955 if (&upper->list == &dev->adj_list.upper) 6956 return NULL; 6957 6958 *iter = &upper->list; 6959 6960 return upper->dev; 6961 } 6962 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu); 6963 6964 static struct net_device *__netdev_next_upper_dev(struct net_device *dev, 6965 struct list_head **iter, 6966 bool *ignore) 6967 { 6968 struct netdev_adjacent *upper; 6969 6970 upper = list_entry((*iter)->next, struct netdev_adjacent, list); 6971 6972 if (&upper->list == &dev->adj_list.upper) 6973 return NULL; 6974 6975 *iter = &upper->list; 6976 *ignore = upper->ignore; 6977 6978 return upper->dev; 6979 } 6980 6981 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev, 6982 struct list_head **iter) 6983 { 6984 struct netdev_adjacent *upper; 6985 6986 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 6987 6988 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 6989 6990 if (&upper->list == &dev->adj_list.upper) 6991 return NULL; 6992 6993 *iter = &upper->list; 6994 6995 return upper->dev; 6996 } 6997 6998 static int __netdev_walk_all_upper_dev(struct net_device *dev, 6999 int (*fn)(struct net_device *dev, 7000 void *data), 7001 void *data) 7002 { 7003 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 7004 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 7005 int ret, cur = 0; 7006 bool ignore; 7007 7008 now = dev; 7009 iter = &dev->adj_list.upper; 7010 7011 while (1) { 7012 if (now != dev) { 7013 ret = fn(now, data); 7014 if (ret) 7015 return ret; 7016 } 7017 7018 next = NULL; 7019 while (1) { 7020 udev = __netdev_next_upper_dev(now, &iter, &ignore); 7021 if (!udev) 7022 break; 7023 if (ignore) 7024 continue; 7025 7026 next = udev; 7027 niter = &udev->adj_list.upper; 7028 dev_stack[cur] = now; 7029 iter_stack[cur++] = iter; 7030 break; 7031 } 7032 7033 if (!next) { 7034 if (!cur) 7035 return 0; 7036 next = dev_stack[--cur]; 7037 niter = iter_stack[cur]; 7038 } 7039 7040 now = next; 7041 iter = niter; 7042 } 7043 7044 return 0; 7045 } 7046 7047 int netdev_walk_all_upper_dev_rcu(struct net_device *dev, 7048 int (*fn)(struct net_device *dev, 7049 void *data), 7050 void *data) 7051 { 7052 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 7053 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 7054 int ret, cur = 0; 7055 7056 now = dev; 7057 iter = &dev->adj_list.upper; 7058 7059 while (1) { 7060 if (now != dev) { 7061 ret = fn(now, data); 7062 if (ret) 7063 return ret; 7064 } 7065 7066 next = NULL; 7067 while (1) { 7068 udev = netdev_next_upper_dev_rcu(now, &iter); 7069 if (!udev) 7070 break; 7071 7072 next = udev; 7073 niter = &udev->adj_list.upper; 7074 dev_stack[cur] = now; 7075 iter_stack[cur++] = iter; 7076 break; 7077 } 7078 7079 if (!next) { 7080 if (!cur) 7081 return 0; 7082 next = dev_stack[--cur]; 7083 niter = iter_stack[cur]; 7084 } 7085 7086 now = next; 7087 iter = niter; 7088 } 7089 7090 return 0; 7091 } 7092 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu); 7093 7094 static bool __netdev_has_upper_dev(struct net_device *dev, 7095 struct net_device *upper_dev) 7096 { 7097 ASSERT_RTNL(); 7098 7099 return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev, 7100 upper_dev); 7101 } 7102 7103 /** 7104 * netdev_lower_get_next_private - Get the next ->private from the 7105 * lower neighbour list 7106 * @dev: device 7107 * @iter: list_head ** of the current position 7108 * 7109 * Gets the next netdev_adjacent->private from the dev's lower neighbour 7110 * list, starting from iter position. The caller must hold either hold the 7111 * RTNL lock or its own locking that guarantees that the neighbour lower 7112 * list will remain unchanged. 7113 */ 7114 void *netdev_lower_get_next_private(struct net_device *dev, 7115 struct list_head **iter) 7116 { 7117 struct netdev_adjacent *lower; 7118 7119 lower = list_entry(*iter, struct netdev_adjacent, list); 7120 7121 if (&lower->list == &dev->adj_list.lower) 7122 return NULL; 7123 7124 *iter = lower->list.next; 7125 7126 return lower->private; 7127 } 7128 EXPORT_SYMBOL(netdev_lower_get_next_private); 7129 7130 /** 7131 * netdev_lower_get_next_private_rcu - Get the next ->private from the 7132 * lower neighbour list, RCU 7133 * variant 7134 * @dev: device 7135 * @iter: list_head ** of the current position 7136 * 7137 * Gets the next netdev_adjacent->private from the dev's lower neighbour 7138 * list, starting from iter position. The caller must hold RCU read lock. 7139 */ 7140 void *netdev_lower_get_next_private_rcu(struct net_device *dev, 7141 struct list_head **iter) 7142 { 7143 struct netdev_adjacent *lower; 7144 7145 WARN_ON_ONCE(!rcu_read_lock_held()); 7146 7147 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 7148 7149 if (&lower->list == &dev->adj_list.lower) 7150 return NULL; 7151 7152 *iter = &lower->list; 7153 7154 return lower->private; 7155 } 7156 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu); 7157 7158 /** 7159 * netdev_lower_get_next - Get the next device from the lower neighbour 7160 * list 7161 * @dev: device 7162 * @iter: list_head ** of the current position 7163 * 7164 * Gets the next netdev_adjacent from the dev's lower neighbour 7165 * list, starting from iter position. The caller must hold RTNL lock or 7166 * its own locking that guarantees that the neighbour lower 7167 * list will remain unchanged. 7168 */ 7169 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter) 7170 { 7171 struct netdev_adjacent *lower; 7172 7173 lower = list_entry(*iter, struct netdev_adjacent, list); 7174 7175 if (&lower->list == &dev->adj_list.lower) 7176 return NULL; 7177 7178 *iter = lower->list.next; 7179 7180 return lower->dev; 7181 } 7182 EXPORT_SYMBOL(netdev_lower_get_next); 7183 7184 static struct net_device *netdev_next_lower_dev(struct net_device *dev, 7185 struct list_head **iter) 7186 { 7187 struct netdev_adjacent *lower; 7188 7189 lower = list_entry((*iter)->next, struct netdev_adjacent, list); 7190 7191 if (&lower->list == &dev->adj_list.lower) 7192 return NULL; 7193 7194 *iter = &lower->list; 7195 7196 return lower->dev; 7197 } 7198 7199 static struct net_device *__netdev_next_lower_dev(struct net_device *dev, 7200 struct list_head **iter, 7201 bool *ignore) 7202 { 7203 struct netdev_adjacent *lower; 7204 7205 lower = list_entry((*iter)->next, struct netdev_adjacent, list); 7206 7207 if (&lower->list == &dev->adj_list.lower) 7208 return NULL; 7209 7210 *iter = &lower->list; 7211 *ignore = lower->ignore; 7212 7213 return lower->dev; 7214 } 7215 7216 int netdev_walk_all_lower_dev(struct net_device *dev, 7217 int (*fn)(struct net_device *dev, 7218 void *data), 7219 void *data) 7220 { 7221 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 7222 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 7223 int ret, cur = 0; 7224 7225 now = dev; 7226 iter = &dev->adj_list.lower; 7227 7228 while (1) { 7229 if (now != dev) { 7230 ret = fn(now, data); 7231 if (ret) 7232 return ret; 7233 } 7234 7235 next = NULL; 7236 while (1) { 7237 ldev = netdev_next_lower_dev(now, &iter); 7238 if (!ldev) 7239 break; 7240 7241 next = ldev; 7242 niter = &ldev->adj_list.lower; 7243 dev_stack[cur] = now; 7244 iter_stack[cur++] = iter; 7245 break; 7246 } 7247 7248 if (!next) { 7249 if (!cur) 7250 return 0; 7251 next = dev_stack[--cur]; 7252 niter = iter_stack[cur]; 7253 } 7254 7255 now = next; 7256 iter = niter; 7257 } 7258 7259 return 0; 7260 } 7261 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev); 7262 7263 static int __netdev_walk_all_lower_dev(struct net_device *dev, 7264 int (*fn)(struct net_device *dev, 7265 void *data), 7266 void *data) 7267 { 7268 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 7269 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 7270 int ret, cur = 0; 7271 bool ignore; 7272 7273 now = dev; 7274 iter = &dev->adj_list.lower; 7275 7276 while (1) { 7277 if (now != dev) { 7278 ret = fn(now, data); 7279 if (ret) 7280 return ret; 7281 } 7282 7283 next = NULL; 7284 while (1) { 7285 ldev = __netdev_next_lower_dev(now, &iter, &ignore); 7286 if (!ldev) 7287 break; 7288 if (ignore) 7289 continue; 7290 7291 next = ldev; 7292 niter = &ldev->adj_list.lower; 7293 dev_stack[cur] = now; 7294 iter_stack[cur++] = iter; 7295 break; 7296 } 7297 7298 if (!next) { 7299 if (!cur) 7300 return 0; 7301 next = dev_stack[--cur]; 7302 niter = iter_stack[cur]; 7303 } 7304 7305 now = next; 7306 iter = niter; 7307 } 7308 7309 return 0; 7310 } 7311 7312 struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev, 7313 struct list_head **iter) 7314 { 7315 struct netdev_adjacent *lower; 7316 7317 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 7318 if (&lower->list == &dev->adj_list.lower) 7319 return NULL; 7320 7321 *iter = &lower->list; 7322 7323 return lower->dev; 7324 } 7325 EXPORT_SYMBOL(netdev_next_lower_dev_rcu); 7326 7327 static u8 __netdev_upper_depth(struct net_device *dev) 7328 { 7329 struct net_device *udev; 7330 struct list_head *iter; 7331 u8 max_depth = 0; 7332 bool ignore; 7333 7334 for (iter = &dev->adj_list.upper, 7335 udev = __netdev_next_upper_dev(dev, &iter, &ignore); 7336 udev; 7337 udev = __netdev_next_upper_dev(dev, &iter, &ignore)) { 7338 if (ignore) 7339 continue; 7340 if (max_depth < udev->upper_level) 7341 max_depth = udev->upper_level; 7342 } 7343 7344 return max_depth; 7345 } 7346 7347 static u8 __netdev_lower_depth(struct net_device *dev) 7348 { 7349 struct net_device *ldev; 7350 struct list_head *iter; 7351 u8 max_depth = 0; 7352 bool ignore; 7353 7354 for (iter = &dev->adj_list.lower, 7355 ldev = __netdev_next_lower_dev(dev, &iter, &ignore); 7356 ldev; 7357 ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) { 7358 if (ignore) 7359 continue; 7360 if (max_depth < ldev->lower_level) 7361 max_depth = ldev->lower_level; 7362 } 7363 7364 return max_depth; 7365 } 7366 7367 static int __netdev_update_upper_level(struct net_device *dev, void *data) 7368 { 7369 dev->upper_level = __netdev_upper_depth(dev) + 1; 7370 return 0; 7371 } 7372 7373 static int __netdev_update_lower_level(struct net_device *dev, void *data) 7374 { 7375 dev->lower_level = __netdev_lower_depth(dev) + 1; 7376 return 0; 7377 } 7378 7379 int netdev_walk_all_lower_dev_rcu(struct net_device *dev, 7380 int (*fn)(struct net_device *dev, 7381 void *data), 7382 void *data) 7383 { 7384 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 7385 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 7386 int ret, cur = 0; 7387 7388 now = dev; 7389 iter = &dev->adj_list.lower; 7390 7391 while (1) { 7392 if (now != dev) { 7393 ret = fn(now, data); 7394 if (ret) 7395 return ret; 7396 } 7397 7398 next = NULL; 7399 while (1) { 7400 ldev = netdev_next_lower_dev_rcu(now, &iter); 7401 if (!ldev) 7402 break; 7403 7404 next = ldev; 7405 niter = &ldev->adj_list.lower; 7406 dev_stack[cur] = now; 7407 iter_stack[cur++] = iter; 7408 break; 7409 } 7410 7411 if (!next) { 7412 if (!cur) 7413 return 0; 7414 next = dev_stack[--cur]; 7415 niter = iter_stack[cur]; 7416 } 7417 7418 now = next; 7419 iter = niter; 7420 } 7421 7422 return 0; 7423 } 7424 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu); 7425 7426 /** 7427 * netdev_lower_get_first_private_rcu - Get the first ->private from the 7428 * lower neighbour list, RCU 7429 * variant 7430 * @dev: device 7431 * 7432 * Gets the first netdev_adjacent->private from the dev's lower neighbour 7433 * list. The caller must hold RCU read lock. 7434 */ 7435 void *netdev_lower_get_first_private_rcu(struct net_device *dev) 7436 { 7437 struct netdev_adjacent *lower; 7438 7439 lower = list_first_or_null_rcu(&dev->adj_list.lower, 7440 struct netdev_adjacent, list); 7441 if (lower) 7442 return lower->private; 7443 return NULL; 7444 } 7445 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu); 7446 7447 /** 7448 * netdev_master_upper_dev_get_rcu - Get master upper device 7449 * @dev: device 7450 * 7451 * Find a master upper device and return pointer to it or NULL in case 7452 * it's not there. The caller must hold the RCU read lock. 7453 */ 7454 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev) 7455 { 7456 struct netdev_adjacent *upper; 7457 7458 upper = list_first_or_null_rcu(&dev->adj_list.upper, 7459 struct netdev_adjacent, list); 7460 if (upper && likely(upper->master)) 7461 return upper->dev; 7462 return NULL; 7463 } 7464 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu); 7465 7466 static int netdev_adjacent_sysfs_add(struct net_device *dev, 7467 struct net_device *adj_dev, 7468 struct list_head *dev_list) 7469 { 7470 char linkname[IFNAMSIZ+7]; 7471 7472 sprintf(linkname, dev_list == &dev->adj_list.upper ? 7473 "upper_%s" : "lower_%s", adj_dev->name); 7474 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), 7475 linkname); 7476 } 7477 static void netdev_adjacent_sysfs_del(struct net_device *dev, 7478 char *name, 7479 struct list_head *dev_list) 7480 { 7481 char linkname[IFNAMSIZ+7]; 7482 7483 sprintf(linkname, dev_list == &dev->adj_list.upper ? 7484 "upper_%s" : "lower_%s", name); 7485 sysfs_remove_link(&(dev->dev.kobj), linkname); 7486 } 7487 7488 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev, 7489 struct net_device *adj_dev, 7490 struct list_head *dev_list) 7491 { 7492 return (dev_list == &dev->adj_list.upper || 7493 dev_list == &dev->adj_list.lower) && 7494 net_eq(dev_net(dev), dev_net(adj_dev)); 7495 } 7496 7497 static int __netdev_adjacent_dev_insert(struct net_device *dev, 7498 struct net_device *adj_dev, 7499 struct list_head *dev_list, 7500 void *private, bool master) 7501 { 7502 struct netdev_adjacent *adj; 7503 int ret; 7504 7505 adj = __netdev_find_adj(adj_dev, dev_list); 7506 7507 if (adj) { 7508 adj->ref_nr += 1; 7509 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n", 7510 dev->name, adj_dev->name, adj->ref_nr); 7511 7512 return 0; 7513 } 7514 7515 adj = kmalloc(sizeof(*adj), GFP_KERNEL); 7516 if (!adj) 7517 return -ENOMEM; 7518 7519 adj->dev = adj_dev; 7520 adj->master = master; 7521 adj->ref_nr = 1; 7522 adj->private = private; 7523 adj->ignore = false; 7524 dev_hold(adj_dev); 7525 7526 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n", 7527 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name); 7528 7529 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) { 7530 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list); 7531 if (ret) 7532 goto free_adj; 7533 } 7534 7535 /* Ensure that master link is always the first item in list. */ 7536 if (master) { 7537 ret = sysfs_create_link(&(dev->dev.kobj), 7538 &(adj_dev->dev.kobj), "master"); 7539 if (ret) 7540 goto remove_symlinks; 7541 7542 list_add_rcu(&adj->list, dev_list); 7543 } else { 7544 list_add_tail_rcu(&adj->list, dev_list); 7545 } 7546 7547 return 0; 7548 7549 remove_symlinks: 7550 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 7551 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 7552 free_adj: 7553 kfree(adj); 7554 dev_put(adj_dev); 7555 7556 return ret; 7557 } 7558 7559 static void __netdev_adjacent_dev_remove(struct net_device *dev, 7560 struct net_device *adj_dev, 7561 u16 ref_nr, 7562 struct list_head *dev_list) 7563 { 7564 struct netdev_adjacent *adj; 7565 7566 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n", 7567 dev->name, adj_dev->name, ref_nr); 7568 7569 adj = __netdev_find_adj(adj_dev, dev_list); 7570 7571 if (!adj) { 7572 pr_err("Adjacency does not exist for device %s from %s\n", 7573 dev->name, adj_dev->name); 7574 WARN_ON(1); 7575 return; 7576 } 7577 7578 if (adj->ref_nr > ref_nr) { 7579 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n", 7580 dev->name, adj_dev->name, ref_nr, 7581 adj->ref_nr - ref_nr); 7582 adj->ref_nr -= ref_nr; 7583 return; 7584 } 7585 7586 if (adj->master) 7587 sysfs_remove_link(&(dev->dev.kobj), "master"); 7588 7589 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 7590 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 7591 7592 list_del_rcu(&adj->list); 7593 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n", 7594 adj_dev->name, dev->name, adj_dev->name); 7595 dev_put(adj_dev); 7596 kfree_rcu(adj, rcu); 7597 } 7598 7599 static int __netdev_adjacent_dev_link_lists(struct net_device *dev, 7600 struct net_device *upper_dev, 7601 struct list_head *up_list, 7602 struct list_head *down_list, 7603 void *private, bool master) 7604 { 7605 int ret; 7606 7607 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, 7608 private, master); 7609 if (ret) 7610 return ret; 7611 7612 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, 7613 private, false); 7614 if (ret) { 7615 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list); 7616 return ret; 7617 } 7618 7619 return 0; 7620 } 7621 7622 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev, 7623 struct net_device *upper_dev, 7624 u16 ref_nr, 7625 struct list_head *up_list, 7626 struct list_head *down_list) 7627 { 7628 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list); 7629 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list); 7630 } 7631 7632 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev, 7633 struct net_device *upper_dev, 7634 void *private, bool master) 7635 { 7636 return __netdev_adjacent_dev_link_lists(dev, upper_dev, 7637 &dev->adj_list.upper, 7638 &upper_dev->adj_list.lower, 7639 private, master); 7640 } 7641 7642 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev, 7643 struct net_device *upper_dev) 7644 { 7645 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1, 7646 &dev->adj_list.upper, 7647 &upper_dev->adj_list.lower); 7648 } 7649 7650 static int __netdev_upper_dev_link(struct net_device *dev, 7651 struct net_device *upper_dev, bool master, 7652 void *upper_priv, void *upper_info, 7653 struct netlink_ext_ack *extack) 7654 { 7655 struct netdev_notifier_changeupper_info changeupper_info = { 7656 .info = { 7657 .dev = dev, 7658 .extack = extack, 7659 }, 7660 .upper_dev = upper_dev, 7661 .master = master, 7662 .linking = true, 7663 .upper_info = upper_info, 7664 }; 7665 struct net_device *master_dev; 7666 int ret = 0; 7667 7668 ASSERT_RTNL(); 7669 7670 if (dev == upper_dev) 7671 return -EBUSY; 7672 7673 /* To prevent loops, check if dev is not upper device to upper_dev. */ 7674 if (__netdev_has_upper_dev(upper_dev, dev)) 7675 return -EBUSY; 7676 7677 if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV) 7678 return -EMLINK; 7679 7680 if (!master) { 7681 if (__netdev_has_upper_dev(dev, upper_dev)) 7682 return -EEXIST; 7683 } else { 7684 master_dev = __netdev_master_upper_dev_get(dev); 7685 if (master_dev) 7686 return master_dev == upper_dev ? -EEXIST : -EBUSY; 7687 } 7688 7689 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, 7690 &changeupper_info.info); 7691 ret = notifier_to_errno(ret); 7692 if (ret) 7693 return ret; 7694 7695 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv, 7696 master); 7697 if (ret) 7698 return ret; 7699 7700 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, 7701 &changeupper_info.info); 7702 ret = notifier_to_errno(ret); 7703 if (ret) 7704 goto rollback; 7705 7706 __netdev_update_upper_level(dev, NULL); 7707 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL); 7708 7709 __netdev_update_lower_level(upper_dev, NULL); 7710 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level, 7711 NULL); 7712 7713 return 0; 7714 7715 rollback: 7716 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 7717 7718 return ret; 7719 } 7720 7721 /** 7722 * netdev_upper_dev_link - Add a link to the upper device 7723 * @dev: device 7724 * @upper_dev: new upper device 7725 * @extack: netlink extended ack 7726 * 7727 * Adds a link to device which is upper to this one. The caller must hold 7728 * the RTNL lock. On a failure a negative errno code is returned. 7729 * On success the reference counts are adjusted and the function 7730 * returns zero. 7731 */ 7732 int netdev_upper_dev_link(struct net_device *dev, 7733 struct net_device *upper_dev, 7734 struct netlink_ext_ack *extack) 7735 { 7736 return __netdev_upper_dev_link(dev, upper_dev, false, 7737 NULL, NULL, extack); 7738 } 7739 EXPORT_SYMBOL(netdev_upper_dev_link); 7740 7741 /** 7742 * netdev_master_upper_dev_link - Add a master link to the upper device 7743 * @dev: device 7744 * @upper_dev: new upper device 7745 * @upper_priv: upper device private 7746 * @upper_info: upper info to be passed down via notifier 7747 * @extack: netlink extended ack 7748 * 7749 * Adds a link to device which is upper to this one. In this case, only 7750 * one master upper device can be linked, although other non-master devices 7751 * might be linked as well. The caller must hold the RTNL lock. 7752 * On a failure a negative errno code is returned. On success the reference 7753 * counts are adjusted and the function returns zero. 7754 */ 7755 int netdev_master_upper_dev_link(struct net_device *dev, 7756 struct net_device *upper_dev, 7757 void *upper_priv, void *upper_info, 7758 struct netlink_ext_ack *extack) 7759 { 7760 return __netdev_upper_dev_link(dev, upper_dev, true, 7761 upper_priv, upper_info, extack); 7762 } 7763 EXPORT_SYMBOL(netdev_master_upper_dev_link); 7764 7765 /** 7766 * netdev_upper_dev_unlink - Removes a link to upper device 7767 * @dev: device 7768 * @upper_dev: new upper device 7769 * 7770 * Removes a link to device which is upper to this one. The caller must hold 7771 * the RTNL lock. 7772 */ 7773 void netdev_upper_dev_unlink(struct net_device *dev, 7774 struct net_device *upper_dev) 7775 { 7776 struct netdev_notifier_changeupper_info changeupper_info = { 7777 .info = { 7778 .dev = dev, 7779 }, 7780 .upper_dev = upper_dev, 7781 .linking = false, 7782 }; 7783 7784 ASSERT_RTNL(); 7785 7786 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev; 7787 7788 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, 7789 &changeupper_info.info); 7790 7791 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 7792 7793 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, 7794 &changeupper_info.info); 7795 7796 __netdev_update_upper_level(dev, NULL); 7797 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL); 7798 7799 __netdev_update_lower_level(upper_dev, NULL); 7800 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level, 7801 NULL); 7802 } 7803 EXPORT_SYMBOL(netdev_upper_dev_unlink); 7804 7805 static void __netdev_adjacent_dev_set(struct net_device *upper_dev, 7806 struct net_device *lower_dev, 7807 bool val) 7808 { 7809 struct netdev_adjacent *adj; 7810 7811 adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower); 7812 if (adj) 7813 adj->ignore = val; 7814 7815 adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper); 7816 if (adj) 7817 adj->ignore = val; 7818 } 7819 7820 static void netdev_adjacent_dev_disable(struct net_device *upper_dev, 7821 struct net_device *lower_dev) 7822 { 7823 __netdev_adjacent_dev_set(upper_dev, lower_dev, true); 7824 } 7825 7826 static void netdev_adjacent_dev_enable(struct net_device *upper_dev, 7827 struct net_device *lower_dev) 7828 { 7829 __netdev_adjacent_dev_set(upper_dev, lower_dev, false); 7830 } 7831 7832 int netdev_adjacent_change_prepare(struct net_device *old_dev, 7833 struct net_device *new_dev, 7834 struct net_device *dev, 7835 struct netlink_ext_ack *extack) 7836 { 7837 int err; 7838 7839 if (!new_dev) 7840 return 0; 7841 7842 if (old_dev && new_dev != old_dev) 7843 netdev_adjacent_dev_disable(dev, old_dev); 7844 7845 err = netdev_upper_dev_link(new_dev, dev, extack); 7846 if (err) { 7847 if (old_dev && new_dev != old_dev) 7848 netdev_adjacent_dev_enable(dev, old_dev); 7849 return err; 7850 } 7851 7852 return 0; 7853 } 7854 EXPORT_SYMBOL(netdev_adjacent_change_prepare); 7855 7856 void netdev_adjacent_change_commit(struct net_device *old_dev, 7857 struct net_device *new_dev, 7858 struct net_device *dev) 7859 { 7860 if (!new_dev || !old_dev) 7861 return; 7862 7863 if (new_dev == old_dev) 7864 return; 7865 7866 netdev_adjacent_dev_enable(dev, old_dev); 7867 netdev_upper_dev_unlink(old_dev, dev); 7868 } 7869 EXPORT_SYMBOL(netdev_adjacent_change_commit); 7870 7871 void netdev_adjacent_change_abort(struct net_device *old_dev, 7872 struct net_device *new_dev, 7873 struct net_device *dev) 7874 { 7875 if (!new_dev) 7876 return; 7877 7878 if (old_dev && new_dev != old_dev) 7879 netdev_adjacent_dev_enable(dev, old_dev); 7880 7881 netdev_upper_dev_unlink(new_dev, dev); 7882 } 7883 EXPORT_SYMBOL(netdev_adjacent_change_abort); 7884 7885 /** 7886 * netdev_bonding_info_change - Dispatch event about slave change 7887 * @dev: device 7888 * @bonding_info: info to dispatch 7889 * 7890 * Send NETDEV_BONDING_INFO to netdev notifiers with info. 7891 * The caller must hold the RTNL lock. 7892 */ 7893 void netdev_bonding_info_change(struct net_device *dev, 7894 struct netdev_bonding_info *bonding_info) 7895 { 7896 struct netdev_notifier_bonding_info info = { 7897 .info.dev = dev, 7898 }; 7899 7900 memcpy(&info.bonding_info, bonding_info, 7901 sizeof(struct netdev_bonding_info)); 7902 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, 7903 &info.info); 7904 } 7905 EXPORT_SYMBOL(netdev_bonding_info_change); 7906 7907 /** 7908 * netdev_get_xmit_slave - Get the xmit slave of master device 7909 * @dev: device 7910 * @skb: The packet 7911 * @all_slaves: assume all the slaves are active 7912 * 7913 * The reference counters are not incremented so the caller must be 7914 * careful with locks. The caller must hold RCU lock. 7915 * %NULL is returned if no slave is found. 7916 */ 7917 7918 struct net_device *netdev_get_xmit_slave(struct net_device *dev, 7919 struct sk_buff *skb, 7920 bool all_slaves) 7921 { 7922 const struct net_device_ops *ops = dev->netdev_ops; 7923 7924 if (!ops->ndo_get_xmit_slave) 7925 return NULL; 7926 return ops->ndo_get_xmit_slave(dev, skb, all_slaves); 7927 } 7928 EXPORT_SYMBOL(netdev_get_xmit_slave); 7929 7930 static void netdev_adjacent_add_links(struct net_device *dev) 7931 { 7932 struct netdev_adjacent *iter; 7933 7934 struct net *net = dev_net(dev); 7935 7936 list_for_each_entry(iter, &dev->adj_list.upper, list) { 7937 if (!net_eq(net, dev_net(iter->dev))) 7938 continue; 7939 netdev_adjacent_sysfs_add(iter->dev, dev, 7940 &iter->dev->adj_list.lower); 7941 netdev_adjacent_sysfs_add(dev, iter->dev, 7942 &dev->adj_list.upper); 7943 } 7944 7945 list_for_each_entry(iter, &dev->adj_list.lower, list) { 7946 if (!net_eq(net, dev_net(iter->dev))) 7947 continue; 7948 netdev_adjacent_sysfs_add(iter->dev, dev, 7949 &iter->dev->adj_list.upper); 7950 netdev_adjacent_sysfs_add(dev, iter->dev, 7951 &dev->adj_list.lower); 7952 } 7953 } 7954 7955 static void netdev_adjacent_del_links(struct net_device *dev) 7956 { 7957 struct netdev_adjacent *iter; 7958 7959 struct net *net = dev_net(dev); 7960 7961 list_for_each_entry(iter, &dev->adj_list.upper, list) { 7962 if (!net_eq(net, dev_net(iter->dev))) 7963 continue; 7964 netdev_adjacent_sysfs_del(iter->dev, dev->name, 7965 &iter->dev->adj_list.lower); 7966 netdev_adjacent_sysfs_del(dev, iter->dev->name, 7967 &dev->adj_list.upper); 7968 } 7969 7970 list_for_each_entry(iter, &dev->adj_list.lower, list) { 7971 if (!net_eq(net, dev_net(iter->dev))) 7972 continue; 7973 netdev_adjacent_sysfs_del(iter->dev, dev->name, 7974 &iter->dev->adj_list.upper); 7975 netdev_adjacent_sysfs_del(dev, iter->dev->name, 7976 &dev->adj_list.lower); 7977 } 7978 } 7979 7980 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname) 7981 { 7982 struct netdev_adjacent *iter; 7983 7984 struct net *net = dev_net(dev); 7985 7986 list_for_each_entry(iter, &dev->adj_list.upper, list) { 7987 if (!net_eq(net, dev_net(iter->dev))) 7988 continue; 7989 netdev_adjacent_sysfs_del(iter->dev, oldname, 7990 &iter->dev->adj_list.lower); 7991 netdev_adjacent_sysfs_add(iter->dev, dev, 7992 &iter->dev->adj_list.lower); 7993 } 7994 7995 list_for_each_entry(iter, &dev->adj_list.lower, list) { 7996 if (!net_eq(net, dev_net(iter->dev))) 7997 continue; 7998 netdev_adjacent_sysfs_del(iter->dev, oldname, 7999 &iter->dev->adj_list.upper); 8000 netdev_adjacent_sysfs_add(iter->dev, dev, 8001 &iter->dev->adj_list.upper); 8002 } 8003 } 8004 8005 void *netdev_lower_dev_get_private(struct net_device *dev, 8006 struct net_device *lower_dev) 8007 { 8008 struct netdev_adjacent *lower; 8009 8010 if (!lower_dev) 8011 return NULL; 8012 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower); 8013 if (!lower) 8014 return NULL; 8015 8016 return lower->private; 8017 } 8018 EXPORT_SYMBOL(netdev_lower_dev_get_private); 8019 8020 8021 /** 8022 * netdev_lower_change - Dispatch event about lower device state change 8023 * @lower_dev: device 8024 * @lower_state_info: state to dispatch 8025 * 8026 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info. 8027 * The caller must hold the RTNL lock. 8028 */ 8029 void netdev_lower_state_changed(struct net_device *lower_dev, 8030 void *lower_state_info) 8031 { 8032 struct netdev_notifier_changelowerstate_info changelowerstate_info = { 8033 .info.dev = lower_dev, 8034 }; 8035 8036 ASSERT_RTNL(); 8037 changelowerstate_info.lower_state_info = lower_state_info; 8038 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, 8039 &changelowerstate_info.info); 8040 } 8041 EXPORT_SYMBOL(netdev_lower_state_changed); 8042 8043 static void dev_change_rx_flags(struct net_device *dev, int flags) 8044 { 8045 const struct net_device_ops *ops = dev->netdev_ops; 8046 8047 if (ops->ndo_change_rx_flags) 8048 ops->ndo_change_rx_flags(dev, flags); 8049 } 8050 8051 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify) 8052 { 8053 unsigned int old_flags = dev->flags; 8054 kuid_t uid; 8055 kgid_t gid; 8056 8057 ASSERT_RTNL(); 8058 8059 dev->flags |= IFF_PROMISC; 8060 dev->promiscuity += inc; 8061 if (dev->promiscuity == 0) { 8062 /* 8063 * Avoid overflow. 8064 * If inc causes overflow, untouch promisc and return error. 8065 */ 8066 if (inc < 0) 8067 dev->flags &= ~IFF_PROMISC; 8068 else { 8069 dev->promiscuity -= inc; 8070 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n", 8071 dev->name); 8072 return -EOVERFLOW; 8073 } 8074 } 8075 if (dev->flags != old_flags) { 8076 pr_info("device %s %s promiscuous mode\n", 8077 dev->name, 8078 dev->flags & IFF_PROMISC ? "entered" : "left"); 8079 if (audit_enabled) { 8080 current_uid_gid(&uid, &gid); 8081 audit_log(audit_context(), GFP_ATOMIC, 8082 AUDIT_ANOM_PROMISCUOUS, 8083 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u", 8084 dev->name, (dev->flags & IFF_PROMISC), 8085 (old_flags & IFF_PROMISC), 8086 from_kuid(&init_user_ns, audit_get_loginuid(current)), 8087 from_kuid(&init_user_ns, uid), 8088 from_kgid(&init_user_ns, gid), 8089 audit_get_sessionid(current)); 8090 } 8091 8092 dev_change_rx_flags(dev, IFF_PROMISC); 8093 } 8094 if (notify) 8095 __dev_notify_flags(dev, old_flags, IFF_PROMISC); 8096 return 0; 8097 } 8098 8099 /** 8100 * dev_set_promiscuity - update promiscuity count on a device 8101 * @dev: device 8102 * @inc: modifier 8103 * 8104 * Add or remove promiscuity from a device. While the count in the device 8105 * remains above zero the interface remains promiscuous. Once it hits zero 8106 * the device reverts back to normal filtering operation. A negative inc 8107 * value is used to drop promiscuity on the device. 8108 * Return 0 if successful or a negative errno code on error. 8109 */ 8110 int dev_set_promiscuity(struct net_device *dev, int inc) 8111 { 8112 unsigned int old_flags = dev->flags; 8113 int err; 8114 8115 err = __dev_set_promiscuity(dev, inc, true); 8116 if (err < 0) 8117 return err; 8118 if (dev->flags != old_flags) 8119 dev_set_rx_mode(dev); 8120 return err; 8121 } 8122 EXPORT_SYMBOL(dev_set_promiscuity); 8123 8124 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify) 8125 { 8126 unsigned int old_flags = dev->flags, old_gflags = dev->gflags; 8127 8128 ASSERT_RTNL(); 8129 8130 dev->flags |= IFF_ALLMULTI; 8131 dev->allmulti += inc; 8132 if (dev->allmulti == 0) { 8133 /* 8134 * Avoid overflow. 8135 * If inc causes overflow, untouch allmulti and return error. 8136 */ 8137 if (inc < 0) 8138 dev->flags &= ~IFF_ALLMULTI; 8139 else { 8140 dev->allmulti -= inc; 8141 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n", 8142 dev->name); 8143 return -EOVERFLOW; 8144 } 8145 } 8146 if (dev->flags ^ old_flags) { 8147 dev_change_rx_flags(dev, IFF_ALLMULTI); 8148 dev_set_rx_mode(dev); 8149 if (notify) 8150 __dev_notify_flags(dev, old_flags, 8151 dev->gflags ^ old_gflags); 8152 } 8153 return 0; 8154 } 8155 8156 /** 8157 * dev_set_allmulti - update allmulti count on a device 8158 * @dev: device 8159 * @inc: modifier 8160 * 8161 * Add or remove reception of all multicast frames to a device. While the 8162 * count in the device remains above zero the interface remains listening 8163 * to all interfaces. Once it hits zero the device reverts back to normal 8164 * filtering operation. A negative @inc value is used to drop the counter 8165 * when releasing a resource needing all multicasts. 8166 * Return 0 if successful or a negative errno code on error. 8167 */ 8168 8169 int dev_set_allmulti(struct net_device *dev, int inc) 8170 { 8171 return __dev_set_allmulti(dev, inc, true); 8172 } 8173 EXPORT_SYMBOL(dev_set_allmulti); 8174 8175 /* 8176 * Upload unicast and multicast address lists to device and 8177 * configure RX filtering. When the device doesn't support unicast 8178 * filtering it is put in promiscuous mode while unicast addresses 8179 * are present. 8180 */ 8181 void __dev_set_rx_mode(struct net_device *dev) 8182 { 8183 const struct net_device_ops *ops = dev->netdev_ops; 8184 8185 /* dev_open will call this function so the list will stay sane. */ 8186 if (!(dev->flags&IFF_UP)) 8187 return; 8188 8189 if (!netif_device_present(dev)) 8190 return; 8191 8192 if (!(dev->priv_flags & IFF_UNICAST_FLT)) { 8193 /* Unicast addresses changes may only happen under the rtnl, 8194 * therefore calling __dev_set_promiscuity here is safe. 8195 */ 8196 if (!netdev_uc_empty(dev) && !dev->uc_promisc) { 8197 __dev_set_promiscuity(dev, 1, false); 8198 dev->uc_promisc = true; 8199 } else if (netdev_uc_empty(dev) && dev->uc_promisc) { 8200 __dev_set_promiscuity(dev, -1, false); 8201 dev->uc_promisc = false; 8202 } 8203 } 8204 8205 if (ops->ndo_set_rx_mode) 8206 ops->ndo_set_rx_mode(dev); 8207 } 8208 8209 void dev_set_rx_mode(struct net_device *dev) 8210 { 8211 netif_addr_lock_bh(dev); 8212 __dev_set_rx_mode(dev); 8213 netif_addr_unlock_bh(dev); 8214 } 8215 8216 /** 8217 * dev_get_flags - get flags reported to userspace 8218 * @dev: device 8219 * 8220 * Get the combination of flag bits exported through APIs to userspace. 8221 */ 8222 unsigned int dev_get_flags(const struct net_device *dev) 8223 { 8224 unsigned int flags; 8225 8226 flags = (dev->flags & ~(IFF_PROMISC | 8227 IFF_ALLMULTI | 8228 IFF_RUNNING | 8229 IFF_LOWER_UP | 8230 IFF_DORMANT)) | 8231 (dev->gflags & (IFF_PROMISC | 8232 IFF_ALLMULTI)); 8233 8234 if (netif_running(dev)) { 8235 if (netif_oper_up(dev)) 8236 flags |= IFF_RUNNING; 8237 if (netif_carrier_ok(dev)) 8238 flags |= IFF_LOWER_UP; 8239 if (netif_dormant(dev)) 8240 flags |= IFF_DORMANT; 8241 } 8242 8243 return flags; 8244 } 8245 EXPORT_SYMBOL(dev_get_flags); 8246 8247 int __dev_change_flags(struct net_device *dev, unsigned int flags, 8248 struct netlink_ext_ack *extack) 8249 { 8250 unsigned int old_flags = dev->flags; 8251 int ret; 8252 8253 ASSERT_RTNL(); 8254 8255 /* 8256 * Set the flags on our device. 8257 */ 8258 8259 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | 8260 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL | 8261 IFF_AUTOMEDIA)) | 8262 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | 8263 IFF_ALLMULTI)); 8264 8265 /* 8266 * Load in the correct multicast list now the flags have changed. 8267 */ 8268 8269 if ((old_flags ^ flags) & IFF_MULTICAST) 8270 dev_change_rx_flags(dev, IFF_MULTICAST); 8271 8272 dev_set_rx_mode(dev); 8273 8274 /* 8275 * Have we downed the interface. We handle IFF_UP ourselves 8276 * according to user attempts to set it, rather than blindly 8277 * setting it. 8278 */ 8279 8280 ret = 0; 8281 if ((old_flags ^ flags) & IFF_UP) { 8282 if (old_flags & IFF_UP) 8283 __dev_close(dev); 8284 else 8285 ret = __dev_open(dev, extack); 8286 } 8287 8288 if ((flags ^ dev->gflags) & IFF_PROMISC) { 8289 int inc = (flags & IFF_PROMISC) ? 1 : -1; 8290 unsigned int old_flags = dev->flags; 8291 8292 dev->gflags ^= IFF_PROMISC; 8293 8294 if (__dev_set_promiscuity(dev, inc, false) >= 0) 8295 if (dev->flags != old_flags) 8296 dev_set_rx_mode(dev); 8297 } 8298 8299 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI 8300 * is important. Some (broken) drivers set IFF_PROMISC, when 8301 * IFF_ALLMULTI is requested not asking us and not reporting. 8302 */ 8303 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { 8304 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; 8305 8306 dev->gflags ^= IFF_ALLMULTI; 8307 __dev_set_allmulti(dev, inc, false); 8308 } 8309 8310 return ret; 8311 } 8312 8313 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, 8314 unsigned int gchanges) 8315 { 8316 unsigned int changes = dev->flags ^ old_flags; 8317 8318 if (gchanges) 8319 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC); 8320 8321 if (changes & IFF_UP) { 8322 if (dev->flags & IFF_UP) 8323 call_netdevice_notifiers(NETDEV_UP, dev); 8324 else 8325 call_netdevice_notifiers(NETDEV_DOWN, dev); 8326 } 8327 8328 if (dev->flags & IFF_UP && 8329 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) { 8330 struct netdev_notifier_change_info change_info = { 8331 .info = { 8332 .dev = dev, 8333 }, 8334 .flags_changed = changes, 8335 }; 8336 8337 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info); 8338 } 8339 } 8340 8341 /** 8342 * dev_change_flags - change device settings 8343 * @dev: device 8344 * @flags: device state flags 8345 * @extack: netlink extended ack 8346 * 8347 * Change settings on device based state flags. The flags are 8348 * in the userspace exported format. 8349 */ 8350 int dev_change_flags(struct net_device *dev, unsigned int flags, 8351 struct netlink_ext_ack *extack) 8352 { 8353 int ret; 8354 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; 8355 8356 ret = __dev_change_flags(dev, flags, extack); 8357 if (ret < 0) 8358 return ret; 8359 8360 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); 8361 __dev_notify_flags(dev, old_flags, changes); 8362 return ret; 8363 } 8364 EXPORT_SYMBOL(dev_change_flags); 8365 8366 int __dev_set_mtu(struct net_device *dev, int new_mtu) 8367 { 8368 const struct net_device_ops *ops = dev->netdev_ops; 8369 8370 if (ops->ndo_change_mtu) 8371 return ops->ndo_change_mtu(dev, new_mtu); 8372 8373 /* Pairs with all the lockless reads of dev->mtu in the stack */ 8374 WRITE_ONCE(dev->mtu, new_mtu); 8375 return 0; 8376 } 8377 EXPORT_SYMBOL(__dev_set_mtu); 8378 8379 int dev_validate_mtu(struct net_device *dev, int new_mtu, 8380 struct netlink_ext_ack *extack) 8381 { 8382 /* MTU must be positive, and in range */ 8383 if (new_mtu < 0 || new_mtu < dev->min_mtu) { 8384 NL_SET_ERR_MSG(extack, "mtu less than device minimum"); 8385 return -EINVAL; 8386 } 8387 8388 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) { 8389 NL_SET_ERR_MSG(extack, "mtu greater than device maximum"); 8390 return -EINVAL; 8391 } 8392 return 0; 8393 } 8394 8395 /** 8396 * dev_set_mtu_ext - Change maximum transfer unit 8397 * @dev: device 8398 * @new_mtu: new transfer unit 8399 * @extack: netlink extended ack 8400 * 8401 * Change the maximum transfer size of the network device. 8402 */ 8403 int dev_set_mtu_ext(struct net_device *dev, int new_mtu, 8404 struct netlink_ext_ack *extack) 8405 { 8406 int err, orig_mtu; 8407 8408 if (new_mtu == dev->mtu) 8409 return 0; 8410 8411 err = dev_validate_mtu(dev, new_mtu, extack); 8412 if (err) 8413 return err; 8414 8415 if (!netif_device_present(dev)) 8416 return -ENODEV; 8417 8418 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev); 8419 err = notifier_to_errno(err); 8420 if (err) 8421 return err; 8422 8423 orig_mtu = dev->mtu; 8424 err = __dev_set_mtu(dev, new_mtu); 8425 8426 if (!err) { 8427 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev, 8428 orig_mtu); 8429 err = notifier_to_errno(err); 8430 if (err) { 8431 /* setting mtu back and notifying everyone again, 8432 * so that they have a chance to revert changes. 8433 */ 8434 __dev_set_mtu(dev, orig_mtu); 8435 call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev, 8436 new_mtu); 8437 } 8438 } 8439 return err; 8440 } 8441 8442 int dev_set_mtu(struct net_device *dev, int new_mtu) 8443 { 8444 struct netlink_ext_ack extack; 8445 int err; 8446 8447 memset(&extack, 0, sizeof(extack)); 8448 err = dev_set_mtu_ext(dev, new_mtu, &extack); 8449 if (err && extack._msg) 8450 net_err_ratelimited("%s: %s\n", dev->name, extack._msg); 8451 return err; 8452 } 8453 EXPORT_SYMBOL(dev_set_mtu); 8454 8455 /** 8456 * dev_change_tx_queue_len - Change TX queue length of a netdevice 8457 * @dev: device 8458 * @new_len: new tx queue length 8459 */ 8460 int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len) 8461 { 8462 unsigned int orig_len = dev->tx_queue_len; 8463 int res; 8464 8465 if (new_len != (unsigned int)new_len) 8466 return -ERANGE; 8467 8468 if (new_len != orig_len) { 8469 dev->tx_queue_len = new_len; 8470 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev); 8471 res = notifier_to_errno(res); 8472 if (res) 8473 goto err_rollback; 8474 res = dev_qdisc_change_tx_queue_len(dev); 8475 if (res) 8476 goto err_rollback; 8477 } 8478 8479 return 0; 8480 8481 err_rollback: 8482 netdev_err(dev, "refused to change device tx_queue_len\n"); 8483 dev->tx_queue_len = orig_len; 8484 return res; 8485 } 8486 8487 /** 8488 * dev_set_group - Change group this device belongs to 8489 * @dev: device 8490 * @new_group: group this device should belong to 8491 */ 8492 void dev_set_group(struct net_device *dev, int new_group) 8493 { 8494 dev->group = new_group; 8495 } 8496 EXPORT_SYMBOL(dev_set_group); 8497 8498 /** 8499 * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR. 8500 * @dev: device 8501 * @addr: new address 8502 * @extack: netlink extended ack 8503 */ 8504 int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr, 8505 struct netlink_ext_ack *extack) 8506 { 8507 struct netdev_notifier_pre_changeaddr_info info = { 8508 .info.dev = dev, 8509 .info.extack = extack, 8510 .dev_addr = addr, 8511 }; 8512 int rc; 8513 8514 rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info); 8515 return notifier_to_errno(rc); 8516 } 8517 EXPORT_SYMBOL(dev_pre_changeaddr_notify); 8518 8519 /** 8520 * dev_set_mac_address - Change Media Access Control Address 8521 * @dev: device 8522 * @sa: new address 8523 * @extack: netlink extended ack 8524 * 8525 * Change the hardware (MAC) address of the device 8526 */ 8527 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa, 8528 struct netlink_ext_ack *extack) 8529 { 8530 const struct net_device_ops *ops = dev->netdev_ops; 8531 int err; 8532 8533 if (!ops->ndo_set_mac_address) 8534 return -EOPNOTSUPP; 8535 if (sa->sa_family != dev->type) 8536 return -EINVAL; 8537 if (!netif_device_present(dev)) 8538 return -ENODEV; 8539 err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack); 8540 if (err) 8541 return err; 8542 err = ops->ndo_set_mac_address(dev, sa); 8543 if (err) 8544 return err; 8545 dev->addr_assign_type = NET_ADDR_SET; 8546 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 8547 add_device_randomness(dev->dev_addr, dev->addr_len); 8548 return 0; 8549 } 8550 EXPORT_SYMBOL(dev_set_mac_address); 8551 8552 /** 8553 * dev_change_carrier - Change device carrier 8554 * @dev: device 8555 * @new_carrier: new value 8556 * 8557 * Change device carrier 8558 */ 8559 int dev_change_carrier(struct net_device *dev, bool new_carrier) 8560 { 8561 const struct net_device_ops *ops = dev->netdev_ops; 8562 8563 if (!ops->ndo_change_carrier) 8564 return -EOPNOTSUPP; 8565 if (!netif_device_present(dev)) 8566 return -ENODEV; 8567 return ops->ndo_change_carrier(dev, new_carrier); 8568 } 8569 EXPORT_SYMBOL(dev_change_carrier); 8570 8571 /** 8572 * dev_get_phys_port_id - Get device physical port ID 8573 * @dev: device 8574 * @ppid: port ID 8575 * 8576 * Get device physical port ID 8577 */ 8578 int dev_get_phys_port_id(struct net_device *dev, 8579 struct netdev_phys_item_id *ppid) 8580 { 8581 const struct net_device_ops *ops = dev->netdev_ops; 8582 8583 if (!ops->ndo_get_phys_port_id) 8584 return -EOPNOTSUPP; 8585 return ops->ndo_get_phys_port_id(dev, ppid); 8586 } 8587 EXPORT_SYMBOL(dev_get_phys_port_id); 8588 8589 /** 8590 * dev_get_phys_port_name - Get device physical port name 8591 * @dev: device 8592 * @name: port name 8593 * @len: limit of bytes to copy to name 8594 * 8595 * Get device physical port name 8596 */ 8597 int dev_get_phys_port_name(struct net_device *dev, 8598 char *name, size_t len) 8599 { 8600 const struct net_device_ops *ops = dev->netdev_ops; 8601 int err; 8602 8603 if (ops->ndo_get_phys_port_name) { 8604 err = ops->ndo_get_phys_port_name(dev, name, len); 8605 if (err != -EOPNOTSUPP) 8606 return err; 8607 } 8608 return devlink_compat_phys_port_name_get(dev, name, len); 8609 } 8610 EXPORT_SYMBOL(dev_get_phys_port_name); 8611 8612 /** 8613 * dev_get_port_parent_id - Get the device's port parent identifier 8614 * @dev: network device 8615 * @ppid: pointer to a storage for the port's parent identifier 8616 * @recurse: allow/disallow recursion to lower devices 8617 * 8618 * Get the devices's port parent identifier 8619 */ 8620 int dev_get_port_parent_id(struct net_device *dev, 8621 struct netdev_phys_item_id *ppid, 8622 bool recurse) 8623 { 8624 const struct net_device_ops *ops = dev->netdev_ops; 8625 struct netdev_phys_item_id first = { }; 8626 struct net_device *lower_dev; 8627 struct list_head *iter; 8628 int err; 8629 8630 if (ops->ndo_get_port_parent_id) { 8631 err = ops->ndo_get_port_parent_id(dev, ppid); 8632 if (err != -EOPNOTSUPP) 8633 return err; 8634 } 8635 8636 err = devlink_compat_switch_id_get(dev, ppid); 8637 if (!err || err != -EOPNOTSUPP) 8638 return err; 8639 8640 if (!recurse) 8641 return -EOPNOTSUPP; 8642 8643 netdev_for_each_lower_dev(dev, lower_dev, iter) { 8644 err = dev_get_port_parent_id(lower_dev, ppid, recurse); 8645 if (err) 8646 break; 8647 if (!first.id_len) 8648 first = *ppid; 8649 else if (memcmp(&first, ppid, sizeof(*ppid))) 8650 return -ENODATA; 8651 } 8652 8653 return err; 8654 } 8655 EXPORT_SYMBOL(dev_get_port_parent_id); 8656 8657 /** 8658 * netdev_port_same_parent_id - Indicate if two network devices have 8659 * the same port parent identifier 8660 * @a: first network device 8661 * @b: second network device 8662 */ 8663 bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b) 8664 { 8665 struct netdev_phys_item_id a_id = { }; 8666 struct netdev_phys_item_id b_id = { }; 8667 8668 if (dev_get_port_parent_id(a, &a_id, true) || 8669 dev_get_port_parent_id(b, &b_id, true)) 8670 return false; 8671 8672 return netdev_phys_item_id_same(&a_id, &b_id); 8673 } 8674 EXPORT_SYMBOL(netdev_port_same_parent_id); 8675 8676 /** 8677 * dev_change_proto_down - update protocol port state information 8678 * @dev: device 8679 * @proto_down: new value 8680 * 8681 * This info can be used by switch drivers to set the phys state of the 8682 * port. 8683 */ 8684 int dev_change_proto_down(struct net_device *dev, bool proto_down) 8685 { 8686 const struct net_device_ops *ops = dev->netdev_ops; 8687 8688 if (!ops->ndo_change_proto_down) 8689 return -EOPNOTSUPP; 8690 if (!netif_device_present(dev)) 8691 return -ENODEV; 8692 return ops->ndo_change_proto_down(dev, proto_down); 8693 } 8694 EXPORT_SYMBOL(dev_change_proto_down); 8695 8696 /** 8697 * dev_change_proto_down_generic - generic implementation for 8698 * ndo_change_proto_down that sets carrier according to 8699 * proto_down. 8700 * 8701 * @dev: device 8702 * @proto_down: new value 8703 */ 8704 int dev_change_proto_down_generic(struct net_device *dev, bool proto_down) 8705 { 8706 if (proto_down) 8707 netif_carrier_off(dev); 8708 else 8709 netif_carrier_on(dev); 8710 dev->proto_down = proto_down; 8711 return 0; 8712 } 8713 EXPORT_SYMBOL(dev_change_proto_down_generic); 8714 8715 /** 8716 * dev_change_proto_down_reason - proto down reason 8717 * 8718 * @dev: device 8719 * @mask: proto down mask 8720 * @value: proto down value 8721 */ 8722 void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask, 8723 u32 value) 8724 { 8725 int b; 8726 8727 if (!mask) { 8728 dev->proto_down_reason = value; 8729 } else { 8730 for_each_set_bit(b, &mask, 32) { 8731 if (value & (1 << b)) 8732 dev->proto_down_reason |= BIT(b); 8733 else 8734 dev->proto_down_reason &= ~BIT(b); 8735 } 8736 } 8737 } 8738 EXPORT_SYMBOL(dev_change_proto_down_reason); 8739 8740 struct bpf_xdp_link { 8741 struct bpf_link link; 8742 struct net_device *dev; /* protected by rtnl_lock, no refcnt held */ 8743 int flags; 8744 }; 8745 8746 static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags) 8747 { 8748 if (flags & XDP_FLAGS_HW_MODE) 8749 return XDP_MODE_HW; 8750 if (flags & XDP_FLAGS_DRV_MODE) 8751 return XDP_MODE_DRV; 8752 if (flags & XDP_FLAGS_SKB_MODE) 8753 return XDP_MODE_SKB; 8754 return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB; 8755 } 8756 8757 static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode) 8758 { 8759 switch (mode) { 8760 case XDP_MODE_SKB: 8761 return generic_xdp_install; 8762 case XDP_MODE_DRV: 8763 case XDP_MODE_HW: 8764 return dev->netdev_ops->ndo_bpf; 8765 default: 8766 return NULL; 8767 }; 8768 } 8769 8770 static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev, 8771 enum bpf_xdp_mode mode) 8772 { 8773 return dev->xdp_state[mode].link; 8774 } 8775 8776 static struct bpf_prog *dev_xdp_prog(struct net_device *dev, 8777 enum bpf_xdp_mode mode) 8778 { 8779 struct bpf_xdp_link *link = dev_xdp_link(dev, mode); 8780 8781 if (link) 8782 return link->link.prog; 8783 return dev->xdp_state[mode].prog; 8784 } 8785 8786 u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode) 8787 { 8788 struct bpf_prog *prog = dev_xdp_prog(dev, mode); 8789 8790 return prog ? prog->aux->id : 0; 8791 } 8792 8793 static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode, 8794 struct bpf_xdp_link *link) 8795 { 8796 dev->xdp_state[mode].link = link; 8797 dev->xdp_state[mode].prog = NULL; 8798 } 8799 8800 static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode, 8801 struct bpf_prog *prog) 8802 { 8803 dev->xdp_state[mode].link = NULL; 8804 dev->xdp_state[mode].prog = prog; 8805 } 8806 8807 static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode, 8808 bpf_op_t bpf_op, struct netlink_ext_ack *extack, 8809 u32 flags, struct bpf_prog *prog) 8810 { 8811 struct netdev_bpf xdp; 8812 int err; 8813 8814 memset(&xdp, 0, sizeof(xdp)); 8815 xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG; 8816 xdp.extack = extack; 8817 xdp.flags = flags; 8818 xdp.prog = prog; 8819 8820 /* Drivers assume refcnt is already incremented (i.e, prog pointer is 8821 * "moved" into driver), so they don't increment it on their own, but 8822 * they do decrement refcnt when program is detached or replaced. 8823 * Given net_device also owns link/prog, we need to bump refcnt here 8824 * to prevent drivers from underflowing it. 8825 */ 8826 if (prog) 8827 bpf_prog_inc(prog); 8828 err = bpf_op(dev, &xdp); 8829 if (err) { 8830 if (prog) 8831 bpf_prog_put(prog); 8832 return err; 8833 } 8834 8835 if (mode != XDP_MODE_HW) 8836 bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog); 8837 8838 return 0; 8839 } 8840 8841 static void dev_xdp_uninstall(struct net_device *dev) 8842 { 8843 struct bpf_xdp_link *link; 8844 struct bpf_prog *prog; 8845 enum bpf_xdp_mode mode; 8846 bpf_op_t bpf_op; 8847 8848 ASSERT_RTNL(); 8849 8850 for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) { 8851 prog = dev_xdp_prog(dev, mode); 8852 if (!prog) 8853 continue; 8854 8855 bpf_op = dev_xdp_bpf_op(dev, mode); 8856 if (!bpf_op) 8857 continue; 8858 8859 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL)); 8860 8861 /* auto-detach link from net device */ 8862 link = dev_xdp_link(dev, mode); 8863 if (link) 8864 link->dev = NULL; 8865 else 8866 bpf_prog_put(prog); 8867 8868 dev_xdp_set_link(dev, mode, NULL); 8869 } 8870 } 8871 8872 static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack, 8873 struct bpf_xdp_link *link, struct bpf_prog *new_prog, 8874 struct bpf_prog *old_prog, u32 flags) 8875 { 8876 struct bpf_prog *cur_prog; 8877 enum bpf_xdp_mode mode; 8878 bpf_op_t bpf_op; 8879 int err; 8880 8881 ASSERT_RTNL(); 8882 8883 /* either link or prog attachment, never both */ 8884 if (link && (new_prog || old_prog)) 8885 return -EINVAL; 8886 /* link supports only XDP mode flags */ 8887 if (link && (flags & ~XDP_FLAGS_MODES)) { 8888 NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment"); 8889 return -EINVAL; 8890 } 8891 /* just one XDP mode bit should be set, zero defaults to SKB mode */ 8892 if (hweight32(flags & XDP_FLAGS_MODES) > 1) { 8893 NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set"); 8894 return -EINVAL; 8895 } 8896 /* old_prog != NULL implies XDP_FLAGS_REPLACE is set */ 8897 if (old_prog && !(flags & XDP_FLAGS_REPLACE)) { 8898 NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified"); 8899 return -EINVAL; 8900 } 8901 8902 mode = dev_xdp_mode(dev, flags); 8903 /* can't replace attached link */ 8904 if (dev_xdp_link(dev, mode)) { 8905 NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link"); 8906 return -EBUSY; 8907 } 8908 8909 cur_prog = dev_xdp_prog(dev, mode); 8910 /* can't replace attached prog with link */ 8911 if (link && cur_prog) { 8912 NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link"); 8913 return -EBUSY; 8914 } 8915 if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) { 8916 NL_SET_ERR_MSG(extack, "Active program does not match expected"); 8917 return -EEXIST; 8918 } 8919 8920 /* put effective new program into new_prog */ 8921 if (link) 8922 new_prog = link->link.prog; 8923 8924 if (new_prog) { 8925 bool offload = mode == XDP_MODE_HW; 8926 enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB 8927 ? XDP_MODE_DRV : XDP_MODE_SKB; 8928 8929 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) { 8930 NL_SET_ERR_MSG(extack, "XDP program already attached"); 8931 return -EBUSY; 8932 } 8933 if (!offload && dev_xdp_prog(dev, other_mode)) { 8934 NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time"); 8935 return -EEXIST; 8936 } 8937 if (!offload && bpf_prog_is_dev_bound(new_prog->aux)) { 8938 NL_SET_ERR_MSG(extack, "Using device-bound program without HW_MODE flag is not supported"); 8939 return -EINVAL; 8940 } 8941 if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) { 8942 NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device"); 8943 return -EINVAL; 8944 } 8945 if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) { 8946 NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device"); 8947 return -EINVAL; 8948 } 8949 } 8950 8951 /* don't call drivers if the effective program didn't change */ 8952 if (new_prog != cur_prog) { 8953 bpf_op = dev_xdp_bpf_op(dev, mode); 8954 if (!bpf_op) { 8955 NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode"); 8956 return -EOPNOTSUPP; 8957 } 8958 8959 err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog); 8960 if (err) 8961 return err; 8962 } 8963 8964 if (link) 8965 dev_xdp_set_link(dev, mode, link); 8966 else 8967 dev_xdp_set_prog(dev, mode, new_prog); 8968 if (cur_prog) 8969 bpf_prog_put(cur_prog); 8970 8971 return 0; 8972 } 8973 8974 static int dev_xdp_attach_link(struct net_device *dev, 8975 struct netlink_ext_ack *extack, 8976 struct bpf_xdp_link *link) 8977 { 8978 return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags); 8979 } 8980 8981 static int dev_xdp_detach_link(struct net_device *dev, 8982 struct netlink_ext_ack *extack, 8983 struct bpf_xdp_link *link) 8984 { 8985 enum bpf_xdp_mode mode; 8986 bpf_op_t bpf_op; 8987 8988 ASSERT_RTNL(); 8989 8990 mode = dev_xdp_mode(dev, link->flags); 8991 if (dev_xdp_link(dev, mode) != link) 8992 return -EINVAL; 8993 8994 bpf_op = dev_xdp_bpf_op(dev, mode); 8995 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL)); 8996 dev_xdp_set_link(dev, mode, NULL); 8997 return 0; 8998 } 8999 9000 static void bpf_xdp_link_release(struct bpf_link *link) 9001 { 9002 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 9003 9004 rtnl_lock(); 9005 9006 /* if racing with net_device's tear down, xdp_link->dev might be 9007 * already NULL, in which case link was already auto-detached 9008 */ 9009 if (xdp_link->dev) { 9010 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link)); 9011 xdp_link->dev = NULL; 9012 } 9013 9014 rtnl_unlock(); 9015 } 9016 9017 static int bpf_xdp_link_detach(struct bpf_link *link) 9018 { 9019 bpf_xdp_link_release(link); 9020 return 0; 9021 } 9022 9023 static void bpf_xdp_link_dealloc(struct bpf_link *link) 9024 { 9025 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 9026 9027 kfree(xdp_link); 9028 } 9029 9030 static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link, 9031 struct seq_file *seq) 9032 { 9033 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 9034 u32 ifindex = 0; 9035 9036 rtnl_lock(); 9037 if (xdp_link->dev) 9038 ifindex = xdp_link->dev->ifindex; 9039 rtnl_unlock(); 9040 9041 seq_printf(seq, "ifindex:\t%u\n", ifindex); 9042 } 9043 9044 static int bpf_xdp_link_fill_link_info(const struct bpf_link *link, 9045 struct bpf_link_info *info) 9046 { 9047 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 9048 u32 ifindex = 0; 9049 9050 rtnl_lock(); 9051 if (xdp_link->dev) 9052 ifindex = xdp_link->dev->ifindex; 9053 rtnl_unlock(); 9054 9055 info->xdp.ifindex = ifindex; 9056 return 0; 9057 } 9058 9059 static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog, 9060 struct bpf_prog *old_prog) 9061 { 9062 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 9063 enum bpf_xdp_mode mode; 9064 bpf_op_t bpf_op; 9065 int err = 0; 9066 9067 rtnl_lock(); 9068 9069 /* link might have been auto-released already, so fail */ 9070 if (!xdp_link->dev) { 9071 err = -ENOLINK; 9072 goto out_unlock; 9073 } 9074 9075 if (old_prog && link->prog != old_prog) { 9076 err = -EPERM; 9077 goto out_unlock; 9078 } 9079 old_prog = link->prog; 9080 if (old_prog == new_prog) { 9081 /* no-op, don't disturb drivers */ 9082 bpf_prog_put(new_prog); 9083 goto out_unlock; 9084 } 9085 9086 mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags); 9087 bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode); 9088 err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL, 9089 xdp_link->flags, new_prog); 9090 if (err) 9091 goto out_unlock; 9092 9093 old_prog = xchg(&link->prog, new_prog); 9094 bpf_prog_put(old_prog); 9095 9096 out_unlock: 9097 rtnl_unlock(); 9098 return err; 9099 } 9100 9101 static const struct bpf_link_ops bpf_xdp_link_lops = { 9102 .release = bpf_xdp_link_release, 9103 .dealloc = bpf_xdp_link_dealloc, 9104 .detach = bpf_xdp_link_detach, 9105 .show_fdinfo = bpf_xdp_link_show_fdinfo, 9106 .fill_link_info = bpf_xdp_link_fill_link_info, 9107 .update_prog = bpf_xdp_link_update, 9108 }; 9109 9110 int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 9111 { 9112 struct net *net = current->nsproxy->net_ns; 9113 struct bpf_link_primer link_primer; 9114 struct bpf_xdp_link *link; 9115 struct net_device *dev; 9116 int err, fd; 9117 9118 dev = dev_get_by_index(net, attr->link_create.target_ifindex); 9119 if (!dev) 9120 return -EINVAL; 9121 9122 link = kzalloc(sizeof(*link), GFP_USER); 9123 if (!link) { 9124 err = -ENOMEM; 9125 goto out_put_dev; 9126 } 9127 9128 bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog); 9129 link->dev = dev; 9130 link->flags = attr->link_create.flags; 9131 9132 err = bpf_link_prime(&link->link, &link_primer); 9133 if (err) { 9134 kfree(link); 9135 goto out_put_dev; 9136 } 9137 9138 rtnl_lock(); 9139 err = dev_xdp_attach_link(dev, NULL, link); 9140 rtnl_unlock(); 9141 9142 if (err) { 9143 bpf_link_cleanup(&link_primer); 9144 goto out_put_dev; 9145 } 9146 9147 fd = bpf_link_settle(&link_primer); 9148 /* link itself doesn't hold dev's refcnt to not complicate shutdown */ 9149 dev_put(dev); 9150 return fd; 9151 9152 out_put_dev: 9153 dev_put(dev); 9154 return err; 9155 } 9156 9157 /** 9158 * dev_change_xdp_fd - set or clear a bpf program for a device rx path 9159 * @dev: device 9160 * @extack: netlink extended ack 9161 * @fd: new program fd or negative value to clear 9162 * @expected_fd: old program fd that userspace expects to replace or clear 9163 * @flags: xdp-related flags 9164 * 9165 * Set or clear a bpf program for a device 9166 */ 9167 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, 9168 int fd, int expected_fd, u32 flags) 9169 { 9170 enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags); 9171 struct bpf_prog *new_prog = NULL, *old_prog = NULL; 9172 int err; 9173 9174 ASSERT_RTNL(); 9175 9176 if (fd >= 0) { 9177 new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP, 9178 mode != XDP_MODE_SKB); 9179 if (IS_ERR(new_prog)) 9180 return PTR_ERR(new_prog); 9181 } 9182 9183 if (expected_fd >= 0) { 9184 old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP, 9185 mode != XDP_MODE_SKB); 9186 if (IS_ERR(old_prog)) { 9187 err = PTR_ERR(old_prog); 9188 old_prog = NULL; 9189 goto err_out; 9190 } 9191 } 9192 9193 err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags); 9194 9195 err_out: 9196 if (err && new_prog) 9197 bpf_prog_put(new_prog); 9198 if (old_prog) 9199 bpf_prog_put(old_prog); 9200 return err; 9201 } 9202 9203 /** 9204 * dev_new_index - allocate an ifindex 9205 * @net: the applicable net namespace 9206 * 9207 * Returns a suitable unique value for a new device interface 9208 * number. The caller must hold the rtnl semaphore or the 9209 * dev_base_lock to be sure it remains unique. 9210 */ 9211 static int dev_new_index(struct net *net) 9212 { 9213 int ifindex = net->ifindex; 9214 9215 for (;;) { 9216 if (++ifindex <= 0) 9217 ifindex = 1; 9218 if (!__dev_get_by_index(net, ifindex)) 9219 return net->ifindex = ifindex; 9220 } 9221 } 9222 9223 /* Delayed registration/unregisteration */ 9224 static LIST_HEAD(net_todo_list); 9225 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq); 9226 9227 static void net_set_todo(struct net_device *dev) 9228 { 9229 list_add_tail(&dev->todo_list, &net_todo_list); 9230 dev_net(dev)->dev_unreg_count++; 9231 } 9232 9233 static void rollback_registered_many(struct list_head *head) 9234 { 9235 struct net_device *dev, *tmp; 9236 LIST_HEAD(close_head); 9237 9238 BUG_ON(dev_boot_phase); 9239 ASSERT_RTNL(); 9240 9241 list_for_each_entry_safe(dev, tmp, head, unreg_list) { 9242 /* Some devices call without registering 9243 * for initialization unwind. Remove those 9244 * devices and proceed with the remaining. 9245 */ 9246 if (dev->reg_state == NETREG_UNINITIALIZED) { 9247 pr_debug("unregister_netdevice: device %s/%p never was registered\n", 9248 dev->name, dev); 9249 9250 WARN_ON(1); 9251 list_del(&dev->unreg_list); 9252 continue; 9253 } 9254 dev->dismantle = true; 9255 BUG_ON(dev->reg_state != NETREG_REGISTERED); 9256 } 9257 9258 /* If device is running, close it first. */ 9259 list_for_each_entry(dev, head, unreg_list) 9260 list_add_tail(&dev->close_list, &close_head); 9261 dev_close_many(&close_head, true); 9262 9263 list_for_each_entry(dev, head, unreg_list) { 9264 /* And unlink it from device chain. */ 9265 unlist_netdevice(dev); 9266 9267 dev->reg_state = NETREG_UNREGISTERING; 9268 } 9269 flush_all_backlogs(); 9270 9271 synchronize_net(); 9272 9273 list_for_each_entry(dev, head, unreg_list) { 9274 struct sk_buff *skb = NULL; 9275 9276 /* Shutdown queueing discipline. */ 9277 dev_shutdown(dev); 9278 9279 dev_xdp_uninstall(dev); 9280 9281 /* Notify protocols, that we are about to destroy 9282 * this device. They should clean all the things. 9283 */ 9284 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 9285 9286 if (!dev->rtnl_link_ops || 9287 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 9288 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0, 9289 GFP_KERNEL, NULL, 0); 9290 9291 /* 9292 * Flush the unicast and multicast chains 9293 */ 9294 dev_uc_flush(dev); 9295 dev_mc_flush(dev); 9296 9297 netdev_name_node_alt_flush(dev); 9298 netdev_name_node_free(dev->name_node); 9299 9300 if (dev->netdev_ops->ndo_uninit) 9301 dev->netdev_ops->ndo_uninit(dev); 9302 9303 if (skb) 9304 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL); 9305 9306 /* Notifier chain MUST detach us all upper devices. */ 9307 WARN_ON(netdev_has_any_upper_dev(dev)); 9308 WARN_ON(netdev_has_any_lower_dev(dev)); 9309 9310 /* Remove entries from kobject tree */ 9311 netdev_unregister_kobject(dev); 9312 #ifdef CONFIG_XPS 9313 /* Remove XPS queueing entries */ 9314 netif_reset_xps_queues_gt(dev, 0); 9315 #endif 9316 } 9317 9318 synchronize_net(); 9319 9320 list_for_each_entry(dev, head, unreg_list) 9321 dev_put(dev); 9322 } 9323 9324 static void rollback_registered(struct net_device *dev) 9325 { 9326 LIST_HEAD(single); 9327 9328 list_add(&dev->unreg_list, &single); 9329 rollback_registered_many(&single); 9330 list_del(&single); 9331 } 9332 9333 static netdev_features_t netdev_sync_upper_features(struct net_device *lower, 9334 struct net_device *upper, netdev_features_t features) 9335 { 9336 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 9337 netdev_features_t feature; 9338 int feature_bit; 9339 9340 for_each_netdev_feature(upper_disables, feature_bit) { 9341 feature = __NETIF_F_BIT(feature_bit); 9342 if (!(upper->wanted_features & feature) 9343 && (features & feature)) { 9344 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n", 9345 &feature, upper->name); 9346 features &= ~feature; 9347 } 9348 } 9349 9350 return features; 9351 } 9352 9353 static void netdev_sync_lower_features(struct net_device *upper, 9354 struct net_device *lower, netdev_features_t features) 9355 { 9356 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 9357 netdev_features_t feature; 9358 int feature_bit; 9359 9360 for_each_netdev_feature(upper_disables, feature_bit) { 9361 feature = __NETIF_F_BIT(feature_bit); 9362 if (!(features & feature) && (lower->features & feature)) { 9363 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n", 9364 &feature, lower->name); 9365 lower->wanted_features &= ~feature; 9366 __netdev_update_features(lower); 9367 9368 if (unlikely(lower->features & feature)) 9369 netdev_WARN(upper, "failed to disable %pNF on %s!\n", 9370 &feature, lower->name); 9371 else 9372 netdev_features_change(lower); 9373 } 9374 } 9375 } 9376 9377 static netdev_features_t netdev_fix_features(struct net_device *dev, 9378 netdev_features_t features) 9379 { 9380 /* Fix illegal checksum combinations */ 9381 if ((features & NETIF_F_HW_CSUM) && 9382 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { 9383 netdev_warn(dev, "mixed HW and IP checksum settings.\n"); 9384 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); 9385 } 9386 9387 /* TSO requires that SG is present as well. */ 9388 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) { 9389 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n"); 9390 features &= ~NETIF_F_ALL_TSO; 9391 } 9392 9393 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) && 9394 !(features & NETIF_F_IP_CSUM)) { 9395 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n"); 9396 features &= ~NETIF_F_TSO; 9397 features &= ~NETIF_F_TSO_ECN; 9398 } 9399 9400 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) && 9401 !(features & NETIF_F_IPV6_CSUM)) { 9402 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n"); 9403 features &= ~NETIF_F_TSO6; 9404 } 9405 9406 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */ 9407 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO)) 9408 features &= ~NETIF_F_TSO_MANGLEID; 9409 9410 /* TSO ECN requires that TSO is present as well. */ 9411 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN) 9412 features &= ~NETIF_F_TSO_ECN; 9413 9414 /* Software GSO depends on SG. */ 9415 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) { 9416 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n"); 9417 features &= ~NETIF_F_GSO; 9418 } 9419 9420 /* GSO partial features require GSO partial be set */ 9421 if ((features & dev->gso_partial_features) && 9422 !(features & NETIF_F_GSO_PARTIAL)) { 9423 netdev_dbg(dev, 9424 "Dropping partially supported GSO features since no GSO partial.\n"); 9425 features &= ~dev->gso_partial_features; 9426 } 9427 9428 if (!(features & NETIF_F_RXCSUM)) { 9429 /* NETIF_F_GRO_HW implies doing RXCSUM since every packet 9430 * successfully merged by hardware must also have the 9431 * checksum verified by hardware. If the user does not 9432 * want to enable RXCSUM, logically, we should disable GRO_HW. 9433 */ 9434 if (features & NETIF_F_GRO_HW) { 9435 netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n"); 9436 features &= ~NETIF_F_GRO_HW; 9437 } 9438 } 9439 9440 /* LRO/HW-GRO features cannot be combined with RX-FCS */ 9441 if (features & NETIF_F_RXFCS) { 9442 if (features & NETIF_F_LRO) { 9443 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n"); 9444 features &= ~NETIF_F_LRO; 9445 } 9446 9447 if (features & NETIF_F_GRO_HW) { 9448 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n"); 9449 features &= ~NETIF_F_GRO_HW; 9450 } 9451 } 9452 9453 return features; 9454 } 9455 9456 int __netdev_update_features(struct net_device *dev) 9457 { 9458 struct net_device *upper, *lower; 9459 netdev_features_t features; 9460 struct list_head *iter; 9461 int err = -1; 9462 9463 ASSERT_RTNL(); 9464 9465 features = netdev_get_wanted_features(dev); 9466 9467 if (dev->netdev_ops->ndo_fix_features) 9468 features = dev->netdev_ops->ndo_fix_features(dev, features); 9469 9470 /* driver might be less strict about feature dependencies */ 9471 features = netdev_fix_features(dev, features); 9472 9473 /* some features can't be enabled if they're off an an upper device */ 9474 netdev_for_each_upper_dev_rcu(dev, upper, iter) 9475 features = netdev_sync_upper_features(dev, upper, features); 9476 9477 if (dev->features == features) 9478 goto sync_lower; 9479 9480 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", 9481 &dev->features, &features); 9482 9483 if (dev->netdev_ops->ndo_set_features) 9484 err = dev->netdev_ops->ndo_set_features(dev, features); 9485 else 9486 err = 0; 9487 9488 if (unlikely(err < 0)) { 9489 netdev_err(dev, 9490 "set_features() failed (%d); wanted %pNF, left %pNF\n", 9491 err, &features, &dev->features); 9492 /* return non-0 since some features might have changed and 9493 * it's better to fire a spurious notification than miss it 9494 */ 9495 return -1; 9496 } 9497 9498 sync_lower: 9499 /* some features must be disabled on lower devices when disabled 9500 * on an upper device (think: bonding master or bridge) 9501 */ 9502 netdev_for_each_lower_dev(dev, lower, iter) 9503 netdev_sync_lower_features(dev, lower, features); 9504 9505 if (!err) { 9506 netdev_features_t diff = features ^ dev->features; 9507 9508 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) { 9509 /* udp_tunnel_{get,drop}_rx_info both need 9510 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the 9511 * device, or they won't do anything. 9512 * Thus we need to update dev->features 9513 * *before* calling udp_tunnel_get_rx_info, 9514 * but *after* calling udp_tunnel_drop_rx_info. 9515 */ 9516 if (features & NETIF_F_RX_UDP_TUNNEL_PORT) { 9517 dev->features = features; 9518 udp_tunnel_get_rx_info(dev); 9519 } else { 9520 udp_tunnel_drop_rx_info(dev); 9521 } 9522 } 9523 9524 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) { 9525 if (features & NETIF_F_HW_VLAN_CTAG_FILTER) { 9526 dev->features = features; 9527 err |= vlan_get_rx_ctag_filter_info(dev); 9528 } else { 9529 vlan_drop_rx_ctag_filter_info(dev); 9530 } 9531 } 9532 9533 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) { 9534 if (features & NETIF_F_HW_VLAN_STAG_FILTER) { 9535 dev->features = features; 9536 err |= vlan_get_rx_stag_filter_info(dev); 9537 } else { 9538 vlan_drop_rx_stag_filter_info(dev); 9539 } 9540 } 9541 9542 dev->features = features; 9543 } 9544 9545 return err < 0 ? 0 : 1; 9546 } 9547 9548 /** 9549 * netdev_update_features - recalculate device features 9550 * @dev: the device to check 9551 * 9552 * Recalculate dev->features set and send notifications if it 9553 * has changed. Should be called after driver or hardware dependent 9554 * conditions might have changed that influence the features. 9555 */ 9556 void netdev_update_features(struct net_device *dev) 9557 { 9558 if (__netdev_update_features(dev)) 9559 netdev_features_change(dev); 9560 } 9561 EXPORT_SYMBOL(netdev_update_features); 9562 9563 /** 9564 * netdev_change_features - recalculate device features 9565 * @dev: the device to check 9566 * 9567 * Recalculate dev->features set and send notifications even 9568 * if they have not changed. Should be called instead of 9569 * netdev_update_features() if also dev->vlan_features might 9570 * have changed to allow the changes to be propagated to stacked 9571 * VLAN devices. 9572 */ 9573 void netdev_change_features(struct net_device *dev) 9574 { 9575 __netdev_update_features(dev); 9576 netdev_features_change(dev); 9577 } 9578 EXPORT_SYMBOL(netdev_change_features); 9579 9580 /** 9581 * netif_stacked_transfer_operstate - transfer operstate 9582 * @rootdev: the root or lower level device to transfer state from 9583 * @dev: the device to transfer operstate to 9584 * 9585 * Transfer operational state from root to device. This is normally 9586 * called when a stacking relationship exists between the root 9587 * device and the device(a leaf device). 9588 */ 9589 void netif_stacked_transfer_operstate(const struct net_device *rootdev, 9590 struct net_device *dev) 9591 { 9592 if (rootdev->operstate == IF_OPER_DORMANT) 9593 netif_dormant_on(dev); 9594 else 9595 netif_dormant_off(dev); 9596 9597 if (rootdev->operstate == IF_OPER_TESTING) 9598 netif_testing_on(dev); 9599 else 9600 netif_testing_off(dev); 9601 9602 if (netif_carrier_ok(rootdev)) 9603 netif_carrier_on(dev); 9604 else 9605 netif_carrier_off(dev); 9606 } 9607 EXPORT_SYMBOL(netif_stacked_transfer_operstate); 9608 9609 static int netif_alloc_rx_queues(struct net_device *dev) 9610 { 9611 unsigned int i, count = dev->num_rx_queues; 9612 struct netdev_rx_queue *rx; 9613 size_t sz = count * sizeof(*rx); 9614 int err = 0; 9615 9616 BUG_ON(count < 1); 9617 9618 rx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL); 9619 if (!rx) 9620 return -ENOMEM; 9621 9622 dev->_rx = rx; 9623 9624 for (i = 0; i < count; i++) { 9625 rx[i].dev = dev; 9626 9627 /* XDP RX-queue setup */ 9628 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i); 9629 if (err < 0) 9630 goto err_rxq_info; 9631 } 9632 return 0; 9633 9634 err_rxq_info: 9635 /* Rollback successful reg's and free other resources */ 9636 while (i--) 9637 xdp_rxq_info_unreg(&rx[i].xdp_rxq); 9638 kvfree(dev->_rx); 9639 dev->_rx = NULL; 9640 return err; 9641 } 9642 9643 static void netif_free_rx_queues(struct net_device *dev) 9644 { 9645 unsigned int i, count = dev->num_rx_queues; 9646 9647 /* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */ 9648 if (!dev->_rx) 9649 return; 9650 9651 for (i = 0; i < count; i++) 9652 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq); 9653 9654 kvfree(dev->_rx); 9655 } 9656 9657 static void netdev_init_one_queue(struct net_device *dev, 9658 struct netdev_queue *queue, void *_unused) 9659 { 9660 /* Initialize queue lock */ 9661 spin_lock_init(&queue->_xmit_lock); 9662 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); 9663 queue->xmit_lock_owner = -1; 9664 netdev_queue_numa_node_write(queue, NUMA_NO_NODE); 9665 queue->dev = dev; 9666 #ifdef CONFIG_BQL 9667 dql_init(&queue->dql, HZ); 9668 #endif 9669 } 9670 9671 static void netif_free_tx_queues(struct net_device *dev) 9672 { 9673 kvfree(dev->_tx); 9674 } 9675 9676 static int netif_alloc_netdev_queues(struct net_device *dev) 9677 { 9678 unsigned int count = dev->num_tx_queues; 9679 struct netdev_queue *tx; 9680 size_t sz = count * sizeof(*tx); 9681 9682 if (count < 1 || count > 0xffff) 9683 return -EINVAL; 9684 9685 tx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL); 9686 if (!tx) 9687 return -ENOMEM; 9688 9689 dev->_tx = tx; 9690 9691 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); 9692 spin_lock_init(&dev->tx_global_lock); 9693 9694 return 0; 9695 } 9696 9697 void netif_tx_stop_all_queues(struct net_device *dev) 9698 { 9699 unsigned int i; 9700 9701 for (i = 0; i < dev->num_tx_queues; i++) { 9702 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 9703 9704 netif_tx_stop_queue(txq); 9705 } 9706 } 9707 EXPORT_SYMBOL(netif_tx_stop_all_queues); 9708 9709 /** 9710 * register_netdevice - register a network device 9711 * @dev: device to register 9712 * 9713 * Take a completed network device structure and add it to the kernel 9714 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 9715 * chain. 0 is returned on success. A negative errno code is returned 9716 * on a failure to set up the device, or if the name is a duplicate. 9717 * 9718 * Callers must hold the rtnl semaphore. You may want 9719 * register_netdev() instead of this. 9720 * 9721 * BUGS: 9722 * The locking appears insufficient to guarantee two parallel registers 9723 * will not get the same name. 9724 */ 9725 9726 int register_netdevice(struct net_device *dev) 9727 { 9728 int ret; 9729 struct net *net = dev_net(dev); 9730 9731 BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE < 9732 NETDEV_FEATURE_COUNT); 9733 BUG_ON(dev_boot_phase); 9734 ASSERT_RTNL(); 9735 9736 might_sleep(); 9737 9738 /* When net_device's are persistent, this will be fatal. */ 9739 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); 9740 BUG_ON(!net); 9741 9742 ret = ethtool_check_ops(dev->ethtool_ops); 9743 if (ret) 9744 return ret; 9745 9746 spin_lock_init(&dev->addr_list_lock); 9747 netdev_set_addr_lockdep_class(dev); 9748 9749 ret = dev_get_valid_name(net, dev, dev->name); 9750 if (ret < 0) 9751 goto out; 9752 9753 ret = -ENOMEM; 9754 dev->name_node = netdev_name_node_head_alloc(dev); 9755 if (!dev->name_node) 9756 goto out; 9757 9758 /* Init, if this function is available */ 9759 if (dev->netdev_ops->ndo_init) { 9760 ret = dev->netdev_ops->ndo_init(dev); 9761 if (ret) { 9762 if (ret > 0) 9763 ret = -EIO; 9764 goto err_free_name; 9765 } 9766 } 9767 9768 if (((dev->hw_features | dev->features) & 9769 NETIF_F_HW_VLAN_CTAG_FILTER) && 9770 (!dev->netdev_ops->ndo_vlan_rx_add_vid || 9771 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { 9772 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n"); 9773 ret = -EINVAL; 9774 goto err_uninit; 9775 } 9776 9777 ret = -EBUSY; 9778 if (!dev->ifindex) 9779 dev->ifindex = dev_new_index(net); 9780 else if (__dev_get_by_index(net, dev->ifindex)) 9781 goto err_uninit; 9782 9783 /* Transfer changeable features to wanted_features and enable 9784 * software offloads (GSO and GRO). 9785 */ 9786 dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF); 9787 dev->features |= NETIF_F_SOFT_FEATURES; 9788 9789 if (dev->netdev_ops->ndo_udp_tunnel_add) { 9790 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT; 9791 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT; 9792 } 9793 9794 dev->wanted_features = dev->features & dev->hw_features; 9795 9796 if (!(dev->flags & IFF_LOOPBACK)) 9797 dev->hw_features |= NETIF_F_NOCACHE_COPY; 9798 9799 /* If IPv4 TCP segmentation offload is supported we should also 9800 * allow the device to enable segmenting the frame with the option 9801 * of ignoring a static IP ID value. This doesn't enable the 9802 * feature itself but allows the user to enable it later. 9803 */ 9804 if (dev->hw_features & NETIF_F_TSO) 9805 dev->hw_features |= NETIF_F_TSO_MANGLEID; 9806 if (dev->vlan_features & NETIF_F_TSO) 9807 dev->vlan_features |= NETIF_F_TSO_MANGLEID; 9808 if (dev->mpls_features & NETIF_F_TSO) 9809 dev->mpls_features |= NETIF_F_TSO_MANGLEID; 9810 if (dev->hw_enc_features & NETIF_F_TSO) 9811 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 9812 9813 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. 9814 */ 9815 dev->vlan_features |= NETIF_F_HIGHDMA; 9816 9817 /* Make NETIF_F_SG inheritable to tunnel devices. 9818 */ 9819 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL; 9820 9821 /* Make NETIF_F_SG inheritable to MPLS. 9822 */ 9823 dev->mpls_features |= NETIF_F_SG; 9824 9825 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); 9826 ret = notifier_to_errno(ret); 9827 if (ret) 9828 goto err_uninit; 9829 9830 ret = netdev_register_kobject(dev); 9831 if (ret) { 9832 dev->reg_state = NETREG_UNREGISTERED; 9833 goto err_uninit; 9834 } 9835 dev->reg_state = NETREG_REGISTERED; 9836 9837 __netdev_update_features(dev); 9838 9839 /* 9840 * Default initial state at registry is that the 9841 * device is present. 9842 */ 9843 9844 set_bit(__LINK_STATE_PRESENT, &dev->state); 9845 9846 linkwatch_init_dev(dev); 9847 9848 dev_init_scheduler(dev); 9849 dev_hold(dev); 9850 list_netdevice(dev); 9851 add_device_randomness(dev->dev_addr, dev->addr_len); 9852 9853 /* If the device has permanent device address, driver should 9854 * set dev_addr and also addr_assign_type should be set to 9855 * NET_ADDR_PERM (default value). 9856 */ 9857 if (dev->addr_assign_type == NET_ADDR_PERM) 9858 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); 9859 9860 /* Notify protocols, that a new device appeared. */ 9861 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); 9862 ret = notifier_to_errno(ret); 9863 if (ret) { 9864 rollback_registered(dev); 9865 rcu_barrier(); 9866 9867 dev->reg_state = NETREG_UNREGISTERED; 9868 /* We should put the kobject that hold in 9869 * netdev_unregister_kobject(), otherwise 9870 * the net device cannot be freed when 9871 * driver calls free_netdev(), because the 9872 * kobject is being hold. 9873 */ 9874 kobject_put(&dev->dev.kobj); 9875 } 9876 /* 9877 * Prevent userspace races by waiting until the network 9878 * device is fully setup before sending notifications. 9879 */ 9880 if (!dev->rtnl_link_ops || 9881 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 9882 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); 9883 9884 out: 9885 return ret; 9886 9887 err_uninit: 9888 if (dev->netdev_ops->ndo_uninit) 9889 dev->netdev_ops->ndo_uninit(dev); 9890 if (dev->priv_destructor) 9891 dev->priv_destructor(dev); 9892 err_free_name: 9893 netdev_name_node_free(dev->name_node); 9894 goto out; 9895 } 9896 EXPORT_SYMBOL(register_netdevice); 9897 9898 /** 9899 * init_dummy_netdev - init a dummy network device for NAPI 9900 * @dev: device to init 9901 * 9902 * This takes a network device structure and initialize the minimum 9903 * amount of fields so it can be used to schedule NAPI polls without 9904 * registering a full blown interface. This is to be used by drivers 9905 * that need to tie several hardware interfaces to a single NAPI 9906 * poll scheduler due to HW limitations. 9907 */ 9908 int init_dummy_netdev(struct net_device *dev) 9909 { 9910 /* Clear everything. Note we don't initialize spinlocks 9911 * are they aren't supposed to be taken by any of the 9912 * NAPI code and this dummy netdev is supposed to be 9913 * only ever used for NAPI polls 9914 */ 9915 memset(dev, 0, sizeof(struct net_device)); 9916 9917 /* make sure we BUG if trying to hit standard 9918 * register/unregister code path 9919 */ 9920 dev->reg_state = NETREG_DUMMY; 9921 9922 /* NAPI wants this */ 9923 INIT_LIST_HEAD(&dev->napi_list); 9924 9925 /* a dummy interface is started by default */ 9926 set_bit(__LINK_STATE_PRESENT, &dev->state); 9927 set_bit(__LINK_STATE_START, &dev->state); 9928 9929 /* napi_busy_loop stats accounting wants this */ 9930 dev_net_set(dev, &init_net); 9931 9932 /* Note : We dont allocate pcpu_refcnt for dummy devices, 9933 * because users of this 'device' dont need to change 9934 * its refcount. 9935 */ 9936 9937 return 0; 9938 } 9939 EXPORT_SYMBOL_GPL(init_dummy_netdev); 9940 9941 9942 /** 9943 * register_netdev - register a network device 9944 * @dev: device to register 9945 * 9946 * Take a completed network device structure and add it to the kernel 9947 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 9948 * chain. 0 is returned on success. A negative errno code is returned 9949 * on a failure to set up the device, or if the name is a duplicate. 9950 * 9951 * This is a wrapper around register_netdevice that takes the rtnl semaphore 9952 * and expands the device name if you passed a format string to 9953 * alloc_netdev. 9954 */ 9955 int register_netdev(struct net_device *dev) 9956 { 9957 int err; 9958 9959 if (rtnl_lock_killable()) 9960 return -EINTR; 9961 err = register_netdevice(dev); 9962 rtnl_unlock(); 9963 return err; 9964 } 9965 EXPORT_SYMBOL(register_netdev); 9966 9967 int netdev_refcnt_read(const struct net_device *dev) 9968 { 9969 int i, refcnt = 0; 9970 9971 for_each_possible_cpu(i) 9972 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); 9973 return refcnt; 9974 } 9975 EXPORT_SYMBOL(netdev_refcnt_read); 9976 9977 /** 9978 * netdev_wait_allrefs - wait until all references are gone. 9979 * @dev: target net_device 9980 * 9981 * This is called when unregistering network devices. 9982 * 9983 * Any protocol or device that holds a reference should register 9984 * for netdevice notification, and cleanup and put back the 9985 * reference if they receive an UNREGISTER event. 9986 * We can get stuck here if buggy protocols don't correctly 9987 * call dev_put. 9988 */ 9989 static void netdev_wait_allrefs(struct net_device *dev) 9990 { 9991 unsigned long rebroadcast_time, warning_time; 9992 int refcnt; 9993 9994 linkwatch_forget_dev(dev); 9995 9996 rebroadcast_time = warning_time = jiffies; 9997 refcnt = netdev_refcnt_read(dev); 9998 9999 while (refcnt != 0) { 10000 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { 10001 rtnl_lock(); 10002 10003 /* Rebroadcast unregister notification */ 10004 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 10005 10006 __rtnl_unlock(); 10007 rcu_barrier(); 10008 rtnl_lock(); 10009 10010 if (test_bit(__LINK_STATE_LINKWATCH_PENDING, 10011 &dev->state)) { 10012 /* We must not have linkwatch events 10013 * pending on unregister. If this 10014 * happens, we simply run the queue 10015 * unscheduled, resulting in a noop 10016 * for this device. 10017 */ 10018 linkwatch_run_queue(); 10019 } 10020 10021 __rtnl_unlock(); 10022 10023 rebroadcast_time = jiffies; 10024 } 10025 10026 msleep(250); 10027 10028 refcnt = netdev_refcnt_read(dev); 10029 10030 if (refcnt && time_after(jiffies, warning_time + 10 * HZ)) { 10031 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", 10032 dev->name, refcnt); 10033 warning_time = jiffies; 10034 } 10035 } 10036 } 10037 10038 /* The sequence is: 10039 * 10040 * rtnl_lock(); 10041 * ... 10042 * register_netdevice(x1); 10043 * register_netdevice(x2); 10044 * ... 10045 * unregister_netdevice(y1); 10046 * unregister_netdevice(y2); 10047 * ... 10048 * rtnl_unlock(); 10049 * free_netdev(y1); 10050 * free_netdev(y2); 10051 * 10052 * We are invoked by rtnl_unlock(). 10053 * This allows us to deal with problems: 10054 * 1) We can delete sysfs objects which invoke hotplug 10055 * without deadlocking with linkwatch via keventd. 10056 * 2) Since we run with the RTNL semaphore not held, we can sleep 10057 * safely in order to wait for the netdev refcnt to drop to zero. 10058 * 10059 * We must not return until all unregister events added during 10060 * the interval the lock was held have been completed. 10061 */ 10062 void netdev_run_todo(void) 10063 { 10064 struct list_head list; 10065 10066 /* Snapshot list, allow later requests */ 10067 list_replace_init(&net_todo_list, &list); 10068 10069 __rtnl_unlock(); 10070 10071 10072 /* Wait for rcu callbacks to finish before next phase */ 10073 if (!list_empty(&list)) 10074 rcu_barrier(); 10075 10076 while (!list_empty(&list)) { 10077 struct net_device *dev 10078 = list_first_entry(&list, struct net_device, todo_list); 10079 list_del(&dev->todo_list); 10080 10081 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { 10082 pr_err("network todo '%s' but state %d\n", 10083 dev->name, dev->reg_state); 10084 dump_stack(); 10085 continue; 10086 } 10087 10088 dev->reg_state = NETREG_UNREGISTERED; 10089 10090 netdev_wait_allrefs(dev); 10091 10092 /* paranoia */ 10093 BUG_ON(netdev_refcnt_read(dev)); 10094 BUG_ON(!list_empty(&dev->ptype_all)); 10095 BUG_ON(!list_empty(&dev->ptype_specific)); 10096 WARN_ON(rcu_access_pointer(dev->ip_ptr)); 10097 WARN_ON(rcu_access_pointer(dev->ip6_ptr)); 10098 #if IS_ENABLED(CONFIG_DECNET) 10099 WARN_ON(dev->dn_ptr); 10100 #endif 10101 if (dev->priv_destructor) 10102 dev->priv_destructor(dev); 10103 if (dev->needs_free_netdev) 10104 free_netdev(dev); 10105 10106 /* Report a network device has been unregistered */ 10107 rtnl_lock(); 10108 dev_net(dev)->dev_unreg_count--; 10109 __rtnl_unlock(); 10110 wake_up(&netdev_unregistering_wq); 10111 10112 /* Free network device */ 10113 kobject_put(&dev->dev.kobj); 10114 } 10115 } 10116 10117 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has 10118 * all the same fields in the same order as net_device_stats, with only 10119 * the type differing, but rtnl_link_stats64 may have additional fields 10120 * at the end for newer counters. 10121 */ 10122 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64, 10123 const struct net_device_stats *netdev_stats) 10124 { 10125 #if BITS_PER_LONG == 64 10126 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats)); 10127 memcpy(stats64, netdev_stats, sizeof(*netdev_stats)); 10128 /* zero out counters that only exist in rtnl_link_stats64 */ 10129 memset((char *)stats64 + sizeof(*netdev_stats), 0, 10130 sizeof(*stats64) - sizeof(*netdev_stats)); 10131 #else 10132 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long); 10133 const unsigned long *src = (const unsigned long *)netdev_stats; 10134 u64 *dst = (u64 *)stats64; 10135 10136 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64)); 10137 for (i = 0; i < n; i++) 10138 dst[i] = src[i]; 10139 /* zero out counters that only exist in rtnl_link_stats64 */ 10140 memset((char *)stats64 + n * sizeof(u64), 0, 10141 sizeof(*stats64) - n * sizeof(u64)); 10142 #endif 10143 } 10144 EXPORT_SYMBOL(netdev_stats_to_stats64); 10145 10146 /** 10147 * dev_get_stats - get network device statistics 10148 * @dev: device to get statistics from 10149 * @storage: place to store stats 10150 * 10151 * Get network statistics from device. Return @storage. 10152 * The device driver may provide its own method by setting 10153 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats; 10154 * otherwise the internal statistics structure is used. 10155 */ 10156 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, 10157 struct rtnl_link_stats64 *storage) 10158 { 10159 const struct net_device_ops *ops = dev->netdev_ops; 10160 10161 if (ops->ndo_get_stats64) { 10162 memset(storage, 0, sizeof(*storage)); 10163 ops->ndo_get_stats64(dev, storage); 10164 } else if (ops->ndo_get_stats) { 10165 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); 10166 } else { 10167 netdev_stats_to_stats64(storage, &dev->stats); 10168 } 10169 storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped); 10170 storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped); 10171 storage->rx_nohandler += (unsigned long)atomic_long_read(&dev->rx_nohandler); 10172 return storage; 10173 } 10174 EXPORT_SYMBOL(dev_get_stats); 10175 10176 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) 10177 { 10178 struct netdev_queue *queue = dev_ingress_queue(dev); 10179 10180 #ifdef CONFIG_NET_CLS_ACT 10181 if (queue) 10182 return queue; 10183 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 10184 if (!queue) 10185 return NULL; 10186 netdev_init_one_queue(dev, queue, NULL); 10187 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); 10188 queue->qdisc_sleeping = &noop_qdisc; 10189 rcu_assign_pointer(dev->ingress_queue, queue); 10190 #endif 10191 return queue; 10192 } 10193 10194 static const struct ethtool_ops default_ethtool_ops; 10195 10196 void netdev_set_default_ethtool_ops(struct net_device *dev, 10197 const struct ethtool_ops *ops) 10198 { 10199 if (dev->ethtool_ops == &default_ethtool_ops) 10200 dev->ethtool_ops = ops; 10201 } 10202 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops); 10203 10204 void netdev_freemem(struct net_device *dev) 10205 { 10206 char *addr = (char *)dev - dev->padded; 10207 10208 kvfree(addr); 10209 } 10210 10211 /** 10212 * alloc_netdev_mqs - allocate network device 10213 * @sizeof_priv: size of private data to allocate space for 10214 * @name: device name format string 10215 * @name_assign_type: origin of device name 10216 * @setup: callback to initialize device 10217 * @txqs: the number of TX subqueues to allocate 10218 * @rxqs: the number of RX subqueues to allocate 10219 * 10220 * Allocates a struct net_device with private data area for driver use 10221 * and performs basic initialization. Also allocates subqueue structs 10222 * for each queue on the device. 10223 */ 10224 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, 10225 unsigned char name_assign_type, 10226 void (*setup)(struct net_device *), 10227 unsigned int txqs, unsigned int rxqs) 10228 { 10229 struct net_device *dev; 10230 unsigned int alloc_size; 10231 struct net_device *p; 10232 10233 BUG_ON(strlen(name) >= sizeof(dev->name)); 10234 10235 if (txqs < 1) { 10236 pr_err("alloc_netdev: Unable to allocate device with zero queues\n"); 10237 return NULL; 10238 } 10239 10240 if (rxqs < 1) { 10241 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n"); 10242 return NULL; 10243 } 10244 10245 alloc_size = sizeof(struct net_device); 10246 if (sizeof_priv) { 10247 /* ensure 32-byte alignment of private area */ 10248 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN); 10249 alloc_size += sizeof_priv; 10250 } 10251 /* ensure 32-byte alignment of whole construct */ 10252 alloc_size += NETDEV_ALIGN - 1; 10253 10254 p = kvzalloc(alloc_size, GFP_KERNEL | __GFP_RETRY_MAYFAIL); 10255 if (!p) 10256 return NULL; 10257 10258 dev = PTR_ALIGN(p, NETDEV_ALIGN); 10259 dev->padded = (char *)dev - (char *)p; 10260 10261 dev->pcpu_refcnt = alloc_percpu(int); 10262 if (!dev->pcpu_refcnt) 10263 goto free_dev; 10264 10265 if (dev_addr_init(dev)) 10266 goto free_pcpu; 10267 10268 dev_mc_init(dev); 10269 dev_uc_init(dev); 10270 10271 dev_net_set(dev, &init_net); 10272 10273 dev->gso_max_size = GSO_MAX_SIZE; 10274 dev->gso_max_segs = GSO_MAX_SEGS; 10275 dev->upper_level = 1; 10276 dev->lower_level = 1; 10277 10278 INIT_LIST_HEAD(&dev->napi_list); 10279 INIT_LIST_HEAD(&dev->unreg_list); 10280 INIT_LIST_HEAD(&dev->close_list); 10281 INIT_LIST_HEAD(&dev->link_watch_list); 10282 INIT_LIST_HEAD(&dev->adj_list.upper); 10283 INIT_LIST_HEAD(&dev->adj_list.lower); 10284 INIT_LIST_HEAD(&dev->ptype_all); 10285 INIT_LIST_HEAD(&dev->ptype_specific); 10286 INIT_LIST_HEAD(&dev->net_notifier_list); 10287 #ifdef CONFIG_NET_SCHED 10288 hash_init(dev->qdisc_hash); 10289 #endif 10290 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; 10291 setup(dev); 10292 10293 if (!dev->tx_queue_len) { 10294 dev->priv_flags |= IFF_NO_QUEUE; 10295 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; 10296 } 10297 10298 dev->num_tx_queues = txqs; 10299 dev->real_num_tx_queues = txqs; 10300 if (netif_alloc_netdev_queues(dev)) 10301 goto free_all; 10302 10303 dev->num_rx_queues = rxqs; 10304 dev->real_num_rx_queues = rxqs; 10305 if (netif_alloc_rx_queues(dev)) 10306 goto free_all; 10307 10308 strcpy(dev->name, name); 10309 dev->name_assign_type = name_assign_type; 10310 dev->group = INIT_NETDEV_GROUP; 10311 if (!dev->ethtool_ops) 10312 dev->ethtool_ops = &default_ethtool_ops; 10313 10314 nf_hook_ingress_init(dev); 10315 10316 return dev; 10317 10318 free_all: 10319 free_netdev(dev); 10320 return NULL; 10321 10322 free_pcpu: 10323 free_percpu(dev->pcpu_refcnt); 10324 free_dev: 10325 netdev_freemem(dev); 10326 return NULL; 10327 } 10328 EXPORT_SYMBOL(alloc_netdev_mqs); 10329 10330 /** 10331 * free_netdev - free network device 10332 * @dev: device 10333 * 10334 * This function does the last stage of destroying an allocated device 10335 * interface. The reference to the device object is released. If this 10336 * is the last reference then it will be freed.Must be called in process 10337 * context. 10338 */ 10339 void free_netdev(struct net_device *dev) 10340 { 10341 struct napi_struct *p, *n; 10342 10343 might_sleep(); 10344 netif_free_tx_queues(dev); 10345 netif_free_rx_queues(dev); 10346 10347 kfree(rcu_dereference_protected(dev->ingress_queue, 1)); 10348 10349 /* Flush device addresses */ 10350 dev_addr_flush(dev); 10351 10352 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) 10353 netif_napi_del(p); 10354 10355 free_percpu(dev->pcpu_refcnt); 10356 dev->pcpu_refcnt = NULL; 10357 free_percpu(dev->xdp_bulkq); 10358 dev->xdp_bulkq = NULL; 10359 10360 /* Compatibility with error handling in drivers */ 10361 if (dev->reg_state == NETREG_UNINITIALIZED) { 10362 netdev_freemem(dev); 10363 return; 10364 } 10365 10366 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); 10367 dev->reg_state = NETREG_RELEASED; 10368 10369 /* will free via device release */ 10370 put_device(&dev->dev); 10371 } 10372 EXPORT_SYMBOL(free_netdev); 10373 10374 /** 10375 * synchronize_net - Synchronize with packet receive processing 10376 * 10377 * Wait for packets currently being received to be done. 10378 * Does not block later packets from starting. 10379 */ 10380 void synchronize_net(void) 10381 { 10382 might_sleep(); 10383 if (rtnl_is_locked()) 10384 synchronize_rcu_expedited(); 10385 else 10386 synchronize_rcu(); 10387 } 10388 EXPORT_SYMBOL(synchronize_net); 10389 10390 /** 10391 * unregister_netdevice_queue - remove device from the kernel 10392 * @dev: device 10393 * @head: list 10394 * 10395 * This function shuts down a device interface and removes it 10396 * from the kernel tables. 10397 * If head not NULL, device is queued to be unregistered later. 10398 * 10399 * Callers must hold the rtnl semaphore. You may want 10400 * unregister_netdev() instead of this. 10401 */ 10402 10403 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) 10404 { 10405 ASSERT_RTNL(); 10406 10407 if (head) { 10408 list_move_tail(&dev->unreg_list, head); 10409 } else { 10410 rollback_registered(dev); 10411 /* Finish processing unregister after unlock */ 10412 net_set_todo(dev); 10413 } 10414 } 10415 EXPORT_SYMBOL(unregister_netdevice_queue); 10416 10417 /** 10418 * unregister_netdevice_many - unregister many devices 10419 * @head: list of devices 10420 * 10421 * Note: As most callers use a stack allocated list_head, 10422 * we force a list_del() to make sure stack wont be corrupted later. 10423 */ 10424 void unregister_netdevice_many(struct list_head *head) 10425 { 10426 struct net_device *dev; 10427 10428 if (!list_empty(head)) { 10429 rollback_registered_many(head); 10430 list_for_each_entry(dev, head, unreg_list) 10431 net_set_todo(dev); 10432 list_del(head); 10433 } 10434 } 10435 EXPORT_SYMBOL(unregister_netdevice_many); 10436 10437 /** 10438 * unregister_netdev - remove device from the kernel 10439 * @dev: device 10440 * 10441 * This function shuts down a device interface and removes it 10442 * from the kernel tables. 10443 * 10444 * This is just a wrapper for unregister_netdevice that takes 10445 * the rtnl semaphore. In general you want to use this and not 10446 * unregister_netdevice. 10447 */ 10448 void unregister_netdev(struct net_device *dev) 10449 { 10450 rtnl_lock(); 10451 unregister_netdevice(dev); 10452 rtnl_unlock(); 10453 } 10454 EXPORT_SYMBOL(unregister_netdev); 10455 10456 /** 10457 * dev_change_net_namespace - move device to different nethost namespace 10458 * @dev: device 10459 * @net: network namespace 10460 * @pat: If not NULL name pattern to try if the current device name 10461 * is already taken in the destination network namespace. 10462 * 10463 * This function shuts down a device interface and moves it 10464 * to a new network namespace. On success 0 is returned, on 10465 * a failure a netagive errno code is returned. 10466 * 10467 * Callers must hold the rtnl semaphore. 10468 */ 10469 10470 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat) 10471 { 10472 struct net *net_old = dev_net(dev); 10473 int err, new_nsid, new_ifindex; 10474 10475 ASSERT_RTNL(); 10476 10477 /* Don't allow namespace local devices to be moved. */ 10478 err = -EINVAL; 10479 if (dev->features & NETIF_F_NETNS_LOCAL) 10480 goto out; 10481 10482 /* Ensure the device has been registrered */ 10483 if (dev->reg_state != NETREG_REGISTERED) 10484 goto out; 10485 10486 /* Get out if there is nothing todo */ 10487 err = 0; 10488 if (net_eq(net_old, net)) 10489 goto out; 10490 10491 /* Pick the destination device name, and ensure 10492 * we can use it in the destination network namespace. 10493 */ 10494 err = -EEXIST; 10495 if (__dev_get_by_name(net, dev->name)) { 10496 /* We get here if we can't use the current device name */ 10497 if (!pat) 10498 goto out; 10499 err = dev_get_valid_name(net, dev, pat); 10500 if (err < 0) 10501 goto out; 10502 } 10503 10504 /* 10505 * And now a mini version of register_netdevice unregister_netdevice. 10506 */ 10507 10508 /* If device is running close it first. */ 10509 dev_close(dev); 10510 10511 /* And unlink it from device chain */ 10512 unlist_netdevice(dev); 10513 10514 synchronize_net(); 10515 10516 /* Shutdown queueing discipline. */ 10517 dev_shutdown(dev); 10518 10519 /* Notify protocols, that we are about to destroy 10520 * this device. They should clean all the things. 10521 * 10522 * Note that dev->reg_state stays at NETREG_REGISTERED. 10523 * This is wanted because this way 8021q and macvlan know 10524 * the device is just moving and can keep their slaves up. 10525 */ 10526 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 10527 rcu_barrier(); 10528 10529 new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL); 10530 /* If there is an ifindex conflict assign a new one */ 10531 if (__dev_get_by_index(net, dev->ifindex)) 10532 new_ifindex = dev_new_index(net); 10533 else 10534 new_ifindex = dev->ifindex; 10535 10536 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid, 10537 new_ifindex); 10538 10539 /* 10540 * Flush the unicast and multicast chains 10541 */ 10542 dev_uc_flush(dev); 10543 dev_mc_flush(dev); 10544 10545 /* Send a netdev-removed uevent to the old namespace */ 10546 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); 10547 netdev_adjacent_del_links(dev); 10548 10549 /* Move per-net netdevice notifiers that are following the netdevice */ 10550 move_netdevice_notifiers_dev_net(dev, net); 10551 10552 /* Actually switch the network namespace */ 10553 dev_net_set(dev, net); 10554 dev->ifindex = new_ifindex; 10555 10556 /* Send a netdev-add uevent to the new namespace */ 10557 kobject_uevent(&dev->dev.kobj, KOBJ_ADD); 10558 netdev_adjacent_add_links(dev); 10559 10560 /* Fixup kobjects */ 10561 err = device_rename(&dev->dev, dev->name); 10562 WARN_ON(err); 10563 10564 /* Adapt owner in case owning user namespace of target network 10565 * namespace is different from the original one. 10566 */ 10567 err = netdev_change_owner(dev, net_old, net); 10568 WARN_ON(err); 10569 10570 /* Add the device back in the hashes */ 10571 list_netdevice(dev); 10572 10573 /* Notify protocols, that a new device appeared. */ 10574 call_netdevice_notifiers(NETDEV_REGISTER, dev); 10575 10576 /* 10577 * Prevent userspace races by waiting until the network 10578 * device is fully setup before sending notifications. 10579 */ 10580 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); 10581 10582 synchronize_net(); 10583 err = 0; 10584 out: 10585 return err; 10586 } 10587 EXPORT_SYMBOL_GPL(dev_change_net_namespace); 10588 10589 static int dev_cpu_dead(unsigned int oldcpu) 10590 { 10591 struct sk_buff **list_skb; 10592 struct sk_buff *skb; 10593 unsigned int cpu; 10594 struct softnet_data *sd, *oldsd, *remsd = NULL; 10595 10596 local_irq_disable(); 10597 cpu = smp_processor_id(); 10598 sd = &per_cpu(softnet_data, cpu); 10599 oldsd = &per_cpu(softnet_data, oldcpu); 10600 10601 /* Find end of our completion_queue. */ 10602 list_skb = &sd->completion_queue; 10603 while (*list_skb) 10604 list_skb = &(*list_skb)->next; 10605 /* Append completion queue from offline CPU. */ 10606 *list_skb = oldsd->completion_queue; 10607 oldsd->completion_queue = NULL; 10608 10609 /* Append output queue from offline CPU. */ 10610 if (oldsd->output_queue) { 10611 *sd->output_queue_tailp = oldsd->output_queue; 10612 sd->output_queue_tailp = oldsd->output_queue_tailp; 10613 oldsd->output_queue = NULL; 10614 oldsd->output_queue_tailp = &oldsd->output_queue; 10615 } 10616 /* Append NAPI poll list from offline CPU, with one exception : 10617 * process_backlog() must be called by cpu owning percpu backlog. 10618 * We properly handle process_queue & input_pkt_queue later. 10619 */ 10620 while (!list_empty(&oldsd->poll_list)) { 10621 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, 10622 struct napi_struct, 10623 poll_list); 10624 10625 list_del_init(&napi->poll_list); 10626 if (napi->poll == process_backlog) 10627 napi->state = 0; 10628 else 10629 ____napi_schedule(sd, napi); 10630 } 10631 10632 raise_softirq_irqoff(NET_TX_SOFTIRQ); 10633 local_irq_enable(); 10634 10635 #ifdef CONFIG_RPS 10636 remsd = oldsd->rps_ipi_list; 10637 oldsd->rps_ipi_list = NULL; 10638 #endif 10639 /* send out pending IPI's on offline CPU */ 10640 net_rps_send_ipi(remsd); 10641 10642 /* Process offline CPU's input_pkt_queue */ 10643 while ((skb = __skb_dequeue(&oldsd->process_queue))) { 10644 netif_rx_ni(skb); 10645 input_queue_head_incr(oldsd); 10646 } 10647 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { 10648 netif_rx_ni(skb); 10649 input_queue_head_incr(oldsd); 10650 } 10651 10652 return 0; 10653 } 10654 10655 /** 10656 * netdev_increment_features - increment feature set by one 10657 * @all: current feature set 10658 * @one: new feature set 10659 * @mask: mask feature set 10660 * 10661 * Computes a new feature set after adding a device with feature set 10662 * @one to the master device with current feature set @all. Will not 10663 * enable anything that is off in @mask. Returns the new feature set. 10664 */ 10665 netdev_features_t netdev_increment_features(netdev_features_t all, 10666 netdev_features_t one, netdev_features_t mask) 10667 { 10668 if (mask & NETIF_F_HW_CSUM) 10669 mask |= NETIF_F_CSUM_MASK; 10670 mask |= NETIF_F_VLAN_CHALLENGED; 10671 10672 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask; 10673 all &= one | ~NETIF_F_ALL_FOR_ALL; 10674 10675 /* If one device supports hw checksumming, set for all. */ 10676 if (all & NETIF_F_HW_CSUM) 10677 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM); 10678 10679 return all; 10680 } 10681 EXPORT_SYMBOL(netdev_increment_features); 10682 10683 static struct hlist_head * __net_init netdev_create_hash(void) 10684 { 10685 int i; 10686 struct hlist_head *hash; 10687 10688 hash = kmalloc_array(NETDEV_HASHENTRIES, sizeof(*hash), GFP_KERNEL); 10689 if (hash != NULL) 10690 for (i = 0; i < NETDEV_HASHENTRIES; i++) 10691 INIT_HLIST_HEAD(&hash[i]); 10692 10693 return hash; 10694 } 10695 10696 /* Initialize per network namespace state */ 10697 static int __net_init netdev_init(struct net *net) 10698 { 10699 BUILD_BUG_ON(GRO_HASH_BUCKETS > 10700 8 * sizeof_field(struct napi_struct, gro_bitmask)); 10701 10702 if (net != &init_net) 10703 INIT_LIST_HEAD(&net->dev_base_head); 10704 10705 net->dev_name_head = netdev_create_hash(); 10706 if (net->dev_name_head == NULL) 10707 goto err_name; 10708 10709 net->dev_index_head = netdev_create_hash(); 10710 if (net->dev_index_head == NULL) 10711 goto err_idx; 10712 10713 RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain); 10714 10715 return 0; 10716 10717 err_idx: 10718 kfree(net->dev_name_head); 10719 err_name: 10720 return -ENOMEM; 10721 } 10722 10723 /** 10724 * netdev_drivername - network driver for the device 10725 * @dev: network device 10726 * 10727 * Determine network driver for device. 10728 */ 10729 const char *netdev_drivername(const struct net_device *dev) 10730 { 10731 const struct device_driver *driver; 10732 const struct device *parent; 10733 const char *empty = ""; 10734 10735 parent = dev->dev.parent; 10736 if (!parent) 10737 return empty; 10738 10739 driver = parent->driver; 10740 if (driver && driver->name) 10741 return driver->name; 10742 return empty; 10743 } 10744 10745 static void __netdev_printk(const char *level, const struct net_device *dev, 10746 struct va_format *vaf) 10747 { 10748 if (dev && dev->dev.parent) { 10749 dev_printk_emit(level[1] - '0', 10750 dev->dev.parent, 10751 "%s %s %s%s: %pV", 10752 dev_driver_string(dev->dev.parent), 10753 dev_name(dev->dev.parent), 10754 netdev_name(dev), netdev_reg_state(dev), 10755 vaf); 10756 } else if (dev) { 10757 printk("%s%s%s: %pV", 10758 level, netdev_name(dev), netdev_reg_state(dev), vaf); 10759 } else { 10760 printk("%s(NULL net_device): %pV", level, vaf); 10761 } 10762 } 10763 10764 void netdev_printk(const char *level, const struct net_device *dev, 10765 const char *format, ...) 10766 { 10767 struct va_format vaf; 10768 va_list args; 10769 10770 va_start(args, format); 10771 10772 vaf.fmt = format; 10773 vaf.va = &args; 10774 10775 __netdev_printk(level, dev, &vaf); 10776 10777 va_end(args); 10778 } 10779 EXPORT_SYMBOL(netdev_printk); 10780 10781 #define define_netdev_printk_level(func, level) \ 10782 void func(const struct net_device *dev, const char *fmt, ...) \ 10783 { \ 10784 struct va_format vaf; \ 10785 va_list args; \ 10786 \ 10787 va_start(args, fmt); \ 10788 \ 10789 vaf.fmt = fmt; \ 10790 vaf.va = &args; \ 10791 \ 10792 __netdev_printk(level, dev, &vaf); \ 10793 \ 10794 va_end(args); \ 10795 } \ 10796 EXPORT_SYMBOL(func); 10797 10798 define_netdev_printk_level(netdev_emerg, KERN_EMERG); 10799 define_netdev_printk_level(netdev_alert, KERN_ALERT); 10800 define_netdev_printk_level(netdev_crit, KERN_CRIT); 10801 define_netdev_printk_level(netdev_err, KERN_ERR); 10802 define_netdev_printk_level(netdev_warn, KERN_WARNING); 10803 define_netdev_printk_level(netdev_notice, KERN_NOTICE); 10804 define_netdev_printk_level(netdev_info, KERN_INFO); 10805 10806 static void __net_exit netdev_exit(struct net *net) 10807 { 10808 kfree(net->dev_name_head); 10809 kfree(net->dev_index_head); 10810 if (net != &init_net) 10811 WARN_ON_ONCE(!list_empty(&net->dev_base_head)); 10812 } 10813 10814 static struct pernet_operations __net_initdata netdev_net_ops = { 10815 .init = netdev_init, 10816 .exit = netdev_exit, 10817 }; 10818 10819 static void __net_exit default_device_exit(struct net *net) 10820 { 10821 struct net_device *dev, *aux; 10822 /* 10823 * Push all migratable network devices back to the 10824 * initial network namespace 10825 */ 10826 rtnl_lock(); 10827 for_each_netdev_safe(net, dev, aux) { 10828 int err; 10829 char fb_name[IFNAMSIZ]; 10830 10831 /* Ignore unmoveable devices (i.e. loopback) */ 10832 if (dev->features & NETIF_F_NETNS_LOCAL) 10833 continue; 10834 10835 /* Leave virtual devices for the generic cleanup */ 10836 if (dev->rtnl_link_ops) 10837 continue; 10838 10839 /* Push remaining network devices to init_net */ 10840 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); 10841 if (__dev_get_by_name(&init_net, fb_name)) 10842 snprintf(fb_name, IFNAMSIZ, "dev%%d"); 10843 err = dev_change_net_namespace(dev, &init_net, fb_name); 10844 if (err) { 10845 pr_emerg("%s: failed to move %s to init_net: %d\n", 10846 __func__, dev->name, err); 10847 BUG(); 10848 } 10849 } 10850 rtnl_unlock(); 10851 } 10852 10853 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list) 10854 { 10855 /* Return with the rtnl_lock held when there are no network 10856 * devices unregistering in any network namespace in net_list. 10857 */ 10858 struct net *net; 10859 bool unregistering; 10860 DEFINE_WAIT_FUNC(wait, woken_wake_function); 10861 10862 add_wait_queue(&netdev_unregistering_wq, &wait); 10863 for (;;) { 10864 unregistering = false; 10865 rtnl_lock(); 10866 list_for_each_entry(net, net_list, exit_list) { 10867 if (net->dev_unreg_count > 0) { 10868 unregistering = true; 10869 break; 10870 } 10871 } 10872 if (!unregistering) 10873 break; 10874 __rtnl_unlock(); 10875 10876 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); 10877 } 10878 remove_wait_queue(&netdev_unregistering_wq, &wait); 10879 } 10880 10881 static void __net_exit default_device_exit_batch(struct list_head *net_list) 10882 { 10883 /* At exit all network devices most be removed from a network 10884 * namespace. Do this in the reverse order of registration. 10885 * Do this across as many network namespaces as possible to 10886 * improve batching efficiency. 10887 */ 10888 struct net_device *dev; 10889 struct net *net; 10890 LIST_HEAD(dev_kill_list); 10891 10892 /* To prevent network device cleanup code from dereferencing 10893 * loopback devices or network devices that have been freed 10894 * wait here for all pending unregistrations to complete, 10895 * before unregistring the loopback device and allowing the 10896 * network namespace be freed. 10897 * 10898 * The netdev todo list containing all network devices 10899 * unregistrations that happen in default_device_exit_batch 10900 * will run in the rtnl_unlock() at the end of 10901 * default_device_exit_batch. 10902 */ 10903 rtnl_lock_unregistering(net_list); 10904 list_for_each_entry(net, net_list, exit_list) { 10905 for_each_netdev_reverse(net, dev) { 10906 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) 10907 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); 10908 else 10909 unregister_netdevice_queue(dev, &dev_kill_list); 10910 } 10911 } 10912 unregister_netdevice_many(&dev_kill_list); 10913 rtnl_unlock(); 10914 } 10915 10916 static struct pernet_operations __net_initdata default_device_ops = { 10917 .exit = default_device_exit, 10918 .exit_batch = default_device_exit_batch, 10919 }; 10920 10921 /* 10922 * Initialize the DEV module. At boot time this walks the device list and 10923 * unhooks any devices that fail to initialise (normally hardware not 10924 * present) and leaves us with a valid list of present and active devices. 10925 * 10926 */ 10927 10928 /* 10929 * This is called single threaded during boot, so no need 10930 * to take the rtnl semaphore. 10931 */ 10932 static int __init net_dev_init(void) 10933 { 10934 int i, rc = -ENOMEM; 10935 10936 BUG_ON(!dev_boot_phase); 10937 10938 if (dev_proc_init()) 10939 goto out; 10940 10941 if (netdev_kobject_init()) 10942 goto out; 10943 10944 INIT_LIST_HEAD(&ptype_all); 10945 for (i = 0; i < PTYPE_HASH_SIZE; i++) 10946 INIT_LIST_HEAD(&ptype_base[i]); 10947 10948 INIT_LIST_HEAD(&offload_base); 10949 10950 if (register_pernet_subsys(&netdev_net_ops)) 10951 goto out; 10952 10953 /* 10954 * Initialise the packet receive queues. 10955 */ 10956 10957 for_each_possible_cpu(i) { 10958 struct work_struct *flush = per_cpu_ptr(&flush_works, i); 10959 struct softnet_data *sd = &per_cpu(softnet_data, i); 10960 10961 INIT_WORK(flush, flush_backlog); 10962 10963 skb_queue_head_init(&sd->input_pkt_queue); 10964 skb_queue_head_init(&sd->process_queue); 10965 #ifdef CONFIG_XFRM_OFFLOAD 10966 skb_queue_head_init(&sd->xfrm_backlog); 10967 #endif 10968 INIT_LIST_HEAD(&sd->poll_list); 10969 sd->output_queue_tailp = &sd->output_queue; 10970 #ifdef CONFIG_RPS 10971 sd->csd.func = rps_trigger_softirq; 10972 sd->csd.info = sd; 10973 sd->cpu = i; 10974 #endif 10975 10976 init_gro_hash(&sd->backlog); 10977 sd->backlog.poll = process_backlog; 10978 sd->backlog.weight = weight_p; 10979 } 10980 10981 dev_boot_phase = 0; 10982 10983 /* The loopback device is special if any other network devices 10984 * is present in a network namespace the loopback device must 10985 * be present. Since we now dynamically allocate and free the 10986 * loopback device ensure this invariant is maintained by 10987 * keeping the loopback device as the first device on the 10988 * list of network devices. Ensuring the loopback devices 10989 * is the first device that appears and the last network device 10990 * that disappears. 10991 */ 10992 if (register_pernet_device(&loopback_net_ops)) 10993 goto out; 10994 10995 if (register_pernet_device(&default_device_ops)) 10996 goto out; 10997 10998 open_softirq(NET_TX_SOFTIRQ, net_tx_action); 10999 open_softirq(NET_RX_SOFTIRQ, net_rx_action); 11000 11001 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead", 11002 NULL, dev_cpu_dead); 11003 WARN_ON(rc < 0); 11004 rc = 0; 11005 out: 11006 return rc; 11007 } 11008 11009 subsys_initcall(net_dev_init); 11010