1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * NET3 Protocol independent device support routines. 4 * 5 * Derived from the non IP parts of dev.c 1.0.19 6 * Authors: Ross Biro 7 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 8 * Mark Evans, <evansmp@uhura.aston.ac.uk> 9 * 10 * Additional Authors: 11 * Florian la Roche <rzsfl@rz.uni-sb.de> 12 * Alan Cox <gw4pts@gw4pts.ampr.org> 13 * David Hinds <dahinds@users.sourceforge.net> 14 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 15 * Adam Sulmicki <adam@cfar.umd.edu> 16 * Pekka Riikonen <priikone@poesidon.pspt.fi> 17 * 18 * Changes: 19 * D.J. Barrow : Fixed bug where dev->refcnt gets set 20 * to 2 if register_netdev gets called 21 * before net_dev_init & also removed a 22 * few lines of code in the process. 23 * Alan Cox : device private ioctl copies fields back. 24 * Alan Cox : Transmit queue code does relevant 25 * stunts to keep the queue safe. 26 * Alan Cox : Fixed double lock. 27 * Alan Cox : Fixed promisc NULL pointer trap 28 * ???????? : Support the full private ioctl range 29 * Alan Cox : Moved ioctl permission check into 30 * drivers 31 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI 32 * Alan Cox : 100 backlog just doesn't cut it when 33 * you start doing multicast video 8) 34 * Alan Cox : Rewrote net_bh and list manager. 35 * Alan Cox : Fix ETH_P_ALL echoback lengths. 36 * Alan Cox : Took out transmit every packet pass 37 * Saved a few bytes in the ioctl handler 38 * Alan Cox : Network driver sets packet type before 39 * calling netif_rx. Saves a function 40 * call a packet. 41 * Alan Cox : Hashed net_bh() 42 * Richard Kooijman: Timestamp fixes. 43 * Alan Cox : Wrong field in SIOCGIFDSTADDR 44 * Alan Cox : Device lock protection. 45 * Alan Cox : Fixed nasty side effect of device close 46 * changes. 47 * Rudi Cilibrasi : Pass the right thing to 48 * set_mac_address() 49 * Dave Miller : 32bit quantity for the device lock to 50 * make it work out on a Sparc. 51 * Bjorn Ekwall : Added KERNELD hack. 52 * Alan Cox : Cleaned up the backlog initialise. 53 * Craig Metz : SIOCGIFCONF fix if space for under 54 * 1 device. 55 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there 56 * is no device open function. 57 * Andi Kleen : Fix error reporting for SIOCGIFCONF 58 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF 59 * Cyrus Durgin : Cleaned for KMOD 60 * Adam Sulmicki : Bug Fix : Network Device Unload 61 * A network device unload needs to purge 62 * the backlog queue. 63 * Paul Rusty Russell : SIOCSIFNAME 64 * Pekka Riikonen : Netdev boot-time settings code 65 * Andrew Morton : Make unregister_netdevice wait 66 * indefinitely on dev->refcnt 67 * J Hadi Salim : - Backlog queue sampling 68 * - netif_rx() feedback 69 */ 70 71 #include <linux/uaccess.h> 72 #include <linux/bitops.h> 73 #include <linux/capability.h> 74 #include <linux/cpu.h> 75 #include <linux/types.h> 76 #include <linux/kernel.h> 77 #include <linux/hash.h> 78 #include <linux/slab.h> 79 #include <linux/sched.h> 80 #include <linux/sched/mm.h> 81 #include <linux/mutex.h> 82 #include <linux/rwsem.h> 83 #include <linux/string.h> 84 #include <linux/mm.h> 85 #include <linux/socket.h> 86 #include <linux/sockios.h> 87 #include <linux/errno.h> 88 #include <linux/interrupt.h> 89 #include <linux/if_ether.h> 90 #include <linux/netdevice.h> 91 #include <linux/etherdevice.h> 92 #include <linux/ethtool.h> 93 #include <linux/skbuff.h> 94 #include <linux/bpf.h> 95 #include <linux/bpf_trace.h> 96 #include <net/net_namespace.h> 97 #include <net/sock.h> 98 #include <net/busy_poll.h> 99 #include <linux/rtnetlink.h> 100 #include <linux/stat.h> 101 #include <net/dst.h> 102 #include <net/dst_metadata.h> 103 #include <net/pkt_sched.h> 104 #include <net/pkt_cls.h> 105 #include <net/checksum.h> 106 #include <net/xfrm.h> 107 #include <linux/highmem.h> 108 #include <linux/init.h> 109 #include <linux/module.h> 110 #include <linux/netpoll.h> 111 #include <linux/rcupdate.h> 112 #include <linux/delay.h> 113 #include <net/iw_handler.h> 114 #include <asm/current.h> 115 #include <linux/audit.h> 116 #include <linux/dmaengine.h> 117 #include <linux/err.h> 118 #include <linux/ctype.h> 119 #include <linux/if_arp.h> 120 #include <linux/if_vlan.h> 121 #include <linux/ip.h> 122 #include <net/ip.h> 123 #include <net/mpls.h> 124 #include <linux/ipv6.h> 125 #include <linux/in.h> 126 #include <linux/jhash.h> 127 #include <linux/random.h> 128 #include <trace/events/napi.h> 129 #include <trace/events/net.h> 130 #include <trace/events/skb.h> 131 #include <linux/inetdevice.h> 132 #include <linux/cpu_rmap.h> 133 #include <linux/static_key.h> 134 #include <linux/hashtable.h> 135 #include <linux/vmalloc.h> 136 #include <linux/if_macvlan.h> 137 #include <linux/errqueue.h> 138 #include <linux/hrtimer.h> 139 #include <linux/netfilter_ingress.h> 140 #include <linux/crash_dump.h> 141 #include <linux/sctp.h> 142 #include <net/udp_tunnel.h> 143 #include <linux/net_namespace.h> 144 #include <linux/indirect_call_wrapper.h> 145 #include <net/devlink.h> 146 #include <linux/pm_runtime.h> 147 148 #include "net-sysfs.h" 149 150 #define MAX_GRO_SKBS 8 151 152 /* This should be increased if a protocol with a bigger head is added. */ 153 #define GRO_MAX_HEAD (MAX_HEADER + 128) 154 155 static DEFINE_SPINLOCK(ptype_lock); 156 static DEFINE_SPINLOCK(offload_lock); 157 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; 158 struct list_head ptype_all __read_mostly; /* Taps */ 159 static struct list_head offload_base __read_mostly; 160 161 static int netif_rx_internal(struct sk_buff *skb); 162 static int call_netdevice_notifiers_info(unsigned long val, 163 struct netdev_notifier_info *info); 164 static int call_netdevice_notifiers_extack(unsigned long val, 165 struct net_device *dev, 166 struct netlink_ext_ack *extack); 167 static struct napi_struct *napi_by_id(unsigned int napi_id); 168 169 /* 170 * The @dev_base_head list is protected by @dev_base_lock and the rtnl 171 * semaphore. 172 * 173 * Pure readers hold dev_base_lock for reading, or rcu_read_lock() 174 * 175 * Writers must hold the rtnl semaphore while they loop through the 176 * dev_base_head list, and hold dev_base_lock for writing when they do the 177 * actual updates. This allows pure readers to access the list even 178 * while a writer is preparing to update it. 179 * 180 * To put it another way, dev_base_lock is held for writing only to 181 * protect against pure readers; the rtnl semaphore provides the 182 * protection against other writers. 183 * 184 * See, for example usages, register_netdevice() and 185 * unregister_netdevice(), which must be called with the rtnl 186 * semaphore held. 187 */ 188 DEFINE_RWLOCK(dev_base_lock); 189 EXPORT_SYMBOL(dev_base_lock); 190 191 static DEFINE_MUTEX(ifalias_mutex); 192 193 /* protects napi_hash addition/deletion and napi_gen_id */ 194 static DEFINE_SPINLOCK(napi_hash_lock); 195 196 static unsigned int napi_gen_id = NR_CPUS; 197 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8); 198 199 static DECLARE_RWSEM(devnet_rename_sem); 200 201 static inline void dev_base_seq_inc(struct net *net) 202 { 203 while (++net->dev_base_seq == 0) 204 ; 205 } 206 207 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name) 208 { 209 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ)); 210 211 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; 212 } 213 214 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) 215 { 216 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; 217 } 218 219 static inline void rps_lock(struct softnet_data *sd) 220 { 221 #ifdef CONFIG_RPS 222 spin_lock(&sd->input_pkt_queue.lock); 223 #endif 224 } 225 226 static inline void rps_unlock(struct softnet_data *sd) 227 { 228 #ifdef CONFIG_RPS 229 spin_unlock(&sd->input_pkt_queue.lock); 230 #endif 231 } 232 233 static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev, 234 const char *name) 235 { 236 struct netdev_name_node *name_node; 237 238 name_node = kmalloc(sizeof(*name_node), GFP_KERNEL); 239 if (!name_node) 240 return NULL; 241 INIT_HLIST_NODE(&name_node->hlist); 242 name_node->dev = dev; 243 name_node->name = name; 244 return name_node; 245 } 246 247 static struct netdev_name_node * 248 netdev_name_node_head_alloc(struct net_device *dev) 249 { 250 struct netdev_name_node *name_node; 251 252 name_node = netdev_name_node_alloc(dev, dev->name); 253 if (!name_node) 254 return NULL; 255 INIT_LIST_HEAD(&name_node->list); 256 return name_node; 257 } 258 259 static void netdev_name_node_free(struct netdev_name_node *name_node) 260 { 261 kfree(name_node); 262 } 263 264 static void netdev_name_node_add(struct net *net, 265 struct netdev_name_node *name_node) 266 { 267 hlist_add_head_rcu(&name_node->hlist, 268 dev_name_hash(net, name_node->name)); 269 } 270 271 static void netdev_name_node_del(struct netdev_name_node *name_node) 272 { 273 hlist_del_rcu(&name_node->hlist); 274 } 275 276 static struct netdev_name_node *netdev_name_node_lookup(struct net *net, 277 const char *name) 278 { 279 struct hlist_head *head = dev_name_hash(net, name); 280 struct netdev_name_node *name_node; 281 282 hlist_for_each_entry(name_node, head, hlist) 283 if (!strcmp(name_node->name, name)) 284 return name_node; 285 return NULL; 286 } 287 288 static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net, 289 const char *name) 290 { 291 struct hlist_head *head = dev_name_hash(net, name); 292 struct netdev_name_node *name_node; 293 294 hlist_for_each_entry_rcu(name_node, head, hlist) 295 if (!strcmp(name_node->name, name)) 296 return name_node; 297 return NULL; 298 } 299 300 int netdev_name_node_alt_create(struct net_device *dev, const char *name) 301 { 302 struct netdev_name_node *name_node; 303 struct net *net = dev_net(dev); 304 305 name_node = netdev_name_node_lookup(net, name); 306 if (name_node) 307 return -EEXIST; 308 name_node = netdev_name_node_alloc(dev, name); 309 if (!name_node) 310 return -ENOMEM; 311 netdev_name_node_add(net, name_node); 312 /* The node that holds dev->name acts as a head of per-device list. */ 313 list_add_tail(&name_node->list, &dev->name_node->list); 314 315 return 0; 316 } 317 EXPORT_SYMBOL(netdev_name_node_alt_create); 318 319 static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node) 320 { 321 list_del(&name_node->list); 322 netdev_name_node_del(name_node); 323 kfree(name_node->name); 324 netdev_name_node_free(name_node); 325 } 326 327 int netdev_name_node_alt_destroy(struct net_device *dev, const char *name) 328 { 329 struct netdev_name_node *name_node; 330 struct net *net = dev_net(dev); 331 332 name_node = netdev_name_node_lookup(net, name); 333 if (!name_node) 334 return -ENOENT; 335 /* lookup might have found our primary name or a name belonging 336 * to another device. 337 */ 338 if (name_node == dev->name_node || name_node->dev != dev) 339 return -EINVAL; 340 341 __netdev_name_node_alt_destroy(name_node); 342 343 return 0; 344 } 345 EXPORT_SYMBOL(netdev_name_node_alt_destroy); 346 347 static void netdev_name_node_alt_flush(struct net_device *dev) 348 { 349 struct netdev_name_node *name_node, *tmp; 350 351 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list) 352 __netdev_name_node_alt_destroy(name_node); 353 } 354 355 /* Device list insertion */ 356 static void list_netdevice(struct net_device *dev) 357 { 358 struct net *net = dev_net(dev); 359 360 ASSERT_RTNL(); 361 362 write_lock_bh(&dev_base_lock); 363 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); 364 netdev_name_node_add(net, dev->name_node); 365 hlist_add_head_rcu(&dev->index_hlist, 366 dev_index_hash(net, dev->ifindex)); 367 write_unlock_bh(&dev_base_lock); 368 369 dev_base_seq_inc(net); 370 } 371 372 /* Device list removal 373 * caller must respect a RCU grace period before freeing/reusing dev 374 */ 375 static void unlist_netdevice(struct net_device *dev) 376 { 377 ASSERT_RTNL(); 378 379 /* Unlink dev from the device chain */ 380 write_lock_bh(&dev_base_lock); 381 list_del_rcu(&dev->dev_list); 382 netdev_name_node_del(dev->name_node); 383 hlist_del_rcu(&dev->index_hlist); 384 write_unlock_bh(&dev_base_lock); 385 386 dev_base_seq_inc(dev_net(dev)); 387 } 388 389 /* 390 * Our notifier list 391 */ 392 393 static RAW_NOTIFIER_HEAD(netdev_chain); 394 395 /* 396 * Device drivers call our routines to queue packets here. We empty the 397 * queue in the local softnet handler. 398 */ 399 400 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data); 401 EXPORT_PER_CPU_SYMBOL(softnet_data); 402 403 #ifdef CONFIG_LOCKDEP 404 /* 405 * register_netdevice() inits txq->_xmit_lock and sets lockdep class 406 * according to dev->type 407 */ 408 static const unsigned short netdev_lock_type[] = { 409 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25, 410 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET, 411 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM, 412 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, 413 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, 414 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, 415 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, 416 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, 417 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, 418 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, 419 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, 420 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, 421 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM, 422 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE, 423 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE}; 424 425 static const char *const netdev_lock_name[] = { 426 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", 427 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET", 428 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM", 429 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", 430 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", 431 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", 432 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", 433 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", 434 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", 435 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", 436 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", 437 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", 438 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM", 439 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE", 440 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"}; 441 442 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; 443 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; 444 445 static inline unsigned short netdev_lock_pos(unsigned short dev_type) 446 { 447 int i; 448 449 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++) 450 if (netdev_lock_type[i] == dev_type) 451 return i; 452 /* the last key is used by default */ 453 return ARRAY_SIZE(netdev_lock_type) - 1; 454 } 455 456 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 457 unsigned short dev_type) 458 { 459 int i; 460 461 i = netdev_lock_pos(dev_type); 462 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i], 463 netdev_lock_name[i]); 464 } 465 466 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 467 { 468 int i; 469 470 i = netdev_lock_pos(dev->type); 471 lockdep_set_class_and_name(&dev->addr_list_lock, 472 &netdev_addr_lock_key[i], 473 netdev_lock_name[i]); 474 } 475 #else 476 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 477 unsigned short dev_type) 478 { 479 } 480 481 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 482 { 483 } 484 #endif 485 486 /******************************************************************************* 487 * 488 * Protocol management and registration routines 489 * 490 *******************************************************************************/ 491 492 493 /* 494 * Add a protocol ID to the list. Now that the input handler is 495 * smarter we can dispense with all the messy stuff that used to be 496 * here. 497 * 498 * BEWARE!!! Protocol handlers, mangling input packets, 499 * MUST BE last in hash buckets and checking protocol handlers 500 * MUST start from promiscuous ptype_all chain in net_bh. 501 * It is true now, do not change it. 502 * Explanation follows: if protocol handler, mangling packet, will 503 * be the first on list, it is not able to sense, that packet 504 * is cloned and should be copied-on-write, so that it will 505 * change it and subsequent readers will get broken packet. 506 * --ANK (980803) 507 */ 508 509 static inline struct list_head *ptype_head(const struct packet_type *pt) 510 { 511 if (pt->type == htons(ETH_P_ALL)) 512 return pt->dev ? &pt->dev->ptype_all : &ptype_all; 513 else 514 return pt->dev ? &pt->dev->ptype_specific : 515 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; 516 } 517 518 /** 519 * dev_add_pack - add packet handler 520 * @pt: packet type declaration 521 * 522 * Add a protocol handler to the networking stack. The passed &packet_type 523 * is linked into kernel lists and may not be freed until it has been 524 * removed from the kernel lists. 525 * 526 * This call does not sleep therefore it can not 527 * guarantee all CPU's that are in middle of receiving packets 528 * will see the new packet type (until the next received packet). 529 */ 530 531 void dev_add_pack(struct packet_type *pt) 532 { 533 struct list_head *head = ptype_head(pt); 534 535 spin_lock(&ptype_lock); 536 list_add_rcu(&pt->list, head); 537 spin_unlock(&ptype_lock); 538 } 539 EXPORT_SYMBOL(dev_add_pack); 540 541 /** 542 * __dev_remove_pack - remove packet handler 543 * @pt: packet type declaration 544 * 545 * Remove a protocol handler that was previously added to the kernel 546 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 547 * from the kernel lists and can be freed or reused once this function 548 * returns. 549 * 550 * The packet type might still be in use by receivers 551 * and must not be freed until after all the CPU's have gone 552 * through a quiescent state. 553 */ 554 void __dev_remove_pack(struct packet_type *pt) 555 { 556 struct list_head *head = ptype_head(pt); 557 struct packet_type *pt1; 558 559 spin_lock(&ptype_lock); 560 561 list_for_each_entry(pt1, head, list) { 562 if (pt == pt1) { 563 list_del_rcu(&pt->list); 564 goto out; 565 } 566 } 567 568 pr_warn("dev_remove_pack: %p not found\n", pt); 569 out: 570 spin_unlock(&ptype_lock); 571 } 572 EXPORT_SYMBOL(__dev_remove_pack); 573 574 /** 575 * dev_remove_pack - remove packet handler 576 * @pt: packet type declaration 577 * 578 * Remove a protocol handler that was previously added to the kernel 579 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 580 * from the kernel lists and can be freed or reused once this function 581 * returns. 582 * 583 * This call sleeps to guarantee that no CPU is looking at the packet 584 * type after return. 585 */ 586 void dev_remove_pack(struct packet_type *pt) 587 { 588 __dev_remove_pack(pt); 589 590 synchronize_net(); 591 } 592 EXPORT_SYMBOL(dev_remove_pack); 593 594 595 /** 596 * dev_add_offload - register offload handlers 597 * @po: protocol offload declaration 598 * 599 * Add protocol offload handlers to the networking stack. The passed 600 * &proto_offload is linked into kernel lists and may not be freed until 601 * it has been removed from the kernel lists. 602 * 603 * This call does not sleep therefore it can not 604 * guarantee all CPU's that are in middle of receiving packets 605 * will see the new offload handlers (until the next received packet). 606 */ 607 void dev_add_offload(struct packet_offload *po) 608 { 609 struct packet_offload *elem; 610 611 spin_lock(&offload_lock); 612 list_for_each_entry(elem, &offload_base, list) { 613 if (po->priority < elem->priority) 614 break; 615 } 616 list_add_rcu(&po->list, elem->list.prev); 617 spin_unlock(&offload_lock); 618 } 619 EXPORT_SYMBOL(dev_add_offload); 620 621 /** 622 * __dev_remove_offload - remove offload handler 623 * @po: packet offload declaration 624 * 625 * Remove a protocol offload handler that was previously added to the 626 * kernel offload handlers by dev_add_offload(). The passed &offload_type 627 * is removed from the kernel lists and can be freed or reused once this 628 * function returns. 629 * 630 * The packet type might still be in use by receivers 631 * and must not be freed until after all the CPU's have gone 632 * through a quiescent state. 633 */ 634 static void __dev_remove_offload(struct packet_offload *po) 635 { 636 struct list_head *head = &offload_base; 637 struct packet_offload *po1; 638 639 spin_lock(&offload_lock); 640 641 list_for_each_entry(po1, head, list) { 642 if (po == po1) { 643 list_del_rcu(&po->list); 644 goto out; 645 } 646 } 647 648 pr_warn("dev_remove_offload: %p not found\n", po); 649 out: 650 spin_unlock(&offload_lock); 651 } 652 653 /** 654 * dev_remove_offload - remove packet offload handler 655 * @po: packet offload declaration 656 * 657 * Remove a packet offload handler that was previously added to the kernel 658 * offload handlers by dev_add_offload(). The passed &offload_type is 659 * removed from the kernel lists and can be freed or reused once this 660 * function returns. 661 * 662 * This call sleeps to guarantee that no CPU is looking at the packet 663 * type after return. 664 */ 665 void dev_remove_offload(struct packet_offload *po) 666 { 667 __dev_remove_offload(po); 668 669 synchronize_net(); 670 } 671 EXPORT_SYMBOL(dev_remove_offload); 672 673 /****************************************************************************** 674 * 675 * Device Boot-time Settings Routines 676 * 677 ******************************************************************************/ 678 679 /* Boot time configuration table */ 680 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX]; 681 682 /** 683 * netdev_boot_setup_add - add new setup entry 684 * @name: name of the device 685 * @map: configured settings for the device 686 * 687 * Adds new setup entry to the dev_boot_setup list. The function 688 * returns 0 on error and 1 on success. This is a generic routine to 689 * all netdevices. 690 */ 691 static int netdev_boot_setup_add(char *name, struct ifmap *map) 692 { 693 struct netdev_boot_setup *s; 694 int i; 695 696 s = dev_boot_setup; 697 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 698 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') { 699 memset(s[i].name, 0, sizeof(s[i].name)); 700 strlcpy(s[i].name, name, IFNAMSIZ); 701 memcpy(&s[i].map, map, sizeof(s[i].map)); 702 break; 703 } 704 } 705 706 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1; 707 } 708 709 /** 710 * netdev_boot_setup_check - check boot time settings 711 * @dev: the netdevice 712 * 713 * Check boot time settings for the device. 714 * The found settings are set for the device to be used 715 * later in the device probing. 716 * Returns 0 if no settings found, 1 if they are. 717 */ 718 int netdev_boot_setup_check(struct net_device *dev) 719 { 720 struct netdev_boot_setup *s = dev_boot_setup; 721 int i; 722 723 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 724 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' && 725 !strcmp(dev->name, s[i].name)) { 726 dev->irq = s[i].map.irq; 727 dev->base_addr = s[i].map.base_addr; 728 dev->mem_start = s[i].map.mem_start; 729 dev->mem_end = s[i].map.mem_end; 730 return 1; 731 } 732 } 733 return 0; 734 } 735 EXPORT_SYMBOL(netdev_boot_setup_check); 736 737 738 /** 739 * netdev_boot_base - get address from boot time settings 740 * @prefix: prefix for network device 741 * @unit: id for network device 742 * 743 * Check boot time settings for the base address of device. 744 * The found settings are set for the device to be used 745 * later in the device probing. 746 * Returns 0 if no settings found. 747 */ 748 unsigned long netdev_boot_base(const char *prefix, int unit) 749 { 750 const struct netdev_boot_setup *s = dev_boot_setup; 751 char name[IFNAMSIZ]; 752 int i; 753 754 sprintf(name, "%s%d", prefix, unit); 755 756 /* 757 * If device already registered then return base of 1 758 * to indicate not to probe for this interface 759 */ 760 if (__dev_get_by_name(&init_net, name)) 761 return 1; 762 763 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) 764 if (!strcmp(name, s[i].name)) 765 return s[i].map.base_addr; 766 return 0; 767 } 768 769 /* 770 * Saves at boot time configured settings for any netdevice. 771 */ 772 int __init netdev_boot_setup(char *str) 773 { 774 int ints[5]; 775 struct ifmap map; 776 777 str = get_options(str, ARRAY_SIZE(ints), ints); 778 if (!str || !*str) 779 return 0; 780 781 /* Save settings */ 782 memset(&map, 0, sizeof(map)); 783 if (ints[0] > 0) 784 map.irq = ints[1]; 785 if (ints[0] > 1) 786 map.base_addr = ints[2]; 787 if (ints[0] > 2) 788 map.mem_start = ints[3]; 789 if (ints[0] > 3) 790 map.mem_end = ints[4]; 791 792 /* Add new entry to the list */ 793 return netdev_boot_setup_add(str, &map); 794 } 795 796 __setup("netdev=", netdev_boot_setup); 797 798 /******************************************************************************* 799 * 800 * Device Interface Subroutines 801 * 802 *******************************************************************************/ 803 804 /** 805 * dev_get_iflink - get 'iflink' value of a interface 806 * @dev: targeted interface 807 * 808 * Indicates the ifindex the interface is linked to. 809 * Physical interfaces have the same 'ifindex' and 'iflink' values. 810 */ 811 812 int dev_get_iflink(const struct net_device *dev) 813 { 814 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) 815 return dev->netdev_ops->ndo_get_iflink(dev); 816 817 return dev->ifindex; 818 } 819 EXPORT_SYMBOL(dev_get_iflink); 820 821 /** 822 * dev_fill_metadata_dst - Retrieve tunnel egress information. 823 * @dev: targeted interface 824 * @skb: The packet. 825 * 826 * For better visibility of tunnel traffic OVS needs to retrieve 827 * egress tunnel information for a packet. Following API allows 828 * user to get this info. 829 */ 830 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb) 831 { 832 struct ip_tunnel_info *info; 833 834 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst) 835 return -EINVAL; 836 837 info = skb_tunnel_info_unclone(skb); 838 if (!info) 839 return -ENOMEM; 840 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX))) 841 return -EINVAL; 842 843 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb); 844 } 845 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst); 846 847 /** 848 * __dev_get_by_name - find a device by its name 849 * @net: the applicable net namespace 850 * @name: name to find 851 * 852 * Find an interface by name. Must be called under RTNL semaphore 853 * or @dev_base_lock. If the name is found a pointer to the device 854 * is returned. If the name is not found then %NULL is returned. The 855 * reference counters are not incremented so the caller must be 856 * careful with locks. 857 */ 858 859 struct net_device *__dev_get_by_name(struct net *net, const char *name) 860 { 861 struct netdev_name_node *node_name; 862 863 node_name = netdev_name_node_lookup(net, name); 864 return node_name ? node_name->dev : NULL; 865 } 866 EXPORT_SYMBOL(__dev_get_by_name); 867 868 /** 869 * dev_get_by_name_rcu - find a device by its name 870 * @net: the applicable net namespace 871 * @name: name to find 872 * 873 * Find an interface by name. 874 * If the name is found a pointer to the device is returned. 875 * If the name is not found then %NULL is returned. 876 * The reference counters are not incremented so the caller must be 877 * careful with locks. The caller must hold RCU lock. 878 */ 879 880 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name) 881 { 882 struct netdev_name_node *node_name; 883 884 node_name = netdev_name_node_lookup_rcu(net, name); 885 return node_name ? node_name->dev : NULL; 886 } 887 EXPORT_SYMBOL(dev_get_by_name_rcu); 888 889 /** 890 * dev_get_by_name - find a device by its name 891 * @net: the applicable net namespace 892 * @name: name to find 893 * 894 * Find an interface by name. This can be called from any 895 * context and does its own locking. The returned handle has 896 * the usage count incremented and the caller must use dev_put() to 897 * release it when it is no longer needed. %NULL is returned if no 898 * matching device is found. 899 */ 900 901 struct net_device *dev_get_by_name(struct net *net, const char *name) 902 { 903 struct net_device *dev; 904 905 rcu_read_lock(); 906 dev = dev_get_by_name_rcu(net, name); 907 if (dev) 908 dev_hold(dev); 909 rcu_read_unlock(); 910 return dev; 911 } 912 EXPORT_SYMBOL(dev_get_by_name); 913 914 /** 915 * __dev_get_by_index - find a device by its ifindex 916 * @net: the applicable net namespace 917 * @ifindex: index of device 918 * 919 * Search for an interface by index. Returns %NULL if the device 920 * is not found or a pointer to the device. The device has not 921 * had its reference counter increased so the caller must be careful 922 * about locking. The caller must hold either the RTNL semaphore 923 * or @dev_base_lock. 924 */ 925 926 struct net_device *__dev_get_by_index(struct net *net, int ifindex) 927 { 928 struct net_device *dev; 929 struct hlist_head *head = dev_index_hash(net, ifindex); 930 931 hlist_for_each_entry(dev, head, index_hlist) 932 if (dev->ifindex == ifindex) 933 return dev; 934 935 return NULL; 936 } 937 EXPORT_SYMBOL(__dev_get_by_index); 938 939 /** 940 * dev_get_by_index_rcu - find a device by its ifindex 941 * @net: the applicable net namespace 942 * @ifindex: index of device 943 * 944 * Search for an interface by index. Returns %NULL if the device 945 * is not found or a pointer to the device. The device has not 946 * had its reference counter increased so the caller must be careful 947 * about locking. The caller must hold RCU lock. 948 */ 949 950 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex) 951 { 952 struct net_device *dev; 953 struct hlist_head *head = dev_index_hash(net, ifindex); 954 955 hlist_for_each_entry_rcu(dev, head, index_hlist) 956 if (dev->ifindex == ifindex) 957 return dev; 958 959 return NULL; 960 } 961 EXPORT_SYMBOL(dev_get_by_index_rcu); 962 963 964 /** 965 * dev_get_by_index - find a device by its ifindex 966 * @net: the applicable net namespace 967 * @ifindex: index of device 968 * 969 * Search for an interface by index. Returns NULL if the device 970 * is not found or a pointer to the device. The device returned has 971 * had a reference added and the pointer is safe until the user calls 972 * dev_put to indicate they have finished with it. 973 */ 974 975 struct net_device *dev_get_by_index(struct net *net, int ifindex) 976 { 977 struct net_device *dev; 978 979 rcu_read_lock(); 980 dev = dev_get_by_index_rcu(net, ifindex); 981 if (dev) 982 dev_hold(dev); 983 rcu_read_unlock(); 984 return dev; 985 } 986 EXPORT_SYMBOL(dev_get_by_index); 987 988 /** 989 * dev_get_by_napi_id - find a device by napi_id 990 * @napi_id: ID of the NAPI struct 991 * 992 * Search for an interface by NAPI ID. Returns %NULL if the device 993 * is not found or a pointer to the device. The device has not had 994 * its reference counter increased so the caller must be careful 995 * about locking. The caller must hold RCU lock. 996 */ 997 998 struct net_device *dev_get_by_napi_id(unsigned int napi_id) 999 { 1000 struct napi_struct *napi; 1001 1002 WARN_ON_ONCE(!rcu_read_lock_held()); 1003 1004 if (napi_id < MIN_NAPI_ID) 1005 return NULL; 1006 1007 napi = napi_by_id(napi_id); 1008 1009 return napi ? napi->dev : NULL; 1010 } 1011 EXPORT_SYMBOL(dev_get_by_napi_id); 1012 1013 /** 1014 * netdev_get_name - get a netdevice name, knowing its ifindex. 1015 * @net: network namespace 1016 * @name: a pointer to the buffer where the name will be stored. 1017 * @ifindex: the ifindex of the interface to get the name from. 1018 */ 1019 int netdev_get_name(struct net *net, char *name, int ifindex) 1020 { 1021 struct net_device *dev; 1022 int ret; 1023 1024 down_read(&devnet_rename_sem); 1025 rcu_read_lock(); 1026 1027 dev = dev_get_by_index_rcu(net, ifindex); 1028 if (!dev) { 1029 ret = -ENODEV; 1030 goto out; 1031 } 1032 1033 strcpy(name, dev->name); 1034 1035 ret = 0; 1036 out: 1037 rcu_read_unlock(); 1038 up_read(&devnet_rename_sem); 1039 return ret; 1040 } 1041 1042 /** 1043 * dev_getbyhwaddr_rcu - find a device by its hardware address 1044 * @net: the applicable net namespace 1045 * @type: media type of device 1046 * @ha: hardware address 1047 * 1048 * Search for an interface by MAC address. Returns NULL if the device 1049 * is not found or a pointer to the device. 1050 * The caller must hold RCU or RTNL. 1051 * The returned device has not had its ref count increased 1052 * and the caller must therefore be careful about locking 1053 * 1054 */ 1055 1056 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type, 1057 const char *ha) 1058 { 1059 struct net_device *dev; 1060 1061 for_each_netdev_rcu(net, dev) 1062 if (dev->type == type && 1063 !memcmp(dev->dev_addr, ha, dev->addr_len)) 1064 return dev; 1065 1066 return NULL; 1067 } 1068 EXPORT_SYMBOL(dev_getbyhwaddr_rcu); 1069 1070 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type) 1071 { 1072 struct net_device *dev; 1073 1074 ASSERT_RTNL(); 1075 for_each_netdev(net, dev) 1076 if (dev->type == type) 1077 return dev; 1078 1079 return NULL; 1080 } 1081 EXPORT_SYMBOL(__dev_getfirstbyhwtype); 1082 1083 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) 1084 { 1085 struct net_device *dev, *ret = NULL; 1086 1087 rcu_read_lock(); 1088 for_each_netdev_rcu(net, dev) 1089 if (dev->type == type) { 1090 dev_hold(dev); 1091 ret = dev; 1092 break; 1093 } 1094 rcu_read_unlock(); 1095 return ret; 1096 } 1097 EXPORT_SYMBOL(dev_getfirstbyhwtype); 1098 1099 /** 1100 * __dev_get_by_flags - find any device with given flags 1101 * @net: the applicable net namespace 1102 * @if_flags: IFF_* values 1103 * @mask: bitmask of bits in if_flags to check 1104 * 1105 * Search for any interface with the given flags. Returns NULL if a device 1106 * is not found or a pointer to the device. Must be called inside 1107 * rtnl_lock(), and result refcount is unchanged. 1108 */ 1109 1110 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags, 1111 unsigned short mask) 1112 { 1113 struct net_device *dev, *ret; 1114 1115 ASSERT_RTNL(); 1116 1117 ret = NULL; 1118 for_each_netdev(net, dev) { 1119 if (((dev->flags ^ if_flags) & mask) == 0) { 1120 ret = dev; 1121 break; 1122 } 1123 } 1124 return ret; 1125 } 1126 EXPORT_SYMBOL(__dev_get_by_flags); 1127 1128 /** 1129 * dev_valid_name - check if name is okay for network device 1130 * @name: name string 1131 * 1132 * Network device names need to be valid file names to 1133 * to allow sysfs to work. We also disallow any kind of 1134 * whitespace. 1135 */ 1136 bool dev_valid_name(const char *name) 1137 { 1138 if (*name == '\0') 1139 return false; 1140 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ) 1141 return false; 1142 if (!strcmp(name, ".") || !strcmp(name, "..")) 1143 return false; 1144 1145 while (*name) { 1146 if (*name == '/' || *name == ':' || isspace(*name)) 1147 return false; 1148 name++; 1149 } 1150 return true; 1151 } 1152 EXPORT_SYMBOL(dev_valid_name); 1153 1154 /** 1155 * __dev_alloc_name - allocate a name for a device 1156 * @net: network namespace to allocate the device name in 1157 * @name: name format string 1158 * @buf: scratch buffer and result name string 1159 * 1160 * Passed a format string - eg "lt%d" it will try and find a suitable 1161 * id. It scans list of devices to build up a free map, then chooses 1162 * the first empty slot. The caller must hold the dev_base or rtnl lock 1163 * while allocating the name and adding the device in order to avoid 1164 * duplicates. 1165 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1166 * Returns the number of the unit assigned or a negative errno code. 1167 */ 1168 1169 static int __dev_alloc_name(struct net *net, const char *name, char *buf) 1170 { 1171 int i = 0; 1172 const char *p; 1173 const int max_netdevices = 8*PAGE_SIZE; 1174 unsigned long *inuse; 1175 struct net_device *d; 1176 1177 if (!dev_valid_name(name)) 1178 return -EINVAL; 1179 1180 p = strchr(name, '%'); 1181 if (p) { 1182 /* 1183 * Verify the string as this thing may have come from 1184 * the user. There must be either one "%d" and no other "%" 1185 * characters. 1186 */ 1187 if (p[1] != 'd' || strchr(p + 2, '%')) 1188 return -EINVAL; 1189 1190 /* Use one page as a bit array of possible slots */ 1191 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC); 1192 if (!inuse) 1193 return -ENOMEM; 1194 1195 for_each_netdev(net, d) { 1196 if (!sscanf(d->name, name, &i)) 1197 continue; 1198 if (i < 0 || i >= max_netdevices) 1199 continue; 1200 1201 /* avoid cases where sscanf is not exact inverse of printf */ 1202 snprintf(buf, IFNAMSIZ, name, i); 1203 if (!strncmp(buf, d->name, IFNAMSIZ)) 1204 set_bit(i, inuse); 1205 } 1206 1207 i = find_first_zero_bit(inuse, max_netdevices); 1208 free_page((unsigned long) inuse); 1209 } 1210 1211 snprintf(buf, IFNAMSIZ, name, i); 1212 if (!__dev_get_by_name(net, buf)) 1213 return i; 1214 1215 /* It is possible to run out of possible slots 1216 * when the name is long and there isn't enough space left 1217 * for the digits, or if all bits are used. 1218 */ 1219 return -ENFILE; 1220 } 1221 1222 static int dev_alloc_name_ns(struct net *net, 1223 struct net_device *dev, 1224 const char *name) 1225 { 1226 char buf[IFNAMSIZ]; 1227 int ret; 1228 1229 BUG_ON(!net); 1230 ret = __dev_alloc_name(net, name, buf); 1231 if (ret >= 0) 1232 strlcpy(dev->name, buf, IFNAMSIZ); 1233 return ret; 1234 } 1235 1236 /** 1237 * dev_alloc_name - allocate a name for a device 1238 * @dev: device 1239 * @name: name format string 1240 * 1241 * Passed a format string - eg "lt%d" it will try and find a suitable 1242 * id. It scans list of devices to build up a free map, then chooses 1243 * the first empty slot. The caller must hold the dev_base or rtnl lock 1244 * while allocating the name and adding the device in order to avoid 1245 * duplicates. 1246 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1247 * Returns the number of the unit assigned or a negative errno code. 1248 */ 1249 1250 int dev_alloc_name(struct net_device *dev, const char *name) 1251 { 1252 return dev_alloc_name_ns(dev_net(dev), dev, name); 1253 } 1254 EXPORT_SYMBOL(dev_alloc_name); 1255 1256 static int dev_get_valid_name(struct net *net, struct net_device *dev, 1257 const char *name) 1258 { 1259 BUG_ON(!net); 1260 1261 if (!dev_valid_name(name)) 1262 return -EINVAL; 1263 1264 if (strchr(name, '%')) 1265 return dev_alloc_name_ns(net, dev, name); 1266 else if (__dev_get_by_name(net, name)) 1267 return -EEXIST; 1268 else if (dev->name != name) 1269 strlcpy(dev->name, name, IFNAMSIZ); 1270 1271 return 0; 1272 } 1273 1274 /** 1275 * dev_change_name - change name of a device 1276 * @dev: device 1277 * @newname: name (or format string) must be at least IFNAMSIZ 1278 * 1279 * Change name of a device, can pass format strings "eth%d". 1280 * for wildcarding. 1281 */ 1282 int dev_change_name(struct net_device *dev, const char *newname) 1283 { 1284 unsigned char old_assign_type; 1285 char oldname[IFNAMSIZ]; 1286 int err = 0; 1287 int ret; 1288 struct net *net; 1289 1290 ASSERT_RTNL(); 1291 BUG_ON(!dev_net(dev)); 1292 1293 net = dev_net(dev); 1294 1295 /* Some auto-enslaved devices e.g. failover slaves are 1296 * special, as userspace might rename the device after 1297 * the interface had been brought up and running since 1298 * the point kernel initiated auto-enslavement. Allow 1299 * live name change even when these slave devices are 1300 * up and running. 1301 * 1302 * Typically, users of these auto-enslaving devices 1303 * don't actually care about slave name change, as 1304 * they are supposed to operate on master interface 1305 * directly. 1306 */ 1307 if (dev->flags & IFF_UP && 1308 likely(!(dev->priv_flags & IFF_LIVE_RENAME_OK))) 1309 return -EBUSY; 1310 1311 down_write(&devnet_rename_sem); 1312 1313 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) { 1314 up_write(&devnet_rename_sem); 1315 return 0; 1316 } 1317 1318 memcpy(oldname, dev->name, IFNAMSIZ); 1319 1320 err = dev_get_valid_name(net, dev, newname); 1321 if (err < 0) { 1322 up_write(&devnet_rename_sem); 1323 return err; 1324 } 1325 1326 if (oldname[0] && !strchr(oldname, '%')) 1327 netdev_info(dev, "renamed from %s\n", oldname); 1328 1329 old_assign_type = dev->name_assign_type; 1330 dev->name_assign_type = NET_NAME_RENAMED; 1331 1332 rollback: 1333 ret = device_rename(&dev->dev, dev->name); 1334 if (ret) { 1335 memcpy(dev->name, oldname, IFNAMSIZ); 1336 dev->name_assign_type = old_assign_type; 1337 up_write(&devnet_rename_sem); 1338 return ret; 1339 } 1340 1341 up_write(&devnet_rename_sem); 1342 1343 netdev_adjacent_rename_links(dev, oldname); 1344 1345 write_lock_bh(&dev_base_lock); 1346 netdev_name_node_del(dev->name_node); 1347 write_unlock_bh(&dev_base_lock); 1348 1349 synchronize_rcu(); 1350 1351 write_lock_bh(&dev_base_lock); 1352 netdev_name_node_add(net, dev->name_node); 1353 write_unlock_bh(&dev_base_lock); 1354 1355 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); 1356 ret = notifier_to_errno(ret); 1357 1358 if (ret) { 1359 /* err >= 0 after dev_alloc_name() or stores the first errno */ 1360 if (err >= 0) { 1361 err = ret; 1362 down_write(&devnet_rename_sem); 1363 memcpy(dev->name, oldname, IFNAMSIZ); 1364 memcpy(oldname, newname, IFNAMSIZ); 1365 dev->name_assign_type = old_assign_type; 1366 old_assign_type = NET_NAME_RENAMED; 1367 goto rollback; 1368 } else { 1369 pr_err("%s: name change rollback failed: %d\n", 1370 dev->name, ret); 1371 } 1372 } 1373 1374 return err; 1375 } 1376 1377 /** 1378 * dev_set_alias - change ifalias of a device 1379 * @dev: device 1380 * @alias: name up to IFALIASZ 1381 * @len: limit of bytes to copy from info 1382 * 1383 * Set ifalias for a device, 1384 */ 1385 int dev_set_alias(struct net_device *dev, const char *alias, size_t len) 1386 { 1387 struct dev_ifalias *new_alias = NULL; 1388 1389 if (len >= IFALIASZ) 1390 return -EINVAL; 1391 1392 if (len) { 1393 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL); 1394 if (!new_alias) 1395 return -ENOMEM; 1396 1397 memcpy(new_alias->ifalias, alias, len); 1398 new_alias->ifalias[len] = 0; 1399 } 1400 1401 mutex_lock(&ifalias_mutex); 1402 new_alias = rcu_replace_pointer(dev->ifalias, new_alias, 1403 mutex_is_locked(&ifalias_mutex)); 1404 mutex_unlock(&ifalias_mutex); 1405 1406 if (new_alias) 1407 kfree_rcu(new_alias, rcuhead); 1408 1409 return len; 1410 } 1411 EXPORT_SYMBOL(dev_set_alias); 1412 1413 /** 1414 * dev_get_alias - get ifalias of a device 1415 * @dev: device 1416 * @name: buffer to store name of ifalias 1417 * @len: size of buffer 1418 * 1419 * get ifalias for a device. Caller must make sure dev cannot go 1420 * away, e.g. rcu read lock or own a reference count to device. 1421 */ 1422 int dev_get_alias(const struct net_device *dev, char *name, size_t len) 1423 { 1424 const struct dev_ifalias *alias; 1425 int ret = 0; 1426 1427 rcu_read_lock(); 1428 alias = rcu_dereference(dev->ifalias); 1429 if (alias) 1430 ret = snprintf(name, len, "%s", alias->ifalias); 1431 rcu_read_unlock(); 1432 1433 return ret; 1434 } 1435 1436 /** 1437 * netdev_features_change - device changes features 1438 * @dev: device to cause notification 1439 * 1440 * Called to indicate a device has changed features. 1441 */ 1442 void netdev_features_change(struct net_device *dev) 1443 { 1444 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); 1445 } 1446 EXPORT_SYMBOL(netdev_features_change); 1447 1448 /** 1449 * netdev_state_change - device changes state 1450 * @dev: device to cause notification 1451 * 1452 * Called to indicate a device has changed state. This function calls 1453 * the notifier chains for netdev_chain and sends a NEWLINK message 1454 * to the routing socket. 1455 */ 1456 void netdev_state_change(struct net_device *dev) 1457 { 1458 if (dev->flags & IFF_UP) { 1459 struct netdev_notifier_change_info change_info = { 1460 .info.dev = dev, 1461 }; 1462 1463 call_netdevice_notifiers_info(NETDEV_CHANGE, 1464 &change_info.info); 1465 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL); 1466 } 1467 } 1468 EXPORT_SYMBOL(netdev_state_change); 1469 1470 /** 1471 * netdev_notify_peers - notify network peers about existence of @dev 1472 * @dev: network device 1473 * 1474 * Generate traffic such that interested network peers are aware of 1475 * @dev, such as by generating a gratuitous ARP. This may be used when 1476 * a device wants to inform the rest of the network about some sort of 1477 * reconfiguration such as a failover event or virtual machine 1478 * migration. 1479 */ 1480 void netdev_notify_peers(struct net_device *dev) 1481 { 1482 rtnl_lock(); 1483 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev); 1484 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev); 1485 rtnl_unlock(); 1486 } 1487 EXPORT_SYMBOL(netdev_notify_peers); 1488 1489 static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack) 1490 { 1491 const struct net_device_ops *ops = dev->netdev_ops; 1492 int ret; 1493 1494 ASSERT_RTNL(); 1495 1496 if (!netif_device_present(dev)) { 1497 /* may be detached because parent is runtime-suspended */ 1498 if (dev->dev.parent) 1499 pm_runtime_resume(dev->dev.parent); 1500 if (!netif_device_present(dev)) 1501 return -ENODEV; 1502 } 1503 1504 /* Block netpoll from trying to do any rx path servicing. 1505 * If we don't do this there is a chance ndo_poll_controller 1506 * or ndo_poll may be running while we open the device 1507 */ 1508 netpoll_poll_disable(dev); 1509 1510 ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack); 1511 ret = notifier_to_errno(ret); 1512 if (ret) 1513 return ret; 1514 1515 set_bit(__LINK_STATE_START, &dev->state); 1516 1517 if (ops->ndo_validate_addr) 1518 ret = ops->ndo_validate_addr(dev); 1519 1520 if (!ret && ops->ndo_open) 1521 ret = ops->ndo_open(dev); 1522 1523 netpoll_poll_enable(dev); 1524 1525 if (ret) 1526 clear_bit(__LINK_STATE_START, &dev->state); 1527 else { 1528 dev->flags |= IFF_UP; 1529 dev_set_rx_mode(dev); 1530 dev_activate(dev); 1531 add_device_randomness(dev->dev_addr, dev->addr_len); 1532 } 1533 1534 return ret; 1535 } 1536 1537 /** 1538 * dev_open - prepare an interface for use. 1539 * @dev: device to open 1540 * @extack: netlink extended ack 1541 * 1542 * Takes a device from down to up state. The device's private open 1543 * function is invoked and then the multicast lists are loaded. Finally 1544 * the device is moved into the up state and a %NETDEV_UP message is 1545 * sent to the netdev notifier chain. 1546 * 1547 * Calling this function on an active interface is a nop. On a failure 1548 * a negative errno code is returned. 1549 */ 1550 int dev_open(struct net_device *dev, struct netlink_ext_ack *extack) 1551 { 1552 int ret; 1553 1554 if (dev->flags & IFF_UP) 1555 return 0; 1556 1557 ret = __dev_open(dev, extack); 1558 if (ret < 0) 1559 return ret; 1560 1561 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); 1562 call_netdevice_notifiers(NETDEV_UP, dev); 1563 1564 return ret; 1565 } 1566 EXPORT_SYMBOL(dev_open); 1567 1568 static void __dev_close_many(struct list_head *head) 1569 { 1570 struct net_device *dev; 1571 1572 ASSERT_RTNL(); 1573 might_sleep(); 1574 1575 list_for_each_entry(dev, head, close_list) { 1576 /* Temporarily disable netpoll until the interface is down */ 1577 netpoll_poll_disable(dev); 1578 1579 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); 1580 1581 clear_bit(__LINK_STATE_START, &dev->state); 1582 1583 /* Synchronize to scheduled poll. We cannot touch poll list, it 1584 * can be even on different cpu. So just clear netif_running(). 1585 * 1586 * dev->stop() will invoke napi_disable() on all of it's 1587 * napi_struct instances on this device. 1588 */ 1589 smp_mb__after_atomic(); /* Commit netif_running(). */ 1590 } 1591 1592 dev_deactivate_many(head); 1593 1594 list_for_each_entry(dev, head, close_list) { 1595 const struct net_device_ops *ops = dev->netdev_ops; 1596 1597 /* 1598 * Call the device specific close. This cannot fail. 1599 * Only if device is UP 1600 * 1601 * We allow it to be called even after a DETACH hot-plug 1602 * event. 1603 */ 1604 if (ops->ndo_stop) 1605 ops->ndo_stop(dev); 1606 1607 dev->flags &= ~IFF_UP; 1608 netpoll_poll_enable(dev); 1609 } 1610 } 1611 1612 static void __dev_close(struct net_device *dev) 1613 { 1614 LIST_HEAD(single); 1615 1616 list_add(&dev->close_list, &single); 1617 __dev_close_many(&single); 1618 list_del(&single); 1619 } 1620 1621 void dev_close_many(struct list_head *head, bool unlink) 1622 { 1623 struct net_device *dev, *tmp; 1624 1625 /* Remove the devices that don't need to be closed */ 1626 list_for_each_entry_safe(dev, tmp, head, close_list) 1627 if (!(dev->flags & IFF_UP)) 1628 list_del_init(&dev->close_list); 1629 1630 __dev_close_many(head); 1631 1632 list_for_each_entry_safe(dev, tmp, head, close_list) { 1633 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); 1634 call_netdevice_notifiers(NETDEV_DOWN, dev); 1635 if (unlink) 1636 list_del_init(&dev->close_list); 1637 } 1638 } 1639 EXPORT_SYMBOL(dev_close_many); 1640 1641 /** 1642 * dev_close - shutdown an interface. 1643 * @dev: device to shutdown 1644 * 1645 * This function moves an active device into down state. A 1646 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device 1647 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier 1648 * chain. 1649 */ 1650 void dev_close(struct net_device *dev) 1651 { 1652 if (dev->flags & IFF_UP) { 1653 LIST_HEAD(single); 1654 1655 list_add(&dev->close_list, &single); 1656 dev_close_many(&single, true); 1657 list_del(&single); 1658 } 1659 } 1660 EXPORT_SYMBOL(dev_close); 1661 1662 1663 /** 1664 * dev_disable_lro - disable Large Receive Offload on a device 1665 * @dev: device 1666 * 1667 * Disable Large Receive Offload (LRO) on a net device. Must be 1668 * called under RTNL. This is needed if received packets may be 1669 * forwarded to another interface. 1670 */ 1671 void dev_disable_lro(struct net_device *dev) 1672 { 1673 struct net_device *lower_dev; 1674 struct list_head *iter; 1675 1676 dev->wanted_features &= ~NETIF_F_LRO; 1677 netdev_update_features(dev); 1678 1679 if (unlikely(dev->features & NETIF_F_LRO)) 1680 netdev_WARN(dev, "failed to disable LRO!\n"); 1681 1682 netdev_for_each_lower_dev(dev, lower_dev, iter) 1683 dev_disable_lro(lower_dev); 1684 } 1685 EXPORT_SYMBOL(dev_disable_lro); 1686 1687 /** 1688 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device 1689 * @dev: device 1690 * 1691 * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be 1692 * called under RTNL. This is needed if Generic XDP is installed on 1693 * the device. 1694 */ 1695 static void dev_disable_gro_hw(struct net_device *dev) 1696 { 1697 dev->wanted_features &= ~NETIF_F_GRO_HW; 1698 netdev_update_features(dev); 1699 1700 if (unlikely(dev->features & NETIF_F_GRO_HW)) 1701 netdev_WARN(dev, "failed to disable GRO_HW!\n"); 1702 } 1703 1704 const char *netdev_cmd_to_name(enum netdev_cmd cmd) 1705 { 1706 #define N(val) \ 1707 case NETDEV_##val: \ 1708 return "NETDEV_" __stringify(val); 1709 switch (cmd) { 1710 N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER) 1711 N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE) 1712 N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE) 1713 N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER) 1714 N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO) 1715 N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO) 1716 N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN) 1717 N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO) 1718 N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO) 1719 N(PRE_CHANGEADDR) 1720 } 1721 #undef N 1722 return "UNKNOWN_NETDEV_EVENT"; 1723 } 1724 EXPORT_SYMBOL_GPL(netdev_cmd_to_name); 1725 1726 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val, 1727 struct net_device *dev) 1728 { 1729 struct netdev_notifier_info info = { 1730 .dev = dev, 1731 }; 1732 1733 return nb->notifier_call(nb, val, &info); 1734 } 1735 1736 static int call_netdevice_register_notifiers(struct notifier_block *nb, 1737 struct net_device *dev) 1738 { 1739 int err; 1740 1741 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev); 1742 err = notifier_to_errno(err); 1743 if (err) 1744 return err; 1745 1746 if (!(dev->flags & IFF_UP)) 1747 return 0; 1748 1749 call_netdevice_notifier(nb, NETDEV_UP, dev); 1750 return 0; 1751 } 1752 1753 static void call_netdevice_unregister_notifiers(struct notifier_block *nb, 1754 struct net_device *dev) 1755 { 1756 if (dev->flags & IFF_UP) { 1757 call_netdevice_notifier(nb, NETDEV_GOING_DOWN, 1758 dev); 1759 call_netdevice_notifier(nb, NETDEV_DOWN, dev); 1760 } 1761 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); 1762 } 1763 1764 static int call_netdevice_register_net_notifiers(struct notifier_block *nb, 1765 struct net *net) 1766 { 1767 struct net_device *dev; 1768 int err; 1769 1770 for_each_netdev(net, dev) { 1771 err = call_netdevice_register_notifiers(nb, dev); 1772 if (err) 1773 goto rollback; 1774 } 1775 return 0; 1776 1777 rollback: 1778 for_each_netdev_continue_reverse(net, dev) 1779 call_netdevice_unregister_notifiers(nb, dev); 1780 return err; 1781 } 1782 1783 static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb, 1784 struct net *net) 1785 { 1786 struct net_device *dev; 1787 1788 for_each_netdev(net, dev) 1789 call_netdevice_unregister_notifiers(nb, dev); 1790 } 1791 1792 static int dev_boot_phase = 1; 1793 1794 /** 1795 * register_netdevice_notifier - register a network notifier block 1796 * @nb: notifier 1797 * 1798 * Register a notifier to be called when network device events occur. 1799 * The notifier passed is linked into the kernel structures and must 1800 * not be reused until it has been unregistered. A negative errno code 1801 * is returned on a failure. 1802 * 1803 * When registered all registration and up events are replayed 1804 * to the new notifier to allow device to have a race free 1805 * view of the network device list. 1806 */ 1807 1808 int register_netdevice_notifier(struct notifier_block *nb) 1809 { 1810 struct net *net; 1811 int err; 1812 1813 /* Close race with setup_net() and cleanup_net() */ 1814 down_write(&pernet_ops_rwsem); 1815 rtnl_lock(); 1816 err = raw_notifier_chain_register(&netdev_chain, nb); 1817 if (err) 1818 goto unlock; 1819 if (dev_boot_phase) 1820 goto unlock; 1821 for_each_net(net) { 1822 err = call_netdevice_register_net_notifiers(nb, net); 1823 if (err) 1824 goto rollback; 1825 } 1826 1827 unlock: 1828 rtnl_unlock(); 1829 up_write(&pernet_ops_rwsem); 1830 return err; 1831 1832 rollback: 1833 for_each_net_continue_reverse(net) 1834 call_netdevice_unregister_net_notifiers(nb, net); 1835 1836 raw_notifier_chain_unregister(&netdev_chain, nb); 1837 goto unlock; 1838 } 1839 EXPORT_SYMBOL(register_netdevice_notifier); 1840 1841 /** 1842 * unregister_netdevice_notifier - unregister a network notifier block 1843 * @nb: notifier 1844 * 1845 * Unregister a notifier previously registered by 1846 * register_netdevice_notifier(). The notifier is unlinked into the 1847 * kernel structures and may then be reused. A negative errno code 1848 * is returned on a failure. 1849 * 1850 * After unregistering unregister and down device events are synthesized 1851 * for all devices on the device list to the removed notifier to remove 1852 * the need for special case cleanup code. 1853 */ 1854 1855 int unregister_netdevice_notifier(struct notifier_block *nb) 1856 { 1857 struct net *net; 1858 int err; 1859 1860 /* Close race with setup_net() and cleanup_net() */ 1861 down_write(&pernet_ops_rwsem); 1862 rtnl_lock(); 1863 err = raw_notifier_chain_unregister(&netdev_chain, nb); 1864 if (err) 1865 goto unlock; 1866 1867 for_each_net(net) 1868 call_netdevice_unregister_net_notifiers(nb, net); 1869 1870 unlock: 1871 rtnl_unlock(); 1872 up_write(&pernet_ops_rwsem); 1873 return err; 1874 } 1875 EXPORT_SYMBOL(unregister_netdevice_notifier); 1876 1877 static int __register_netdevice_notifier_net(struct net *net, 1878 struct notifier_block *nb, 1879 bool ignore_call_fail) 1880 { 1881 int err; 1882 1883 err = raw_notifier_chain_register(&net->netdev_chain, nb); 1884 if (err) 1885 return err; 1886 if (dev_boot_phase) 1887 return 0; 1888 1889 err = call_netdevice_register_net_notifiers(nb, net); 1890 if (err && !ignore_call_fail) 1891 goto chain_unregister; 1892 1893 return 0; 1894 1895 chain_unregister: 1896 raw_notifier_chain_unregister(&net->netdev_chain, nb); 1897 return err; 1898 } 1899 1900 static int __unregister_netdevice_notifier_net(struct net *net, 1901 struct notifier_block *nb) 1902 { 1903 int err; 1904 1905 err = raw_notifier_chain_unregister(&net->netdev_chain, nb); 1906 if (err) 1907 return err; 1908 1909 call_netdevice_unregister_net_notifiers(nb, net); 1910 return 0; 1911 } 1912 1913 /** 1914 * register_netdevice_notifier_net - register a per-netns network notifier block 1915 * @net: network namespace 1916 * @nb: notifier 1917 * 1918 * Register a notifier to be called when network device events occur. 1919 * The notifier passed is linked into the kernel structures and must 1920 * not be reused until it has been unregistered. A negative errno code 1921 * is returned on a failure. 1922 * 1923 * When registered all registration and up events are replayed 1924 * to the new notifier to allow device to have a race free 1925 * view of the network device list. 1926 */ 1927 1928 int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb) 1929 { 1930 int err; 1931 1932 rtnl_lock(); 1933 err = __register_netdevice_notifier_net(net, nb, false); 1934 rtnl_unlock(); 1935 return err; 1936 } 1937 EXPORT_SYMBOL(register_netdevice_notifier_net); 1938 1939 /** 1940 * unregister_netdevice_notifier_net - unregister a per-netns 1941 * network notifier block 1942 * @net: network namespace 1943 * @nb: notifier 1944 * 1945 * Unregister a notifier previously registered by 1946 * register_netdevice_notifier(). The notifier is unlinked into the 1947 * kernel structures and may then be reused. A negative errno code 1948 * is returned on a failure. 1949 * 1950 * After unregistering unregister and down device events are synthesized 1951 * for all devices on the device list to the removed notifier to remove 1952 * the need for special case cleanup code. 1953 */ 1954 1955 int unregister_netdevice_notifier_net(struct net *net, 1956 struct notifier_block *nb) 1957 { 1958 int err; 1959 1960 rtnl_lock(); 1961 err = __unregister_netdevice_notifier_net(net, nb); 1962 rtnl_unlock(); 1963 return err; 1964 } 1965 EXPORT_SYMBOL(unregister_netdevice_notifier_net); 1966 1967 int register_netdevice_notifier_dev_net(struct net_device *dev, 1968 struct notifier_block *nb, 1969 struct netdev_net_notifier *nn) 1970 { 1971 int err; 1972 1973 rtnl_lock(); 1974 err = __register_netdevice_notifier_net(dev_net(dev), nb, false); 1975 if (!err) { 1976 nn->nb = nb; 1977 list_add(&nn->list, &dev->net_notifier_list); 1978 } 1979 rtnl_unlock(); 1980 return err; 1981 } 1982 EXPORT_SYMBOL(register_netdevice_notifier_dev_net); 1983 1984 int unregister_netdevice_notifier_dev_net(struct net_device *dev, 1985 struct notifier_block *nb, 1986 struct netdev_net_notifier *nn) 1987 { 1988 int err; 1989 1990 rtnl_lock(); 1991 list_del(&nn->list); 1992 err = __unregister_netdevice_notifier_net(dev_net(dev), nb); 1993 rtnl_unlock(); 1994 return err; 1995 } 1996 EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net); 1997 1998 static void move_netdevice_notifiers_dev_net(struct net_device *dev, 1999 struct net *net) 2000 { 2001 struct netdev_net_notifier *nn; 2002 2003 list_for_each_entry(nn, &dev->net_notifier_list, list) { 2004 __unregister_netdevice_notifier_net(dev_net(dev), nn->nb); 2005 __register_netdevice_notifier_net(net, nn->nb, true); 2006 } 2007 } 2008 2009 /** 2010 * call_netdevice_notifiers_info - call all network notifier blocks 2011 * @val: value passed unmodified to notifier function 2012 * @info: notifier information data 2013 * 2014 * Call all network notifier blocks. Parameters and return value 2015 * are as for raw_notifier_call_chain(). 2016 */ 2017 2018 static int call_netdevice_notifiers_info(unsigned long val, 2019 struct netdev_notifier_info *info) 2020 { 2021 struct net *net = dev_net(info->dev); 2022 int ret; 2023 2024 ASSERT_RTNL(); 2025 2026 /* Run per-netns notifier block chain first, then run the global one. 2027 * Hopefully, one day, the global one is going to be removed after 2028 * all notifier block registrators get converted to be per-netns. 2029 */ 2030 ret = raw_notifier_call_chain(&net->netdev_chain, val, info); 2031 if (ret & NOTIFY_STOP_MASK) 2032 return ret; 2033 return raw_notifier_call_chain(&netdev_chain, val, info); 2034 } 2035 2036 static int call_netdevice_notifiers_extack(unsigned long val, 2037 struct net_device *dev, 2038 struct netlink_ext_ack *extack) 2039 { 2040 struct netdev_notifier_info info = { 2041 .dev = dev, 2042 .extack = extack, 2043 }; 2044 2045 return call_netdevice_notifiers_info(val, &info); 2046 } 2047 2048 /** 2049 * call_netdevice_notifiers - call all network notifier blocks 2050 * @val: value passed unmodified to notifier function 2051 * @dev: net_device pointer passed unmodified to notifier function 2052 * 2053 * Call all network notifier blocks. Parameters and return value 2054 * are as for raw_notifier_call_chain(). 2055 */ 2056 2057 int call_netdevice_notifiers(unsigned long val, struct net_device *dev) 2058 { 2059 return call_netdevice_notifiers_extack(val, dev, NULL); 2060 } 2061 EXPORT_SYMBOL(call_netdevice_notifiers); 2062 2063 /** 2064 * call_netdevice_notifiers_mtu - call all network notifier blocks 2065 * @val: value passed unmodified to notifier function 2066 * @dev: net_device pointer passed unmodified to notifier function 2067 * @arg: additional u32 argument passed to the notifier function 2068 * 2069 * Call all network notifier blocks. Parameters and return value 2070 * are as for raw_notifier_call_chain(). 2071 */ 2072 static int call_netdevice_notifiers_mtu(unsigned long val, 2073 struct net_device *dev, u32 arg) 2074 { 2075 struct netdev_notifier_info_ext info = { 2076 .info.dev = dev, 2077 .ext.mtu = arg, 2078 }; 2079 2080 BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0); 2081 2082 return call_netdevice_notifiers_info(val, &info.info); 2083 } 2084 2085 #ifdef CONFIG_NET_INGRESS 2086 static DEFINE_STATIC_KEY_FALSE(ingress_needed_key); 2087 2088 void net_inc_ingress_queue(void) 2089 { 2090 static_branch_inc(&ingress_needed_key); 2091 } 2092 EXPORT_SYMBOL_GPL(net_inc_ingress_queue); 2093 2094 void net_dec_ingress_queue(void) 2095 { 2096 static_branch_dec(&ingress_needed_key); 2097 } 2098 EXPORT_SYMBOL_GPL(net_dec_ingress_queue); 2099 #endif 2100 2101 #ifdef CONFIG_NET_EGRESS 2102 static DEFINE_STATIC_KEY_FALSE(egress_needed_key); 2103 2104 void net_inc_egress_queue(void) 2105 { 2106 static_branch_inc(&egress_needed_key); 2107 } 2108 EXPORT_SYMBOL_GPL(net_inc_egress_queue); 2109 2110 void net_dec_egress_queue(void) 2111 { 2112 static_branch_dec(&egress_needed_key); 2113 } 2114 EXPORT_SYMBOL_GPL(net_dec_egress_queue); 2115 #endif 2116 2117 static DEFINE_STATIC_KEY_FALSE(netstamp_needed_key); 2118 #ifdef CONFIG_JUMP_LABEL 2119 static atomic_t netstamp_needed_deferred; 2120 static atomic_t netstamp_wanted; 2121 static void netstamp_clear(struct work_struct *work) 2122 { 2123 int deferred = atomic_xchg(&netstamp_needed_deferred, 0); 2124 int wanted; 2125 2126 wanted = atomic_add_return(deferred, &netstamp_wanted); 2127 if (wanted > 0) 2128 static_branch_enable(&netstamp_needed_key); 2129 else 2130 static_branch_disable(&netstamp_needed_key); 2131 } 2132 static DECLARE_WORK(netstamp_work, netstamp_clear); 2133 #endif 2134 2135 void net_enable_timestamp(void) 2136 { 2137 #ifdef CONFIG_JUMP_LABEL 2138 int wanted; 2139 2140 while (1) { 2141 wanted = atomic_read(&netstamp_wanted); 2142 if (wanted <= 0) 2143 break; 2144 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted) 2145 return; 2146 } 2147 atomic_inc(&netstamp_needed_deferred); 2148 schedule_work(&netstamp_work); 2149 #else 2150 static_branch_inc(&netstamp_needed_key); 2151 #endif 2152 } 2153 EXPORT_SYMBOL(net_enable_timestamp); 2154 2155 void net_disable_timestamp(void) 2156 { 2157 #ifdef CONFIG_JUMP_LABEL 2158 int wanted; 2159 2160 while (1) { 2161 wanted = atomic_read(&netstamp_wanted); 2162 if (wanted <= 1) 2163 break; 2164 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted) 2165 return; 2166 } 2167 atomic_dec(&netstamp_needed_deferred); 2168 schedule_work(&netstamp_work); 2169 #else 2170 static_branch_dec(&netstamp_needed_key); 2171 #endif 2172 } 2173 EXPORT_SYMBOL(net_disable_timestamp); 2174 2175 static inline void net_timestamp_set(struct sk_buff *skb) 2176 { 2177 skb->tstamp = 0; 2178 if (static_branch_unlikely(&netstamp_needed_key)) 2179 __net_timestamp(skb); 2180 } 2181 2182 #define net_timestamp_check(COND, SKB) \ 2183 if (static_branch_unlikely(&netstamp_needed_key)) { \ 2184 if ((COND) && !(SKB)->tstamp) \ 2185 __net_timestamp(SKB); \ 2186 } \ 2187 2188 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb) 2189 { 2190 unsigned int len; 2191 2192 if (!(dev->flags & IFF_UP)) 2193 return false; 2194 2195 len = dev->mtu + dev->hard_header_len + VLAN_HLEN; 2196 if (skb->len <= len) 2197 return true; 2198 2199 /* if TSO is enabled, we don't care about the length as the packet 2200 * could be forwarded without being segmented before 2201 */ 2202 if (skb_is_gso(skb)) 2203 return true; 2204 2205 return false; 2206 } 2207 EXPORT_SYMBOL_GPL(is_skb_forwardable); 2208 2209 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 2210 { 2211 int ret = ____dev_forward_skb(dev, skb); 2212 2213 if (likely(!ret)) { 2214 skb->protocol = eth_type_trans(skb, dev); 2215 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); 2216 } 2217 2218 return ret; 2219 } 2220 EXPORT_SYMBOL_GPL(__dev_forward_skb); 2221 2222 /** 2223 * dev_forward_skb - loopback an skb to another netif 2224 * 2225 * @dev: destination network device 2226 * @skb: buffer to forward 2227 * 2228 * return values: 2229 * NET_RX_SUCCESS (no congestion) 2230 * NET_RX_DROP (packet was dropped, but freed) 2231 * 2232 * dev_forward_skb can be used for injecting an skb from the 2233 * start_xmit function of one device into the receive queue 2234 * of another device. 2235 * 2236 * The receiving device may be in another namespace, so 2237 * we have to clear all information in the skb that could 2238 * impact namespace isolation. 2239 */ 2240 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 2241 { 2242 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb); 2243 } 2244 EXPORT_SYMBOL_GPL(dev_forward_skb); 2245 2246 static inline int deliver_skb(struct sk_buff *skb, 2247 struct packet_type *pt_prev, 2248 struct net_device *orig_dev) 2249 { 2250 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) 2251 return -ENOMEM; 2252 refcount_inc(&skb->users); 2253 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 2254 } 2255 2256 static inline void deliver_ptype_list_skb(struct sk_buff *skb, 2257 struct packet_type **pt, 2258 struct net_device *orig_dev, 2259 __be16 type, 2260 struct list_head *ptype_list) 2261 { 2262 struct packet_type *ptype, *pt_prev = *pt; 2263 2264 list_for_each_entry_rcu(ptype, ptype_list, list) { 2265 if (ptype->type != type) 2266 continue; 2267 if (pt_prev) 2268 deliver_skb(skb, pt_prev, orig_dev); 2269 pt_prev = ptype; 2270 } 2271 *pt = pt_prev; 2272 } 2273 2274 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) 2275 { 2276 if (!ptype->af_packet_priv || !skb->sk) 2277 return false; 2278 2279 if (ptype->id_match) 2280 return ptype->id_match(ptype, skb->sk); 2281 else if ((struct sock *)ptype->af_packet_priv == skb->sk) 2282 return true; 2283 2284 return false; 2285 } 2286 2287 /** 2288 * dev_nit_active - return true if any network interface taps are in use 2289 * 2290 * @dev: network device to check for the presence of taps 2291 */ 2292 bool dev_nit_active(struct net_device *dev) 2293 { 2294 return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all); 2295 } 2296 EXPORT_SYMBOL_GPL(dev_nit_active); 2297 2298 /* 2299 * Support routine. Sends outgoing frames to any network 2300 * taps currently in use. 2301 */ 2302 2303 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) 2304 { 2305 struct packet_type *ptype; 2306 struct sk_buff *skb2 = NULL; 2307 struct packet_type *pt_prev = NULL; 2308 struct list_head *ptype_list = &ptype_all; 2309 2310 rcu_read_lock(); 2311 again: 2312 list_for_each_entry_rcu(ptype, ptype_list, list) { 2313 if (ptype->ignore_outgoing) 2314 continue; 2315 2316 /* Never send packets back to the socket 2317 * they originated from - MvS (miquels@drinkel.ow.org) 2318 */ 2319 if (skb_loop_sk(ptype, skb)) 2320 continue; 2321 2322 if (pt_prev) { 2323 deliver_skb(skb2, pt_prev, skb->dev); 2324 pt_prev = ptype; 2325 continue; 2326 } 2327 2328 /* need to clone skb, done only once */ 2329 skb2 = skb_clone(skb, GFP_ATOMIC); 2330 if (!skb2) 2331 goto out_unlock; 2332 2333 net_timestamp_set(skb2); 2334 2335 /* skb->nh should be correctly 2336 * set by sender, so that the second statement is 2337 * just protection against buggy protocols. 2338 */ 2339 skb_reset_mac_header(skb2); 2340 2341 if (skb_network_header(skb2) < skb2->data || 2342 skb_network_header(skb2) > skb_tail_pointer(skb2)) { 2343 net_crit_ratelimited("protocol %04x is buggy, dev %s\n", 2344 ntohs(skb2->protocol), 2345 dev->name); 2346 skb_reset_network_header(skb2); 2347 } 2348 2349 skb2->transport_header = skb2->network_header; 2350 skb2->pkt_type = PACKET_OUTGOING; 2351 pt_prev = ptype; 2352 } 2353 2354 if (ptype_list == &ptype_all) { 2355 ptype_list = &dev->ptype_all; 2356 goto again; 2357 } 2358 out_unlock: 2359 if (pt_prev) { 2360 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC)) 2361 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); 2362 else 2363 kfree_skb(skb2); 2364 } 2365 rcu_read_unlock(); 2366 } 2367 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit); 2368 2369 /** 2370 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change 2371 * @dev: Network device 2372 * @txq: number of queues available 2373 * 2374 * If real_num_tx_queues is changed the tc mappings may no longer be 2375 * valid. To resolve this verify the tc mapping remains valid and if 2376 * not NULL the mapping. With no priorities mapping to this 2377 * offset/count pair it will no longer be used. In the worst case TC0 2378 * is invalid nothing can be done so disable priority mappings. If is 2379 * expected that drivers will fix this mapping if they can before 2380 * calling netif_set_real_num_tx_queues. 2381 */ 2382 static void netif_setup_tc(struct net_device *dev, unsigned int txq) 2383 { 2384 int i; 2385 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 2386 2387 /* If TC0 is invalidated disable TC mapping */ 2388 if (tc->offset + tc->count > txq) { 2389 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n"); 2390 dev->num_tc = 0; 2391 return; 2392 } 2393 2394 /* Invalidated prio to tc mappings set to TC0 */ 2395 for (i = 1; i < TC_BITMASK + 1; i++) { 2396 int q = netdev_get_prio_tc_map(dev, i); 2397 2398 tc = &dev->tc_to_txq[q]; 2399 if (tc->offset + tc->count > txq) { 2400 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n", 2401 i, q); 2402 netdev_set_prio_tc_map(dev, i, 0); 2403 } 2404 } 2405 } 2406 2407 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq) 2408 { 2409 if (dev->num_tc) { 2410 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 2411 int i; 2412 2413 /* walk through the TCs and see if it falls into any of them */ 2414 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) { 2415 if ((txq - tc->offset) < tc->count) 2416 return i; 2417 } 2418 2419 /* didn't find it, just return -1 to indicate no match */ 2420 return -1; 2421 } 2422 2423 return 0; 2424 } 2425 EXPORT_SYMBOL(netdev_txq_to_tc); 2426 2427 #ifdef CONFIG_XPS 2428 struct static_key xps_needed __read_mostly; 2429 EXPORT_SYMBOL(xps_needed); 2430 struct static_key xps_rxqs_needed __read_mostly; 2431 EXPORT_SYMBOL(xps_rxqs_needed); 2432 static DEFINE_MUTEX(xps_map_mutex); 2433 #define xmap_dereference(P) \ 2434 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex)) 2435 2436 static bool remove_xps_queue(struct xps_dev_maps *dev_maps, 2437 int tci, u16 index) 2438 { 2439 struct xps_map *map = NULL; 2440 int pos; 2441 2442 if (dev_maps) 2443 map = xmap_dereference(dev_maps->attr_map[tci]); 2444 if (!map) 2445 return false; 2446 2447 for (pos = map->len; pos--;) { 2448 if (map->queues[pos] != index) 2449 continue; 2450 2451 if (map->len > 1) { 2452 map->queues[pos] = map->queues[--map->len]; 2453 break; 2454 } 2455 2456 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL); 2457 kfree_rcu(map, rcu); 2458 return false; 2459 } 2460 2461 return true; 2462 } 2463 2464 static bool remove_xps_queue_cpu(struct net_device *dev, 2465 struct xps_dev_maps *dev_maps, 2466 int cpu, u16 offset, u16 count) 2467 { 2468 int num_tc = dev->num_tc ? : 1; 2469 bool active = false; 2470 int tci; 2471 2472 for (tci = cpu * num_tc; num_tc--; tci++) { 2473 int i, j; 2474 2475 for (i = count, j = offset; i--; j++) { 2476 if (!remove_xps_queue(dev_maps, tci, j)) 2477 break; 2478 } 2479 2480 active |= i < 0; 2481 } 2482 2483 return active; 2484 } 2485 2486 static void reset_xps_maps(struct net_device *dev, 2487 struct xps_dev_maps *dev_maps, 2488 bool is_rxqs_map) 2489 { 2490 if (is_rxqs_map) { 2491 static_key_slow_dec_cpuslocked(&xps_rxqs_needed); 2492 RCU_INIT_POINTER(dev->xps_rxqs_map, NULL); 2493 } else { 2494 RCU_INIT_POINTER(dev->xps_cpus_map, NULL); 2495 } 2496 static_key_slow_dec_cpuslocked(&xps_needed); 2497 kfree_rcu(dev_maps, rcu); 2498 } 2499 2500 static void clean_xps_maps(struct net_device *dev, const unsigned long *mask, 2501 struct xps_dev_maps *dev_maps, unsigned int nr_ids, 2502 u16 offset, u16 count, bool is_rxqs_map) 2503 { 2504 bool active = false; 2505 int i, j; 2506 2507 for (j = -1; j = netif_attrmask_next(j, mask, nr_ids), 2508 j < nr_ids;) 2509 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, 2510 count); 2511 if (!active) 2512 reset_xps_maps(dev, dev_maps, is_rxqs_map); 2513 2514 if (!is_rxqs_map) { 2515 for (i = offset + (count - 1); count--; i--) { 2516 netdev_queue_numa_node_write( 2517 netdev_get_tx_queue(dev, i), 2518 NUMA_NO_NODE); 2519 } 2520 } 2521 } 2522 2523 static void netif_reset_xps_queues(struct net_device *dev, u16 offset, 2524 u16 count) 2525 { 2526 const unsigned long *possible_mask = NULL; 2527 struct xps_dev_maps *dev_maps; 2528 unsigned int nr_ids; 2529 2530 if (!static_key_false(&xps_needed)) 2531 return; 2532 2533 cpus_read_lock(); 2534 mutex_lock(&xps_map_mutex); 2535 2536 if (static_key_false(&xps_rxqs_needed)) { 2537 dev_maps = xmap_dereference(dev->xps_rxqs_map); 2538 if (dev_maps) { 2539 nr_ids = dev->num_rx_queues; 2540 clean_xps_maps(dev, possible_mask, dev_maps, nr_ids, 2541 offset, count, true); 2542 } 2543 } 2544 2545 dev_maps = xmap_dereference(dev->xps_cpus_map); 2546 if (!dev_maps) 2547 goto out_no_maps; 2548 2549 if (num_possible_cpus() > 1) 2550 possible_mask = cpumask_bits(cpu_possible_mask); 2551 nr_ids = nr_cpu_ids; 2552 clean_xps_maps(dev, possible_mask, dev_maps, nr_ids, offset, count, 2553 false); 2554 2555 out_no_maps: 2556 mutex_unlock(&xps_map_mutex); 2557 cpus_read_unlock(); 2558 } 2559 2560 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index) 2561 { 2562 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index); 2563 } 2564 2565 static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index, 2566 u16 index, bool is_rxqs_map) 2567 { 2568 struct xps_map *new_map; 2569 int alloc_len = XPS_MIN_MAP_ALLOC; 2570 int i, pos; 2571 2572 for (pos = 0; map && pos < map->len; pos++) { 2573 if (map->queues[pos] != index) 2574 continue; 2575 return map; 2576 } 2577 2578 /* Need to add tx-queue to this CPU's/rx-queue's existing map */ 2579 if (map) { 2580 if (pos < map->alloc_len) 2581 return map; 2582 2583 alloc_len = map->alloc_len * 2; 2584 } 2585 2586 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's 2587 * map 2588 */ 2589 if (is_rxqs_map) 2590 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL); 2591 else 2592 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL, 2593 cpu_to_node(attr_index)); 2594 if (!new_map) 2595 return NULL; 2596 2597 for (i = 0; i < pos; i++) 2598 new_map->queues[i] = map->queues[i]; 2599 new_map->alloc_len = alloc_len; 2600 new_map->len = pos; 2601 2602 return new_map; 2603 } 2604 2605 /* Must be called under cpus_read_lock */ 2606 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask, 2607 u16 index, bool is_rxqs_map) 2608 { 2609 const unsigned long *online_mask = NULL, *possible_mask = NULL; 2610 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL; 2611 int i, j, tci, numa_node_id = -2; 2612 int maps_sz, num_tc = 1, tc = 0; 2613 struct xps_map *map, *new_map; 2614 bool active = false; 2615 unsigned int nr_ids; 2616 2617 if (dev->num_tc) { 2618 /* Do not allow XPS on subordinate device directly */ 2619 num_tc = dev->num_tc; 2620 if (num_tc < 0) 2621 return -EINVAL; 2622 2623 /* If queue belongs to subordinate dev use its map */ 2624 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev; 2625 2626 tc = netdev_txq_to_tc(dev, index); 2627 if (tc < 0) 2628 return -EINVAL; 2629 } 2630 2631 mutex_lock(&xps_map_mutex); 2632 if (is_rxqs_map) { 2633 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues); 2634 dev_maps = xmap_dereference(dev->xps_rxqs_map); 2635 nr_ids = dev->num_rx_queues; 2636 } else { 2637 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc); 2638 if (num_possible_cpus() > 1) { 2639 online_mask = cpumask_bits(cpu_online_mask); 2640 possible_mask = cpumask_bits(cpu_possible_mask); 2641 } 2642 dev_maps = xmap_dereference(dev->xps_cpus_map); 2643 nr_ids = nr_cpu_ids; 2644 } 2645 2646 if (maps_sz < L1_CACHE_BYTES) 2647 maps_sz = L1_CACHE_BYTES; 2648 2649 /* allocate memory for queue storage */ 2650 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids), 2651 j < nr_ids;) { 2652 if (!new_dev_maps) 2653 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL); 2654 if (!new_dev_maps) { 2655 mutex_unlock(&xps_map_mutex); 2656 return -ENOMEM; 2657 } 2658 2659 tci = j * num_tc + tc; 2660 map = dev_maps ? xmap_dereference(dev_maps->attr_map[tci]) : 2661 NULL; 2662 2663 map = expand_xps_map(map, j, index, is_rxqs_map); 2664 if (!map) 2665 goto error; 2666 2667 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); 2668 } 2669 2670 if (!new_dev_maps) 2671 goto out_no_new_maps; 2672 2673 if (!dev_maps) { 2674 /* Increment static keys at most once per type */ 2675 static_key_slow_inc_cpuslocked(&xps_needed); 2676 if (is_rxqs_map) 2677 static_key_slow_inc_cpuslocked(&xps_rxqs_needed); 2678 } 2679 2680 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids), 2681 j < nr_ids;) { 2682 /* copy maps belonging to foreign traffic classes */ 2683 for (i = tc, tci = j * num_tc; dev_maps && i--; tci++) { 2684 /* fill in the new device map from the old device map */ 2685 map = xmap_dereference(dev_maps->attr_map[tci]); 2686 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); 2687 } 2688 2689 /* We need to explicitly update tci as prevous loop 2690 * could break out early if dev_maps is NULL. 2691 */ 2692 tci = j * num_tc + tc; 2693 2694 if (netif_attr_test_mask(j, mask, nr_ids) && 2695 netif_attr_test_online(j, online_mask, nr_ids)) { 2696 /* add tx-queue to CPU/rx-queue maps */ 2697 int pos = 0; 2698 2699 map = xmap_dereference(new_dev_maps->attr_map[tci]); 2700 while ((pos < map->len) && (map->queues[pos] != index)) 2701 pos++; 2702 2703 if (pos == map->len) 2704 map->queues[map->len++] = index; 2705 #ifdef CONFIG_NUMA 2706 if (!is_rxqs_map) { 2707 if (numa_node_id == -2) 2708 numa_node_id = cpu_to_node(j); 2709 else if (numa_node_id != cpu_to_node(j)) 2710 numa_node_id = -1; 2711 } 2712 #endif 2713 } else if (dev_maps) { 2714 /* fill in the new device map from the old device map */ 2715 map = xmap_dereference(dev_maps->attr_map[tci]); 2716 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); 2717 } 2718 2719 /* copy maps belonging to foreign traffic classes */ 2720 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) { 2721 /* fill in the new device map from the old device map */ 2722 map = xmap_dereference(dev_maps->attr_map[tci]); 2723 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); 2724 } 2725 } 2726 2727 if (is_rxqs_map) 2728 rcu_assign_pointer(dev->xps_rxqs_map, new_dev_maps); 2729 else 2730 rcu_assign_pointer(dev->xps_cpus_map, new_dev_maps); 2731 2732 /* Cleanup old maps */ 2733 if (!dev_maps) 2734 goto out_no_old_maps; 2735 2736 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids), 2737 j < nr_ids;) { 2738 for (i = num_tc, tci = j * num_tc; i--; tci++) { 2739 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); 2740 map = xmap_dereference(dev_maps->attr_map[tci]); 2741 if (map && map != new_map) 2742 kfree_rcu(map, rcu); 2743 } 2744 } 2745 2746 kfree_rcu(dev_maps, rcu); 2747 2748 out_no_old_maps: 2749 dev_maps = new_dev_maps; 2750 active = true; 2751 2752 out_no_new_maps: 2753 if (!is_rxqs_map) { 2754 /* update Tx queue numa node */ 2755 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index), 2756 (numa_node_id >= 0) ? 2757 numa_node_id : NUMA_NO_NODE); 2758 } 2759 2760 if (!dev_maps) 2761 goto out_no_maps; 2762 2763 /* removes tx-queue from unused CPUs/rx-queues */ 2764 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids), 2765 j < nr_ids;) { 2766 for (i = tc, tci = j * num_tc; i--; tci++) 2767 active |= remove_xps_queue(dev_maps, tci, index); 2768 if (!netif_attr_test_mask(j, mask, nr_ids) || 2769 !netif_attr_test_online(j, online_mask, nr_ids)) 2770 active |= remove_xps_queue(dev_maps, tci, index); 2771 for (i = num_tc - tc, tci++; --i; tci++) 2772 active |= remove_xps_queue(dev_maps, tci, index); 2773 } 2774 2775 /* free map if not active */ 2776 if (!active) 2777 reset_xps_maps(dev, dev_maps, is_rxqs_map); 2778 2779 out_no_maps: 2780 mutex_unlock(&xps_map_mutex); 2781 2782 return 0; 2783 error: 2784 /* remove any maps that we added */ 2785 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids), 2786 j < nr_ids;) { 2787 for (i = num_tc, tci = j * num_tc; i--; tci++) { 2788 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); 2789 map = dev_maps ? 2790 xmap_dereference(dev_maps->attr_map[tci]) : 2791 NULL; 2792 if (new_map && new_map != map) 2793 kfree(new_map); 2794 } 2795 } 2796 2797 mutex_unlock(&xps_map_mutex); 2798 2799 kfree(new_dev_maps); 2800 return -ENOMEM; 2801 } 2802 EXPORT_SYMBOL_GPL(__netif_set_xps_queue); 2803 2804 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask, 2805 u16 index) 2806 { 2807 int ret; 2808 2809 cpus_read_lock(); 2810 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, false); 2811 cpus_read_unlock(); 2812 2813 return ret; 2814 } 2815 EXPORT_SYMBOL(netif_set_xps_queue); 2816 2817 #endif 2818 static void netdev_unbind_all_sb_channels(struct net_device *dev) 2819 { 2820 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; 2821 2822 /* Unbind any subordinate channels */ 2823 while (txq-- != &dev->_tx[0]) { 2824 if (txq->sb_dev) 2825 netdev_unbind_sb_channel(dev, txq->sb_dev); 2826 } 2827 } 2828 2829 void netdev_reset_tc(struct net_device *dev) 2830 { 2831 #ifdef CONFIG_XPS 2832 netif_reset_xps_queues_gt(dev, 0); 2833 #endif 2834 netdev_unbind_all_sb_channels(dev); 2835 2836 /* Reset TC configuration of device */ 2837 dev->num_tc = 0; 2838 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq)); 2839 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map)); 2840 } 2841 EXPORT_SYMBOL(netdev_reset_tc); 2842 2843 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset) 2844 { 2845 if (tc >= dev->num_tc) 2846 return -EINVAL; 2847 2848 #ifdef CONFIG_XPS 2849 netif_reset_xps_queues(dev, offset, count); 2850 #endif 2851 dev->tc_to_txq[tc].count = count; 2852 dev->tc_to_txq[tc].offset = offset; 2853 return 0; 2854 } 2855 EXPORT_SYMBOL(netdev_set_tc_queue); 2856 2857 int netdev_set_num_tc(struct net_device *dev, u8 num_tc) 2858 { 2859 if (num_tc > TC_MAX_QUEUE) 2860 return -EINVAL; 2861 2862 #ifdef CONFIG_XPS 2863 netif_reset_xps_queues_gt(dev, 0); 2864 #endif 2865 netdev_unbind_all_sb_channels(dev); 2866 2867 dev->num_tc = num_tc; 2868 return 0; 2869 } 2870 EXPORT_SYMBOL(netdev_set_num_tc); 2871 2872 void netdev_unbind_sb_channel(struct net_device *dev, 2873 struct net_device *sb_dev) 2874 { 2875 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; 2876 2877 #ifdef CONFIG_XPS 2878 netif_reset_xps_queues_gt(sb_dev, 0); 2879 #endif 2880 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq)); 2881 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map)); 2882 2883 while (txq-- != &dev->_tx[0]) { 2884 if (txq->sb_dev == sb_dev) 2885 txq->sb_dev = NULL; 2886 } 2887 } 2888 EXPORT_SYMBOL(netdev_unbind_sb_channel); 2889 2890 int netdev_bind_sb_channel_queue(struct net_device *dev, 2891 struct net_device *sb_dev, 2892 u8 tc, u16 count, u16 offset) 2893 { 2894 /* Make certain the sb_dev and dev are already configured */ 2895 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc) 2896 return -EINVAL; 2897 2898 /* We cannot hand out queues we don't have */ 2899 if ((offset + count) > dev->real_num_tx_queues) 2900 return -EINVAL; 2901 2902 /* Record the mapping */ 2903 sb_dev->tc_to_txq[tc].count = count; 2904 sb_dev->tc_to_txq[tc].offset = offset; 2905 2906 /* Provide a way for Tx queue to find the tc_to_txq map or 2907 * XPS map for itself. 2908 */ 2909 while (count--) 2910 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev; 2911 2912 return 0; 2913 } 2914 EXPORT_SYMBOL(netdev_bind_sb_channel_queue); 2915 2916 int netdev_set_sb_channel(struct net_device *dev, u16 channel) 2917 { 2918 /* Do not use a multiqueue device to represent a subordinate channel */ 2919 if (netif_is_multiqueue(dev)) 2920 return -ENODEV; 2921 2922 /* We allow channels 1 - 32767 to be used for subordinate channels. 2923 * Channel 0 is meant to be "native" mode and used only to represent 2924 * the main root device. We allow writing 0 to reset the device back 2925 * to normal mode after being used as a subordinate channel. 2926 */ 2927 if (channel > S16_MAX) 2928 return -EINVAL; 2929 2930 dev->num_tc = -channel; 2931 2932 return 0; 2933 } 2934 EXPORT_SYMBOL(netdev_set_sb_channel); 2935 2936 /* 2937 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues 2938 * greater than real_num_tx_queues stale skbs on the qdisc must be flushed. 2939 */ 2940 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) 2941 { 2942 bool disabling; 2943 int rc; 2944 2945 disabling = txq < dev->real_num_tx_queues; 2946 2947 if (txq < 1 || txq > dev->num_tx_queues) 2948 return -EINVAL; 2949 2950 if (dev->reg_state == NETREG_REGISTERED || 2951 dev->reg_state == NETREG_UNREGISTERING) { 2952 ASSERT_RTNL(); 2953 2954 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, 2955 txq); 2956 if (rc) 2957 return rc; 2958 2959 if (dev->num_tc) 2960 netif_setup_tc(dev, txq); 2961 2962 dev->real_num_tx_queues = txq; 2963 2964 if (disabling) { 2965 synchronize_net(); 2966 qdisc_reset_all_tx_gt(dev, txq); 2967 #ifdef CONFIG_XPS 2968 netif_reset_xps_queues_gt(dev, txq); 2969 #endif 2970 } 2971 } else { 2972 dev->real_num_tx_queues = txq; 2973 } 2974 2975 return 0; 2976 } 2977 EXPORT_SYMBOL(netif_set_real_num_tx_queues); 2978 2979 #ifdef CONFIG_SYSFS 2980 /** 2981 * netif_set_real_num_rx_queues - set actual number of RX queues used 2982 * @dev: Network device 2983 * @rxq: Actual number of RX queues 2984 * 2985 * This must be called either with the rtnl_lock held or before 2986 * registration of the net device. Returns 0 on success, or a 2987 * negative error code. If called before registration, it always 2988 * succeeds. 2989 */ 2990 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) 2991 { 2992 int rc; 2993 2994 if (rxq < 1 || rxq > dev->num_rx_queues) 2995 return -EINVAL; 2996 2997 if (dev->reg_state == NETREG_REGISTERED) { 2998 ASSERT_RTNL(); 2999 3000 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, 3001 rxq); 3002 if (rc) 3003 return rc; 3004 } 3005 3006 dev->real_num_rx_queues = rxq; 3007 return 0; 3008 } 3009 EXPORT_SYMBOL(netif_set_real_num_rx_queues); 3010 #endif 3011 3012 /** 3013 * netif_get_num_default_rss_queues - default number of RSS queues 3014 * 3015 * This routine should set an upper limit on the number of RSS queues 3016 * used by default by multiqueue devices. 3017 */ 3018 int netif_get_num_default_rss_queues(void) 3019 { 3020 return is_kdump_kernel() ? 3021 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus()); 3022 } 3023 EXPORT_SYMBOL(netif_get_num_default_rss_queues); 3024 3025 static void __netif_reschedule(struct Qdisc *q) 3026 { 3027 struct softnet_data *sd; 3028 unsigned long flags; 3029 3030 local_irq_save(flags); 3031 sd = this_cpu_ptr(&softnet_data); 3032 q->next_sched = NULL; 3033 *sd->output_queue_tailp = q; 3034 sd->output_queue_tailp = &q->next_sched; 3035 raise_softirq_irqoff(NET_TX_SOFTIRQ); 3036 local_irq_restore(flags); 3037 } 3038 3039 void __netif_schedule(struct Qdisc *q) 3040 { 3041 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) 3042 __netif_reschedule(q); 3043 } 3044 EXPORT_SYMBOL(__netif_schedule); 3045 3046 struct dev_kfree_skb_cb { 3047 enum skb_free_reason reason; 3048 }; 3049 3050 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb) 3051 { 3052 return (struct dev_kfree_skb_cb *)skb->cb; 3053 } 3054 3055 void netif_schedule_queue(struct netdev_queue *txq) 3056 { 3057 rcu_read_lock(); 3058 if (!netif_xmit_stopped(txq)) { 3059 struct Qdisc *q = rcu_dereference(txq->qdisc); 3060 3061 __netif_schedule(q); 3062 } 3063 rcu_read_unlock(); 3064 } 3065 EXPORT_SYMBOL(netif_schedule_queue); 3066 3067 void netif_tx_wake_queue(struct netdev_queue *dev_queue) 3068 { 3069 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { 3070 struct Qdisc *q; 3071 3072 rcu_read_lock(); 3073 q = rcu_dereference(dev_queue->qdisc); 3074 __netif_schedule(q); 3075 rcu_read_unlock(); 3076 } 3077 } 3078 EXPORT_SYMBOL(netif_tx_wake_queue); 3079 3080 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason) 3081 { 3082 unsigned long flags; 3083 3084 if (unlikely(!skb)) 3085 return; 3086 3087 if (likely(refcount_read(&skb->users) == 1)) { 3088 smp_rmb(); 3089 refcount_set(&skb->users, 0); 3090 } else if (likely(!refcount_dec_and_test(&skb->users))) { 3091 return; 3092 } 3093 get_kfree_skb_cb(skb)->reason = reason; 3094 local_irq_save(flags); 3095 skb->next = __this_cpu_read(softnet_data.completion_queue); 3096 __this_cpu_write(softnet_data.completion_queue, skb); 3097 raise_softirq_irqoff(NET_TX_SOFTIRQ); 3098 local_irq_restore(flags); 3099 } 3100 EXPORT_SYMBOL(__dev_kfree_skb_irq); 3101 3102 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason) 3103 { 3104 if (in_irq() || irqs_disabled()) 3105 __dev_kfree_skb_irq(skb, reason); 3106 else 3107 dev_kfree_skb(skb); 3108 } 3109 EXPORT_SYMBOL(__dev_kfree_skb_any); 3110 3111 3112 /** 3113 * netif_device_detach - mark device as removed 3114 * @dev: network device 3115 * 3116 * Mark device as removed from system and therefore no longer available. 3117 */ 3118 void netif_device_detach(struct net_device *dev) 3119 { 3120 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && 3121 netif_running(dev)) { 3122 netif_tx_stop_all_queues(dev); 3123 } 3124 } 3125 EXPORT_SYMBOL(netif_device_detach); 3126 3127 /** 3128 * netif_device_attach - mark device as attached 3129 * @dev: network device 3130 * 3131 * Mark device as attached from system and restart if needed. 3132 */ 3133 void netif_device_attach(struct net_device *dev) 3134 { 3135 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && 3136 netif_running(dev)) { 3137 netif_tx_wake_all_queues(dev); 3138 __netdev_watchdog_up(dev); 3139 } 3140 } 3141 EXPORT_SYMBOL(netif_device_attach); 3142 3143 /* 3144 * Returns a Tx hash based on the given packet descriptor a Tx queues' number 3145 * to be used as a distribution range. 3146 */ 3147 static u16 skb_tx_hash(const struct net_device *dev, 3148 const struct net_device *sb_dev, 3149 struct sk_buff *skb) 3150 { 3151 u32 hash; 3152 u16 qoffset = 0; 3153 u16 qcount = dev->real_num_tx_queues; 3154 3155 if (dev->num_tc) { 3156 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); 3157 3158 qoffset = sb_dev->tc_to_txq[tc].offset; 3159 qcount = sb_dev->tc_to_txq[tc].count; 3160 } 3161 3162 if (skb_rx_queue_recorded(skb)) { 3163 hash = skb_get_rx_queue(skb); 3164 if (hash >= qoffset) 3165 hash -= qoffset; 3166 while (unlikely(hash >= qcount)) 3167 hash -= qcount; 3168 return hash + qoffset; 3169 } 3170 3171 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset; 3172 } 3173 3174 static void skb_warn_bad_offload(const struct sk_buff *skb) 3175 { 3176 static const netdev_features_t null_features; 3177 struct net_device *dev = skb->dev; 3178 const char *name = ""; 3179 3180 if (!net_ratelimit()) 3181 return; 3182 3183 if (dev) { 3184 if (dev->dev.parent) 3185 name = dev_driver_string(dev->dev.parent); 3186 else 3187 name = netdev_name(dev); 3188 } 3189 skb_dump(KERN_WARNING, skb, false); 3190 WARN(1, "%s: caps=(%pNF, %pNF)\n", 3191 name, dev ? &dev->features : &null_features, 3192 skb->sk ? &skb->sk->sk_route_caps : &null_features); 3193 } 3194 3195 /* 3196 * Invalidate hardware checksum when packet is to be mangled, and 3197 * complete checksum manually on outgoing path. 3198 */ 3199 int skb_checksum_help(struct sk_buff *skb) 3200 { 3201 __wsum csum; 3202 int ret = 0, offset; 3203 3204 if (skb->ip_summed == CHECKSUM_COMPLETE) 3205 goto out_set_summed; 3206 3207 if (unlikely(skb_shinfo(skb)->gso_size)) { 3208 skb_warn_bad_offload(skb); 3209 return -EINVAL; 3210 } 3211 3212 /* Before computing a checksum, we should make sure no frag could 3213 * be modified by an external entity : checksum could be wrong. 3214 */ 3215 if (skb_has_shared_frag(skb)) { 3216 ret = __skb_linearize(skb); 3217 if (ret) 3218 goto out; 3219 } 3220 3221 offset = skb_checksum_start_offset(skb); 3222 BUG_ON(offset >= skb_headlen(skb)); 3223 csum = skb_checksum(skb, offset, skb->len - offset, 0); 3224 3225 offset += skb->csum_offset; 3226 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb)); 3227 3228 ret = skb_ensure_writable(skb, offset + sizeof(__sum16)); 3229 if (ret) 3230 goto out; 3231 3232 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0; 3233 out_set_summed: 3234 skb->ip_summed = CHECKSUM_NONE; 3235 out: 3236 return ret; 3237 } 3238 EXPORT_SYMBOL(skb_checksum_help); 3239 3240 int skb_crc32c_csum_help(struct sk_buff *skb) 3241 { 3242 __le32 crc32c_csum; 3243 int ret = 0, offset, start; 3244 3245 if (skb->ip_summed != CHECKSUM_PARTIAL) 3246 goto out; 3247 3248 if (unlikely(skb_is_gso(skb))) 3249 goto out; 3250 3251 /* Before computing a checksum, we should make sure no frag could 3252 * be modified by an external entity : checksum could be wrong. 3253 */ 3254 if (unlikely(skb_has_shared_frag(skb))) { 3255 ret = __skb_linearize(skb); 3256 if (ret) 3257 goto out; 3258 } 3259 start = skb_checksum_start_offset(skb); 3260 offset = start + offsetof(struct sctphdr, checksum); 3261 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) { 3262 ret = -EINVAL; 3263 goto out; 3264 } 3265 3266 ret = skb_ensure_writable(skb, offset + sizeof(__le32)); 3267 if (ret) 3268 goto out; 3269 3270 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start, 3271 skb->len - start, ~(__u32)0, 3272 crc32c_csum_stub)); 3273 *(__le32 *)(skb->data + offset) = crc32c_csum; 3274 skb->ip_summed = CHECKSUM_NONE; 3275 skb->csum_not_inet = 0; 3276 out: 3277 return ret; 3278 } 3279 3280 __be16 skb_network_protocol(struct sk_buff *skb, int *depth) 3281 { 3282 __be16 type = skb->protocol; 3283 3284 /* Tunnel gso handlers can set protocol to ethernet. */ 3285 if (type == htons(ETH_P_TEB)) { 3286 struct ethhdr *eth; 3287 3288 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr)))) 3289 return 0; 3290 3291 eth = (struct ethhdr *)skb->data; 3292 type = eth->h_proto; 3293 } 3294 3295 return __vlan_get_protocol(skb, type, depth); 3296 } 3297 3298 /** 3299 * skb_mac_gso_segment - mac layer segmentation handler. 3300 * @skb: buffer to segment 3301 * @features: features for the output path (see dev->features) 3302 */ 3303 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, 3304 netdev_features_t features) 3305 { 3306 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); 3307 struct packet_offload *ptype; 3308 int vlan_depth = skb->mac_len; 3309 __be16 type = skb_network_protocol(skb, &vlan_depth); 3310 3311 if (unlikely(!type)) 3312 return ERR_PTR(-EINVAL); 3313 3314 __skb_pull(skb, vlan_depth); 3315 3316 rcu_read_lock(); 3317 list_for_each_entry_rcu(ptype, &offload_base, list) { 3318 if (ptype->type == type && ptype->callbacks.gso_segment) { 3319 segs = ptype->callbacks.gso_segment(skb, features); 3320 break; 3321 } 3322 } 3323 rcu_read_unlock(); 3324 3325 __skb_push(skb, skb->data - skb_mac_header(skb)); 3326 3327 return segs; 3328 } 3329 EXPORT_SYMBOL(skb_mac_gso_segment); 3330 3331 3332 /* openvswitch calls this on rx path, so we need a different check. 3333 */ 3334 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) 3335 { 3336 if (tx_path) 3337 return skb->ip_summed != CHECKSUM_PARTIAL && 3338 skb->ip_summed != CHECKSUM_UNNECESSARY; 3339 3340 return skb->ip_summed == CHECKSUM_NONE; 3341 } 3342 3343 /** 3344 * __skb_gso_segment - Perform segmentation on skb. 3345 * @skb: buffer to segment 3346 * @features: features for the output path (see dev->features) 3347 * @tx_path: whether it is called in TX path 3348 * 3349 * This function segments the given skb and returns a list of segments. 3350 * 3351 * It may return NULL if the skb requires no segmentation. This is 3352 * only possible when GSO is used for verifying header integrity. 3353 * 3354 * Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb. 3355 */ 3356 struct sk_buff *__skb_gso_segment(struct sk_buff *skb, 3357 netdev_features_t features, bool tx_path) 3358 { 3359 struct sk_buff *segs; 3360 3361 if (unlikely(skb_needs_check(skb, tx_path))) { 3362 int err; 3363 3364 /* We're going to init ->check field in TCP or UDP header */ 3365 err = skb_cow_head(skb, 0); 3366 if (err < 0) 3367 return ERR_PTR(err); 3368 } 3369 3370 /* Only report GSO partial support if it will enable us to 3371 * support segmentation on this frame without needing additional 3372 * work. 3373 */ 3374 if (features & NETIF_F_GSO_PARTIAL) { 3375 netdev_features_t partial_features = NETIF_F_GSO_ROBUST; 3376 struct net_device *dev = skb->dev; 3377 3378 partial_features |= dev->features & dev->gso_partial_features; 3379 if (!skb_gso_ok(skb, features | partial_features)) 3380 features &= ~NETIF_F_GSO_PARTIAL; 3381 } 3382 3383 BUILD_BUG_ON(SKB_GSO_CB_OFFSET + 3384 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb)); 3385 3386 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb); 3387 SKB_GSO_CB(skb)->encap_level = 0; 3388 3389 skb_reset_mac_header(skb); 3390 skb_reset_mac_len(skb); 3391 3392 segs = skb_mac_gso_segment(skb, features); 3393 3394 if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs))) 3395 skb_warn_bad_offload(skb); 3396 3397 return segs; 3398 } 3399 EXPORT_SYMBOL(__skb_gso_segment); 3400 3401 /* Take action when hardware reception checksum errors are detected. */ 3402 #ifdef CONFIG_BUG 3403 void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb) 3404 { 3405 if (net_ratelimit()) { 3406 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>"); 3407 skb_dump(KERN_ERR, skb, true); 3408 dump_stack(); 3409 } 3410 } 3411 EXPORT_SYMBOL(netdev_rx_csum_fault); 3412 #endif 3413 3414 /* XXX: check that highmem exists at all on the given machine. */ 3415 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) 3416 { 3417 #ifdef CONFIG_HIGHMEM 3418 int i; 3419 3420 if (!(dev->features & NETIF_F_HIGHDMA)) { 3421 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3422 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3423 3424 if (PageHighMem(skb_frag_page(frag))) 3425 return 1; 3426 } 3427 } 3428 #endif 3429 return 0; 3430 } 3431 3432 /* If MPLS offload request, verify we are testing hardware MPLS features 3433 * instead of standard features for the netdev. 3434 */ 3435 #if IS_ENABLED(CONFIG_NET_MPLS_GSO) 3436 static netdev_features_t net_mpls_features(struct sk_buff *skb, 3437 netdev_features_t features, 3438 __be16 type) 3439 { 3440 if (eth_p_mpls(type)) 3441 features &= skb->dev->mpls_features; 3442 3443 return features; 3444 } 3445 #else 3446 static netdev_features_t net_mpls_features(struct sk_buff *skb, 3447 netdev_features_t features, 3448 __be16 type) 3449 { 3450 return features; 3451 } 3452 #endif 3453 3454 static netdev_features_t harmonize_features(struct sk_buff *skb, 3455 netdev_features_t features) 3456 { 3457 int tmp; 3458 __be16 type; 3459 3460 type = skb_network_protocol(skb, &tmp); 3461 features = net_mpls_features(skb, features, type); 3462 3463 if (skb->ip_summed != CHECKSUM_NONE && 3464 !can_checksum_protocol(features, type)) { 3465 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 3466 } 3467 if (illegal_highdma(skb->dev, skb)) 3468 features &= ~NETIF_F_SG; 3469 3470 return features; 3471 } 3472 3473 netdev_features_t passthru_features_check(struct sk_buff *skb, 3474 struct net_device *dev, 3475 netdev_features_t features) 3476 { 3477 return features; 3478 } 3479 EXPORT_SYMBOL(passthru_features_check); 3480 3481 static netdev_features_t dflt_features_check(struct sk_buff *skb, 3482 struct net_device *dev, 3483 netdev_features_t features) 3484 { 3485 return vlan_features_check(skb, features); 3486 } 3487 3488 static netdev_features_t gso_features_check(const struct sk_buff *skb, 3489 struct net_device *dev, 3490 netdev_features_t features) 3491 { 3492 u16 gso_segs = skb_shinfo(skb)->gso_segs; 3493 3494 if (gso_segs > dev->gso_max_segs) 3495 return features & ~NETIF_F_GSO_MASK; 3496 3497 /* Support for GSO partial features requires software 3498 * intervention before we can actually process the packets 3499 * so we need to strip support for any partial features now 3500 * and we can pull them back in after we have partially 3501 * segmented the frame. 3502 */ 3503 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)) 3504 features &= ~dev->gso_partial_features; 3505 3506 /* Make sure to clear the IPv4 ID mangling feature if the 3507 * IPv4 header has the potential to be fragmented. 3508 */ 3509 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { 3510 struct iphdr *iph = skb->encapsulation ? 3511 inner_ip_hdr(skb) : ip_hdr(skb); 3512 3513 if (!(iph->frag_off & htons(IP_DF))) 3514 features &= ~NETIF_F_TSO_MANGLEID; 3515 } 3516 3517 return features; 3518 } 3519 3520 netdev_features_t netif_skb_features(struct sk_buff *skb) 3521 { 3522 struct net_device *dev = skb->dev; 3523 netdev_features_t features = dev->features; 3524 3525 if (skb_is_gso(skb)) 3526 features = gso_features_check(skb, dev, features); 3527 3528 /* If encapsulation offload request, verify we are testing 3529 * hardware encapsulation features instead of standard 3530 * features for the netdev 3531 */ 3532 if (skb->encapsulation) 3533 features &= dev->hw_enc_features; 3534 3535 if (skb_vlan_tagged(skb)) 3536 features = netdev_intersect_features(features, 3537 dev->vlan_features | 3538 NETIF_F_HW_VLAN_CTAG_TX | 3539 NETIF_F_HW_VLAN_STAG_TX); 3540 3541 if (dev->netdev_ops->ndo_features_check) 3542 features &= dev->netdev_ops->ndo_features_check(skb, dev, 3543 features); 3544 else 3545 features &= dflt_features_check(skb, dev, features); 3546 3547 return harmonize_features(skb, features); 3548 } 3549 EXPORT_SYMBOL(netif_skb_features); 3550 3551 static int xmit_one(struct sk_buff *skb, struct net_device *dev, 3552 struct netdev_queue *txq, bool more) 3553 { 3554 unsigned int len; 3555 int rc; 3556 3557 if (dev_nit_active(dev)) 3558 dev_queue_xmit_nit(skb, dev); 3559 3560 len = skb->len; 3561 trace_net_dev_start_xmit(skb, dev); 3562 rc = netdev_start_xmit(skb, dev, txq, more); 3563 trace_net_dev_xmit(skb, rc, dev, len); 3564 3565 return rc; 3566 } 3567 3568 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev, 3569 struct netdev_queue *txq, int *ret) 3570 { 3571 struct sk_buff *skb = first; 3572 int rc = NETDEV_TX_OK; 3573 3574 while (skb) { 3575 struct sk_buff *next = skb->next; 3576 3577 skb_mark_not_on_list(skb); 3578 rc = xmit_one(skb, dev, txq, next != NULL); 3579 if (unlikely(!dev_xmit_complete(rc))) { 3580 skb->next = next; 3581 goto out; 3582 } 3583 3584 skb = next; 3585 if (netif_tx_queue_stopped(txq) && skb) { 3586 rc = NETDEV_TX_BUSY; 3587 break; 3588 } 3589 } 3590 3591 out: 3592 *ret = rc; 3593 return skb; 3594 } 3595 3596 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, 3597 netdev_features_t features) 3598 { 3599 if (skb_vlan_tag_present(skb) && 3600 !vlan_hw_offload_capable(features, skb->vlan_proto)) 3601 skb = __vlan_hwaccel_push_inside(skb); 3602 return skb; 3603 } 3604 3605 int skb_csum_hwoffload_help(struct sk_buff *skb, 3606 const netdev_features_t features) 3607 { 3608 if (unlikely(skb->csum_not_inet)) 3609 return !!(features & NETIF_F_SCTP_CRC) ? 0 : 3610 skb_crc32c_csum_help(skb); 3611 3612 return !!(features & NETIF_F_CSUM_MASK) ? 0 : skb_checksum_help(skb); 3613 } 3614 EXPORT_SYMBOL(skb_csum_hwoffload_help); 3615 3616 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again) 3617 { 3618 netdev_features_t features; 3619 3620 features = netif_skb_features(skb); 3621 skb = validate_xmit_vlan(skb, features); 3622 if (unlikely(!skb)) 3623 goto out_null; 3624 3625 skb = sk_validate_xmit_skb(skb, dev); 3626 if (unlikely(!skb)) 3627 goto out_null; 3628 3629 if (netif_needs_gso(skb, features)) { 3630 struct sk_buff *segs; 3631 3632 segs = skb_gso_segment(skb, features); 3633 if (IS_ERR(segs)) { 3634 goto out_kfree_skb; 3635 } else if (segs) { 3636 consume_skb(skb); 3637 skb = segs; 3638 } 3639 } else { 3640 if (skb_needs_linearize(skb, features) && 3641 __skb_linearize(skb)) 3642 goto out_kfree_skb; 3643 3644 /* If packet is not checksummed and device does not 3645 * support checksumming for this protocol, complete 3646 * checksumming here. 3647 */ 3648 if (skb->ip_summed == CHECKSUM_PARTIAL) { 3649 if (skb->encapsulation) 3650 skb_set_inner_transport_header(skb, 3651 skb_checksum_start_offset(skb)); 3652 else 3653 skb_set_transport_header(skb, 3654 skb_checksum_start_offset(skb)); 3655 if (skb_csum_hwoffload_help(skb, features)) 3656 goto out_kfree_skb; 3657 } 3658 } 3659 3660 skb = validate_xmit_xfrm(skb, features, again); 3661 3662 return skb; 3663 3664 out_kfree_skb: 3665 kfree_skb(skb); 3666 out_null: 3667 atomic_long_inc(&dev->tx_dropped); 3668 return NULL; 3669 } 3670 3671 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again) 3672 { 3673 struct sk_buff *next, *head = NULL, *tail; 3674 3675 for (; skb != NULL; skb = next) { 3676 next = skb->next; 3677 skb_mark_not_on_list(skb); 3678 3679 /* in case skb wont be segmented, point to itself */ 3680 skb->prev = skb; 3681 3682 skb = validate_xmit_skb(skb, dev, again); 3683 if (!skb) 3684 continue; 3685 3686 if (!head) 3687 head = skb; 3688 else 3689 tail->next = skb; 3690 /* If skb was segmented, skb->prev points to 3691 * the last segment. If not, it still contains skb. 3692 */ 3693 tail = skb->prev; 3694 } 3695 return head; 3696 } 3697 EXPORT_SYMBOL_GPL(validate_xmit_skb_list); 3698 3699 static void qdisc_pkt_len_init(struct sk_buff *skb) 3700 { 3701 const struct skb_shared_info *shinfo = skb_shinfo(skb); 3702 3703 qdisc_skb_cb(skb)->pkt_len = skb->len; 3704 3705 /* To get more precise estimation of bytes sent on wire, 3706 * we add to pkt_len the headers size of all segments 3707 */ 3708 if (shinfo->gso_size && skb_transport_header_was_set(skb)) { 3709 unsigned int hdr_len; 3710 u16 gso_segs = shinfo->gso_segs; 3711 3712 /* mac layer + network layer */ 3713 hdr_len = skb_transport_header(skb) - skb_mac_header(skb); 3714 3715 /* + transport layer */ 3716 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { 3717 const struct tcphdr *th; 3718 struct tcphdr _tcphdr; 3719 3720 th = skb_header_pointer(skb, skb_transport_offset(skb), 3721 sizeof(_tcphdr), &_tcphdr); 3722 if (likely(th)) 3723 hdr_len += __tcp_hdrlen(th); 3724 } else { 3725 struct udphdr _udphdr; 3726 3727 if (skb_header_pointer(skb, skb_transport_offset(skb), 3728 sizeof(_udphdr), &_udphdr)) 3729 hdr_len += sizeof(struct udphdr); 3730 } 3731 3732 if (shinfo->gso_type & SKB_GSO_DODGY) 3733 gso_segs = DIV_ROUND_UP(skb->len - hdr_len, 3734 shinfo->gso_size); 3735 3736 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; 3737 } 3738 } 3739 3740 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 3741 struct net_device *dev, 3742 struct netdev_queue *txq) 3743 { 3744 spinlock_t *root_lock = qdisc_lock(q); 3745 struct sk_buff *to_free = NULL; 3746 bool contended; 3747 int rc; 3748 3749 qdisc_calculate_pkt_len(skb, q); 3750 3751 if (q->flags & TCQ_F_NOLOCK) { 3752 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK; 3753 qdisc_run(q); 3754 3755 if (unlikely(to_free)) 3756 kfree_skb_list(to_free); 3757 return rc; 3758 } 3759 3760 /* 3761 * Heuristic to force contended enqueues to serialize on a 3762 * separate lock before trying to get qdisc main lock. 3763 * This permits qdisc->running owner to get the lock more 3764 * often and dequeue packets faster. 3765 */ 3766 contended = qdisc_is_running(q); 3767 if (unlikely(contended)) 3768 spin_lock(&q->busylock); 3769 3770 spin_lock(root_lock); 3771 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { 3772 __qdisc_drop(skb, &to_free); 3773 rc = NET_XMIT_DROP; 3774 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && 3775 qdisc_run_begin(q)) { 3776 /* 3777 * This is a work-conserving queue; there are no old skbs 3778 * waiting to be sent out; and the qdisc is not running - 3779 * xmit the skb directly. 3780 */ 3781 3782 qdisc_bstats_update(q, skb); 3783 3784 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) { 3785 if (unlikely(contended)) { 3786 spin_unlock(&q->busylock); 3787 contended = false; 3788 } 3789 __qdisc_run(q); 3790 } 3791 3792 qdisc_run_end(q); 3793 rc = NET_XMIT_SUCCESS; 3794 } else { 3795 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK; 3796 if (qdisc_run_begin(q)) { 3797 if (unlikely(contended)) { 3798 spin_unlock(&q->busylock); 3799 contended = false; 3800 } 3801 __qdisc_run(q); 3802 qdisc_run_end(q); 3803 } 3804 } 3805 spin_unlock(root_lock); 3806 if (unlikely(to_free)) 3807 kfree_skb_list(to_free); 3808 if (unlikely(contended)) 3809 spin_unlock(&q->busylock); 3810 return rc; 3811 } 3812 3813 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO) 3814 static void skb_update_prio(struct sk_buff *skb) 3815 { 3816 const struct netprio_map *map; 3817 const struct sock *sk; 3818 unsigned int prioidx; 3819 3820 if (skb->priority) 3821 return; 3822 map = rcu_dereference_bh(skb->dev->priomap); 3823 if (!map) 3824 return; 3825 sk = skb_to_full_sk(skb); 3826 if (!sk) 3827 return; 3828 3829 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data); 3830 3831 if (prioidx < map->priomap_len) 3832 skb->priority = map->priomap[prioidx]; 3833 } 3834 #else 3835 #define skb_update_prio(skb) 3836 #endif 3837 3838 /** 3839 * dev_loopback_xmit - loop back @skb 3840 * @net: network namespace this loopback is happening in 3841 * @sk: sk needed to be a netfilter okfn 3842 * @skb: buffer to transmit 3843 */ 3844 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb) 3845 { 3846 skb_reset_mac_header(skb); 3847 __skb_pull(skb, skb_network_offset(skb)); 3848 skb->pkt_type = PACKET_LOOPBACK; 3849 skb->ip_summed = CHECKSUM_UNNECESSARY; 3850 WARN_ON(!skb_dst(skb)); 3851 skb_dst_force(skb); 3852 netif_rx_ni(skb); 3853 return 0; 3854 } 3855 EXPORT_SYMBOL(dev_loopback_xmit); 3856 3857 #ifdef CONFIG_NET_EGRESS 3858 static struct sk_buff * 3859 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) 3860 { 3861 struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress); 3862 struct tcf_result cl_res; 3863 3864 if (!miniq) 3865 return skb; 3866 3867 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */ 3868 mini_qdisc_bstats_cpu_update(miniq, skb); 3869 3870 switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) { 3871 case TC_ACT_OK: 3872 case TC_ACT_RECLASSIFY: 3873 skb->tc_index = TC_H_MIN(cl_res.classid); 3874 break; 3875 case TC_ACT_SHOT: 3876 mini_qdisc_qstats_cpu_drop(miniq); 3877 *ret = NET_XMIT_DROP; 3878 kfree_skb(skb); 3879 return NULL; 3880 case TC_ACT_STOLEN: 3881 case TC_ACT_QUEUED: 3882 case TC_ACT_TRAP: 3883 *ret = NET_XMIT_SUCCESS; 3884 consume_skb(skb); 3885 return NULL; 3886 case TC_ACT_REDIRECT: 3887 /* No need to push/pop skb's mac_header here on egress! */ 3888 skb_do_redirect(skb); 3889 *ret = NET_XMIT_SUCCESS; 3890 return NULL; 3891 default: 3892 break; 3893 } 3894 3895 return skb; 3896 } 3897 #endif /* CONFIG_NET_EGRESS */ 3898 3899 #ifdef CONFIG_XPS 3900 static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb, 3901 struct xps_dev_maps *dev_maps, unsigned int tci) 3902 { 3903 struct xps_map *map; 3904 int queue_index = -1; 3905 3906 if (dev->num_tc) { 3907 tci *= dev->num_tc; 3908 tci += netdev_get_prio_tc_map(dev, skb->priority); 3909 } 3910 3911 map = rcu_dereference(dev_maps->attr_map[tci]); 3912 if (map) { 3913 if (map->len == 1) 3914 queue_index = map->queues[0]; 3915 else 3916 queue_index = map->queues[reciprocal_scale( 3917 skb_get_hash(skb), map->len)]; 3918 if (unlikely(queue_index >= dev->real_num_tx_queues)) 3919 queue_index = -1; 3920 } 3921 return queue_index; 3922 } 3923 #endif 3924 3925 static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev, 3926 struct sk_buff *skb) 3927 { 3928 #ifdef CONFIG_XPS 3929 struct xps_dev_maps *dev_maps; 3930 struct sock *sk = skb->sk; 3931 int queue_index = -1; 3932 3933 if (!static_key_false(&xps_needed)) 3934 return -1; 3935 3936 rcu_read_lock(); 3937 if (!static_key_false(&xps_rxqs_needed)) 3938 goto get_cpus_map; 3939 3940 dev_maps = rcu_dereference(sb_dev->xps_rxqs_map); 3941 if (dev_maps) { 3942 int tci = sk_rx_queue_get(sk); 3943 3944 if (tci >= 0 && tci < dev->num_rx_queues) 3945 queue_index = __get_xps_queue_idx(dev, skb, dev_maps, 3946 tci); 3947 } 3948 3949 get_cpus_map: 3950 if (queue_index < 0) { 3951 dev_maps = rcu_dereference(sb_dev->xps_cpus_map); 3952 if (dev_maps) { 3953 unsigned int tci = skb->sender_cpu - 1; 3954 3955 queue_index = __get_xps_queue_idx(dev, skb, dev_maps, 3956 tci); 3957 } 3958 } 3959 rcu_read_unlock(); 3960 3961 return queue_index; 3962 #else 3963 return -1; 3964 #endif 3965 } 3966 3967 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb, 3968 struct net_device *sb_dev) 3969 { 3970 return 0; 3971 } 3972 EXPORT_SYMBOL(dev_pick_tx_zero); 3973 3974 u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb, 3975 struct net_device *sb_dev) 3976 { 3977 return (u16)raw_smp_processor_id() % dev->real_num_tx_queues; 3978 } 3979 EXPORT_SYMBOL(dev_pick_tx_cpu_id); 3980 3981 u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb, 3982 struct net_device *sb_dev) 3983 { 3984 struct sock *sk = skb->sk; 3985 int queue_index = sk_tx_queue_get(sk); 3986 3987 sb_dev = sb_dev ? : dev; 3988 3989 if (queue_index < 0 || skb->ooo_okay || 3990 queue_index >= dev->real_num_tx_queues) { 3991 int new_index = get_xps_queue(dev, sb_dev, skb); 3992 3993 if (new_index < 0) 3994 new_index = skb_tx_hash(dev, sb_dev, skb); 3995 3996 if (queue_index != new_index && sk && 3997 sk_fullsock(sk) && 3998 rcu_access_pointer(sk->sk_dst_cache)) 3999 sk_tx_queue_set(sk, new_index); 4000 4001 queue_index = new_index; 4002 } 4003 4004 return queue_index; 4005 } 4006 EXPORT_SYMBOL(netdev_pick_tx); 4007 4008 struct netdev_queue *netdev_core_pick_tx(struct net_device *dev, 4009 struct sk_buff *skb, 4010 struct net_device *sb_dev) 4011 { 4012 int queue_index = 0; 4013 4014 #ifdef CONFIG_XPS 4015 u32 sender_cpu = skb->sender_cpu - 1; 4016 4017 if (sender_cpu >= (u32)NR_CPUS) 4018 skb->sender_cpu = raw_smp_processor_id() + 1; 4019 #endif 4020 4021 if (dev->real_num_tx_queues != 1) { 4022 const struct net_device_ops *ops = dev->netdev_ops; 4023 4024 if (ops->ndo_select_queue) 4025 queue_index = ops->ndo_select_queue(dev, skb, sb_dev); 4026 else 4027 queue_index = netdev_pick_tx(dev, skb, sb_dev); 4028 4029 queue_index = netdev_cap_txqueue(dev, queue_index); 4030 } 4031 4032 skb_set_queue_mapping(skb, queue_index); 4033 return netdev_get_tx_queue(dev, queue_index); 4034 } 4035 4036 /** 4037 * __dev_queue_xmit - transmit a buffer 4038 * @skb: buffer to transmit 4039 * @sb_dev: suboordinate device used for L2 forwarding offload 4040 * 4041 * Queue a buffer for transmission to a network device. The caller must 4042 * have set the device and priority and built the buffer before calling 4043 * this function. The function can be called from an interrupt. 4044 * 4045 * A negative errno code is returned on a failure. A success does not 4046 * guarantee the frame will be transmitted as it may be dropped due 4047 * to congestion or traffic shaping. 4048 * 4049 * ----------------------------------------------------------------------------------- 4050 * I notice this method can also return errors from the queue disciplines, 4051 * including NET_XMIT_DROP, which is a positive value. So, errors can also 4052 * be positive. 4053 * 4054 * Regardless of the return value, the skb is consumed, so it is currently 4055 * difficult to retry a send to this method. (You can bump the ref count 4056 * before sending to hold a reference for retry if you are careful.) 4057 * 4058 * When calling this method, interrupts MUST be enabled. This is because 4059 * the BH enable code must have IRQs enabled so that it will not deadlock. 4060 * --BLG 4061 */ 4062 static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) 4063 { 4064 struct net_device *dev = skb->dev; 4065 struct netdev_queue *txq; 4066 struct Qdisc *q; 4067 int rc = -ENOMEM; 4068 bool again = false; 4069 4070 skb_reset_mac_header(skb); 4071 4072 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP)) 4073 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED); 4074 4075 /* Disable soft irqs for various locks below. Also 4076 * stops preemption for RCU. 4077 */ 4078 rcu_read_lock_bh(); 4079 4080 skb_update_prio(skb); 4081 4082 qdisc_pkt_len_init(skb); 4083 #ifdef CONFIG_NET_CLS_ACT 4084 skb->tc_at_ingress = 0; 4085 # ifdef CONFIG_NET_EGRESS 4086 if (static_branch_unlikely(&egress_needed_key)) { 4087 skb = sch_handle_egress(skb, &rc, dev); 4088 if (!skb) 4089 goto out; 4090 } 4091 # endif 4092 #endif 4093 /* If device/qdisc don't need skb->dst, release it right now while 4094 * its hot in this cpu cache. 4095 */ 4096 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) 4097 skb_dst_drop(skb); 4098 else 4099 skb_dst_force(skb); 4100 4101 txq = netdev_core_pick_tx(dev, skb, sb_dev); 4102 q = rcu_dereference_bh(txq->qdisc); 4103 4104 trace_net_dev_queue(skb); 4105 if (q->enqueue) { 4106 rc = __dev_xmit_skb(skb, q, dev, txq); 4107 goto out; 4108 } 4109 4110 /* The device has no queue. Common case for software devices: 4111 * loopback, all the sorts of tunnels... 4112 4113 * Really, it is unlikely that netif_tx_lock protection is necessary 4114 * here. (f.e. loopback and IP tunnels are clean ignoring statistics 4115 * counters.) 4116 * However, it is possible, that they rely on protection 4117 * made by us here. 4118 4119 * Check this and shot the lock. It is not prone from deadlocks. 4120 *Either shot noqueue qdisc, it is even simpler 8) 4121 */ 4122 if (dev->flags & IFF_UP) { 4123 int cpu = smp_processor_id(); /* ok because BHs are off */ 4124 4125 if (txq->xmit_lock_owner != cpu) { 4126 if (dev_xmit_recursion()) 4127 goto recursion_alert; 4128 4129 skb = validate_xmit_skb(skb, dev, &again); 4130 if (!skb) 4131 goto out; 4132 4133 HARD_TX_LOCK(dev, txq, cpu); 4134 4135 if (!netif_xmit_stopped(txq)) { 4136 dev_xmit_recursion_inc(); 4137 skb = dev_hard_start_xmit(skb, dev, txq, &rc); 4138 dev_xmit_recursion_dec(); 4139 if (dev_xmit_complete(rc)) { 4140 HARD_TX_UNLOCK(dev, txq); 4141 goto out; 4142 } 4143 } 4144 HARD_TX_UNLOCK(dev, txq); 4145 net_crit_ratelimited("Virtual device %s asks to queue packet!\n", 4146 dev->name); 4147 } else { 4148 /* Recursion is detected! It is possible, 4149 * unfortunately 4150 */ 4151 recursion_alert: 4152 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", 4153 dev->name); 4154 } 4155 } 4156 4157 rc = -ENETDOWN; 4158 rcu_read_unlock_bh(); 4159 4160 atomic_long_inc(&dev->tx_dropped); 4161 kfree_skb_list(skb); 4162 return rc; 4163 out: 4164 rcu_read_unlock_bh(); 4165 return rc; 4166 } 4167 4168 int dev_queue_xmit(struct sk_buff *skb) 4169 { 4170 return __dev_queue_xmit(skb, NULL); 4171 } 4172 EXPORT_SYMBOL(dev_queue_xmit); 4173 4174 int dev_queue_xmit_accel(struct sk_buff *skb, struct net_device *sb_dev) 4175 { 4176 return __dev_queue_xmit(skb, sb_dev); 4177 } 4178 EXPORT_SYMBOL(dev_queue_xmit_accel); 4179 4180 int dev_direct_xmit(struct sk_buff *skb, u16 queue_id) 4181 { 4182 struct net_device *dev = skb->dev; 4183 struct sk_buff *orig_skb = skb; 4184 struct netdev_queue *txq; 4185 int ret = NETDEV_TX_BUSY; 4186 bool again = false; 4187 4188 if (unlikely(!netif_running(dev) || 4189 !netif_carrier_ok(dev))) 4190 goto drop; 4191 4192 skb = validate_xmit_skb_list(skb, dev, &again); 4193 if (skb != orig_skb) 4194 goto drop; 4195 4196 skb_set_queue_mapping(skb, queue_id); 4197 txq = skb_get_tx_queue(dev, skb); 4198 4199 local_bh_disable(); 4200 4201 dev_xmit_recursion_inc(); 4202 HARD_TX_LOCK(dev, txq, smp_processor_id()); 4203 if (!netif_xmit_frozen_or_drv_stopped(txq)) 4204 ret = netdev_start_xmit(skb, dev, txq, false); 4205 HARD_TX_UNLOCK(dev, txq); 4206 dev_xmit_recursion_dec(); 4207 4208 local_bh_enable(); 4209 4210 if (!dev_xmit_complete(ret)) 4211 kfree_skb(skb); 4212 4213 return ret; 4214 drop: 4215 atomic_long_inc(&dev->tx_dropped); 4216 kfree_skb_list(skb); 4217 return NET_XMIT_DROP; 4218 } 4219 EXPORT_SYMBOL(dev_direct_xmit); 4220 4221 /************************************************************************* 4222 * Receiver routines 4223 *************************************************************************/ 4224 4225 int netdev_max_backlog __read_mostly = 1000; 4226 EXPORT_SYMBOL(netdev_max_backlog); 4227 4228 int netdev_tstamp_prequeue __read_mostly = 1; 4229 int netdev_budget __read_mostly = 300; 4230 /* Must be at least 2 jiffes to guarantee 1 jiffy timeout */ 4231 unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ; 4232 int weight_p __read_mostly = 64; /* old backlog weight */ 4233 int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */ 4234 int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */ 4235 int dev_rx_weight __read_mostly = 64; 4236 int dev_tx_weight __read_mostly = 64; 4237 /* Maximum number of GRO_NORMAL skbs to batch up for list-RX */ 4238 int gro_normal_batch __read_mostly = 8; 4239 4240 /* Called with irq disabled */ 4241 static inline void ____napi_schedule(struct softnet_data *sd, 4242 struct napi_struct *napi) 4243 { 4244 list_add_tail(&napi->poll_list, &sd->poll_list); 4245 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 4246 } 4247 4248 #ifdef CONFIG_RPS 4249 4250 /* One global table that all flow-based protocols share. */ 4251 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; 4252 EXPORT_SYMBOL(rps_sock_flow_table); 4253 u32 rps_cpu_mask __read_mostly; 4254 EXPORT_SYMBOL(rps_cpu_mask); 4255 4256 struct static_key_false rps_needed __read_mostly; 4257 EXPORT_SYMBOL(rps_needed); 4258 struct static_key_false rfs_needed __read_mostly; 4259 EXPORT_SYMBOL(rfs_needed); 4260 4261 static struct rps_dev_flow * 4262 set_rps_cpu(struct net_device *dev, struct sk_buff *skb, 4263 struct rps_dev_flow *rflow, u16 next_cpu) 4264 { 4265 if (next_cpu < nr_cpu_ids) { 4266 #ifdef CONFIG_RFS_ACCEL 4267 struct netdev_rx_queue *rxqueue; 4268 struct rps_dev_flow_table *flow_table; 4269 struct rps_dev_flow *old_rflow; 4270 u32 flow_id; 4271 u16 rxq_index; 4272 int rc; 4273 4274 /* Should we steer this flow to a different hardware queue? */ 4275 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || 4276 !(dev->features & NETIF_F_NTUPLE)) 4277 goto out; 4278 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); 4279 if (rxq_index == skb_get_rx_queue(skb)) 4280 goto out; 4281 4282 rxqueue = dev->_rx + rxq_index; 4283 flow_table = rcu_dereference(rxqueue->rps_flow_table); 4284 if (!flow_table) 4285 goto out; 4286 flow_id = skb_get_hash(skb) & flow_table->mask; 4287 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, 4288 rxq_index, flow_id); 4289 if (rc < 0) 4290 goto out; 4291 old_rflow = rflow; 4292 rflow = &flow_table->flows[flow_id]; 4293 rflow->filter = rc; 4294 if (old_rflow->filter == rflow->filter) 4295 old_rflow->filter = RPS_NO_FILTER; 4296 out: 4297 #endif 4298 rflow->last_qtail = 4299 per_cpu(softnet_data, next_cpu).input_queue_head; 4300 } 4301 4302 rflow->cpu = next_cpu; 4303 return rflow; 4304 } 4305 4306 /* 4307 * get_rps_cpu is called from netif_receive_skb and returns the target 4308 * CPU from the RPS map of the receiving queue for a given skb. 4309 * rcu_read_lock must be held on entry. 4310 */ 4311 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, 4312 struct rps_dev_flow **rflowp) 4313 { 4314 const struct rps_sock_flow_table *sock_flow_table; 4315 struct netdev_rx_queue *rxqueue = dev->_rx; 4316 struct rps_dev_flow_table *flow_table; 4317 struct rps_map *map; 4318 int cpu = -1; 4319 u32 tcpu; 4320 u32 hash; 4321 4322 if (skb_rx_queue_recorded(skb)) { 4323 u16 index = skb_get_rx_queue(skb); 4324 4325 if (unlikely(index >= dev->real_num_rx_queues)) { 4326 WARN_ONCE(dev->real_num_rx_queues > 1, 4327 "%s received packet on queue %u, but number " 4328 "of RX queues is %u\n", 4329 dev->name, index, dev->real_num_rx_queues); 4330 goto done; 4331 } 4332 rxqueue += index; 4333 } 4334 4335 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */ 4336 4337 flow_table = rcu_dereference(rxqueue->rps_flow_table); 4338 map = rcu_dereference(rxqueue->rps_map); 4339 if (!flow_table && !map) 4340 goto done; 4341 4342 skb_reset_network_header(skb); 4343 hash = skb_get_hash(skb); 4344 if (!hash) 4345 goto done; 4346 4347 sock_flow_table = rcu_dereference(rps_sock_flow_table); 4348 if (flow_table && sock_flow_table) { 4349 struct rps_dev_flow *rflow; 4350 u32 next_cpu; 4351 u32 ident; 4352 4353 /* First check into global flow table if there is a match */ 4354 ident = sock_flow_table->ents[hash & sock_flow_table->mask]; 4355 if ((ident ^ hash) & ~rps_cpu_mask) 4356 goto try_rps; 4357 4358 next_cpu = ident & rps_cpu_mask; 4359 4360 /* OK, now we know there is a match, 4361 * we can look at the local (per receive queue) flow table 4362 */ 4363 rflow = &flow_table->flows[hash & flow_table->mask]; 4364 tcpu = rflow->cpu; 4365 4366 /* 4367 * If the desired CPU (where last recvmsg was done) is 4368 * different from current CPU (one in the rx-queue flow 4369 * table entry), switch if one of the following holds: 4370 * - Current CPU is unset (>= nr_cpu_ids). 4371 * - Current CPU is offline. 4372 * - The current CPU's queue tail has advanced beyond the 4373 * last packet that was enqueued using this table entry. 4374 * This guarantees that all previous packets for the flow 4375 * have been dequeued, thus preserving in order delivery. 4376 */ 4377 if (unlikely(tcpu != next_cpu) && 4378 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) || 4379 ((int)(per_cpu(softnet_data, tcpu).input_queue_head - 4380 rflow->last_qtail)) >= 0)) { 4381 tcpu = next_cpu; 4382 rflow = set_rps_cpu(dev, skb, rflow, next_cpu); 4383 } 4384 4385 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { 4386 *rflowp = rflow; 4387 cpu = tcpu; 4388 goto done; 4389 } 4390 } 4391 4392 try_rps: 4393 4394 if (map) { 4395 tcpu = map->cpus[reciprocal_scale(hash, map->len)]; 4396 if (cpu_online(tcpu)) { 4397 cpu = tcpu; 4398 goto done; 4399 } 4400 } 4401 4402 done: 4403 return cpu; 4404 } 4405 4406 #ifdef CONFIG_RFS_ACCEL 4407 4408 /** 4409 * rps_may_expire_flow - check whether an RFS hardware filter may be removed 4410 * @dev: Device on which the filter was set 4411 * @rxq_index: RX queue index 4412 * @flow_id: Flow ID passed to ndo_rx_flow_steer() 4413 * @filter_id: Filter ID returned by ndo_rx_flow_steer() 4414 * 4415 * Drivers that implement ndo_rx_flow_steer() should periodically call 4416 * this function for each installed filter and remove the filters for 4417 * which it returns %true. 4418 */ 4419 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, 4420 u32 flow_id, u16 filter_id) 4421 { 4422 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; 4423 struct rps_dev_flow_table *flow_table; 4424 struct rps_dev_flow *rflow; 4425 bool expire = true; 4426 unsigned int cpu; 4427 4428 rcu_read_lock(); 4429 flow_table = rcu_dereference(rxqueue->rps_flow_table); 4430 if (flow_table && flow_id <= flow_table->mask) { 4431 rflow = &flow_table->flows[flow_id]; 4432 cpu = READ_ONCE(rflow->cpu); 4433 if (rflow->filter == filter_id && cpu < nr_cpu_ids && 4434 ((int)(per_cpu(softnet_data, cpu).input_queue_head - 4435 rflow->last_qtail) < 4436 (int)(10 * flow_table->mask))) 4437 expire = false; 4438 } 4439 rcu_read_unlock(); 4440 return expire; 4441 } 4442 EXPORT_SYMBOL(rps_may_expire_flow); 4443 4444 #endif /* CONFIG_RFS_ACCEL */ 4445 4446 /* Called from hardirq (IPI) context */ 4447 static void rps_trigger_softirq(void *data) 4448 { 4449 struct softnet_data *sd = data; 4450 4451 ____napi_schedule(sd, &sd->backlog); 4452 sd->received_rps++; 4453 } 4454 4455 #endif /* CONFIG_RPS */ 4456 4457 /* 4458 * Check if this softnet_data structure is another cpu one 4459 * If yes, queue it to our IPI list and return 1 4460 * If no, return 0 4461 */ 4462 static int rps_ipi_queued(struct softnet_data *sd) 4463 { 4464 #ifdef CONFIG_RPS 4465 struct softnet_data *mysd = this_cpu_ptr(&softnet_data); 4466 4467 if (sd != mysd) { 4468 sd->rps_ipi_next = mysd->rps_ipi_list; 4469 mysd->rps_ipi_list = sd; 4470 4471 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 4472 return 1; 4473 } 4474 #endif /* CONFIG_RPS */ 4475 return 0; 4476 } 4477 4478 #ifdef CONFIG_NET_FLOW_LIMIT 4479 int netdev_flow_limit_table_len __read_mostly = (1 << 12); 4480 #endif 4481 4482 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) 4483 { 4484 #ifdef CONFIG_NET_FLOW_LIMIT 4485 struct sd_flow_limit *fl; 4486 struct softnet_data *sd; 4487 unsigned int old_flow, new_flow; 4488 4489 if (qlen < (netdev_max_backlog >> 1)) 4490 return false; 4491 4492 sd = this_cpu_ptr(&softnet_data); 4493 4494 rcu_read_lock(); 4495 fl = rcu_dereference(sd->flow_limit); 4496 if (fl) { 4497 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1); 4498 old_flow = fl->history[fl->history_head]; 4499 fl->history[fl->history_head] = new_flow; 4500 4501 fl->history_head++; 4502 fl->history_head &= FLOW_LIMIT_HISTORY - 1; 4503 4504 if (likely(fl->buckets[old_flow])) 4505 fl->buckets[old_flow]--; 4506 4507 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { 4508 fl->count++; 4509 rcu_read_unlock(); 4510 return true; 4511 } 4512 } 4513 rcu_read_unlock(); 4514 #endif 4515 return false; 4516 } 4517 4518 /* 4519 * enqueue_to_backlog is called to queue an skb to a per CPU backlog 4520 * queue (may be a remote CPU queue). 4521 */ 4522 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, 4523 unsigned int *qtail) 4524 { 4525 struct softnet_data *sd; 4526 unsigned long flags; 4527 unsigned int qlen; 4528 4529 sd = &per_cpu(softnet_data, cpu); 4530 4531 local_irq_save(flags); 4532 4533 rps_lock(sd); 4534 if (!netif_running(skb->dev)) 4535 goto drop; 4536 qlen = skb_queue_len(&sd->input_pkt_queue); 4537 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) { 4538 if (qlen) { 4539 enqueue: 4540 __skb_queue_tail(&sd->input_pkt_queue, skb); 4541 input_queue_tail_incr_save(sd, qtail); 4542 rps_unlock(sd); 4543 local_irq_restore(flags); 4544 return NET_RX_SUCCESS; 4545 } 4546 4547 /* Schedule NAPI for backlog device 4548 * We can use non atomic operation since we own the queue lock 4549 */ 4550 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) { 4551 if (!rps_ipi_queued(sd)) 4552 ____napi_schedule(sd, &sd->backlog); 4553 } 4554 goto enqueue; 4555 } 4556 4557 drop: 4558 sd->dropped++; 4559 rps_unlock(sd); 4560 4561 local_irq_restore(flags); 4562 4563 atomic_long_inc(&skb->dev->rx_dropped); 4564 kfree_skb(skb); 4565 return NET_RX_DROP; 4566 } 4567 4568 static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb) 4569 { 4570 struct net_device *dev = skb->dev; 4571 struct netdev_rx_queue *rxqueue; 4572 4573 rxqueue = dev->_rx; 4574 4575 if (skb_rx_queue_recorded(skb)) { 4576 u16 index = skb_get_rx_queue(skb); 4577 4578 if (unlikely(index >= dev->real_num_rx_queues)) { 4579 WARN_ONCE(dev->real_num_rx_queues > 1, 4580 "%s received packet on queue %u, but number " 4581 "of RX queues is %u\n", 4582 dev->name, index, dev->real_num_rx_queues); 4583 4584 return rxqueue; /* Return first rxqueue */ 4585 } 4586 rxqueue += index; 4587 } 4588 return rxqueue; 4589 } 4590 4591 static u32 netif_receive_generic_xdp(struct sk_buff *skb, 4592 struct xdp_buff *xdp, 4593 struct bpf_prog *xdp_prog) 4594 { 4595 struct netdev_rx_queue *rxqueue; 4596 void *orig_data, *orig_data_end; 4597 u32 metalen, act = XDP_DROP; 4598 __be16 orig_eth_type; 4599 struct ethhdr *eth; 4600 bool orig_bcast; 4601 int hlen, off; 4602 u32 mac_len; 4603 4604 /* Reinjected packets coming from act_mirred or similar should 4605 * not get XDP generic processing. 4606 */ 4607 if (skb_is_redirected(skb)) 4608 return XDP_PASS; 4609 4610 /* XDP packets must be linear and must have sufficient headroom 4611 * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also 4612 * native XDP provides, thus we need to do it here as well. 4613 */ 4614 if (skb_cloned(skb) || skb_is_nonlinear(skb) || 4615 skb_headroom(skb) < XDP_PACKET_HEADROOM) { 4616 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb); 4617 int troom = skb->tail + skb->data_len - skb->end; 4618 4619 /* In case we have to go down the path and also linearize, 4620 * then lets do the pskb_expand_head() work just once here. 4621 */ 4622 if (pskb_expand_head(skb, 4623 hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0, 4624 troom > 0 ? troom + 128 : 0, GFP_ATOMIC)) 4625 goto do_drop; 4626 if (skb_linearize(skb)) 4627 goto do_drop; 4628 } 4629 4630 /* The XDP program wants to see the packet starting at the MAC 4631 * header. 4632 */ 4633 mac_len = skb->data - skb_mac_header(skb); 4634 hlen = skb_headlen(skb) + mac_len; 4635 xdp->data = skb->data - mac_len; 4636 xdp->data_meta = xdp->data; 4637 xdp->data_end = xdp->data + hlen; 4638 xdp->data_hard_start = skb->data - skb_headroom(skb); 4639 4640 /* SKB "head" area always have tailroom for skb_shared_info */ 4641 xdp->frame_sz = (void *)skb_end_pointer(skb) - xdp->data_hard_start; 4642 xdp->frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 4643 4644 orig_data_end = xdp->data_end; 4645 orig_data = xdp->data; 4646 eth = (struct ethhdr *)xdp->data; 4647 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest); 4648 orig_eth_type = eth->h_proto; 4649 4650 rxqueue = netif_get_rxqueue(skb); 4651 xdp->rxq = &rxqueue->xdp_rxq; 4652 4653 act = bpf_prog_run_xdp(xdp_prog, xdp); 4654 4655 /* check if bpf_xdp_adjust_head was used */ 4656 off = xdp->data - orig_data; 4657 if (off) { 4658 if (off > 0) 4659 __skb_pull(skb, off); 4660 else if (off < 0) 4661 __skb_push(skb, -off); 4662 4663 skb->mac_header += off; 4664 skb_reset_network_header(skb); 4665 } 4666 4667 /* check if bpf_xdp_adjust_tail was used */ 4668 off = xdp->data_end - orig_data_end; 4669 if (off != 0) { 4670 skb_set_tail_pointer(skb, xdp->data_end - xdp->data); 4671 skb->len += off; /* positive on grow, negative on shrink */ 4672 } 4673 4674 /* check if XDP changed eth hdr such SKB needs update */ 4675 eth = (struct ethhdr *)xdp->data; 4676 if ((orig_eth_type != eth->h_proto) || 4677 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) { 4678 __skb_push(skb, ETH_HLEN); 4679 skb->protocol = eth_type_trans(skb, skb->dev); 4680 } 4681 4682 switch (act) { 4683 case XDP_REDIRECT: 4684 case XDP_TX: 4685 __skb_push(skb, mac_len); 4686 break; 4687 case XDP_PASS: 4688 metalen = xdp->data - xdp->data_meta; 4689 if (metalen) 4690 skb_metadata_set(skb, metalen); 4691 break; 4692 default: 4693 bpf_warn_invalid_xdp_action(act); 4694 /* fall through */ 4695 case XDP_ABORTED: 4696 trace_xdp_exception(skb->dev, xdp_prog, act); 4697 /* fall through */ 4698 case XDP_DROP: 4699 do_drop: 4700 kfree_skb(skb); 4701 break; 4702 } 4703 4704 return act; 4705 } 4706 4707 /* When doing generic XDP we have to bypass the qdisc layer and the 4708 * network taps in order to match in-driver-XDP behavior. 4709 */ 4710 void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog) 4711 { 4712 struct net_device *dev = skb->dev; 4713 struct netdev_queue *txq; 4714 bool free_skb = true; 4715 int cpu, rc; 4716 4717 txq = netdev_core_pick_tx(dev, skb, NULL); 4718 cpu = smp_processor_id(); 4719 HARD_TX_LOCK(dev, txq, cpu); 4720 if (!netif_xmit_stopped(txq)) { 4721 rc = netdev_start_xmit(skb, dev, txq, 0); 4722 if (dev_xmit_complete(rc)) 4723 free_skb = false; 4724 } 4725 HARD_TX_UNLOCK(dev, txq); 4726 if (free_skb) { 4727 trace_xdp_exception(dev, xdp_prog, XDP_TX); 4728 kfree_skb(skb); 4729 } 4730 } 4731 4732 static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key); 4733 4734 int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb) 4735 { 4736 if (xdp_prog) { 4737 struct xdp_buff xdp; 4738 u32 act; 4739 int err; 4740 4741 act = netif_receive_generic_xdp(skb, &xdp, xdp_prog); 4742 if (act != XDP_PASS) { 4743 switch (act) { 4744 case XDP_REDIRECT: 4745 err = xdp_do_generic_redirect(skb->dev, skb, 4746 &xdp, xdp_prog); 4747 if (err) 4748 goto out_redir; 4749 break; 4750 case XDP_TX: 4751 generic_xdp_tx(skb, xdp_prog); 4752 break; 4753 } 4754 return XDP_DROP; 4755 } 4756 } 4757 return XDP_PASS; 4758 out_redir: 4759 kfree_skb(skb); 4760 return XDP_DROP; 4761 } 4762 EXPORT_SYMBOL_GPL(do_xdp_generic); 4763 4764 static int netif_rx_internal(struct sk_buff *skb) 4765 { 4766 int ret; 4767 4768 net_timestamp_check(netdev_tstamp_prequeue, skb); 4769 4770 trace_netif_rx(skb); 4771 4772 #ifdef CONFIG_RPS 4773 if (static_branch_unlikely(&rps_needed)) { 4774 struct rps_dev_flow voidflow, *rflow = &voidflow; 4775 int cpu; 4776 4777 preempt_disable(); 4778 rcu_read_lock(); 4779 4780 cpu = get_rps_cpu(skb->dev, skb, &rflow); 4781 if (cpu < 0) 4782 cpu = smp_processor_id(); 4783 4784 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 4785 4786 rcu_read_unlock(); 4787 preempt_enable(); 4788 } else 4789 #endif 4790 { 4791 unsigned int qtail; 4792 4793 ret = enqueue_to_backlog(skb, get_cpu(), &qtail); 4794 put_cpu(); 4795 } 4796 return ret; 4797 } 4798 4799 /** 4800 * netif_rx - post buffer to the network code 4801 * @skb: buffer to post 4802 * 4803 * This function receives a packet from a device driver and queues it for 4804 * the upper (protocol) levels to process. It always succeeds. The buffer 4805 * may be dropped during processing for congestion control or by the 4806 * protocol layers. 4807 * 4808 * return values: 4809 * NET_RX_SUCCESS (no congestion) 4810 * NET_RX_DROP (packet was dropped) 4811 * 4812 */ 4813 4814 int netif_rx(struct sk_buff *skb) 4815 { 4816 int ret; 4817 4818 trace_netif_rx_entry(skb); 4819 4820 ret = netif_rx_internal(skb); 4821 trace_netif_rx_exit(ret); 4822 4823 return ret; 4824 } 4825 EXPORT_SYMBOL(netif_rx); 4826 4827 int netif_rx_ni(struct sk_buff *skb) 4828 { 4829 int err; 4830 4831 trace_netif_rx_ni_entry(skb); 4832 4833 preempt_disable(); 4834 err = netif_rx_internal(skb); 4835 if (local_softirq_pending()) 4836 do_softirq(); 4837 preempt_enable(); 4838 trace_netif_rx_ni_exit(err); 4839 4840 return err; 4841 } 4842 EXPORT_SYMBOL(netif_rx_ni); 4843 4844 static __latent_entropy void net_tx_action(struct softirq_action *h) 4845 { 4846 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 4847 4848 if (sd->completion_queue) { 4849 struct sk_buff *clist; 4850 4851 local_irq_disable(); 4852 clist = sd->completion_queue; 4853 sd->completion_queue = NULL; 4854 local_irq_enable(); 4855 4856 while (clist) { 4857 struct sk_buff *skb = clist; 4858 4859 clist = clist->next; 4860 4861 WARN_ON(refcount_read(&skb->users)); 4862 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED)) 4863 trace_consume_skb(skb); 4864 else 4865 trace_kfree_skb(skb, net_tx_action); 4866 4867 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) 4868 __kfree_skb(skb); 4869 else 4870 __kfree_skb_defer(skb); 4871 } 4872 4873 __kfree_skb_flush(); 4874 } 4875 4876 if (sd->output_queue) { 4877 struct Qdisc *head; 4878 4879 local_irq_disable(); 4880 head = sd->output_queue; 4881 sd->output_queue = NULL; 4882 sd->output_queue_tailp = &sd->output_queue; 4883 local_irq_enable(); 4884 4885 while (head) { 4886 struct Qdisc *q = head; 4887 spinlock_t *root_lock = NULL; 4888 4889 head = head->next_sched; 4890 4891 if (!(q->flags & TCQ_F_NOLOCK)) { 4892 root_lock = qdisc_lock(q); 4893 spin_lock(root_lock); 4894 } 4895 /* We need to make sure head->next_sched is read 4896 * before clearing __QDISC_STATE_SCHED 4897 */ 4898 smp_mb__before_atomic(); 4899 clear_bit(__QDISC_STATE_SCHED, &q->state); 4900 qdisc_run(q); 4901 if (root_lock) 4902 spin_unlock(root_lock); 4903 } 4904 } 4905 4906 xfrm_dev_backlog(sd); 4907 } 4908 4909 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE) 4910 /* This hook is defined here for ATM LANE */ 4911 int (*br_fdb_test_addr_hook)(struct net_device *dev, 4912 unsigned char *addr) __read_mostly; 4913 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); 4914 #endif 4915 4916 static inline struct sk_buff * 4917 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, 4918 struct net_device *orig_dev) 4919 { 4920 #ifdef CONFIG_NET_CLS_ACT 4921 struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress); 4922 struct tcf_result cl_res; 4923 4924 /* If there's at least one ingress present somewhere (so 4925 * we get here via enabled static key), remaining devices 4926 * that are not configured with an ingress qdisc will bail 4927 * out here. 4928 */ 4929 if (!miniq) 4930 return skb; 4931 4932 if (*pt_prev) { 4933 *ret = deliver_skb(skb, *pt_prev, orig_dev); 4934 *pt_prev = NULL; 4935 } 4936 4937 qdisc_skb_cb(skb)->pkt_len = skb->len; 4938 skb->tc_at_ingress = 1; 4939 mini_qdisc_bstats_cpu_update(miniq, skb); 4940 4941 switch (tcf_classify_ingress(skb, miniq->block, miniq->filter_list, 4942 &cl_res, false)) { 4943 case TC_ACT_OK: 4944 case TC_ACT_RECLASSIFY: 4945 skb->tc_index = TC_H_MIN(cl_res.classid); 4946 break; 4947 case TC_ACT_SHOT: 4948 mini_qdisc_qstats_cpu_drop(miniq); 4949 kfree_skb(skb); 4950 return NULL; 4951 case TC_ACT_STOLEN: 4952 case TC_ACT_QUEUED: 4953 case TC_ACT_TRAP: 4954 consume_skb(skb); 4955 return NULL; 4956 case TC_ACT_REDIRECT: 4957 /* skb_mac_header check was done by cls/act_bpf, so 4958 * we can safely push the L2 header back before 4959 * redirecting to another netdev 4960 */ 4961 __skb_push(skb, skb->mac_len); 4962 skb_do_redirect(skb); 4963 return NULL; 4964 case TC_ACT_CONSUMED: 4965 return NULL; 4966 default: 4967 break; 4968 } 4969 #endif /* CONFIG_NET_CLS_ACT */ 4970 return skb; 4971 } 4972 4973 /** 4974 * netdev_is_rx_handler_busy - check if receive handler is registered 4975 * @dev: device to check 4976 * 4977 * Check if a receive handler is already registered for a given device. 4978 * Return true if there one. 4979 * 4980 * The caller must hold the rtnl_mutex. 4981 */ 4982 bool netdev_is_rx_handler_busy(struct net_device *dev) 4983 { 4984 ASSERT_RTNL(); 4985 return dev && rtnl_dereference(dev->rx_handler); 4986 } 4987 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy); 4988 4989 /** 4990 * netdev_rx_handler_register - register receive handler 4991 * @dev: device to register a handler for 4992 * @rx_handler: receive handler to register 4993 * @rx_handler_data: data pointer that is used by rx handler 4994 * 4995 * Register a receive handler for a device. This handler will then be 4996 * called from __netif_receive_skb. A negative errno code is returned 4997 * on a failure. 4998 * 4999 * The caller must hold the rtnl_mutex. 5000 * 5001 * For a general description of rx_handler, see enum rx_handler_result. 5002 */ 5003 int netdev_rx_handler_register(struct net_device *dev, 5004 rx_handler_func_t *rx_handler, 5005 void *rx_handler_data) 5006 { 5007 if (netdev_is_rx_handler_busy(dev)) 5008 return -EBUSY; 5009 5010 if (dev->priv_flags & IFF_NO_RX_HANDLER) 5011 return -EINVAL; 5012 5013 /* Note: rx_handler_data must be set before rx_handler */ 5014 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); 5015 rcu_assign_pointer(dev->rx_handler, rx_handler); 5016 5017 return 0; 5018 } 5019 EXPORT_SYMBOL_GPL(netdev_rx_handler_register); 5020 5021 /** 5022 * netdev_rx_handler_unregister - unregister receive handler 5023 * @dev: device to unregister a handler from 5024 * 5025 * Unregister a receive handler from a device. 5026 * 5027 * The caller must hold the rtnl_mutex. 5028 */ 5029 void netdev_rx_handler_unregister(struct net_device *dev) 5030 { 5031 5032 ASSERT_RTNL(); 5033 RCU_INIT_POINTER(dev->rx_handler, NULL); 5034 /* a reader seeing a non NULL rx_handler in a rcu_read_lock() 5035 * section has a guarantee to see a non NULL rx_handler_data 5036 * as well. 5037 */ 5038 synchronize_net(); 5039 RCU_INIT_POINTER(dev->rx_handler_data, NULL); 5040 } 5041 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); 5042 5043 /* 5044 * Limit the use of PFMEMALLOC reserves to those protocols that implement 5045 * the special handling of PFMEMALLOC skbs. 5046 */ 5047 static bool skb_pfmemalloc_protocol(struct sk_buff *skb) 5048 { 5049 switch (skb->protocol) { 5050 case htons(ETH_P_ARP): 5051 case htons(ETH_P_IP): 5052 case htons(ETH_P_IPV6): 5053 case htons(ETH_P_8021Q): 5054 case htons(ETH_P_8021AD): 5055 return true; 5056 default: 5057 return false; 5058 } 5059 } 5060 5061 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev, 5062 int *ret, struct net_device *orig_dev) 5063 { 5064 if (nf_hook_ingress_active(skb)) { 5065 int ingress_retval; 5066 5067 if (*pt_prev) { 5068 *ret = deliver_skb(skb, *pt_prev, orig_dev); 5069 *pt_prev = NULL; 5070 } 5071 5072 rcu_read_lock(); 5073 ingress_retval = nf_hook_ingress(skb); 5074 rcu_read_unlock(); 5075 return ingress_retval; 5076 } 5077 return 0; 5078 } 5079 5080 static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc, 5081 struct packet_type **ppt_prev) 5082 { 5083 struct packet_type *ptype, *pt_prev; 5084 rx_handler_func_t *rx_handler; 5085 struct sk_buff *skb = *pskb; 5086 struct net_device *orig_dev; 5087 bool deliver_exact = false; 5088 int ret = NET_RX_DROP; 5089 __be16 type; 5090 5091 net_timestamp_check(!netdev_tstamp_prequeue, skb); 5092 5093 trace_netif_receive_skb(skb); 5094 5095 orig_dev = skb->dev; 5096 5097 skb_reset_network_header(skb); 5098 if (!skb_transport_header_was_set(skb)) 5099 skb_reset_transport_header(skb); 5100 skb_reset_mac_len(skb); 5101 5102 pt_prev = NULL; 5103 5104 another_round: 5105 skb->skb_iif = skb->dev->ifindex; 5106 5107 __this_cpu_inc(softnet_data.processed); 5108 5109 if (static_branch_unlikely(&generic_xdp_needed_key)) { 5110 int ret2; 5111 5112 preempt_disable(); 5113 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb); 5114 preempt_enable(); 5115 5116 if (ret2 != XDP_PASS) { 5117 ret = NET_RX_DROP; 5118 goto out; 5119 } 5120 skb_reset_mac_len(skb); 5121 } 5122 5123 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 5124 skb->protocol == cpu_to_be16(ETH_P_8021AD)) { 5125 skb = skb_vlan_untag(skb); 5126 if (unlikely(!skb)) 5127 goto out; 5128 } 5129 5130 if (skb_skip_tc_classify(skb)) 5131 goto skip_classify; 5132 5133 if (pfmemalloc) 5134 goto skip_taps; 5135 5136 list_for_each_entry_rcu(ptype, &ptype_all, list) { 5137 if (pt_prev) 5138 ret = deliver_skb(skb, pt_prev, orig_dev); 5139 pt_prev = ptype; 5140 } 5141 5142 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { 5143 if (pt_prev) 5144 ret = deliver_skb(skb, pt_prev, orig_dev); 5145 pt_prev = ptype; 5146 } 5147 5148 skip_taps: 5149 #ifdef CONFIG_NET_INGRESS 5150 if (static_branch_unlikely(&ingress_needed_key)) { 5151 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev); 5152 if (!skb) 5153 goto out; 5154 5155 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0) 5156 goto out; 5157 } 5158 #endif 5159 skb_reset_redirect(skb); 5160 skip_classify: 5161 if (pfmemalloc && !skb_pfmemalloc_protocol(skb)) 5162 goto drop; 5163 5164 if (skb_vlan_tag_present(skb)) { 5165 if (pt_prev) { 5166 ret = deliver_skb(skb, pt_prev, orig_dev); 5167 pt_prev = NULL; 5168 } 5169 if (vlan_do_receive(&skb)) 5170 goto another_round; 5171 else if (unlikely(!skb)) 5172 goto out; 5173 } 5174 5175 rx_handler = rcu_dereference(skb->dev->rx_handler); 5176 if (rx_handler) { 5177 if (pt_prev) { 5178 ret = deliver_skb(skb, pt_prev, orig_dev); 5179 pt_prev = NULL; 5180 } 5181 switch (rx_handler(&skb)) { 5182 case RX_HANDLER_CONSUMED: 5183 ret = NET_RX_SUCCESS; 5184 goto out; 5185 case RX_HANDLER_ANOTHER: 5186 goto another_round; 5187 case RX_HANDLER_EXACT: 5188 deliver_exact = true; 5189 case RX_HANDLER_PASS: 5190 break; 5191 default: 5192 BUG(); 5193 } 5194 } 5195 5196 if (unlikely(skb_vlan_tag_present(skb))) { 5197 check_vlan_id: 5198 if (skb_vlan_tag_get_id(skb)) { 5199 /* Vlan id is non 0 and vlan_do_receive() above couldn't 5200 * find vlan device. 5201 */ 5202 skb->pkt_type = PACKET_OTHERHOST; 5203 } else if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 5204 skb->protocol == cpu_to_be16(ETH_P_8021AD)) { 5205 /* Outer header is 802.1P with vlan 0, inner header is 5206 * 802.1Q or 802.1AD and vlan_do_receive() above could 5207 * not find vlan dev for vlan id 0. 5208 */ 5209 __vlan_hwaccel_clear_tag(skb); 5210 skb = skb_vlan_untag(skb); 5211 if (unlikely(!skb)) 5212 goto out; 5213 if (vlan_do_receive(&skb)) 5214 /* After stripping off 802.1P header with vlan 0 5215 * vlan dev is found for inner header. 5216 */ 5217 goto another_round; 5218 else if (unlikely(!skb)) 5219 goto out; 5220 else 5221 /* We have stripped outer 802.1P vlan 0 header. 5222 * But could not find vlan dev. 5223 * check again for vlan id to set OTHERHOST. 5224 */ 5225 goto check_vlan_id; 5226 } 5227 /* Note: we might in the future use prio bits 5228 * and set skb->priority like in vlan_do_receive() 5229 * For the time being, just ignore Priority Code Point 5230 */ 5231 __vlan_hwaccel_clear_tag(skb); 5232 } 5233 5234 type = skb->protocol; 5235 5236 /* deliver only exact match when indicated */ 5237 if (likely(!deliver_exact)) { 5238 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 5239 &ptype_base[ntohs(type) & 5240 PTYPE_HASH_MASK]); 5241 } 5242 5243 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 5244 &orig_dev->ptype_specific); 5245 5246 if (unlikely(skb->dev != orig_dev)) { 5247 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 5248 &skb->dev->ptype_specific); 5249 } 5250 5251 if (pt_prev) { 5252 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) 5253 goto drop; 5254 *ppt_prev = pt_prev; 5255 } else { 5256 drop: 5257 if (!deliver_exact) 5258 atomic_long_inc(&skb->dev->rx_dropped); 5259 else 5260 atomic_long_inc(&skb->dev->rx_nohandler); 5261 kfree_skb(skb); 5262 /* Jamal, now you will not able to escape explaining 5263 * me how you were going to use this. :-) 5264 */ 5265 ret = NET_RX_DROP; 5266 } 5267 5268 out: 5269 /* The invariant here is that if *ppt_prev is not NULL 5270 * then skb should also be non-NULL. 5271 * 5272 * Apparently *ppt_prev assignment above holds this invariant due to 5273 * skb dereferencing near it. 5274 */ 5275 *pskb = skb; 5276 return ret; 5277 } 5278 5279 static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc) 5280 { 5281 struct net_device *orig_dev = skb->dev; 5282 struct packet_type *pt_prev = NULL; 5283 int ret; 5284 5285 ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev); 5286 if (pt_prev) 5287 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb, 5288 skb->dev, pt_prev, orig_dev); 5289 return ret; 5290 } 5291 5292 /** 5293 * netif_receive_skb_core - special purpose version of netif_receive_skb 5294 * @skb: buffer to process 5295 * 5296 * More direct receive version of netif_receive_skb(). It should 5297 * only be used by callers that have a need to skip RPS and Generic XDP. 5298 * Caller must also take care of handling if ``(page_is_)pfmemalloc``. 5299 * 5300 * This function may only be called from softirq context and interrupts 5301 * should be enabled. 5302 * 5303 * Return values (usually ignored): 5304 * NET_RX_SUCCESS: no congestion 5305 * NET_RX_DROP: packet was dropped 5306 */ 5307 int netif_receive_skb_core(struct sk_buff *skb) 5308 { 5309 int ret; 5310 5311 rcu_read_lock(); 5312 ret = __netif_receive_skb_one_core(skb, false); 5313 rcu_read_unlock(); 5314 5315 return ret; 5316 } 5317 EXPORT_SYMBOL(netif_receive_skb_core); 5318 5319 static inline void __netif_receive_skb_list_ptype(struct list_head *head, 5320 struct packet_type *pt_prev, 5321 struct net_device *orig_dev) 5322 { 5323 struct sk_buff *skb, *next; 5324 5325 if (!pt_prev) 5326 return; 5327 if (list_empty(head)) 5328 return; 5329 if (pt_prev->list_func != NULL) 5330 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv, 5331 ip_list_rcv, head, pt_prev, orig_dev); 5332 else 5333 list_for_each_entry_safe(skb, next, head, list) { 5334 skb_list_del_init(skb); 5335 pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 5336 } 5337 } 5338 5339 static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc) 5340 { 5341 /* Fast-path assumptions: 5342 * - There is no RX handler. 5343 * - Only one packet_type matches. 5344 * If either of these fails, we will end up doing some per-packet 5345 * processing in-line, then handling the 'last ptype' for the whole 5346 * sublist. This can't cause out-of-order delivery to any single ptype, 5347 * because the 'last ptype' must be constant across the sublist, and all 5348 * other ptypes are handled per-packet. 5349 */ 5350 /* Current (common) ptype of sublist */ 5351 struct packet_type *pt_curr = NULL; 5352 /* Current (common) orig_dev of sublist */ 5353 struct net_device *od_curr = NULL; 5354 struct list_head sublist; 5355 struct sk_buff *skb, *next; 5356 5357 INIT_LIST_HEAD(&sublist); 5358 list_for_each_entry_safe(skb, next, head, list) { 5359 struct net_device *orig_dev = skb->dev; 5360 struct packet_type *pt_prev = NULL; 5361 5362 skb_list_del_init(skb); 5363 __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev); 5364 if (!pt_prev) 5365 continue; 5366 if (pt_curr != pt_prev || od_curr != orig_dev) { 5367 /* dispatch old sublist */ 5368 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr); 5369 /* start new sublist */ 5370 INIT_LIST_HEAD(&sublist); 5371 pt_curr = pt_prev; 5372 od_curr = orig_dev; 5373 } 5374 list_add_tail(&skb->list, &sublist); 5375 } 5376 5377 /* dispatch final sublist */ 5378 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr); 5379 } 5380 5381 static int __netif_receive_skb(struct sk_buff *skb) 5382 { 5383 int ret; 5384 5385 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) { 5386 unsigned int noreclaim_flag; 5387 5388 /* 5389 * PFMEMALLOC skbs are special, they should 5390 * - be delivered to SOCK_MEMALLOC sockets only 5391 * - stay away from userspace 5392 * - have bounded memory usage 5393 * 5394 * Use PF_MEMALLOC as this saves us from propagating the allocation 5395 * context down to all allocation sites. 5396 */ 5397 noreclaim_flag = memalloc_noreclaim_save(); 5398 ret = __netif_receive_skb_one_core(skb, true); 5399 memalloc_noreclaim_restore(noreclaim_flag); 5400 } else 5401 ret = __netif_receive_skb_one_core(skb, false); 5402 5403 return ret; 5404 } 5405 5406 static void __netif_receive_skb_list(struct list_head *head) 5407 { 5408 unsigned long noreclaim_flag = 0; 5409 struct sk_buff *skb, *next; 5410 bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */ 5411 5412 list_for_each_entry_safe(skb, next, head, list) { 5413 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) { 5414 struct list_head sublist; 5415 5416 /* Handle the previous sublist */ 5417 list_cut_before(&sublist, head, &skb->list); 5418 if (!list_empty(&sublist)) 5419 __netif_receive_skb_list_core(&sublist, pfmemalloc); 5420 pfmemalloc = !pfmemalloc; 5421 /* See comments in __netif_receive_skb */ 5422 if (pfmemalloc) 5423 noreclaim_flag = memalloc_noreclaim_save(); 5424 else 5425 memalloc_noreclaim_restore(noreclaim_flag); 5426 } 5427 } 5428 /* Handle the remaining sublist */ 5429 if (!list_empty(head)) 5430 __netif_receive_skb_list_core(head, pfmemalloc); 5431 /* Restore pflags */ 5432 if (pfmemalloc) 5433 memalloc_noreclaim_restore(noreclaim_flag); 5434 } 5435 5436 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp) 5437 { 5438 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog); 5439 struct bpf_prog *new = xdp->prog; 5440 int ret = 0; 5441 5442 if (new) { 5443 u32 i; 5444 5445 /* generic XDP does not work with DEVMAPs that can 5446 * have a bpf_prog installed on an entry 5447 */ 5448 for (i = 0; i < new->aux->used_map_cnt; i++) { 5449 if (dev_map_can_have_prog(new->aux->used_maps[i])) 5450 return -EINVAL; 5451 } 5452 } 5453 5454 switch (xdp->command) { 5455 case XDP_SETUP_PROG: 5456 rcu_assign_pointer(dev->xdp_prog, new); 5457 if (old) 5458 bpf_prog_put(old); 5459 5460 if (old && !new) { 5461 static_branch_dec(&generic_xdp_needed_key); 5462 } else if (new && !old) { 5463 static_branch_inc(&generic_xdp_needed_key); 5464 dev_disable_lro(dev); 5465 dev_disable_gro_hw(dev); 5466 } 5467 break; 5468 5469 case XDP_QUERY_PROG: 5470 xdp->prog_id = old ? old->aux->id : 0; 5471 break; 5472 5473 default: 5474 ret = -EINVAL; 5475 break; 5476 } 5477 5478 return ret; 5479 } 5480 5481 static int netif_receive_skb_internal(struct sk_buff *skb) 5482 { 5483 int ret; 5484 5485 net_timestamp_check(netdev_tstamp_prequeue, skb); 5486 5487 if (skb_defer_rx_timestamp(skb)) 5488 return NET_RX_SUCCESS; 5489 5490 rcu_read_lock(); 5491 #ifdef CONFIG_RPS 5492 if (static_branch_unlikely(&rps_needed)) { 5493 struct rps_dev_flow voidflow, *rflow = &voidflow; 5494 int cpu = get_rps_cpu(skb->dev, skb, &rflow); 5495 5496 if (cpu >= 0) { 5497 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 5498 rcu_read_unlock(); 5499 return ret; 5500 } 5501 } 5502 #endif 5503 ret = __netif_receive_skb(skb); 5504 rcu_read_unlock(); 5505 return ret; 5506 } 5507 5508 static void netif_receive_skb_list_internal(struct list_head *head) 5509 { 5510 struct sk_buff *skb, *next; 5511 struct list_head sublist; 5512 5513 INIT_LIST_HEAD(&sublist); 5514 list_for_each_entry_safe(skb, next, head, list) { 5515 net_timestamp_check(netdev_tstamp_prequeue, skb); 5516 skb_list_del_init(skb); 5517 if (!skb_defer_rx_timestamp(skb)) 5518 list_add_tail(&skb->list, &sublist); 5519 } 5520 list_splice_init(&sublist, head); 5521 5522 rcu_read_lock(); 5523 #ifdef CONFIG_RPS 5524 if (static_branch_unlikely(&rps_needed)) { 5525 list_for_each_entry_safe(skb, next, head, list) { 5526 struct rps_dev_flow voidflow, *rflow = &voidflow; 5527 int cpu = get_rps_cpu(skb->dev, skb, &rflow); 5528 5529 if (cpu >= 0) { 5530 /* Will be handled, remove from list */ 5531 skb_list_del_init(skb); 5532 enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 5533 } 5534 } 5535 } 5536 #endif 5537 __netif_receive_skb_list(head); 5538 rcu_read_unlock(); 5539 } 5540 5541 /** 5542 * netif_receive_skb - process receive buffer from network 5543 * @skb: buffer to process 5544 * 5545 * netif_receive_skb() is the main receive data processing function. 5546 * It always succeeds. The buffer may be dropped during processing 5547 * for congestion control or by the protocol layers. 5548 * 5549 * This function may only be called from softirq context and interrupts 5550 * should be enabled. 5551 * 5552 * Return values (usually ignored): 5553 * NET_RX_SUCCESS: no congestion 5554 * NET_RX_DROP: packet was dropped 5555 */ 5556 int netif_receive_skb(struct sk_buff *skb) 5557 { 5558 int ret; 5559 5560 trace_netif_receive_skb_entry(skb); 5561 5562 ret = netif_receive_skb_internal(skb); 5563 trace_netif_receive_skb_exit(ret); 5564 5565 return ret; 5566 } 5567 EXPORT_SYMBOL(netif_receive_skb); 5568 5569 /** 5570 * netif_receive_skb_list - process many receive buffers from network 5571 * @head: list of skbs to process. 5572 * 5573 * Since return value of netif_receive_skb() is normally ignored, and 5574 * wouldn't be meaningful for a list, this function returns void. 5575 * 5576 * This function may only be called from softirq context and interrupts 5577 * should be enabled. 5578 */ 5579 void netif_receive_skb_list(struct list_head *head) 5580 { 5581 struct sk_buff *skb; 5582 5583 if (list_empty(head)) 5584 return; 5585 if (trace_netif_receive_skb_list_entry_enabled()) { 5586 list_for_each_entry(skb, head, list) 5587 trace_netif_receive_skb_list_entry(skb); 5588 } 5589 netif_receive_skb_list_internal(head); 5590 trace_netif_receive_skb_list_exit(0); 5591 } 5592 EXPORT_SYMBOL(netif_receive_skb_list); 5593 5594 DEFINE_PER_CPU(struct work_struct, flush_works); 5595 5596 /* Network device is going away, flush any packets still pending */ 5597 static void flush_backlog(struct work_struct *work) 5598 { 5599 struct sk_buff *skb, *tmp; 5600 struct softnet_data *sd; 5601 5602 local_bh_disable(); 5603 sd = this_cpu_ptr(&softnet_data); 5604 5605 local_irq_disable(); 5606 rps_lock(sd); 5607 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { 5608 if (skb->dev->reg_state == NETREG_UNREGISTERING) { 5609 __skb_unlink(skb, &sd->input_pkt_queue); 5610 kfree_skb(skb); 5611 input_queue_head_incr(sd); 5612 } 5613 } 5614 rps_unlock(sd); 5615 local_irq_enable(); 5616 5617 skb_queue_walk_safe(&sd->process_queue, skb, tmp) { 5618 if (skb->dev->reg_state == NETREG_UNREGISTERING) { 5619 __skb_unlink(skb, &sd->process_queue); 5620 kfree_skb(skb); 5621 input_queue_head_incr(sd); 5622 } 5623 } 5624 local_bh_enable(); 5625 } 5626 5627 static void flush_all_backlogs(void) 5628 { 5629 unsigned int cpu; 5630 5631 get_online_cpus(); 5632 5633 for_each_online_cpu(cpu) 5634 queue_work_on(cpu, system_highpri_wq, 5635 per_cpu_ptr(&flush_works, cpu)); 5636 5637 for_each_online_cpu(cpu) 5638 flush_work(per_cpu_ptr(&flush_works, cpu)); 5639 5640 put_online_cpus(); 5641 } 5642 5643 /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */ 5644 static void gro_normal_list(struct napi_struct *napi) 5645 { 5646 if (!napi->rx_count) 5647 return; 5648 netif_receive_skb_list_internal(&napi->rx_list); 5649 INIT_LIST_HEAD(&napi->rx_list); 5650 napi->rx_count = 0; 5651 } 5652 5653 /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded, 5654 * pass the whole batch up to the stack. 5655 */ 5656 static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb) 5657 { 5658 list_add_tail(&skb->list, &napi->rx_list); 5659 if (++napi->rx_count >= gro_normal_batch) 5660 gro_normal_list(napi); 5661 } 5662 5663 INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int)); 5664 INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int)); 5665 static int napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb) 5666 { 5667 struct packet_offload *ptype; 5668 __be16 type = skb->protocol; 5669 struct list_head *head = &offload_base; 5670 int err = -ENOENT; 5671 5672 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); 5673 5674 if (NAPI_GRO_CB(skb)->count == 1) { 5675 skb_shinfo(skb)->gso_size = 0; 5676 goto out; 5677 } 5678 5679 rcu_read_lock(); 5680 list_for_each_entry_rcu(ptype, head, list) { 5681 if (ptype->type != type || !ptype->callbacks.gro_complete) 5682 continue; 5683 5684 err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete, 5685 ipv6_gro_complete, inet_gro_complete, 5686 skb, 0); 5687 break; 5688 } 5689 rcu_read_unlock(); 5690 5691 if (err) { 5692 WARN_ON(&ptype->list == head); 5693 kfree_skb(skb); 5694 return NET_RX_SUCCESS; 5695 } 5696 5697 out: 5698 gro_normal_one(napi, skb); 5699 return NET_RX_SUCCESS; 5700 } 5701 5702 static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index, 5703 bool flush_old) 5704 { 5705 struct list_head *head = &napi->gro_hash[index].list; 5706 struct sk_buff *skb, *p; 5707 5708 list_for_each_entry_safe_reverse(skb, p, head, list) { 5709 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) 5710 return; 5711 skb_list_del_init(skb); 5712 napi_gro_complete(napi, skb); 5713 napi->gro_hash[index].count--; 5714 } 5715 5716 if (!napi->gro_hash[index].count) 5717 __clear_bit(index, &napi->gro_bitmask); 5718 } 5719 5720 /* napi->gro_hash[].list contains packets ordered by age. 5721 * youngest packets at the head of it. 5722 * Complete skbs in reverse order to reduce latencies. 5723 */ 5724 void napi_gro_flush(struct napi_struct *napi, bool flush_old) 5725 { 5726 unsigned long bitmask = napi->gro_bitmask; 5727 unsigned int i, base = ~0U; 5728 5729 while ((i = ffs(bitmask)) != 0) { 5730 bitmask >>= i; 5731 base += i; 5732 __napi_gro_flush_chain(napi, base, flush_old); 5733 } 5734 } 5735 EXPORT_SYMBOL(napi_gro_flush); 5736 5737 static struct list_head *gro_list_prepare(struct napi_struct *napi, 5738 struct sk_buff *skb) 5739 { 5740 unsigned int maclen = skb->dev->hard_header_len; 5741 u32 hash = skb_get_hash_raw(skb); 5742 struct list_head *head; 5743 struct sk_buff *p; 5744 5745 head = &napi->gro_hash[hash & (GRO_HASH_BUCKETS - 1)].list; 5746 list_for_each_entry(p, head, list) { 5747 unsigned long diffs; 5748 5749 NAPI_GRO_CB(p)->flush = 0; 5750 5751 if (hash != skb_get_hash_raw(p)) { 5752 NAPI_GRO_CB(p)->same_flow = 0; 5753 continue; 5754 } 5755 5756 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; 5757 diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb); 5758 if (skb_vlan_tag_present(p)) 5759 diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb); 5760 diffs |= skb_metadata_dst_cmp(p, skb); 5761 diffs |= skb_metadata_differs(p, skb); 5762 if (maclen == ETH_HLEN) 5763 diffs |= compare_ether_header(skb_mac_header(p), 5764 skb_mac_header(skb)); 5765 else if (!diffs) 5766 diffs = memcmp(skb_mac_header(p), 5767 skb_mac_header(skb), 5768 maclen); 5769 NAPI_GRO_CB(p)->same_flow = !diffs; 5770 } 5771 5772 return head; 5773 } 5774 5775 static void skb_gro_reset_offset(struct sk_buff *skb) 5776 { 5777 const struct skb_shared_info *pinfo = skb_shinfo(skb); 5778 const skb_frag_t *frag0 = &pinfo->frags[0]; 5779 5780 NAPI_GRO_CB(skb)->data_offset = 0; 5781 NAPI_GRO_CB(skb)->frag0 = NULL; 5782 NAPI_GRO_CB(skb)->frag0_len = 0; 5783 5784 if (!skb_headlen(skb) && pinfo->nr_frags && 5785 !PageHighMem(skb_frag_page(frag0))) { 5786 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); 5787 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int, 5788 skb_frag_size(frag0), 5789 skb->end - skb->tail); 5790 } 5791 } 5792 5793 static void gro_pull_from_frag0(struct sk_buff *skb, int grow) 5794 { 5795 struct skb_shared_info *pinfo = skb_shinfo(skb); 5796 5797 BUG_ON(skb->end - skb->tail < grow); 5798 5799 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); 5800 5801 skb->data_len -= grow; 5802 skb->tail += grow; 5803 5804 skb_frag_off_add(&pinfo->frags[0], grow); 5805 skb_frag_size_sub(&pinfo->frags[0], grow); 5806 5807 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { 5808 skb_frag_unref(skb, 0); 5809 memmove(pinfo->frags, pinfo->frags + 1, 5810 --pinfo->nr_frags * sizeof(pinfo->frags[0])); 5811 } 5812 } 5813 5814 static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head) 5815 { 5816 struct sk_buff *oldest; 5817 5818 oldest = list_last_entry(head, struct sk_buff, list); 5819 5820 /* We are called with head length >= MAX_GRO_SKBS, so this is 5821 * impossible. 5822 */ 5823 if (WARN_ON_ONCE(!oldest)) 5824 return; 5825 5826 /* Do not adjust napi->gro_hash[].count, caller is adding a new 5827 * SKB to the chain. 5828 */ 5829 skb_list_del_init(oldest); 5830 napi_gro_complete(napi, oldest); 5831 } 5832 5833 INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *, 5834 struct sk_buff *)); 5835 INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *, 5836 struct sk_buff *)); 5837 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 5838 { 5839 u32 hash = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1); 5840 struct list_head *head = &offload_base; 5841 struct packet_offload *ptype; 5842 __be16 type = skb->protocol; 5843 struct list_head *gro_head; 5844 struct sk_buff *pp = NULL; 5845 enum gro_result ret; 5846 int same_flow; 5847 int grow; 5848 5849 if (netif_elide_gro(skb->dev)) 5850 goto normal; 5851 5852 gro_head = gro_list_prepare(napi, skb); 5853 5854 rcu_read_lock(); 5855 list_for_each_entry_rcu(ptype, head, list) { 5856 if (ptype->type != type || !ptype->callbacks.gro_receive) 5857 continue; 5858 5859 skb_set_network_header(skb, skb_gro_offset(skb)); 5860 skb_reset_mac_len(skb); 5861 NAPI_GRO_CB(skb)->same_flow = 0; 5862 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb); 5863 NAPI_GRO_CB(skb)->free = 0; 5864 NAPI_GRO_CB(skb)->encap_mark = 0; 5865 NAPI_GRO_CB(skb)->recursion_counter = 0; 5866 NAPI_GRO_CB(skb)->is_fou = 0; 5867 NAPI_GRO_CB(skb)->is_atomic = 1; 5868 NAPI_GRO_CB(skb)->gro_remcsum_start = 0; 5869 5870 /* Setup for GRO checksum validation */ 5871 switch (skb->ip_summed) { 5872 case CHECKSUM_COMPLETE: 5873 NAPI_GRO_CB(skb)->csum = skb->csum; 5874 NAPI_GRO_CB(skb)->csum_valid = 1; 5875 NAPI_GRO_CB(skb)->csum_cnt = 0; 5876 break; 5877 case CHECKSUM_UNNECESSARY: 5878 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; 5879 NAPI_GRO_CB(skb)->csum_valid = 0; 5880 break; 5881 default: 5882 NAPI_GRO_CB(skb)->csum_cnt = 0; 5883 NAPI_GRO_CB(skb)->csum_valid = 0; 5884 } 5885 5886 pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive, 5887 ipv6_gro_receive, inet_gro_receive, 5888 gro_head, skb); 5889 break; 5890 } 5891 rcu_read_unlock(); 5892 5893 if (&ptype->list == head) 5894 goto normal; 5895 5896 if (PTR_ERR(pp) == -EINPROGRESS) { 5897 ret = GRO_CONSUMED; 5898 goto ok; 5899 } 5900 5901 same_flow = NAPI_GRO_CB(skb)->same_flow; 5902 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; 5903 5904 if (pp) { 5905 skb_list_del_init(pp); 5906 napi_gro_complete(napi, pp); 5907 napi->gro_hash[hash].count--; 5908 } 5909 5910 if (same_flow) 5911 goto ok; 5912 5913 if (NAPI_GRO_CB(skb)->flush) 5914 goto normal; 5915 5916 if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) { 5917 gro_flush_oldest(napi, gro_head); 5918 } else { 5919 napi->gro_hash[hash].count++; 5920 } 5921 NAPI_GRO_CB(skb)->count = 1; 5922 NAPI_GRO_CB(skb)->age = jiffies; 5923 NAPI_GRO_CB(skb)->last = skb; 5924 skb_shinfo(skb)->gso_size = skb_gro_len(skb); 5925 list_add(&skb->list, gro_head); 5926 ret = GRO_HELD; 5927 5928 pull: 5929 grow = skb_gro_offset(skb) - skb_headlen(skb); 5930 if (grow > 0) 5931 gro_pull_from_frag0(skb, grow); 5932 ok: 5933 if (napi->gro_hash[hash].count) { 5934 if (!test_bit(hash, &napi->gro_bitmask)) 5935 __set_bit(hash, &napi->gro_bitmask); 5936 } else if (test_bit(hash, &napi->gro_bitmask)) { 5937 __clear_bit(hash, &napi->gro_bitmask); 5938 } 5939 5940 return ret; 5941 5942 normal: 5943 ret = GRO_NORMAL; 5944 goto pull; 5945 } 5946 5947 struct packet_offload *gro_find_receive_by_type(__be16 type) 5948 { 5949 struct list_head *offload_head = &offload_base; 5950 struct packet_offload *ptype; 5951 5952 list_for_each_entry_rcu(ptype, offload_head, list) { 5953 if (ptype->type != type || !ptype->callbacks.gro_receive) 5954 continue; 5955 return ptype; 5956 } 5957 return NULL; 5958 } 5959 EXPORT_SYMBOL(gro_find_receive_by_type); 5960 5961 struct packet_offload *gro_find_complete_by_type(__be16 type) 5962 { 5963 struct list_head *offload_head = &offload_base; 5964 struct packet_offload *ptype; 5965 5966 list_for_each_entry_rcu(ptype, offload_head, list) { 5967 if (ptype->type != type || !ptype->callbacks.gro_complete) 5968 continue; 5969 return ptype; 5970 } 5971 return NULL; 5972 } 5973 EXPORT_SYMBOL(gro_find_complete_by_type); 5974 5975 static void napi_skb_free_stolen_head(struct sk_buff *skb) 5976 { 5977 skb_dst_drop(skb); 5978 skb_ext_put(skb); 5979 kmem_cache_free(skbuff_head_cache, skb); 5980 } 5981 5982 static gro_result_t napi_skb_finish(struct napi_struct *napi, 5983 struct sk_buff *skb, 5984 gro_result_t ret) 5985 { 5986 switch (ret) { 5987 case GRO_NORMAL: 5988 gro_normal_one(napi, skb); 5989 break; 5990 5991 case GRO_DROP: 5992 kfree_skb(skb); 5993 break; 5994 5995 case GRO_MERGED_FREE: 5996 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) 5997 napi_skb_free_stolen_head(skb); 5998 else 5999 __kfree_skb(skb); 6000 break; 6001 6002 case GRO_HELD: 6003 case GRO_MERGED: 6004 case GRO_CONSUMED: 6005 break; 6006 } 6007 6008 return ret; 6009 } 6010 6011 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 6012 { 6013 gro_result_t ret; 6014 6015 skb_mark_napi_id(skb, napi); 6016 trace_napi_gro_receive_entry(skb); 6017 6018 skb_gro_reset_offset(skb); 6019 6020 ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb)); 6021 trace_napi_gro_receive_exit(ret); 6022 6023 return ret; 6024 } 6025 EXPORT_SYMBOL(napi_gro_receive); 6026 6027 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) 6028 { 6029 if (unlikely(skb->pfmemalloc)) { 6030 consume_skb(skb); 6031 return; 6032 } 6033 __skb_pull(skb, skb_headlen(skb)); 6034 /* restore the reserve we had after netdev_alloc_skb_ip_align() */ 6035 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); 6036 __vlan_hwaccel_clear_tag(skb); 6037 skb->dev = napi->dev; 6038 skb->skb_iif = 0; 6039 6040 /* eth_type_trans() assumes pkt_type is PACKET_HOST */ 6041 skb->pkt_type = PACKET_HOST; 6042 6043 skb->encapsulation = 0; 6044 skb_shinfo(skb)->gso_type = 0; 6045 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 6046 skb_ext_reset(skb); 6047 6048 napi->skb = skb; 6049 } 6050 6051 struct sk_buff *napi_get_frags(struct napi_struct *napi) 6052 { 6053 struct sk_buff *skb = napi->skb; 6054 6055 if (!skb) { 6056 skb = napi_alloc_skb(napi, GRO_MAX_HEAD); 6057 if (skb) { 6058 napi->skb = skb; 6059 skb_mark_napi_id(skb, napi); 6060 } 6061 } 6062 return skb; 6063 } 6064 EXPORT_SYMBOL(napi_get_frags); 6065 6066 static gro_result_t napi_frags_finish(struct napi_struct *napi, 6067 struct sk_buff *skb, 6068 gro_result_t ret) 6069 { 6070 switch (ret) { 6071 case GRO_NORMAL: 6072 case GRO_HELD: 6073 __skb_push(skb, ETH_HLEN); 6074 skb->protocol = eth_type_trans(skb, skb->dev); 6075 if (ret == GRO_NORMAL) 6076 gro_normal_one(napi, skb); 6077 break; 6078 6079 case GRO_DROP: 6080 napi_reuse_skb(napi, skb); 6081 break; 6082 6083 case GRO_MERGED_FREE: 6084 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) 6085 napi_skb_free_stolen_head(skb); 6086 else 6087 napi_reuse_skb(napi, skb); 6088 break; 6089 6090 case GRO_MERGED: 6091 case GRO_CONSUMED: 6092 break; 6093 } 6094 6095 return ret; 6096 } 6097 6098 /* Upper GRO stack assumes network header starts at gro_offset=0 6099 * Drivers could call both napi_gro_frags() and napi_gro_receive() 6100 * We copy ethernet header into skb->data to have a common layout. 6101 */ 6102 static struct sk_buff *napi_frags_skb(struct napi_struct *napi) 6103 { 6104 struct sk_buff *skb = napi->skb; 6105 const struct ethhdr *eth; 6106 unsigned int hlen = sizeof(*eth); 6107 6108 napi->skb = NULL; 6109 6110 skb_reset_mac_header(skb); 6111 skb_gro_reset_offset(skb); 6112 6113 if (unlikely(skb_gro_header_hard(skb, hlen))) { 6114 eth = skb_gro_header_slow(skb, hlen, 0); 6115 if (unlikely(!eth)) { 6116 net_warn_ratelimited("%s: dropping impossible skb from %s\n", 6117 __func__, napi->dev->name); 6118 napi_reuse_skb(napi, skb); 6119 return NULL; 6120 } 6121 } else { 6122 eth = (const struct ethhdr *)skb->data; 6123 gro_pull_from_frag0(skb, hlen); 6124 NAPI_GRO_CB(skb)->frag0 += hlen; 6125 NAPI_GRO_CB(skb)->frag0_len -= hlen; 6126 } 6127 __skb_pull(skb, hlen); 6128 6129 /* 6130 * This works because the only protocols we care about don't require 6131 * special handling. 6132 * We'll fix it up properly in napi_frags_finish() 6133 */ 6134 skb->protocol = eth->h_proto; 6135 6136 return skb; 6137 } 6138 6139 gro_result_t napi_gro_frags(struct napi_struct *napi) 6140 { 6141 gro_result_t ret; 6142 struct sk_buff *skb = napi_frags_skb(napi); 6143 6144 if (!skb) 6145 return GRO_DROP; 6146 6147 trace_napi_gro_frags_entry(skb); 6148 6149 ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); 6150 trace_napi_gro_frags_exit(ret); 6151 6152 return ret; 6153 } 6154 EXPORT_SYMBOL(napi_gro_frags); 6155 6156 /* Compute the checksum from gro_offset and return the folded value 6157 * after adding in any pseudo checksum. 6158 */ 6159 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb) 6160 { 6161 __wsum wsum; 6162 __sum16 sum; 6163 6164 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); 6165 6166 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ 6167 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); 6168 /* See comments in __skb_checksum_complete(). */ 6169 if (likely(!sum)) { 6170 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 6171 !skb->csum_complete_sw) 6172 netdev_rx_csum_fault(skb->dev, skb); 6173 } 6174 6175 NAPI_GRO_CB(skb)->csum = wsum; 6176 NAPI_GRO_CB(skb)->csum_valid = 1; 6177 6178 return sum; 6179 } 6180 EXPORT_SYMBOL(__skb_gro_checksum_complete); 6181 6182 static void net_rps_send_ipi(struct softnet_data *remsd) 6183 { 6184 #ifdef CONFIG_RPS 6185 while (remsd) { 6186 struct softnet_data *next = remsd->rps_ipi_next; 6187 6188 if (cpu_online(remsd->cpu)) 6189 smp_call_function_single_async(remsd->cpu, &remsd->csd); 6190 remsd = next; 6191 } 6192 #endif 6193 } 6194 6195 /* 6196 * net_rps_action_and_irq_enable sends any pending IPI's for rps. 6197 * Note: called with local irq disabled, but exits with local irq enabled. 6198 */ 6199 static void net_rps_action_and_irq_enable(struct softnet_data *sd) 6200 { 6201 #ifdef CONFIG_RPS 6202 struct softnet_data *remsd = sd->rps_ipi_list; 6203 6204 if (remsd) { 6205 sd->rps_ipi_list = NULL; 6206 6207 local_irq_enable(); 6208 6209 /* Send pending IPI's to kick RPS processing on remote cpus. */ 6210 net_rps_send_ipi(remsd); 6211 } else 6212 #endif 6213 local_irq_enable(); 6214 } 6215 6216 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) 6217 { 6218 #ifdef CONFIG_RPS 6219 return sd->rps_ipi_list != NULL; 6220 #else 6221 return false; 6222 #endif 6223 } 6224 6225 static int process_backlog(struct napi_struct *napi, int quota) 6226 { 6227 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); 6228 bool again = true; 6229 int work = 0; 6230 6231 /* Check if we have pending ipi, its better to send them now, 6232 * not waiting net_rx_action() end. 6233 */ 6234 if (sd_has_rps_ipi_waiting(sd)) { 6235 local_irq_disable(); 6236 net_rps_action_and_irq_enable(sd); 6237 } 6238 6239 napi->weight = dev_rx_weight; 6240 while (again) { 6241 struct sk_buff *skb; 6242 6243 while ((skb = __skb_dequeue(&sd->process_queue))) { 6244 rcu_read_lock(); 6245 __netif_receive_skb(skb); 6246 rcu_read_unlock(); 6247 input_queue_head_incr(sd); 6248 if (++work >= quota) 6249 return work; 6250 6251 } 6252 6253 local_irq_disable(); 6254 rps_lock(sd); 6255 if (skb_queue_empty(&sd->input_pkt_queue)) { 6256 /* 6257 * Inline a custom version of __napi_complete(). 6258 * only current cpu owns and manipulates this napi, 6259 * and NAPI_STATE_SCHED is the only possible flag set 6260 * on backlog. 6261 * We can use a plain write instead of clear_bit(), 6262 * and we dont need an smp_mb() memory barrier. 6263 */ 6264 napi->state = 0; 6265 again = false; 6266 } else { 6267 skb_queue_splice_tail_init(&sd->input_pkt_queue, 6268 &sd->process_queue); 6269 } 6270 rps_unlock(sd); 6271 local_irq_enable(); 6272 } 6273 6274 return work; 6275 } 6276 6277 /** 6278 * __napi_schedule - schedule for receive 6279 * @n: entry to schedule 6280 * 6281 * The entry's receive function will be scheduled to run. 6282 * Consider using __napi_schedule_irqoff() if hard irqs are masked. 6283 */ 6284 void __napi_schedule(struct napi_struct *n) 6285 { 6286 unsigned long flags; 6287 6288 local_irq_save(flags); 6289 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 6290 local_irq_restore(flags); 6291 } 6292 EXPORT_SYMBOL(__napi_schedule); 6293 6294 /** 6295 * napi_schedule_prep - check if napi can be scheduled 6296 * @n: napi context 6297 * 6298 * Test if NAPI routine is already running, and if not mark 6299 * it as running. This is used as a condition variable 6300 * insure only one NAPI poll instance runs. We also make 6301 * sure there is no pending NAPI disable. 6302 */ 6303 bool napi_schedule_prep(struct napi_struct *n) 6304 { 6305 unsigned long val, new; 6306 6307 do { 6308 val = READ_ONCE(n->state); 6309 if (unlikely(val & NAPIF_STATE_DISABLE)) 6310 return false; 6311 new = val | NAPIF_STATE_SCHED; 6312 6313 /* Sets STATE_MISSED bit if STATE_SCHED was already set 6314 * This was suggested by Alexander Duyck, as compiler 6315 * emits better code than : 6316 * if (val & NAPIF_STATE_SCHED) 6317 * new |= NAPIF_STATE_MISSED; 6318 */ 6319 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED * 6320 NAPIF_STATE_MISSED; 6321 } while (cmpxchg(&n->state, val, new) != val); 6322 6323 return !(val & NAPIF_STATE_SCHED); 6324 } 6325 EXPORT_SYMBOL(napi_schedule_prep); 6326 6327 /** 6328 * __napi_schedule_irqoff - schedule for receive 6329 * @n: entry to schedule 6330 * 6331 * Variant of __napi_schedule() assuming hard irqs are masked 6332 */ 6333 void __napi_schedule_irqoff(struct napi_struct *n) 6334 { 6335 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 6336 } 6337 EXPORT_SYMBOL(__napi_schedule_irqoff); 6338 6339 bool napi_complete_done(struct napi_struct *n, int work_done) 6340 { 6341 unsigned long flags, val, new, timeout = 0; 6342 bool ret = true; 6343 6344 /* 6345 * 1) Don't let napi dequeue from the cpu poll list 6346 * just in case its running on a different cpu. 6347 * 2) If we are busy polling, do nothing here, we have 6348 * the guarantee we will be called later. 6349 */ 6350 if (unlikely(n->state & (NAPIF_STATE_NPSVC | 6351 NAPIF_STATE_IN_BUSY_POLL))) 6352 return false; 6353 6354 if (work_done) { 6355 if (n->gro_bitmask) 6356 timeout = READ_ONCE(n->dev->gro_flush_timeout); 6357 n->defer_hard_irqs_count = READ_ONCE(n->dev->napi_defer_hard_irqs); 6358 } 6359 if (n->defer_hard_irqs_count > 0) { 6360 n->defer_hard_irqs_count--; 6361 timeout = READ_ONCE(n->dev->gro_flush_timeout); 6362 if (timeout) 6363 ret = false; 6364 } 6365 if (n->gro_bitmask) { 6366 /* When the NAPI instance uses a timeout and keeps postponing 6367 * it, we need to bound somehow the time packets are kept in 6368 * the GRO layer 6369 */ 6370 napi_gro_flush(n, !!timeout); 6371 } 6372 6373 gro_normal_list(n); 6374 6375 if (unlikely(!list_empty(&n->poll_list))) { 6376 /* If n->poll_list is not empty, we need to mask irqs */ 6377 local_irq_save(flags); 6378 list_del_init(&n->poll_list); 6379 local_irq_restore(flags); 6380 } 6381 6382 do { 6383 val = READ_ONCE(n->state); 6384 6385 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED)); 6386 6387 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED); 6388 6389 /* If STATE_MISSED was set, leave STATE_SCHED set, 6390 * because we will call napi->poll() one more time. 6391 * This C code was suggested by Alexander Duyck to help gcc. 6392 */ 6393 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED * 6394 NAPIF_STATE_SCHED; 6395 } while (cmpxchg(&n->state, val, new) != val); 6396 6397 if (unlikely(val & NAPIF_STATE_MISSED)) { 6398 __napi_schedule(n); 6399 return false; 6400 } 6401 6402 if (timeout) 6403 hrtimer_start(&n->timer, ns_to_ktime(timeout), 6404 HRTIMER_MODE_REL_PINNED); 6405 return ret; 6406 } 6407 EXPORT_SYMBOL(napi_complete_done); 6408 6409 /* must be called under rcu_read_lock(), as we dont take a reference */ 6410 static struct napi_struct *napi_by_id(unsigned int napi_id) 6411 { 6412 unsigned int hash = napi_id % HASH_SIZE(napi_hash); 6413 struct napi_struct *napi; 6414 6415 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node) 6416 if (napi->napi_id == napi_id) 6417 return napi; 6418 6419 return NULL; 6420 } 6421 6422 #if defined(CONFIG_NET_RX_BUSY_POLL) 6423 6424 #define BUSY_POLL_BUDGET 8 6425 6426 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock) 6427 { 6428 int rc; 6429 6430 /* Busy polling means there is a high chance device driver hard irq 6431 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was 6432 * set in napi_schedule_prep(). 6433 * Since we are about to call napi->poll() once more, we can safely 6434 * clear NAPI_STATE_MISSED. 6435 * 6436 * Note: x86 could use a single "lock and ..." instruction 6437 * to perform these two clear_bit() 6438 */ 6439 clear_bit(NAPI_STATE_MISSED, &napi->state); 6440 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state); 6441 6442 local_bh_disable(); 6443 6444 /* All we really want here is to re-enable device interrupts. 6445 * Ideally, a new ndo_busy_poll_stop() could avoid another round. 6446 */ 6447 rc = napi->poll(napi, BUSY_POLL_BUDGET); 6448 /* We can't gro_normal_list() here, because napi->poll() might have 6449 * rearmed the napi (napi_complete_done()) in which case it could 6450 * already be running on another CPU. 6451 */ 6452 trace_napi_poll(napi, rc, BUSY_POLL_BUDGET); 6453 netpoll_poll_unlock(have_poll_lock); 6454 if (rc == BUSY_POLL_BUDGET) { 6455 /* As the whole budget was spent, we still own the napi so can 6456 * safely handle the rx_list. 6457 */ 6458 gro_normal_list(napi); 6459 __napi_schedule(napi); 6460 } 6461 local_bh_enable(); 6462 } 6463 6464 void napi_busy_loop(unsigned int napi_id, 6465 bool (*loop_end)(void *, unsigned long), 6466 void *loop_end_arg) 6467 { 6468 unsigned long start_time = loop_end ? busy_loop_current_time() : 0; 6469 int (*napi_poll)(struct napi_struct *napi, int budget); 6470 void *have_poll_lock = NULL; 6471 struct napi_struct *napi; 6472 6473 restart: 6474 napi_poll = NULL; 6475 6476 rcu_read_lock(); 6477 6478 napi = napi_by_id(napi_id); 6479 if (!napi) 6480 goto out; 6481 6482 preempt_disable(); 6483 for (;;) { 6484 int work = 0; 6485 6486 local_bh_disable(); 6487 if (!napi_poll) { 6488 unsigned long val = READ_ONCE(napi->state); 6489 6490 /* If multiple threads are competing for this napi, 6491 * we avoid dirtying napi->state as much as we can. 6492 */ 6493 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED | 6494 NAPIF_STATE_IN_BUSY_POLL)) 6495 goto count; 6496 if (cmpxchg(&napi->state, val, 6497 val | NAPIF_STATE_IN_BUSY_POLL | 6498 NAPIF_STATE_SCHED) != val) 6499 goto count; 6500 have_poll_lock = netpoll_poll_lock(napi); 6501 napi_poll = napi->poll; 6502 } 6503 work = napi_poll(napi, BUSY_POLL_BUDGET); 6504 trace_napi_poll(napi, work, BUSY_POLL_BUDGET); 6505 gro_normal_list(napi); 6506 count: 6507 if (work > 0) 6508 __NET_ADD_STATS(dev_net(napi->dev), 6509 LINUX_MIB_BUSYPOLLRXPACKETS, work); 6510 local_bh_enable(); 6511 6512 if (!loop_end || loop_end(loop_end_arg, start_time)) 6513 break; 6514 6515 if (unlikely(need_resched())) { 6516 if (napi_poll) 6517 busy_poll_stop(napi, have_poll_lock); 6518 preempt_enable(); 6519 rcu_read_unlock(); 6520 cond_resched(); 6521 if (loop_end(loop_end_arg, start_time)) 6522 return; 6523 goto restart; 6524 } 6525 cpu_relax(); 6526 } 6527 if (napi_poll) 6528 busy_poll_stop(napi, have_poll_lock); 6529 preempt_enable(); 6530 out: 6531 rcu_read_unlock(); 6532 } 6533 EXPORT_SYMBOL(napi_busy_loop); 6534 6535 #endif /* CONFIG_NET_RX_BUSY_POLL */ 6536 6537 static void napi_hash_add(struct napi_struct *napi) 6538 { 6539 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) || 6540 test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) 6541 return; 6542 6543 spin_lock(&napi_hash_lock); 6544 6545 /* 0..NR_CPUS range is reserved for sender_cpu use */ 6546 do { 6547 if (unlikely(++napi_gen_id < MIN_NAPI_ID)) 6548 napi_gen_id = MIN_NAPI_ID; 6549 } while (napi_by_id(napi_gen_id)); 6550 napi->napi_id = napi_gen_id; 6551 6552 hlist_add_head_rcu(&napi->napi_hash_node, 6553 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); 6554 6555 spin_unlock(&napi_hash_lock); 6556 } 6557 6558 /* Warning : caller is responsible to make sure rcu grace period 6559 * is respected before freeing memory containing @napi 6560 */ 6561 bool napi_hash_del(struct napi_struct *napi) 6562 { 6563 bool rcu_sync_needed = false; 6564 6565 spin_lock(&napi_hash_lock); 6566 6567 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) { 6568 rcu_sync_needed = true; 6569 hlist_del_rcu(&napi->napi_hash_node); 6570 } 6571 spin_unlock(&napi_hash_lock); 6572 return rcu_sync_needed; 6573 } 6574 EXPORT_SYMBOL_GPL(napi_hash_del); 6575 6576 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer) 6577 { 6578 struct napi_struct *napi; 6579 6580 napi = container_of(timer, struct napi_struct, timer); 6581 6582 /* Note : we use a relaxed variant of napi_schedule_prep() not setting 6583 * NAPI_STATE_MISSED, since we do not react to a device IRQ. 6584 */ 6585 if (!napi_disable_pending(napi) && 6586 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) 6587 __napi_schedule_irqoff(napi); 6588 6589 return HRTIMER_NORESTART; 6590 } 6591 6592 static void init_gro_hash(struct napi_struct *napi) 6593 { 6594 int i; 6595 6596 for (i = 0; i < GRO_HASH_BUCKETS; i++) { 6597 INIT_LIST_HEAD(&napi->gro_hash[i].list); 6598 napi->gro_hash[i].count = 0; 6599 } 6600 napi->gro_bitmask = 0; 6601 } 6602 6603 void netif_napi_add(struct net_device *dev, struct napi_struct *napi, 6604 int (*poll)(struct napi_struct *, int), int weight) 6605 { 6606 INIT_LIST_HEAD(&napi->poll_list); 6607 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); 6608 napi->timer.function = napi_watchdog; 6609 init_gro_hash(napi); 6610 napi->skb = NULL; 6611 INIT_LIST_HEAD(&napi->rx_list); 6612 napi->rx_count = 0; 6613 napi->poll = poll; 6614 if (weight > NAPI_POLL_WEIGHT) 6615 netdev_err_once(dev, "%s() called with weight %d\n", __func__, 6616 weight); 6617 napi->weight = weight; 6618 list_add(&napi->dev_list, &dev->napi_list); 6619 napi->dev = dev; 6620 #ifdef CONFIG_NETPOLL 6621 napi->poll_owner = -1; 6622 #endif 6623 set_bit(NAPI_STATE_SCHED, &napi->state); 6624 napi_hash_add(napi); 6625 } 6626 EXPORT_SYMBOL(netif_napi_add); 6627 6628 void napi_disable(struct napi_struct *n) 6629 { 6630 might_sleep(); 6631 set_bit(NAPI_STATE_DISABLE, &n->state); 6632 6633 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state)) 6634 msleep(1); 6635 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state)) 6636 msleep(1); 6637 6638 hrtimer_cancel(&n->timer); 6639 6640 clear_bit(NAPI_STATE_DISABLE, &n->state); 6641 } 6642 EXPORT_SYMBOL(napi_disable); 6643 6644 static void flush_gro_hash(struct napi_struct *napi) 6645 { 6646 int i; 6647 6648 for (i = 0; i < GRO_HASH_BUCKETS; i++) { 6649 struct sk_buff *skb, *n; 6650 6651 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list) 6652 kfree_skb(skb); 6653 napi->gro_hash[i].count = 0; 6654 } 6655 } 6656 6657 /* Must be called in process context */ 6658 void netif_napi_del(struct napi_struct *napi) 6659 { 6660 might_sleep(); 6661 if (napi_hash_del(napi)) 6662 synchronize_net(); 6663 list_del_init(&napi->dev_list); 6664 napi_free_frags(napi); 6665 6666 flush_gro_hash(napi); 6667 napi->gro_bitmask = 0; 6668 } 6669 EXPORT_SYMBOL(netif_napi_del); 6670 6671 static int napi_poll(struct napi_struct *n, struct list_head *repoll) 6672 { 6673 void *have; 6674 int work, weight; 6675 6676 list_del_init(&n->poll_list); 6677 6678 have = netpoll_poll_lock(n); 6679 6680 weight = n->weight; 6681 6682 /* This NAPI_STATE_SCHED test is for avoiding a race 6683 * with netpoll's poll_napi(). Only the entity which 6684 * obtains the lock and sees NAPI_STATE_SCHED set will 6685 * actually make the ->poll() call. Therefore we avoid 6686 * accidentally calling ->poll() when NAPI is not scheduled. 6687 */ 6688 work = 0; 6689 if (test_bit(NAPI_STATE_SCHED, &n->state)) { 6690 work = n->poll(n, weight); 6691 trace_napi_poll(n, work, weight); 6692 } 6693 6694 if (unlikely(work > weight)) 6695 pr_err_once("NAPI poll function %pS returned %d, exceeding its budget of %d.\n", 6696 n->poll, work, weight); 6697 6698 if (likely(work < weight)) 6699 goto out_unlock; 6700 6701 /* Drivers must not modify the NAPI state if they 6702 * consume the entire weight. In such cases this code 6703 * still "owns" the NAPI instance and therefore can 6704 * move the instance around on the list at-will. 6705 */ 6706 if (unlikely(napi_disable_pending(n))) { 6707 napi_complete(n); 6708 goto out_unlock; 6709 } 6710 6711 if (n->gro_bitmask) { 6712 /* flush too old packets 6713 * If HZ < 1000, flush all packets. 6714 */ 6715 napi_gro_flush(n, HZ >= 1000); 6716 } 6717 6718 gro_normal_list(n); 6719 6720 /* Some drivers may have called napi_schedule 6721 * prior to exhausting their budget. 6722 */ 6723 if (unlikely(!list_empty(&n->poll_list))) { 6724 pr_warn_once("%s: Budget exhausted after napi rescheduled\n", 6725 n->dev ? n->dev->name : "backlog"); 6726 goto out_unlock; 6727 } 6728 6729 list_add_tail(&n->poll_list, repoll); 6730 6731 out_unlock: 6732 netpoll_poll_unlock(have); 6733 6734 return work; 6735 } 6736 6737 static __latent_entropy void net_rx_action(struct softirq_action *h) 6738 { 6739 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 6740 unsigned long time_limit = jiffies + 6741 usecs_to_jiffies(netdev_budget_usecs); 6742 int budget = netdev_budget; 6743 LIST_HEAD(list); 6744 LIST_HEAD(repoll); 6745 6746 local_irq_disable(); 6747 list_splice_init(&sd->poll_list, &list); 6748 local_irq_enable(); 6749 6750 for (;;) { 6751 struct napi_struct *n; 6752 6753 if (list_empty(&list)) { 6754 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll)) 6755 goto out; 6756 break; 6757 } 6758 6759 n = list_first_entry(&list, struct napi_struct, poll_list); 6760 budget -= napi_poll(n, &repoll); 6761 6762 /* If softirq window is exhausted then punt. 6763 * Allow this to run for 2 jiffies since which will allow 6764 * an average latency of 1.5/HZ. 6765 */ 6766 if (unlikely(budget <= 0 || 6767 time_after_eq(jiffies, time_limit))) { 6768 sd->time_squeeze++; 6769 break; 6770 } 6771 } 6772 6773 local_irq_disable(); 6774 6775 list_splice_tail_init(&sd->poll_list, &list); 6776 list_splice_tail(&repoll, &list); 6777 list_splice(&list, &sd->poll_list); 6778 if (!list_empty(&sd->poll_list)) 6779 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 6780 6781 net_rps_action_and_irq_enable(sd); 6782 out: 6783 __kfree_skb_flush(); 6784 } 6785 6786 struct netdev_adjacent { 6787 struct net_device *dev; 6788 6789 /* upper master flag, there can only be one master device per list */ 6790 bool master; 6791 6792 /* lookup ignore flag */ 6793 bool ignore; 6794 6795 /* counter for the number of times this device was added to us */ 6796 u16 ref_nr; 6797 6798 /* private field for the users */ 6799 void *private; 6800 6801 struct list_head list; 6802 struct rcu_head rcu; 6803 }; 6804 6805 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev, 6806 struct list_head *adj_list) 6807 { 6808 struct netdev_adjacent *adj; 6809 6810 list_for_each_entry(adj, adj_list, list) { 6811 if (adj->dev == adj_dev) 6812 return adj; 6813 } 6814 return NULL; 6815 } 6816 6817 static int ____netdev_has_upper_dev(struct net_device *upper_dev, void *data) 6818 { 6819 struct net_device *dev = data; 6820 6821 return upper_dev == dev; 6822 } 6823 6824 /** 6825 * netdev_has_upper_dev - Check if device is linked to an upper device 6826 * @dev: device 6827 * @upper_dev: upper device to check 6828 * 6829 * Find out if a device is linked to specified upper device and return true 6830 * in case it is. Note that this checks only immediate upper device, 6831 * not through a complete stack of devices. The caller must hold the RTNL lock. 6832 */ 6833 bool netdev_has_upper_dev(struct net_device *dev, 6834 struct net_device *upper_dev) 6835 { 6836 ASSERT_RTNL(); 6837 6838 return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev, 6839 upper_dev); 6840 } 6841 EXPORT_SYMBOL(netdev_has_upper_dev); 6842 6843 /** 6844 * netdev_has_upper_dev_all - Check if device is linked to an upper device 6845 * @dev: device 6846 * @upper_dev: upper device to check 6847 * 6848 * Find out if a device is linked to specified upper device and return true 6849 * in case it is. Note that this checks the entire upper device chain. 6850 * The caller must hold rcu lock. 6851 */ 6852 6853 bool netdev_has_upper_dev_all_rcu(struct net_device *dev, 6854 struct net_device *upper_dev) 6855 { 6856 return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev, 6857 upper_dev); 6858 } 6859 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu); 6860 6861 /** 6862 * netdev_has_any_upper_dev - Check if device is linked to some device 6863 * @dev: device 6864 * 6865 * Find out if a device is linked to an upper device and return true in case 6866 * it is. The caller must hold the RTNL lock. 6867 */ 6868 bool netdev_has_any_upper_dev(struct net_device *dev) 6869 { 6870 ASSERT_RTNL(); 6871 6872 return !list_empty(&dev->adj_list.upper); 6873 } 6874 EXPORT_SYMBOL(netdev_has_any_upper_dev); 6875 6876 /** 6877 * netdev_master_upper_dev_get - Get master upper device 6878 * @dev: device 6879 * 6880 * Find a master upper device and return pointer to it or NULL in case 6881 * it's not there. The caller must hold the RTNL lock. 6882 */ 6883 struct net_device *netdev_master_upper_dev_get(struct net_device *dev) 6884 { 6885 struct netdev_adjacent *upper; 6886 6887 ASSERT_RTNL(); 6888 6889 if (list_empty(&dev->adj_list.upper)) 6890 return NULL; 6891 6892 upper = list_first_entry(&dev->adj_list.upper, 6893 struct netdev_adjacent, list); 6894 if (likely(upper->master)) 6895 return upper->dev; 6896 return NULL; 6897 } 6898 EXPORT_SYMBOL(netdev_master_upper_dev_get); 6899 6900 static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev) 6901 { 6902 struct netdev_adjacent *upper; 6903 6904 ASSERT_RTNL(); 6905 6906 if (list_empty(&dev->adj_list.upper)) 6907 return NULL; 6908 6909 upper = list_first_entry(&dev->adj_list.upper, 6910 struct netdev_adjacent, list); 6911 if (likely(upper->master) && !upper->ignore) 6912 return upper->dev; 6913 return NULL; 6914 } 6915 6916 /** 6917 * netdev_has_any_lower_dev - Check if device is linked to some device 6918 * @dev: device 6919 * 6920 * Find out if a device is linked to a lower device and return true in case 6921 * it is. The caller must hold the RTNL lock. 6922 */ 6923 static bool netdev_has_any_lower_dev(struct net_device *dev) 6924 { 6925 ASSERT_RTNL(); 6926 6927 return !list_empty(&dev->adj_list.lower); 6928 } 6929 6930 void *netdev_adjacent_get_private(struct list_head *adj_list) 6931 { 6932 struct netdev_adjacent *adj; 6933 6934 adj = list_entry(adj_list, struct netdev_adjacent, list); 6935 6936 return adj->private; 6937 } 6938 EXPORT_SYMBOL(netdev_adjacent_get_private); 6939 6940 /** 6941 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list 6942 * @dev: device 6943 * @iter: list_head ** of the current position 6944 * 6945 * Gets the next device from the dev's upper list, starting from iter 6946 * position. The caller must hold RCU read lock. 6947 */ 6948 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev, 6949 struct list_head **iter) 6950 { 6951 struct netdev_adjacent *upper; 6952 6953 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 6954 6955 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 6956 6957 if (&upper->list == &dev->adj_list.upper) 6958 return NULL; 6959 6960 *iter = &upper->list; 6961 6962 return upper->dev; 6963 } 6964 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu); 6965 6966 static struct net_device *__netdev_next_upper_dev(struct net_device *dev, 6967 struct list_head **iter, 6968 bool *ignore) 6969 { 6970 struct netdev_adjacent *upper; 6971 6972 upper = list_entry((*iter)->next, struct netdev_adjacent, list); 6973 6974 if (&upper->list == &dev->adj_list.upper) 6975 return NULL; 6976 6977 *iter = &upper->list; 6978 *ignore = upper->ignore; 6979 6980 return upper->dev; 6981 } 6982 6983 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev, 6984 struct list_head **iter) 6985 { 6986 struct netdev_adjacent *upper; 6987 6988 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 6989 6990 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 6991 6992 if (&upper->list == &dev->adj_list.upper) 6993 return NULL; 6994 6995 *iter = &upper->list; 6996 6997 return upper->dev; 6998 } 6999 7000 static int __netdev_walk_all_upper_dev(struct net_device *dev, 7001 int (*fn)(struct net_device *dev, 7002 void *data), 7003 void *data) 7004 { 7005 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 7006 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 7007 int ret, cur = 0; 7008 bool ignore; 7009 7010 now = dev; 7011 iter = &dev->adj_list.upper; 7012 7013 while (1) { 7014 if (now != dev) { 7015 ret = fn(now, data); 7016 if (ret) 7017 return ret; 7018 } 7019 7020 next = NULL; 7021 while (1) { 7022 udev = __netdev_next_upper_dev(now, &iter, &ignore); 7023 if (!udev) 7024 break; 7025 if (ignore) 7026 continue; 7027 7028 next = udev; 7029 niter = &udev->adj_list.upper; 7030 dev_stack[cur] = now; 7031 iter_stack[cur++] = iter; 7032 break; 7033 } 7034 7035 if (!next) { 7036 if (!cur) 7037 return 0; 7038 next = dev_stack[--cur]; 7039 niter = iter_stack[cur]; 7040 } 7041 7042 now = next; 7043 iter = niter; 7044 } 7045 7046 return 0; 7047 } 7048 7049 int netdev_walk_all_upper_dev_rcu(struct net_device *dev, 7050 int (*fn)(struct net_device *dev, 7051 void *data), 7052 void *data) 7053 { 7054 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 7055 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 7056 int ret, cur = 0; 7057 7058 now = dev; 7059 iter = &dev->adj_list.upper; 7060 7061 while (1) { 7062 if (now != dev) { 7063 ret = fn(now, data); 7064 if (ret) 7065 return ret; 7066 } 7067 7068 next = NULL; 7069 while (1) { 7070 udev = netdev_next_upper_dev_rcu(now, &iter); 7071 if (!udev) 7072 break; 7073 7074 next = udev; 7075 niter = &udev->adj_list.upper; 7076 dev_stack[cur] = now; 7077 iter_stack[cur++] = iter; 7078 break; 7079 } 7080 7081 if (!next) { 7082 if (!cur) 7083 return 0; 7084 next = dev_stack[--cur]; 7085 niter = iter_stack[cur]; 7086 } 7087 7088 now = next; 7089 iter = niter; 7090 } 7091 7092 return 0; 7093 } 7094 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu); 7095 7096 static bool __netdev_has_upper_dev(struct net_device *dev, 7097 struct net_device *upper_dev) 7098 { 7099 ASSERT_RTNL(); 7100 7101 return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev, 7102 upper_dev); 7103 } 7104 7105 /** 7106 * netdev_lower_get_next_private - Get the next ->private from the 7107 * lower neighbour list 7108 * @dev: device 7109 * @iter: list_head ** of the current position 7110 * 7111 * Gets the next netdev_adjacent->private from the dev's lower neighbour 7112 * list, starting from iter position. The caller must hold either hold the 7113 * RTNL lock or its own locking that guarantees that the neighbour lower 7114 * list will remain unchanged. 7115 */ 7116 void *netdev_lower_get_next_private(struct net_device *dev, 7117 struct list_head **iter) 7118 { 7119 struct netdev_adjacent *lower; 7120 7121 lower = list_entry(*iter, struct netdev_adjacent, list); 7122 7123 if (&lower->list == &dev->adj_list.lower) 7124 return NULL; 7125 7126 *iter = lower->list.next; 7127 7128 return lower->private; 7129 } 7130 EXPORT_SYMBOL(netdev_lower_get_next_private); 7131 7132 /** 7133 * netdev_lower_get_next_private_rcu - Get the next ->private from the 7134 * lower neighbour list, RCU 7135 * variant 7136 * @dev: device 7137 * @iter: list_head ** of the current position 7138 * 7139 * Gets the next netdev_adjacent->private from the dev's lower neighbour 7140 * list, starting from iter position. The caller must hold RCU read lock. 7141 */ 7142 void *netdev_lower_get_next_private_rcu(struct net_device *dev, 7143 struct list_head **iter) 7144 { 7145 struct netdev_adjacent *lower; 7146 7147 WARN_ON_ONCE(!rcu_read_lock_held()); 7148 7149 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 7150 7151 if (&lower->list == &dev->adj_list.lower) 7152 return NULL; 7153 7154 *iter = &lower->list; 7155 7156 return lower->private; 7157 } 7158 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu); 7159 7160 /** 7161 * netdev_lower_get_next - Get the next device from the lower neighbour 7162 * list 7163 * @dev: device 7164 * @iter: list_head ** of the current position 7165 * 7166 * Gets the next netdev_adjacent from the dev's lower neighbour 7167 * list, starting from iter position. The caller must hold RTNL lock or 7168 * its own locking that guarantees that the neighbour lower 7169 * list will remain unchanged. 7170 */ 7171 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter) 7172 { 7173 struct netdev_adjacent *lower; 7174 7175 lower = list_entry(*iter, struct netdev_adjacent, list); 7176 7177 if (&lower->list == &dev->adj_list.lower) 7178 return NULL; 7179 7180 *iter = lower->list.next; 7181 7182 return lower->dev; 7183 } 7184 EXPORT_SYMBOL(netdev_lower_get_next); 7185 7186 static struct net_device *netdev_next_lower_dev(struct net_device *dev, 7187 struct list_head **iter) 7188 { 7189 struct netdev_adjacent *lower; 7190 7191 lower = list_entry((*iter)->next, struct netdev_adjacent, list); 7192 7193 if (&lower->list == &dev->adj_list.lower) 7194 return NULL; 7195 7196 *iter = &lower->list; 7197 7198 return lower->dev; 7199 } 7200 7201 static struct net_device *__netdev_next_lower_dev(struct net_device *dev, 7202 struct list_head **iter, 7203 bool *ignore) 7204 { 7205 struct netdev_adjacent *lower; 7206 7207 lower = list_entry((*iter)->next, struct netdev_adjacent, list); 7208 7209 if (&lower->list == &dev->adj_list.lower) 7210 return NULL; 7211 7212 *iter = &lower->list; 7213 *ignore = lower->ignore; 7214 7215 return lower->dev; 7216 } 7217 7218 int netdev_walk_all_lower_dev(struct net_device *dev, 7219 int (*fn)(struct net_device *dev, 7220 void *data), 7221 void *data) 7222 { 7223 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 7224 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 7225 int ret, cur = 0; 7226 7227 now = dev; 7228 iter = &dev->adj_list.lower; 7229 7230 while (1) { 7231 if (now != dev) { 7232 ret = fn(now, data); 7233 if (ret) 7234 return ret; 7235 } 7236 7237 next = NULL; 7238 while (1) { 7239 ldev = netdev_next_lower_dev(now, &iter); 7240 if (!ldev) 7241 break; 7242 7243 next = ldev; 7244 niter = &ldev->adj_list.lower; 7245 dev_stack[cur] = now; 7246 iter_stack[cur++] = iter; 7247 break; 7248 } 7249 7250 if (!next) { 7251 if (!cur) 7252 return 0; 7253 next = dev_stack[--cur]; 7254 niter = iter_stack[cur]; 7255 } 7256 7257 now = next; 7258 iter = niter; 7259 } 7260 7261 return 0; 7262 } 7263 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev); 7264 7265 static int __netdev_walk_all_lower_dev(struct net_device *dev, 7266 int (*fn)(struct net_device *dev, 7267 void *data), 7268 void *data) 7269 { 7270 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 7271 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 7272 int ret, cur = 0; 7273 bool ignore; 7274 7275 now = dev; 7276 iter = &dev->adj_list.lower; 7277 7278 while (1) { 7279 if (now != dev) { 7280 ret = fn(now, data); 7281 if (ret) 7282 return ret; 7283 } 7284 7285 next = NULL; 7286 while (1) { 7287 ldev = __netdev_next_lower_dev(now, &iter, &ignore); 7288 if (!ldev) 7289 break; 7290 if (ignore) 7291 continue; 7292 7293 next = ldev; 7294 niter = &ldev->adj_list.lower; 7295 dev_stack[cur] = now; 7296 iter_stack[cur++] = iter; 7297 break; 7298 } 7299 7300 if (!next) { 7301 if (!cur) 7302 return 0; 7303 next = dev_stack[--cur]; 7304 niter = iter_stack[cur]; 7305 } 7306 7307 now = next; 7308 iter = niter; 7309 } 7310 7311 return 0; 7312 } 7313 7314 struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev, 7315 struct list_head **iter) 7316 { 7317 struct netdev_adjacent *lower; 7318 7319 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 7320 if (&lower->list == &dev->adj_list.lower) 7321 return NULL; 7322 7323 *iter = &lower->list; 7324 7325 return lower->dev; 7326 } 7327 EXPORT_SYMBOL(netdev_next_lower_dev_rcu); 7328 7329 static u8 __netdev_upper_depth(struct net_device *dev) 7330 { 7331 struct net_device *udev; 7332 struct list_head *iter; 7333 u8 max_depth = 0; 7334 bool ignore; 7335 7336 for (iter = &dev->adj_list.upper, 7337 udev = __netdev_next_upper_dev(dev, &iter, &ignore); 7338 udev; 7339 udev = __netdev_next_upper_dev(dev, &iter, &ignore)) { 7340 if (ignore) 7341 continue; 7342 if (max_depth < udev->upper_level) 7343 max_depth = udev->upper_level; 7344 } 7345 7346 return max_depth; 7347 } 7348 7349 static u8 __netdev_lower_depth(struct net_device *dev) 7350 { 7351 struct net_device *ldev; 7352 struct list_head *iter; 7353 u8 max_depth = 0; 7354 bool ignore; 7355 7356 for (iter = &dev->adj_list.lower, 7357 ldev = __netdev_next_lower_dev(dev, &iter, &ignore); 7358 ldev; 7359 ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) { 7360 if (ignore) 7361 continue; 7362 if (max_depth < ldev->lower_level) 7363 max_depth = ldev->lower_level; 7364 } 7365 7366 return max_depth; 7367 } 7368 7369 static int __netdev_update_upper_level(struct net_device *dev, void *data) 7370 { 7371 dev->upper_level = __netdev_upper_depth(dev) + 1; 7372 return 0; 7373 } 7374 7375 static int __netdev_update_lower_level(struct net_device *dev, void *data) 7376 { 7377 dev->lower_level = __netdev_lower_depth(dev) + 1; 7378 return 0; 7379 } 7380 7381 int netdev_walk_all_lower_dev_rcu(struct net_device *dev, 7382 int (*fn)(struct net_device *dev, 7383 void *data), 7384 void *data) 7385 { 7386 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 7387 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 7388 int ret, cur = 0; 7389 7390 now = dev; 7391 iter = &dev->adj_list.lower; 7392 7393 while (1) { 7394 if (now != dev) { 7395 ret = fn(now, data); 7396 if (ret) 7397 return ret; 7398 } 7399 7400 next = NULL; 7401 while (1) { 7402 ldev = netdev_next_lower_dev_rcu(now, &iter); 7403 if (!ldev) 7404 break; 7405 7406 next = ldev; 7407 niter = &ldev->adj_list.lower; 7408 dev_stack[cur] = now; 7409 iter_stack[cur++] = iter; 7410 break; 7411 } 7412 7413 if (!next) { 7414 if (!cur) 7415 return 0; 7416 next = dev_stack[--cur]; 7417 niter = iter_stack[cur]; 7418 } 7419 7420 now = next; 7421 iter = niter; 7422 } 7423 7424 return 0; 7425 } 7426 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu); 7427 7428 /** 7429 * netdev_lower_get_first_private_rcu - Get the first ->private from the 7430 * lower neighbour list, RCU 7431 * variant 7432 * @dev: device 7433 * 7434 * Gets the first netdev_adjacent->private from the dev's lower neighbour 7435 * list. The caller must hold RCU read lock. 7436 */ 7437 void *netdev_lower_get_first_private_rcu(struct net_device *dev) 7438 { 7439 struct netdev_adjacent *lower; 7440 7441 lower = list_first_or_null_rcu(&dev->adj_list.lower, 7442 struct netdev_adjacent, list); 7443 if (lower) 7444 return lower->private; 7445 return NULL; 7446 } 7447 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu); 7448 7449 /** 7450 * netdev_master_upper_dev_get_rcu - Get master upper device 7451 * @dev: device 7452 * 7453 * Find a master upper device and return pointer to it or NULL in case 7454 * it's not there. The caller must hold the RCU read lock. 7455 */ 7456 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev) 7457 { 7458 struct netdev_adjacent *upper; 7459 7460 upper = list_first_or_null_rcu(&dev->adj_list.upper, 7461 struct netdev_adjacent, list); 7462 if (upper && likely(upper->master)) 7463 return upper->dev; 7464 return NULL; 7465 } 7466 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu); 7467 7468 static int netdev_adjacent_sysfs_add(struct net_device *dev, 7469 struct net_device *adj_dev, 7470 struct list_head *dev_list) 7471 { 7472 char linkname[IFNAMSIZ+7]; 7473 7474 sprintf(linkname, dev_list == &dev->adj_list.upper ? 7475 "upper_%s" : "lower_%s", adj_dev->name); 7476 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), 7477 linkname); 7478 } 7479 static void netdev_adjacent_sysfs_del(struct net_device *dev, 7480 char *name, 7481 struct list_head *dev_list) 7482 { 7483 char linkname[IFNAMSIZ+7]; 7484 7485 sprintf(linkname, dev_list == &dev->adj_list.upper ? 7486 "upper_%s" : "lower_%s", name); 7487 sysfs_remove_link(&(dev->dev.kobj), linkname); 7488 } 7489 7490 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev, 7491 struct net_device *adj_dev, 7492 struct list_head *dev_list) 7493 { 7494 return (dev_list == &dev->adj_list.upper || 7495 dev_list == &dev->adj_list.lower) && 7496 net_eq(dev_net(dev), dev_net(adj_dev)); 7497 } 7498 7499 static int __netdev_adjacent_dev_insert(struct net_device *dev, 7500 struct net_device *adj_dev, 7501 struct list_head *dev_list, 7502 void *private, bool master) 7503 { 7504 struct netdev_adjacent *adj; 7505 int ret; 7506 7507 adj = __netdev_find_adj(adj_dev, dev_list); 7508 7509 if (adj) { 7510 adj->ref_nr += 1; 7511 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n", 7512 dev->name, adj_dev->name, adj->ref_nr); 7513 7514 return 0; 7515 } 7516 7517 adj = kmalloc(sizeof(*adj), GFP_KERNEL); 7518 if (!adj) 7519 return -ENOMEM; 7520 7521 adj->dev = adj_dev; 7522 adj->master = master; 7523 adj->ref_nr = 1; 7524 adj->private = private; 7525 adj->ignore = false; 7526 dev_hold(adj_dev); 7527 7528 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n", 7529 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name); 7530 7531 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) { 7532 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list); 7533 if (ret) 7534 goto free_adj; 7535 } 7536 7537 /* Ensure that master link is always the first item in list. */ 7538 if (master) { 7539 ret = sysfs_create_link(&(dev->dev.kobj), 7540 &(adj_dev->dev.kobj), "master"); 7541 if (ret) 7542 goto remove_symlinks; 7543 7544 list_add_rcu(&adj->list, dev_list); 7545 } else { 7546 list_add_tail_rcu(&adj->list, dev_list); 7547 } 7548 7549 return 0; 7550 7551 remove_symlinks: 7552 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 7553 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 7554 free_adj: 7555 kfree(adj); 7556 dev_put(adj_dev); 7557 7558 return ret; 7559 } 7560 7561 static void __netdev_adjacent_dev_remove(struct net_device *dev, 7562 struct net_device *adj_dev, 7563 u16 ref_nr, 7564 struct list_head *dev_list) 7565 { 7566 struct netdev_adjacent *adj; 7567 7568 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n", 7569 dev->name, adj_dev->name, ref_nr); 7570 7571 adj = __netdev_find_adj(adj_dev, dev_list); 7572 7573 if (!adj) { 7574 pr_err("Adjacency does not exist for device %s from %s\n", 7575 dev->name, adj_dev->name); 7576 WARN_ON(1); 7577 return; 7578 } 7579 7580 if (adj->ref_nr > ref_nr) { 7581 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n", 7582 dev->name, adj_dev->name, ref_nr, 7583 adj->ref_nr - ref_nr); 7584 adj->ref_nr -= ref_nr; 7585 return; 7586 } 7587 7588 if (adj->master) 7589 sysfs_remove_link(&(dev->dev.kobj), "master"); 7590 7591 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 7592 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 7593 7594 list_del_rcu(&adj->list); 7595 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n", 7596 adj_dev->name, dev->name, adj_dev->name); 7597 dev_put(adj_dev); 7598 kfree_rcu(adj, rcu); 7599 } 7600 7601 static int __netdev_adjacent_dev_link_lists(struct net_device *dev, 7602 struct net_device *upper_dev, 7603 struct list_head *up_list, 7604 struct list_head *down_list, 7605 void *private, bool master) 7606 { 7607 int ret; 7608 7609 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, 7610 private, master); 7611 if (ret) 7612 return ret; 7613 7614 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, 7615 private, false); 7616 if (ret) { 7617 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list); 7618 return ret; 7619 } 7620 7621 return 0; 7622 } 7623 7624 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev, 7625 struct net_device *upper_dev, 7626 u16 ref_nr, 7627 struct list_head *up_list, 7628 struct list_head *down_list) 7629 { 7630 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list); 7631 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list); 7632 } 7633 7634 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev, 7635 struct net_device *upper_dev, 7636 void *private, bool master) 7637 { 7638 return __netdev_adjacent_dev_link_lists(dev, upper_dev, 7639 &dev->adj_list.upper, 7640 &upper_dev->adj_list.lower, 7641 private, master); 7642 } 7643 7644 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev, 7645 struct net_device *upper_dev) 7646 { 7647 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1, 7648 &dev->adj_list.upper, 7649 &upper_dev->adj_list.lower); 7650 } 7651 7652 static int __netdev_upper_dev_link(struct net_device *dev, 7653 struct net_device *upper_dev, bool master, 7654 void *upper_priv, void *upper_info, 7655 struct netlink_ext_ack *extack) 7656 { 7657 struct netdev_notifier_changeupper_info changeupper_info = { 7658 .info = { 7659 .dev = dev, 7660 .extack = extack, 7661 }, 7662 .upper_dev = upper_dev, 7663 .master = master, 7664 .linking = true, 7665 .upper_info = upper_info, 7666 }; 7667 struct net_device *master_dev; 7668 int ret = 0; 7669 7670 ASSERT_RTNL(); 7671 7672 if (dev == upper_dev) 7673 return -EBUSY; 7674 7675 /* To prevent loops, check if dev is not upper device to upper_dev. */ 7676 if (__netdev_has_upper_dev(upper_dev, dev)) 7677 return -EBUSY; 7678 7679 if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV) 7680 return -EMLINK; 7681 7682 if (!master) { 7683 if (__netdev_has_upper_dev(dev, upper_dev)) 7684 return -EEXIST; 7685 } else { 7686 master_dev = __netdev_master_upper_dev_get(dev); 7687 if (master_dev) 7688 return master_dev == upper_dev ? -EEXIST : -EBUSY; 7689 } 7690 7691 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, 7692 &changeupper_info.info); 7693 ret = notifier_to_errno(ret); 7694 if (ret) 7695 return ret; 7696 7697 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv, 7698 master); 7699 if (ret) 7700 return ret; 7701 7702 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, 7703 &changeupper_info.info); 7704 ret = notifier_to_errno(ret); 7705 if (ret) 7706 goto rollback; 7707 7708 __netdev_update_upper_level(dev, NULL); 7709 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL); 7710 7711 __netdev_update_lower_level(upper_dev, NULL); 7712 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level, 7713 NULL); 7714 7715 return 0; 7716 7717 rollback: 7718 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 7719 7720 return ret; 7721 } 7722 7723 /** 7724 * netdev_upper_dev_link - Add a link to the upper device 7725 * @dev: device 7726 * @upper_dev: new upper device 7727 * @extack: netlink extended ack 7728 * 7729 * Adds a link to device which is upper to this one. The caller must hold 7730 * the RTNL lock. On a failure a negative errno code is returned. 7731 * On success the reference counts are adjusted and the function 7732 * returns zero. 7733 */ 7734 int netdev_upper_dev_link(struct net_device *dev, 7735 struct net_device *upper_dev, 7736 struct netlink_ext_ack *extack) 7737 { 7738 return __netdev_upper_dev_link(dev, upper_dev, false, 7739 NULL, NULL, extack); 7740 } 7741 EXPORT_SYMBOL(netdev_upper_dev_link); 7742 7743 /** 7744 * netdev_master_upper_dev_link - Add a master link to the upper device 7745 * @dev: device 7746 * @upper_dev: new upper device 7747 * @upper_priv: upper device private 7748 * @upper_info: upper info to be passed down via notifier 7749 * @extack: netlink extended ack 7750 * 7751 * Adds a link to device which is upper to this one. In this case, only 7752 * one master upper device can be linked, although other non-master devices 7753 * might be linked as well. The caller must hold the RTNL lock. 7754 * On a failure a negative errno code is returned. On success the reference 7755 * counts are adjusted and the function returns zero. 7756 */ 7757 int netdev_master_upper_dev_link(struct net_device *dev, 7758 struct net_device *upper_dev, 7759 void *upper_priv, void *upper_info, 7760 struct netlink_ext_ack *extack) 7761 { 7762 return __netdev_upper_dev_link(dev, upper_dev, true, 7763 upper_priv, upper_info, extack); 7764 } 7765 EXPORT_SYMBOL(netdev_master_upper_dev_link); 7766 7767 /** 7768 * netdev_upper_dev_unlink - Removes a link to upper device 7769 * @dev: device 7770 * @upper_dev: new upper device 7771 * 7772 * Removes a link to device which is upper to this one. The caller must hold 7773 * the RTNL lock. 7774 */ 7775 void netdev_upper_dev_unlink(struct net_device *dev, 7776 struct net_device *upper_dev) 7777 { 7778 struct netdev_notifier_changeupper_info changeupper_info = { 7779 .info = { 7780 .dev = dev, 7781 }, 7782 .upper_dev = upper_dev, 7783 .linking = false, 7784 }; 7785 7786 ASSERT_RTNL(); 7787 7788 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev; 7789 7790 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, 7791 &changeupper_info.info); 7792 7793 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 7794 7795 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, 7796 &changeupper_info.info); 7797 7798 __netdev_update_upper_level(dev, NULL); 7799 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL); 7800 7801 __netdev_update_lower_level(upper_dev, NULL); 7802 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level, 7803 NULL); 7804 } 7805 EXPORT_SYMBOL(netdev_upper_dev_unlink); 7806 7807 static void __netdev_adjacent_dev_set(struct net_device *upper_dev, 7808 struct net_device *lower_dev, 7809 bool val) 7810 { 7811 struct netdev_adjacent *adj; 7812 7813 adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower); 7814 if (adj) 7815 adj->ignore = val; 7816 7817 adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper); 7818 if (adj) 7819 adj->ignore = val; 7820 } 7821 7822 static void netdev_adjacent_dev_disable(struct net_device *upper_dev, 7823 struct net_device *lower_dev) 7824 { 7825 __netdev_adjacent_dev_set(upper_dev, lower_dev, true); 7826 } 7827 7828 static void netdev_adjacent_dev_enable(struct net_device *upper_dev, 7829 struct net_device *lower_dev) 7830 { 7831 __netdev_adjacent_dev_set(upper_dev, lower_dev, false); 7832 } 7833 7834 int netdev_adjacent_change_prepare(struct net_device *old_dev, 7835 struct net_device *new_dev, 7836 struct net_device *dev, 7837 struct netlink_ext_ack *extack) 7838 { 7839 int err; 7840 7841 if (!new_dev) 7842 return 0; 7843 7844 if (old_dev && new_dev != old_dev) 7845 netdev_adjacent_dev_disable(dev, old_dev); 7846 7847 err = netdev_upper_dev_link(new_dev, dev, extack); 7848 if (err) { 7849 if (old_dev && new_dev != old_dev) 7850 netdev_adjacent_dev_enable(dev, old_dev); 7851 return err; 7852 } 7853 7854 return 0; 7855 } 7856 EXPORT_SYMBOL(netdev_adjacent_change_prepare); 7857 7858 void netdev_adjacent_change_commit(struct net_device *old_dev, 7859 struct net_device *new_dev, 7860 struct net_device *dev) 7861 { 7862 if (!new_dev || !old_dev) 7863 return; 7864 7865 if (new_dev == old_dev) 7866 return; 7867 7868 netdev_adjacent_dev_enable(dev, old_dev); 7869 netdev_upper_dev_unlink(old_dev, dev); 7870 } 7871 EXPORT_SYMBOL(netdev_adjacent_change_commit); 7872 7873 void netdev_adjacent_change_abort(struct net_device *old_dev, 7874 struct net_device *new_dev, 7875 struct net_device *dev) 7876 { 7877 if (!new_dev) 7878 return; 7879 7880 if (old_dev && new_dev != old_dev) 7881 netdev_adjacent_dev_enable(dev, old_dev); 7882 7883 netdev_upper_dev_unlink(new_dev, dev); 7884 } 7885 EXPORT_SYMBOL(netdev_adjacent_change_abort); 7886 7887 /** 7888 * netdev_bonding_info_change - Dispatch event about slave change 7889 * @dev: device 7890 * @bonding_info: info to dispatch 7891 * 7892 * Send NETDEV_BONDING_INFO to netdev notifiers with info. 7893 * The caller must hold the RTNL lock. 7894 */ 7895 void netdev_bonding_info_change(struct net_device *dev, 7896 struct netdev_bonding_info *bonding_info) 7897 { 7898 struct netdev_notifier_bonding_info info = { 7899 .info.dev = dev, 7900 }; 7901 7902 memcpy(&info.bonding_info, bonding_info, 7903 sizeof(struct netdev_bonding_info)); 7904 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, 7905 &info.info); 7906 } 7907 EXPORT_SYMBOL(netdev_bonding_info_change); 7908 7909 /** 7910 * netdev_get_xmit_slave - Get the xmit slave of master device 7911 * @skb: The packet 7912 * @all_slaves: assume all the slaves are active 7913 * 7914 * The reference counters are not incremented so the caller must be 7915 * careful with locks. The caller must hold RCU lock. 7916 * %NULL is returned if no slave is found. 7917 */ 7918 7919 struct net_device *netdev_get_xmit_slave(struct net_device *dev, 7920 struct sk_buff *skb, 7921 bool all_slaves) 7922 { 7923 const struct net_device_ops *ops = dev->netdev_ops; 7924 7925 if (!ops->ndo_get_xmit_slave) 7926 return NULL; 7927 return ops->ndo_get_xmit_slave(dev, skb, all_slaves); 7928 } 7929 EXPORT_SYMBOL(netdev_get_xmit_slave); 7930 7931 static void netdev_adjacent_add_links(struct net_device *dev) 7932 { 7933 struct netdev_adjacent *iter; 7934 7935 struct net *net = dev_net(dev); 7936 7937 list_for_each_entry(iter, &dev->adj_list.upper, list) { 7938 if (!net_eq(net, dev_net(iter->dev))) 7939 continue; 7940 netdev_adjacent_sysfs_add(iter->dev, dev, 7941 &iter->dev->adj_list.lower); 7942 netdev_adjacent_sysfs_add(dev, iter->dev, 7943 &dev->adj_list.upper); 7944 } 7945 7946 list_for_each_entry(iter, &dev->adj_list.lower, list) { 7947 if (!net_eq(net, dev_net(iter->dev))) 7948 continue; 7949 netdev_adjacent_sysfs_add(iter->dev, dev, 7950 &iter->dev->adj_list.upper); 7951 netdev_adjacent_sysfs_add(dev, iter->dev, 7952 &dev->adj_list.lower); 7953 } 7954 } 7955 7956 static void netdev_adjacent_del_links(struct net_device *dev) 7957 { 7958 struct netdev_adjacent *iter; 7959 7960 struct net *net = dev_net(dev); 7961 7962 list_for_each_entry(iter, &dev->adj_list.upper, list) { 7963 if (!net_eq(net, dev_net(iter->dev))) 7964 continue; 7965 netdev_adjacent_sysfs_del(iter->dev, dev->name, 7966 &iter->dev->adj_list.lower); 7967 netdev_adjacent_sysfs_del(dev, iter->dev->name, 7968 &dev->adj_list.upper); 7969 } 7970 7971 list_for_each_entry(iter, &dev->adj_list.lower, list) { 7972 if (!net_eq(net, dev_net(iter->dev))) 7973 continue; 7974 netdev_adjacent_sysfs_del(iter->dev, dev->name, 7975 &iter->dev->adj_list.upper); 7976 netdev_adjacent_sysfs_del(dev, iter->dev->name, 7977 &dev->adj_list.lower); 7978 } 7979 } 7980 7981 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname) 7982 { 7983 struct netdev_adjacent *iter; 7984 7985 struct net *net = dev_net(dev); 7986 7987 list_for_each_entry(iter, &dev->adj_list.upper, list) { 7988 if (!net_eq(net, dev_net(iter->dev))) 7989 continue; 7990 netdev_adjacent_sysfs_del(iter->dev, oldname, 7991 &iter->dev->adj_list.lower); 7992 netdev_adjacent_sysfs_add(iter->dev, dev, 7993 &iter->dev->adj_list.lower); 7994 } 7995 7996 list_for_each_entry(iter, &dev->adj_list.lower, list) { 7997 if (!net_eq(net, dev_net(iter->dev))) 7998 continue; 7999 netdev_adjacent_sysfs_del(iter->dev, oldname, 8000 &iter->dev->adj_list.upper); 8001 netdev_adjacent_sysfs_add(iter->dev, dev, 8002 &iter->dev->adj_list.upper); 8003 } 8004 } 8005 8006 void *netdev_lower_dev_get_private(struct net_device *dev, 8007 struct net_device *lower_dev) 8008 { 8009 struct netdev_adjacent *lower; 8010 8011 if (!lower_dev) 8012 return NULL; 8013 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower); 8014 if (!lower) 8015 return NULL; 8016 8017 return lower->private; 8018 } 8019 EXPORT_SYMBOL(netdev_lower_dev_get_private); 8020 8021 8022 /** 8023 * netdev_lower_change - Dispatch event about lower device state change 8024 * @lower_dev: device 8025 * @lower_state_info: state to dispatch 8026 * 8027 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info. 8028 * The caller must hold the RTNL lock. 8029 */ 8030 void netdev_lower_state_changed(struct net_device *lower_dev, 8031 void *lower_state_info) 8032 { 8033 struct netdev_notifier_changelowerstate_info changelowerstate_info = { 8034 .info.dev = lower_dev, 8035 }; 8036 8037 ASSERT_RTNL(); 8038 changelowerstate_info.lower_state_info = lower_state_info; 8039 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, 8040 &changelowerstate_info.info); 8041 } 8042 EXPORT_SYMBOL(netdev_lower_state_changed); 8043 8044 static void dev_change_rx_flags(struct net_device *dev, int flags) 8045 { 8046 const struct net_device_ops *ops = dev->netdev_ops; 8047 8048 if (ops->ndo_change_rx_flags) 8049 ops->ndo_change_rx_flags(dev, flags); 8050 } 8051 8052 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify) 8053 { 8054 unsigned int old_flags = dev->flags; 8055 kuid_t uid; 8056 kgid_t gid; 8057 8058 ASSERT_RTNL(); 8059 8060 dev->flags |= IFF_PROMISC; 8061 dev->promiscuity += inc; 8062 if (dev->promiscuity == 0) { 8063 /* 8064 * Avoid overflow. 8065 * If inc causes overflow, untouch promisc and return error. 8066 */ 8067 if (inc < 0) 8068 dev->flags &= ~IFF_PROMISC; 8069 else { 8070 dev->promiscuity -= inc; 8071 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n", 8072 dev->name); 8073 return -EOVERFLOW; 8074 } 8075 } 8076 if (dev->flags != old_flags) { 8077 pr_info("device %s %s promiscuous mode\n", 8078 dev->name, 8079 dev->flags & IFF_PROMISC ? "entered" : "left"); 8080 if (audit_enabled) { 8081 current_uid_gid(&uid, &gid); 8082 audit_log(audit_context(), GFP_ATOMIC, 8083 AUDIT_ANOM_PROMISCUOUS, 8084 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u", 8085 dev->name, (dev->flags & IFF_PROMISC), 8086 (old_flags & IFF_PROMISC), 8087 from_kuid(&init_user_ns, audit_get_loginuid(current)), 8088 from_kuid(&init_user_ns, uid), 8089 from_kgid(&init_user_ns, gid), 8090 audit_get_sessionid(current)); 8091 } 8092 8093 dev_change_rx_flags(dev, IFF_PROMISC); 8094 } 8095 if (notify) 8096 __dev_notify_flags(dev, old_flags, IFF_PROMISC); 8097 return 0; 8098 } 8099 8100 /** 8101 * dev_set_promiscuity - update promiscuity count on a device 8102 * @dev: device 8103 * @inc: modifier 8104 * 8105 * Add or remove promiscuity from a device. While the count in the device 8106 * remains above zero the interface remains promiscuous. Once it hits zero 8107 * the device reverts back to normal filtering operation. A negative inc 8108 * value is used to drop promiscuity on the device. 8109 * Return 0 if successful or a negative errno code on error. 8110 */ 8111 int dev_set_promiscuity(struct net_device *dev, int inc) 8112 { 8113 unsigned int old_flags = dev->flags; 8114 int err; 8115 8116 err = __dev_set_promiscuity(dev, inc, true); 8117 if (err < 0) 8118 return err; 8119 if (dev->flags != old_flags) 8120 dev_set_rx_mode(dev); 8121 return err; 8122 } 8123 EXPORT_SYMBOL(dev_set_promiscuity); 8124 8125 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify) 8126 { 8127 unsigned int old_flags = dev->flags, old_gflags = dev->gflags; 8128 8129 ASSERT_RTNL(); 8130 8131 dev->flags |= IFF_ALLMULTI; 8132 dev->allmulti += inc; 8133 if (dev->allmulti == 0) { 8134 /* 8135 * Avoid overflow. 8136 * If inc causes overflow, untouch allmulti and return error. 8137 */ 8138 if (inc < 0) 8139 dev->flags &= ~IFF_ALLMULTI; 8140 else { 8141 dev->allmulti -= inc; 8142 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n", 8143 dev->name); 8144 return -EOVERFLOW; 8145 } 8146 } 8147 if (dev->flags ^ old_flags) { 8148 dev_change_rx_flags(dev, IFF_ALLMULTI); 8149 dev_set_rx_mode(dev); 8150 if (notify) 8151 __dev_notify_flags(dev, old_flags, 8152 dev->gflags ^ old_gflags); 8153 } 8154 return 0; 8155 } 8156 8157 /** 8158 * dev_set_allmulti - update allmulti count on a device 8159 * @dev: device 8160 * @inc: modifier 8161 * 8162 * Add or remove reception of all multicast frames to a device. While the 8163 * count in the device remains above zero the interface remains listening 8164 * to all interfaces. Once it hits zero the device reverts back to normal 8165 * filtering operation. A negative @inc value is used to drop the counter 8166 * when releasing a resource needing all multicasts. 8167 * Return 0 if successful or a negative errno code on error. 8168 */ 8169 8170 int dev_set_allmulti(struct net_device *dev, int inc) 8171 { 8172 return __dev_set_allmulti(dev, inc, true); 8173 } 8174 EXPORT_SYMBOL(dev_set_allmulti); 8175 8176 /* 8177 * Upload unicast and multicast address lists to device and 8178 * configure RX filtering. When the device doesn't support unicast 8179 * filtering it is put in promiscuous mode while unicast addresses 8180 * are present. 8181 */ 8182 void __dev_set_rx_mode(struct net_device *dev) 8183 { 8184 const struct net_device_ops *ops = dev->netdev_ops; 8185 8186 /* dev_open will call this function so the list will stay sane. */ 8187 if (!(dev->flags&IFF_UP)) 8188 return; 8189 8190 if (!netif_device_present(dev)) 8191 return; 8192 8193 if (!(dev->priv_flags & IFF_UNICAST_FLT)) { 8194 /* Unicast addresses changes may only happen under the rtnl, 8195 * therefore calling __dev_set_promiscuity here is safe. 8196 */ 8197 if (!netdev_uc_empty(dev) && !dev->uc_promisc) { 8198 __dev_set_promiscuity(dev, 1, false); 8199 dev->uc_promisc = true; 8200 } else if (netdev_uc_empty(dev) && dev->uc_promisc) { 8201 __dev_set_promiscuity(dev, -1, false); 8202 dev->uc_promisc = false; 8203 } 8204 } 8205 8206 if (ops->ndo_set_rx_mode) 8207 ops->ndo_set_rx_mode(dev); 8208 } 8209 8210 void dev_set_rx_mode(struct net_device *dev) 8211 { 8212 netif_addr_lock_bh(dev); 8213 __dev_set_rx_mode(dev); 8214 netif_addr_unlock_bh(dev); 8215 } 8216 8217 /** 8218 * dev_get_flags - get flags reported to userspace 8219 * @dev: device 8220 * 8221 * Get the combination of flag bits exported through APIs to userspace. 8222 */ 8223 unsigned int dev_get_flags(const struct net_device *dev) 8224 { 8225 unsigned int flags; 8226 8227 flags = (dev->flags & ~(IFF_PROMISC | 8228 IFF_ALLMULTI | 8229 IFF_RUNNING | 8230 IFF_LOWER_UP | 8231 IFF_DORMANT)) | 8232 (dev->gflags & (IFF_PROMISC | 8233 IFF_ALLMULTI)); 8234 8235 if (netif_running(dev)) { 8236 if (netif_oper_up(dev)) 8237 flags |= IFF_RUNNING; 8238 if (netif_carrier_ok(dev)) 8239 flags |= IFF_LOWER_UP; 8240 if (netif_dormant(dev)) 8241 flags |= IFF_DORMANT; 8242 } 8243 8244 return flags; 8245 } 8246 EXPORT_SYMBOL(dev_get_flags); 8247 8248 int __dev_change_flags(struct net_device *dev, unsigned int flags, 8249 struct netlink_ext_ack *extack) 8250 { 8251 unsigned int old_flags = dev->flags; 8252 int ret; 8253 8254 ASSERT_RTNL(); 8255 8256 /* 8257 * Set the flags on our device. 8258 */ 8259 8260 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | 8261 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL | 8262 IFF_AUTOMEDIA)) | 8263 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | 8264 IFF_ALLMULTI)); 8265 8266 /* 8267 * Load in the correct multicast list now the flags have changed. 8268 */ 8269 8270 if ((old_flags ^ flags) & IFF_MULTICAST) 8271 dev_change_rx_flags(dev, IFF_MULTICAST); 8272 8273 dev_set_rx_mode(dev); 8274 8275 /* 8276 * Have we downed the interface. We handle IFF_UP ourselves 8277 * according to user attempts to set it, rather than blindly 8278 * setting it. 8279 */ 8280 8281 ret = 0; 8282 if ((old_flags ^ flags) & IFF_UP) { 8283 if (old_flags & IFF_UP) 8284 __dev_close(dev); 8285 else 8286 ret = __dev_open(dev, extack); 8287 } 8288 8289 if ((flags ^ dev->gflags) & IFF_PROMISC) { 8290 int inc = (flags & IFF_PROMISC) ? 1 : -1; 8291 unsigned int old_flags = dev->flags; 8292 8293 dev->gflags ^= IFF_PROMISC; 8294 8295 if (__dev_set_promiscuity(dev, inc, false) >= 0) 8296 if (dev->flags != old_flags) 8297 dev_set_rx_mode(dev); 8298 } 8299 8300 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI 8301 * is important. Some (broken) drivers set IFF_PROMISC, when 8302 * IFF_ALLMULTI is requested not asking us and not reporting. 8303 */ 8304 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { 8305 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; 8306 8307 dev->gflags ^= IFF_ALLMULTI; 8308 __dev_set_allmulti(dev, inc, false); 8309 } 8310 8311 return ret; 8312 } 8313 8314 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, 8315 unsigned int gchanges) 8316 { 8317 unsigned int changes = dev->flags ^ old_flags; 8318 8319 if (gchanges) 8320 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC); 8321 8322 if (changes & IFF_UP) { 8323 if (dev->flags & IFF_UP) 8324 call_netdevice_notifiers(NETDEV_UP, dev); 8325 else 8326 call_netdevice_notifiers(NETDEV_DOWN, dev); 8327 } 8328 8329 if (dev->flags & IFF_UP && 8330 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) { 8331 struct netdev_notifier_change_info change_info = { 8332 .info = { 8333 .dev = dev, 8334 }, 8335 .flags_changed = changes, 8336 }; 8337 8338 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info); 8339 } 8340 } 8341 8342 /** 8343 * dev_change_flags - change device settings 8344 * @dev: device 8345 * @flags: device state flags 8346 * @extack: netlink extended ack 8347 * 8348 * Change settings on device based state flags. The flags are 8349 * in the userspace exported format. 8350 */ 8351 int dev_change_flags(struct net_device *dev, unsigned int flags, 8352 struct netlink_ext_ack *extack) 8353 { 8354 int ret; 8355 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; 8356 8357 ret = __dev_change_flags(dev, flags, extack); 8358 if (ret < 0) 8359 return ret; 8360 8361 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); 8362 __dev_notify_flags(dev, old_flags, changes); 8363 return ret; 8364 } 8365 EXPORT_SYMBOL(dev_change_flags); 8366 8367 int __dev_set_mtu(struct net_device *dev, int new_mtu) 8368 { 8369 const struct net_device_ops *ops = dev->netdev_ops; 8370 8371 if (ops->ndo_change_mtu) 8372 return ops->ndo_change_mtu(dev, new_mtu); 8373 8374 /* Pairs with all the lockless reads of dev->mtu in the stack */ 8375 WRITE_ONCE(dev->mtu, new_mtu); 8376 return 0; 8377 } 8378 EXPORT_SYMBOL(__dev_set_mtu); 8379 8380 int dev_validate_mtu(struct net_device *dev, int new_mtu, 8381 struct netlink_ext_ack *extack) 8382 { 8383 /* MTU must be positive, and in range */ 8384 if (new_mtu < 0 || new_mtu < dev->min_mtu) { 8385 NL_SET_ERR_MSG(extack, "mtu less than device minimum"); 8386 return -EINVAL; 8387 } 8388 8389 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) { 8390 NL_SET_ERR_MSG(extack, "mtu greater than device maximum"); 8391 return -EINVAL; 8392 } 8393 return 0; 8394 } 8395 8396 /** 8397 * dev_set_mtu_ext - Change maximum transfer unit 8398 * @dev: device 8399 * @new_mtu: new transfer unit 8400 * @extack: netlink extended ack 8401 * 8402 * Change the maximum transfer size of the network device. 8403 */ 8404 int dev_set_mtu_ext(struct net_device *dev, int new_mtu, 8405 struct netlink_ext_ack *extack) 8406 { 8407 int err, orig_mtu; 8408 8409 if (new_mtu == dev->mtu) 8410 return 0; 8411 8412 err = dev_validate_mtu(dev, new_mtu, extack); 8413 if (err) 8414 return err; 8415 8416 if (!netif_device_present(dev)) 8417 return -ENODEV; 8418 8419 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev); 8420 err = notifier_to_errno(err); 8421 if (err) 8422 return err; 8423 8424 orig_mtu = dev->mtu; 8425 err = __dev_set_mtu(dev, new_mtu); 8426 8427 if (!err) { 8428 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev, 8429 orig_mtu); 8430 err = notifier_to_errno(err); 8431 if (err) { 8432 /* setting mtu back and notifying everyone again, 8433 * so that they have a chance to revert changes. 8434 */ 8435 __dev_set_mtu(dev, orig_mtu); 8436 call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev, 8437 new_mtu); 8438 } 8439 } 8440 return err; 8441 } 8442 8443 int dev_set_mtu(struct net_device *dev, int new_mtu) 8444 { 8445 struct netlink_ext_ack extack; 8446 int err; 8447 8448 memset(&extack, 0, sizeof(extack)); 8449 err = dev_set_mtu_ext(dev, new_mtu, &extack); 8450 if (err && extack._msg) 8451 net_err_ratelimited("%s: %s\n", dev->name, extack._msg); 8452 return err; 8453 } 8454 EXPORT_SYMBOL(dev_set_mtu); 8455 8456 /** 8457 * dev_change_tx_queue_len - Change TX queue length of a netdevice 8458 * @dev: device 8459 * @new_len: new tx queue length 8460 */ 8461 int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len) 8462 { 8463 unsigned int orig_len = dev->tx_queue_len; 8464 int res; 8465 8466 if (new_len != (unsigned int)new_len) 8467 return -ERANGE; 8468 8469 if (new_len != orig_len) { 8470 dev->tx_queue_len = new_len; 8471 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev); 8472 res = notifier_to_errno(res); 8473 if (res) 8474 goto err_rollback; 8475 res = dev_qdisc_change_tx_queue_len(dev); 8476 if (res) 8477 goto err_rollback; 8478 } 8479 8480 return 0; 8481 8482 err_rollback: 8483 netdev_err(dev, "refused to change device tx_queue_len\n"); 8484 dev->tx_queue_len = orig_len; 8485 return res; 8486 } 8487 8488 /** 8489 * dev_set_group - Change group this device belongs to 8490 * @dev: device 8491 * @new_group: group this device should belong to 8492 */ 8493 void dev_set_group(struct net_device *dev, int new_group) 8494 { 8495 dev->group = new_group; 8496 } 8497 EXPORT_SYMBOL(dev_set_group); 8498 8499 /** 8500 * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR. 8501 * @dev: device 8502 * @addr: new address 8503 * @extack: netlink extended ack 8504 */ 8505 int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr, 8506 struct netlink_ext_ack *extack) 8507 { 8508 struct netdev_notifier_pre_changeaddr_info info = { 8509 .info.dev = dev, 8510 .info.extack = extack, 8511 .dev_addr = addr, 8512 }; 8513 int rc; 8514 8515 rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info); 8516 return notifier_to_errno(rc); 8517 } 8518 EXPORT_SYMBOL(dev_pre_changeaddr_notify); 8519 8520 /** 8521 * dev_set_mac_address - Change Media Access Control Address 8522 * @dev: device 8523 * @sa: new address 8524 * @extack: netlink extended ack 8525 * 8526 * Change the hardware (MAC) address of the device 8527 */ 8528 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa, 8529 struct netlink_ext_ack *extack) 8530 { 8531 const struct net_device_ops *ops = dev->netdev_ops; 8532 int err; 8533 8534 if (!ops->ndo_set_mac_address) 8535 return -EOPNOTSUPP; 8536 if (sa->sa_family != dev->type) 8537 return -EINVAL; 8538 if (!netif_device_present(dev)) 8539 return -ENODEV; 8540 err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack); 8541 if (err) 8542 return err; 8543 err = ops->ndo_set_mac_address(dev, sa); 8544 if (err) 8545 return err; 8546 dev->addr_assign_type = NET_ADDR_SET; 8547 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 8548 add_device_randomness(dev->dev_addr, dev->addr_len); 8549 return 0; 8550 } 8551 EXPORT_SYMBOL(dev_set_mac_address); 8552 8553 /** 8554 * dev_change_carrier - Change device carrier 8555 * @dev: device 8556 * @new_carrier: new value 8557 * 8558 * Change device carrier 8559 */ 8560 int dev_change_carrier(struct net_device *dev, bool new_carrier) 8561 { 8562 const struct net_device_ops *ops = dev->netdev_ops; 8563 8564 if (!ops->ndo_change_carrier) 8565 return -EOPNOTSUPP; 8566 if (!netif_device_present(dev)) 8567 return -ENODEV; 8568 return ops->ndo_change_carrier(dev, new_carrier); 8569 } 8570 EXPORT_SYMBOL(dev_change_carrier); 8571 8572 /** 8573 * dev_get_phys_port_id - Get device physical port ID 8574 * @dev: device 8575 * @ppid: port ID 8576 * 8577 * Get device physical port ID 8578 */ 8579 int dev_get_phys_port_id(struct net_device *dev, 8580 struct netdev_phys_item_id *ppid) 8581 { 8582 const struct net_device_ops *ops = dev->netdev_ops; 8583 8584 if (!ops->ndo_get_phys_port_id) 8585 return -EOPNOTSUPP; 8586 return ops->ndo_get_phys_port_id(dev, ppid); 8587 } 8588 EXPORT_SYMBOL(dev_get_phys_port_id); 8589 8590 /** 8591 * dev_get_phys_port_name - Get device physical port name 8592 * @dev: device 8593 * @name: port name 8594 * @len: limit of bytes to copy to name 8595 * 8596 * Get device physical port name 8597 */ 8598 int dev_get_phys_port_name(struct net_device *dev, 8599 char *name, size_t len) 8600 { 8601 const struct net_device_ops *ops = dev->netdev_ops; 8602 int err; 8603 8604 if (ops->ndo_get_phys_port_name) { 8605 err = ops->ndo_get_phys_port_name(dev, name, len); 8606 if (err != -EOPNOTSUPP) 8607 return err; 8608 } 8609 return devlink_compat_phys_port_name_get(dev, name, len); 8610 } 8611 EXPORT_SYMBOL(dev_get_phys_port_name); 8612 8613 /** 8614 * dev_get_port_parent_id - Get the device's port parent identifier 8615 * @dev: network device 8616 * @ppid: pointer to a storage for the port's parent identifier 8617 * @recurse: allow/disallow recursion to lower devices 8618 * 8619 * Get the devices's port parent identifier 8620 */ 8621 int dev_get_port_parent_id(struct net_device *dev, 8622 struct netdev_phys_item_id *ppid, 8623 bool recurse) 8624 { 8625 const struct net_device_ops *ops = dev->netdev_ops; 8626 struct netdev_phys_item_id first = { }; 8627 struct net_device *lower_dev; 8628 struct list_head *iter; 8629 int err; 8630 8631 if (ops->ndo_get_port_parent_id) { 8632 err = ops->ndo_get_port_parent_id(dev, ppid); 8633 if (err != -EOPNOTSUPP) 8634 return err; 8635 } 8636 8637 err = devlink_compat_switch_id_get(dev, ppid); 8638 if (!err || err != -EOPNOTSUPP) 8639 return err; 8640 8641 if (!recurse) 8642 return -EOPNOTSUPP; 8643 8644 netdev_for_each_lower_dev(dev, lower_dev, iter) { 8645 err = dev_get_port_parent_id(lower_dev, ppid, recurse); 8646 if (err) 8647 break; 8648 if (!first.id_len) 8649 first = *ppid; 8650 else if (memcmp(&first, ppid, sizeof(*ppid))) 8651 return -ENODATA; 8652 } 8653 8654 return err; 8655 } 8656 EXPORT_SYMBOL(dev_get_port_parent_id); 8657 8658 /** 8659 * netdev_port_same_parent_id - Indicate if two network devices have 8660 * the same port parent identifier 8661 * @a: first network device 8662 * @b: second network device 8663 */ 8664 bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b) 8665 { 8666 struct netdev_phys_item_id a_id = { }; 8667 struct netdev_phys_item_id b_id = { }; 8668 8669 if (dev_get_port_parent_id(a, &a_id, true) || 8670 dev_get_port_parent_id(b, &b_id, true)) 8671 return false; 8672 8673 return netdev_phys_item_id_same(&a_id, &b_id); 8674 } 8675 EXPORT_SYMBOL(netdev_port_same_parent_id); 8676 8677 /** 8678 * dev_change_proto_down - update protocol port state information 8679 * @dev: device 8680 * @proto_down: new value 8681 * 8682 * This info can be used by switch drivers to set the phys state of the 8683 * port. 8684 */ 8685 int dev_change_proto_down(struct net_device *dev, bool proto_down) 8686 { 8687 const struct net_device_ops *ops = dev->netdev_ops; 8688 8689 if (!ops->ndo_change_proto_down) 8690 return -EOPNOTSUPP; 8691 if (!netif_device_present(dev)) 8692 return -ENODEV; 8693 return ops->ndo_change_proto_down(dev, proto_down); 8694 } 8695 EXPORT_SYMBOL(dev_change_proto_down); 8696 8697 /** 8698 * dev_change_proto_down_generic - generic implementation for 8699 * ndo_change_proto_down that sets carrier according to 8700 * proto_down. 8701 * 8702 * @dev: device 8703 * @proto_down: new value 8704 */ 8705 int dev_change_proto_down_generic(struct net_device *dev, bool proto_down) 8706 { 8707 if (proto_down) 8708 netif_carrier_off(dev); 8709 else 8710 netif_carrier_on(dev); 8711 dev->proto_down = proto_down; 8712 return 0; 8713 } 8714 EXPORT_SYMBOL(dev_change_proto_down_generic); 8715 8716 u32 __dev_xdp_query(struct net_device *dev, bpf_op_t bpf_op, 8717 enum bpf_netdev_command cmd) 8718 { 8719 struct netdev_bpf xdp; 8720 8721 if (!bpf_op) 8722 return 0; 8723 8724 memset(&xdp, 0, sizeof(xdp)); 8725 xdp.command = cmd; 8726 8727 /* Query must always succeed. */ 8728 WARN_ON(bpf_op(dev, &xdp) < 0 && cmd == XDP_QUERY_PROG); 8729 8730 return xdp.prog_id; 8731 } 8732 8733 static int dev_xdp_install(struct net_device *dev, bpf_op_t bpf_op, 8734 struct netlink_ext_ack *extack, u32 flags, 8735 struct bpf_prog *prog) 8736 { 8737 bool non_hw = !(flags & XDP_FLAGS_HW_MODE); 8738 struct bpf_prog *prev_prog = NULL; 8739 struct netdev_bpf xdp; 8740 int err; 8741 8742 if (non_hw) { 8743 prev_prog = bpf_prog_by_id(__dev_xdp_query(dev, bpf_op, 8744 XDP_QUERY_PROG)); 8745 if (IS_ERR(prev_prog)) 8746 prev_prog = NULL; 8747 } 8748 8749 memset(&xdp, 0, sizeof(xdp)); 8750 if (flags & XDP_FLAGS_HW_MODE) 8751 xdp.command = XDP_SETUP_PROG_HW; 8752 else 8753 xdp.command = XDP_SETUP_PROG; 8754 xdp.extack = extack; 8755 xdp.flags = flags; 8756 xdp.prog = prog; 8757 8758 err = bpf_op(dev, &xdp); 8759 if (!err && non_hw) 8760 bpf_prog_change_xdp(prev_prog, prog); 8761 8762 if (prev_prog) 8763 bpf_prog_put(prev_prog); 8764 8765 return err; 8766 } 8767 8768 static void dev_xdp_uninstall(struct net_device *dev) 8769 { 8770 struct netdev_bpf xdp; 8771 bpf_op_t ndo_bpf; 8772 8773 /* Remove generic XDP */ 8774 WARN_ON(dev_xdp_install(dev, generic_xdp_install, NULL, 0, NULL)); 8775 8776 /* Remove from the driver */ 8777 ndo_bpf = dev->netdev_ops->ndo_bpf; 8778 if (!ndo_bpf) 8779 return; 8780 8781 memset(&xdp, 0, sizeof(xdp)); 8782 xdp.command = XDP_QUERY_PROG; 8783 WARN_ON(ndo_bpf(dev, &xdp)); 8784 if (xdp.prog_id) 8785 WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags, 8786 NULL)); 8787 8788 /* Remove HW offload */ 8789 memset(&xdp, 0, sizeof(xdp)); 8790 xdp.command = XDP_QUERY_PROG_HW; 8791 if (!ndo_bpf(dev, &xdp) && xdp.prog_id) 8792 WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags, 8793 NULL)); 8794 } 8795 8796 /** 8797 * dev_change_xdp_fd - set or clear a bpf program for a device rx path 8798 * @dev: device 8799 * @extack: netlink extended ack 8800 * @fd: new program fd or negative value to clear 8801 * @expected_fd: old program fd that userspace expects to replace or clear 8802 * @flags: xdp-related flags 8803 * 8804 * Set or clear a bpf program for a device 8805 */ 8806 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, 8807 int fd, int expected_fd, u32 flags) 8808 { 8809 const struct net_device_ops *ops = dev->netdev_ops; 8810 enum bpf_netdev_command query; 8811 u32 prog_id, expected_id = 0; 8812 bpf_op_t bpf_op, bpf_chk; 8813 struct bpf_prog *prog; 8814 bool offload; 8815 int err; 8816 8817 ASSERT_RTNL(); 8818 8819 offload = flags & XDP_FLAGS_HW_MODE; 8820 query = offload ? XDP_QUERY_PROG_HW : XDP_QUERY_PROG; 8821 8822 bpf_op = bpf_chk = ops->ndo_bpf; 8823 if (!bpf_op && (flags & (XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE))) { 8824 NL_SET_ERR_MSG(extack, "underlying driver does not support XDP in native mode"); 8825 return -EOPNOTSUPP; 8826 } 8827 if (!bpf_op || (flags & XDP_FLAGS_SKB_MODE)) 8828 bpf_op = generic_xdp_install; 8829 if (bpf_op == bpf_chk) 8830 bpf_chk = generic_xdp_install; 8831 8832 prog_id = __dev_xdp_query(dev, bpf_op, query); 8833 if (flags & XDP_FLAGS_REPLACE) { 8834 if (expected_fd >= 0) { 8835 prog = bpf_prog_get_type_dev(expected_fd, 8836 BPF_PROG_TYPE_XDP, 8837 bpf_op == ops->ndo_bpf); 8838 if (IS_ERR(prog)) 8839 return PTR_ERR(prog); 8840 expected_id = prog->aux->id; 8841 bpf_prog_put(prog); 8842 } 8843 8844 if (prog_id != expected_id) { 8845 NL_SET_ERR_MSG(extack, "Active program does not match expected"); 8846 return -EEXIST; 8847 } 8848 } 8849 if (fd >= 0) { 8850 if (!offload && __dev_xdp_query(dev, bpf_chk, XDP_QUERY_PROG)) { 8851 NL_SET_ERR_MSG(extack, "native and generic XDP can't be active at the same time"); 8852 return -EEXIST; 8853 } 8854 8855 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && prog_id) { 8856 NL_SET_ERR_MSG(extack, "XDP program already attached"); 8857 return -EBUSY; 8858 } 8859 8860 prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP, 8861 bpf_op == ops->ndo_bpf); 8862 if (IS_ERR(prog)) 8863 return PTR_ERR(prog); 8864 8865 if (!offload && bpf_prog_is_dev_bound(prog->aux)) { 8866 NL_SET_ERR_MSG(extack, "using device-bound program without HW_MODE flag is not supported"); 8867 bpf_prog_put(prog); 8868 return -EINVAL; 8869 } 8870 8871 if (prog->expected_attach_type == BPF_XDP_DEVMAP) { 8872 NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device"); 8873 bpf_prog_put(prog); 8874 return -EINVAL; 8875 } 8876 8877 /* prog->aux->id may be 0 for orphaned device-bound progs */ 8878 if (prog->aux->id && prog->aux->id == prog_id) { 8879 bpf_prog_put(prog); 8880 return 0; 8881 } 8882 } else { 8883 if (!prog_id) 8884 return 0; 8885 prog = NULL; 8886 } 8887 8888 err = dev_xdp_install(dev, bpf_op, extack, flags, prog); 8889 if (err < 0 && prog) 8890 bpf_prog_put(prog); 8891 8892 return err; 8893 } 8894 8895 /** 8896 * dev_new_index - allocate an ifindex 8897 * @net: the applicable net namespace 8898 * 8899 * Returns a suitable unique value for a new device interface 8900 * number. The caller must hold the rtnl semaphore or the 8901 * dev_base_lock to be sure it remains unique. 8902 */ 8903 static int dev_new_index(struct net *net) 8904 { 8905 int ifindex = net->ifindex; 8906 8907 for (;;) { 8908 if (++ifindex <= 0) 8909 ifindex = 1; 8910 if (!__dev_get_by_index(net, ifindex)) 8911 return net->ifindex = ifindex; 8912 } 8913 } 8914 8915 /* Delayed registration/unregisteration */ 8916 static LIST_HEAD(net_todo_list); 8917 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq); 8918 8919 static void net_set_todo(struct net_device *dev) 8920 { 8921 list_add_tail(&dev->todo_list, &net_todo_list); 8922 dev_net(dev)->dev_unreg_count++; 8923 } 8924 8925 static void rollback_registered_many(struct list_head *head) 8926 { 8927 struct net_device *dev, *tmp; 8928 LIST_HEAD(close_head); 8929 8930 BUG_ON(dev_boot_phase); 8931 ASSERT_RTNL(); 8932 8933 list_for_each_entry_safe(dev, tmp, head, unreg_list) { 8934 /* Some devices call without registering 8935 * for initialization unwind. Remove those 8936 * devices and proceed with the remaining. 8937 */ 8938 if (dev->reg_state == NETREG_UNINITIALIZED) { 8939 pr_debug("unregister_netdevice: device %s/%p never was registered\n", 8940 dev->name, dev); 8941 8942 WARN_ON(1); 8943 list_del(&dev->unreg_list); 8944 continue; 8945 } 8946 dev->dismantle = true; 8947 BUG_ON(dev->reg_state != NETREG_REGISTERED); 8948 } 8949 8950 /* If device is running, close it first. */ 8951 list_for_each_entry(dev, head, unreg_list) 8952 list_add_tail(&dev->close_list, &close_head); 8953 dev_close_many(&close_head, true); 8954 8955 list_for_each_entry(dev, head, unreg_list) { 8956 /* And unlink it from device chain. */ 8957 unlist_netdevice(dev); 8958 8959 dev->reg_state = NETREG_UNREGISTERING; 8960 } 8961 flush_all_backlogs(); 8962 8963 synchronize_net(); 8964 8965 list_for_each_entry(dev, head, unreg_list) { 8966 struct sk_buff *skb = NULL; 8967 8968 /* Shutdown queueing discipline. */ 8969 dev_shutdown(dev); 8970 8971 dev_xdp_uninstall(dev); 8972 8973 /* Notify protocols, that we are about to destroy 8974 * this device. They should clean all the things. 8975 */ 8976 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 8977 8978 if (!dev->rtnl_link_ops || 8979 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 8980 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0, 8981 GFP_KERNEL, NULL, 0); 8982 8983 /* 8984 * Flush the unicast and multicast chains 8985 */ 8986 dev_uc_flush(dev); 8987 dev_mc_flush(dev); 8988 8989 netdev_name_node_alt_flush(dev); 8990 netdev_name_node_free(dev->name_node); 8991 8992 if (dev->netdev_ops->ndo_uninit) 8993 dev->netdev_ops->ndo_uninit(dev); 8994 8995 if (skb) 8996 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL); 8997 8998 /* Notifier chain MUST detach us all upper devices. */ 8999 WARN_ON(netdev_has_any_upper_dev(dev)); 9000 WARN_ON(netdev_has_any_lower_dev(dev)); 9001 9002 /* Remove entries from kobject tree */ 9003 netdev_unregister_kobject(dev); 9004 #ifdef CONFIG_XPS 9005 /* Remove XPS queueing entries */ 9006 netif_reset_xps_queues_gt(dev, 0); 9007 #endif 9008 } 9009 9010 synchronize_net(); 9011 9012 list_for_each_entry(dev, head, unreg_list) 9013 dev_put(dev); 9014 } 9015 9016 static void rollback_registered(struct net_device *dev) 9017 { 9018 LIST_HEAD(single); 9019 9020 list_add(&dev->unreg_list, &single); 9021 rollback_registered_many(&single); 9022 list_del(&single); 9023 } 9024 9025 static netdev_features_t netdev_sync_upper_features(struct net_device *lower, 9026 struct net_device *upper, netdev_features_t features) 9027 { 9028 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 9029 netdev_features_t feature; 9030 int feature_bit; 9031 9032 for_each_netdev_feature(upper_disables, feature_bit) { 9033 feature = __NETIF_F_BIT(feature_bit); 9034 if (!(upper->wanted_features & feature) 9035 && (features & feature)) { 9036 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n", 9037 &feature, upper->name); 9038 features &= ~feature; 9039 } 9040 } 9041 9042 return features; 9043 } 9044 9045 static void netdev_sync_lower_features(struct net_device *upper, 9046 struct net_device *lower, netdev_features_t features) 9047 { 9048 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 9049 netdev_features_t feature; 9050 int feature_bit; 9051 9052 for_each_netdev_feature(upper_disables, feature_bit) { 9053 feature = __NETIF_F_BIT(feature_bit); 9054 if (!(features & feature) && (lower->features & feature)) { 9055 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n", 9056 &feature, lower->name); 9057 lower->wanted_features &= ~feature; 9058 __netdev_update_features(lower); 9059 9060 if (unlikely(lower->features & feature)) 9061 netdev_WARN(upper, "failed to disable %pNF on %s!\n", 9062 &feature, lower->name); 9063 else 9064 netdev_features_change(lower); 9065 } 9066 } 9067 } 9068 9069 static netdev_features_t netdev_fix_features(struct net_device *dev, 9070 netdev_features_t features) 9071 { 9072 /* Fix illegal checksum combinations */ 9073 if ((features & NETIF_F_HW_CSUM) && 9074 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { 9075 netdev_warn(dev, "mixed HW and IP checksum settings.\n"); 9076 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); 9077 } 9078 9079 /* TSO requires that SG is present as well. */ 9080 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) { 9081 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n"); 9082 features &= ~NETIF_F_ALL_TSO; 9083 } 9084 9085 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) && 9086 !(features & NETIF_F_IP_CSUM)) { 9087 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n"); 9088 features &= ~NETIF_F_TSO; 9089 features &= ~NETIF_F_TSO_ECN; 9090 } 9091 9092 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) && 9093 !(features & NETIF_F_IPV6_CSUM)) { 9094 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n"); 9095 features &= ~NETIF_F_TSO6; 9096 } 9097 9098 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */ 9099 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO)) 9100 features &= ~NETIF_F_TSO_MANGLEID; 9101 9102 /* TSO ECN requires that TSO is present as well. */ 9103 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN) 9104 features &= ~NETIF_F_TSO_ECN; 9105 9106 /* Software GSO depends on SG. */ 9107 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) { 9108 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n"); 9109 features &= ~NETIF_F_GSO; 9110 } 9111 9112 /* GSO partial features require GSO partial be set */ 9113 if ((features & dev->gso_partial_features) && 9114 !(features & NETIF_F_GSO_PARTIAL)) { 9115 netdev_dbg(dev, 9116 "Dropping partially supported GSO features since no GSO partial.\n"); 9117 features &= ~dev->gso_partial_features; 9118 } 9119 9120 if (!(features & NETIF_F_RXCSUM)) { 9121 /* NETIF_F_GRO_HW implies doing RXCSUM since every packet 9122 * successfully merged by hardware must also have the 9123 * checksum verified by hardware. If the user does not 9124 * want to enable RXCSUM, logically, we should disable GRO_HW. 9125 */ 9126 if (features & NETIF_F_GRO_HW) { 9127 netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n"); 9128 features &= ~NETIF_F_GRO_HW; 9129 } 9130 } 9131 9132 /* LRO/HW-GRO features cannot be combined with RX-FCS */ 9133 if (features & NETIF_F_RXFCS) { 9134 if (features & NETIF_F_LRO) { 9135 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n"); 9136 features &= ~NETIF_F_LRO; 9137 } 9138 9139 if (features & NETIF_F_GRO_HW) { 9140 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n"); 9141 features &= ~NETIF_F_GRO_HW; 9142 } 9143 } 9144 9145 return features; 9146 } 9147 9148 int __netdev_update_features(struct net_device *dev) 9149 { 9150 struct net_device *upper, *lower; 9151 netdev_features_t features; 9152 struct list_head *iter; 9153 int err = -1; 9154 9155 ASSERT_RTNL(); 9156 9157 features = netdev_get_wanted_features(dev); 9158 9159 if (dev->netdev_ops->ndo_fix_features) 9160 features = dev->netdev_ops->ndo_fix_features(dev, features); 9161 9162 /* driver might be less strict about feature dependencies */ 9163 features = netdev_fix_features(dev, features); 9164 9165 /* some features can't be enabled if they're off an an upper device */ 9166 netdev_for_each_upper_dev_rcu(dev, upper, iter) 9167 features = netdev_sync_upper_features(dev, upper, features); 9168 9169 if (dev->features == features) 9170 goto sync_lower; 9171 9172 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", 9173 &dev->features, &features); 9174 9175 if (dev->netdev_ops->ndo_set_features) 9176 err = dev->netdev_ops->ndo_set_features(dev, features); 9177 else 9178 err = 0; 9179 9180 if (unlikely(err < 0)) { 9181 netdev_err(dev, 9182 "set_features() failed (%d); wanted %pNF, left %pNF\n", 9183 err, &features, &dev->features); 9184 /* return non-0 since some features might have changed and 9185 * it's better to fire a spurious notification than miss it 9186 */ 9187 return -1; 9188 } 9189 9190 sync_lower: 9191 /* some features must be disabled on lower devices when disabled 9192 * on an upper device (think: bonding master or bridge) 9193 */ 9194 netdev_for_each_lower_dev(dev, lower, iter) 9195 netdev_sync_lower_features(dev, lower, features); 9196 9197 if (!err) { 9198 netdev_features_t diff = features ^ dev->features; 9199 9200 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) { 9201 /* udp_tunnel_{get,drop}_rx_info both need 9202 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the 9203 * device, or they won't do anything. 9204 * Thus we need to update dev->features 9205 * *before* calling udp_tunnel_get_rx_info, 9206 * but *after* calling udp_tunnel_drop_rx_info. 9207 */ 9208 if (features & NETIF_F_RX_UDP_TUNNEL_PORT) { 9209 dev->features = features; 9210 udp_tunnel_get_rx_info(dev); 9211 } else { 9212 udp_tunnel_drop_rx_info(dev); 9213 } 9214 } 9215 9216 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) { 9217 if (features & NETIF_F_HW_VLAN_CTAG_FILTER) { 9218 dev->features = features; 9219 err |= vlan_get_rx_ctag_filter_info(dev); 9220 } else { 9221 vlan_drop_rx_ctag_filter_info(dev); 9222 } 9223 } 9224 9225 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) { 9226 if (features & NETIF_F_HW_VLAN_STAG_FILTER) { 9227 dev->features = features; 9228 err |= vlan_get_rx_stag_filter_info(dev); 9229 } else { 9230 vlan_drop_rx_stag_filter_info(dev); 9231 } 9232 } 9233 9234 dev->features = features; 9235 } 9236 9237 return err < 0 ? 0 : 1; 9238 } 9239 9240 /** 9241 * netdev_update_features - recalculate device features 9242 * @dev: the device to check 9243 * 9244 * Recalculate dev->features set and send notifications if it 9245 * has changed. Should be called after driver or hardware dependent 9246 * conditions might have changed that influence the features. 9247 */ 9248 void netdev_update_features(struct net_device *dev) 9249 { 9250 if (__netdev_update_features(dev)) 9251 netdev_features_change(dev); 9252 } 9253 EXPORT_SYMBOL(netdev_update_features); 9254 9255 /** 9256 * netdev_change_features - recalculate device features 9257 * @dev: the device to check 9258 * 9259 * Recalculate dev->features set and send notifications even 9260 * if they have not changed. Should be called instead of 9261 * netdev_update_features() if also dev->vlan_features might 9262 * have changed to allow the changes to be propagated to stacked 9263 * VLAN devices. 9264 */ 9265 void netdev_change_features(struct net_device *dev) 9266 { 9267 __netdev_update_features(dev); 9268 netdev_features_change(dev); 9269 } 9270 EXPORT_SYMBOL(netdev_change_features); 9271 9272 /** 9273 * netif_stacked_transfer_operstate - transfer operstate 9274 * @rootdev: the root or lower level device to transfer state from 9275 * @dev: the device to transfer operstate to 9276 * 9277 * Transfer operational state from root to device. This is normally 9278 * called when a stacking relationship exists between the root 9279 * device and the device(a leaf device). 9280 */ 9281 void netif_stacked_transfer_operstate(const struct net_device *rootdev, 9282 struct net_device *dev) 9283 { 9284 if (rootdev->operstate == IF_OPER_DORMANT) 9285 netif_dormant_on(dev); 9286 else 9287 netif_dormant_off(dev); 9288 9289 if (rootdev->operstate == IF_OPER_TESTING) 9290 netif_testing_on(dev); 9291 else 9292 netif_testing_off(dev); 9293 9294 if (netif_carrier_ok(rootdev)) 9295 netif_carrier_on(dev); 9296 else 9297 netif_carrier_off(dev); 9298 } 9299 EXPORT_SYMBOL(netif_stacked_transfer_operstate); 9300 9301 static int netif_alloc_rx_queues(struct net_device *dev) 9302 { 9303 unsigned int i, count = dev->num_rx_queues; 9304 struct netdev_rx_queue *rx; 9305 size_t sz = count * sizeof(*rx); 9306 int err = 0; 9307 9308 BUG_ON(count < 1); 9309 9310 rx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL); 9311 if (!rx) 9312 return -ENOMEM; 9313 9314 dev->_rx = rx; 9315 9316 for (i = 0; i < count; i++) { 9317 rx[i].dev = dev; 9318 9319 /* XDP RX-queue setup */ 9320 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i); 9321 if (err < 0) 9322 goto err_rxq_info; 9323 } 9324 return 0; 9325 9326 err_rxq_info: 9327 /* Rollback successful reg's and free other resources */ 9328 while (i--) 9329 xdp_rxq_info_unreg(&rx[i].xdp_rxq); 9330 kvfree(dev->_rx); 9331 dev->_rx = NULL; 9332 return err; 9333 } 9334 9335 static void netif_free_rx_queues(struct net_device *dev) 9336 { 9337 unsigned int i, count = dev->num_rx_queues; 9338 9339 /* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */ 9340 if (!dev->_rx) 9341 return; 9342 9343 for (i = 0; i < count; i++) 9344 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq); 9345 9346 kvfree(dev->_rx); 9347 } 9348 9349 static void netdev_init_one_queue(struct net_device *dev, 9350 struct netdev_queue *queue, void *_unused) 9351 { 9352 /* Initialize queue lock */ 9353 spin_lock_init(&queue->_xmit_lock); 9354 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); 9355 queue->xmit_lock_owner = -1; 9356 netdev_queue_numa_node_write(queue, NUMA_NO_NODE); 9357 queue->dev = dev; 9358 #ifdef CONFIG_BQL 9359 dql_init(&queue->dql, HZ); 9360 #endif 9361 } 9362 9363 static void netif_free_tx_queues(struct net_device *dev) 9364 { 9365 kvfree(dev->_tx); 9366 } 9367 9368 static int netif_alloc_netdev_queues(struct net_device *dev) 9369 { 9370 unsigned int count = dev->num_tx_queues; 9371 struct netdev_queue *tx; 9372 size_t sz = count * sizeof(*tx); 9373 9374 if (count < 1 || count > 0xffff) 9375 return -EINVAL; 9376 9377 tx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL); 9378 if (!tx) 9379 return -ENOMEM; 9380 9381 dev->_tx = tx; 9382 9383 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); 9384 spin_lock_init(&dev->tx_global_lock); 9385 9386 return 0; 9387 } 9388 9389 void netif_tx_stop_all_queues(struct net_device *dev) 9390 { 9391 unsigned int i; 9392 9393 for (i = 0; i < dev->num_tx_queues; i++) { 9394 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 9395 9396 netif_tx_stop_queue(txq); 9397 } 9398 } 9399 EXPORT_SYMBOL(netif_tx_stop_all_queues); 9400 9401 /** 9402 * register_netdevice - register a network device 9403 * @dev: device to register 9404 * 9405 * Take a completed network device structure and add it to the kernel 9406 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 9407 * chain. 0 is returned on success. A negative errno code is returned 9408 * on a failure to set up the device, or if the name is a duplicate. 9409 * 9410 * Callers must hold the rtnl semaphore. You may want 9411 * register_netdev() instead of this. 9412 * 9413 * BUGS: 9414 * The locking appears insufficient to guarantee two parallel registers 9415 * will not get the same name. 9416 */ 9417 9418 int register_netdevice(struct net_device *dev) 9419 { 9420 int ret; 9421 struct net *net = dev_net(dev); 9422 9423 BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE < 9424 NETDEV_FEATURE_COUNT); 9425 BUG_ON(dev_boot_phase); 9426 ASSERT_RTNL(); 9427 9428 might_sleep(); 9429 9430 /* When net_device's are persistent, this will be fatal. */ 9431 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); 9432 BUG_ON(!net); 9433 9434 ret = ethtool_check_ops(dev->ethtool_ops); 9435 if (ret) 9436 return ret; 9437 9438 spin_lock_init(&dev->addr_list_lock); 9439 netdev_set_addr_lockdep_class(dev); 9440 9441 ret = dev_get_valid_name(net, dev, dev->name); 9442 if (ret < 0) 9443 goto out; 9444 9445 ret = -ENOMEM; 9446 dev->name_node = netdev_name_node_head_alloc(dev); 9447 if (!dev->name_node) 9448 goto out; 9449 9450 /* Init, if this function is available */ 9451 if (dev->netdev_ops->ndo_init) { 9452 ret = dev->netdev_ops->ndo_init(dev); 9453 if (ret) { 9454 if (ret > 0) 9455 ret = -EIO; 9456 goto err_free_name; 9457 } 9458 } 9459 9460 if (((dev->hw_features | dev->features) & 9461 NETIF_F_HW_VLAN_CTAG_FILTER) && 9462 (!dev->netdev_ops->ndo_vlan_rx_add_vid || 9463 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { 9464 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n"); 9465 ret = -EINVAL; 9466 goto err_uninit; 9467 } 9468 9469 ret = -EBUSY; 9470 if (!dev->ifindex) 9471 dev->ifindex = dev_new_index(net); 9472 else if (__dev_get_by_index(net, dev->ifindex)) 9473 goto err_uninit; 9474 9475 /* Transfer changeable features to wanted_features and enable 9476 * software offloads (GSO and GRO). 9477 */ 9478 dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF); 9479 dev->features |= NETIF_F_SOFT_FEATURES; 9480 9481 if (dev->netdev_ops->ndo_udp_tunnel_add) { 9482 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT; 9483 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT; 9484 } 9485 9486 dev->wanted_features = dev->features & dev->hw_features; 9487 9488 if (!(dev->flags & IFF_LOOPBACK)) 9489 dev->hw_features |= NETIF_F_NOCACHE_COPY; 9490 9491 /* If IPv4 TCP segmentation offload is supported we should also 9492 * allow the device to enable segmenting the frame with the option 9493 * of ignoring a static IP ID value. This doesn't enable the 9494 * feature itself but allows the user to enable it later. 9495 */ 9496 if (dev->hw_features & NETIF_F_TSO) 9497 dev->hw_features |= NETIF_F_TSO_MANGLEID; 9498 if (dev->vlan_features & NETIF_F_TSO) 9499 dev->vlan_features |= NETIF_F_TSO_MANGLEID; 9500 if (dev->mpls_features & NETIF_F_TSO) 9501 dev->mpls_features |= NETIF_F_TSO_MANGLEID; 9502 if (dev->hw_enc_features & NETIF_F_TSO) 9503 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 9504 9505 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. 9506 */ 9507 dev->vlan_features |= NETIF_F_HIGHDMA; 9508 9509 /* Make NETIF_F_SG inheritable to tunnel devices. 9510 */ 9511 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL; 9512 9513 /* Make NETIF_F_SG inheritable to MPLS. 9514 */ 9515 dev->mpls_features |= NETIF_F_SG; 9516 9517 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); 9518 ret = notifier_to_errno(ret); 9519 if (ret) 9520 goto err_uninit; 9521 9522 ret = netdev_register_kobject(dev); 9523 if (ret) { 9524 dev->reg_state = NETREG_UNREGISTERED; 9525 goto err_uninit; 9526 } 9527 dev->reg_state = NETREG_REGISTERED; 9528 9529 __netdev_update_features(dev); 9530 9531 /* 9532 * Default initial state at registry is that the 9533 * device is present. 9534 */ 9535 9536 set_bit(__LINK_STATE_PRESENT, &dev->state); 9537 9538 linkwatch_init_dev(dev); 9539 9540 dev_init_scheduler(dev); 9541 dev_hold(dev); 9542 list_netdevice(dev); 9543 add_device_randomness(dev->dev_addr, dev->addr_len); 9544 9545 /* If the device has permanent device address, driver should 9546 * set dev_addr and also addr_assign_type should be set to 9547 * NET_ADDR_PERM (default value). 9548 */ 9549 if (dev->addr_assign_type == NET_ADDR_PERM) 9550 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); 9551 9552 /* Notify protocols, that a new device appeared. */ 9553 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); 9554 ret = notifier_to_errno(ret); 9555 if (ret) { 9556 rollback_registered(dev); 9557 rcu_barrier(); 9558 9559 dev->reg_state = NETREG_UNREGISTERED; 9560 /* We should put the kobject that hold in 9561 * netdev_unregister_kobject(), otherwise 9562 * the net device cannot be freed when 9563 * driver calls free_netdev(), because the 9564 * kobject is being hold. 9565 */ 9566 kobject_put(&dev->dev.kobj); 9567 } 9568 /* 9569 * Prevent userspace races by waiting until the network 9570 * device is fully setup before sending notifications. 9571 */ 9572 if (!dev->rtnl_link_ops || 9573 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 9574 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); 9575 9576 out: 9577 return ret; 9578 9579 err_uninit: 9580 if (dev->netdev_ops->ndo_uninit) 9581 dev->netdev_ops->ndo_uninit(dev); 9582 if (dev->priv_destructor) 9583 dev->priv_destructor(dev); 9584 err_free_name: 9585 netdev_name_node_free(dev->name_node); 9586 goto out; 9587 } 9588 EXPORT_SYMBOL(register_netdevice); 9589 9590 /** 9591 * init_dummy_netdev - init a dummy network device for NAPI 9592 * @dev: device to init 9593 * 9594 * This takes a network device structure and initialize the minimum 9595 * amount of fields so it can be used to schedule NAPI polls without 9596 * registering a full blown interface. This is to be used by drivers 9597 * that need to tie several hardware interfaces to a single NAPI 9598 * poll scheduler due to HW limitations. 9599 */ 9600 int init_dummy_netdev(struct net_device *dev) 9601 { 9602 /* Clear everything. Note we don't initialize spinlocks 9603 * are they aren't supposed to be taken by any of the 9604 * NAPI code and this dummy netdev is supposed to be 9605 * only ever used for NAPI polls 9606 */ 9607 memset(dev, 0, sizeof(struct net_device)); 9608 9609 /* make sure we BUG if trying to hit standard 9610 * register/unregister code path 9611 */ 9612 dev->reg_state = NETREG_DUMMY; 9613 9614 /* NAPI wants this */ 9615 INIT_LIST_HEAD(&dev->napi_list); 9616 9617 /* a dummy interface is started by default */ 9618 set_bit(__LINK_STATE_PRESENT, &dev->state); 9619 set_bit(__LINK_STATE_START, &dev->state); 9620 9621 /* napi_busy_loop stats accounting wants this */ 9622 dev_net_set(dev, &init_net); 9623 9624 /* Note : We dont allocate pcpu_refcnt for dummy devices, 9625 * because users of this 'device' dont need to change 9626 * its refcount. 9627 */ 9628 9629 return 0; 9630 } 9631 EXPORT_SYMBOL_GPL(init_dummy_netdev); 9632 9633 9634 /** 9635 * register_netdev - register a network device 9636 * @dev: device to register 9637 * 9638 * Take a completed network device structure and add it to the kernel 9639 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 9640 * chain. 0 is returned on success. A negative errno code is returned 9641 * on a failure to set up the device, or if the name is a duplicate. 9642 * 9643 * This is a wrapper around register_netdevice that takes the rtnl semaphore 9644 * and expands the device name if you passed a format string to 9645 * alloc_netdev. 9646 */ 9647 int register_netdev(struct net_device *dev) 9648 { 9649 int err; 9650 9651 if (rtnl_lock_killable()) 9652 return -EINTR; 9653 err = register_netdevice(dev); 9654 rtnl_unlock(); 9655 return err; 9656 } 9657 EXPORT_SYMBOL(register_netdev); 9658 9659 int netdev_refcnt_read(const struct net_device *dev) 9660 { 9661 int i, refcnt = 0; 9662 9663 for_each_possible_cpu(i) 9664 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); 9665 return refcnt; 9666 } 9667 EXPORT_SYMBOL(netdev_refcnt_read); 9668 9669 /** 9670 * netdev_wait_allrefs - wait until all references are gone. 9671 * @dev: target net_device 9672 * 9673 * This is called when unregistering network devices. 9674 * 9675 * Any protocol or device that holds a reference should register 9676 * for netdevice notification, and cleanup and put back the 9677 * reference if they receive an UNREGISTER event. 9678 * We can get stuck here if buggy protocols don't correctly 9679 * call dev_put. 9680 */ 9681 static void netdev_wait_allrefs(struct net_device *dev) 9682 { 9683 unsigned long rebroadcast_time, warning_time; 9684 int refcnt; 9685 9686 linkwatch_forget_dev(dev); 9687 9688 rebroadcast_time = warning_time = jiffies; 9689 refcnt = netdev_refcnt_read(dev); 9690 9691 while (refcnt != 0) { 9692 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { 9693 rtnl_lock(); 9694 9695 /* Rebroadcast unregister notification */ 9696 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 9697 9698 __rtnl_unlock(); 9699 rcu_barrier(); 9700 rtnl_lock(); 9701 9702 if (test_bit(__LINK_STATE_LINKWATCH_PENDING, 9703 &dev->state)) { 9704 /* We must not have linkwatch events 9705 * pending on unregister. If this 9706 * happens, we simply run the queue 9707 * unscheduled, resulting in a noop 9708 * for this device. 9709 */ 9710 linkwatch_run_queue(); 9711 } 9712 9713 __rtnl_unlock(); 9714 9715 rebroadcast_time = jiffies; 9716 } 9717 9718 msleep(250); 9719 9720 refcnt = netdev_refcnt_read(dev); 9721 9722 if (refcnt && time_after(jiffies, warning_time + 10 * HZ)) { 9723 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", 9724 dev->name, refcnt); 9725 warning_time = jiffies; 9726 } 9727 } 9728 } 9729 9730 /* The sequence is: 9731 * 9732 * rtnl_lock(); 9733 * ... 9734 * register_netdevice(x1); 9735 * register_netdevice(x2); 9736 * ... 9737 * unregister_netdevice(y1); 9738 * unregister_netdevice(y2); 9739 * ... 9740 * rtnl_unlock(); 9741 * free_netdev(y1); 9742 * free_netdev(y2); 9743 * 9744 * We are invoked by rtnl_unlock(). 9745 * This allows us to deal with problems: 9746 * 1) We can delete sysfs objects which invoke hotplug 9747 * without deadlocking with linkwatch via keventd. 9748 * 2) Since we run with the RTNL semaphore not held, we can sleep 9749 * safely in order to wait for the netdev refcnt to drop to zero. 9750 * 9751 * We must not return until all unregister events added during 9752 * the interval the lock was held have been completed. 9753 */ 9754 void netdev_run_todo(void) 9755 { 9756 struct list_head list; 9757 9758 /* Snapshot list, allow later requests */ 9759 list_replace_init(&net_todo_list, &list); 9760 9761 __rtnl_unlock(); 9762 9763 9764 /* Wait for rcu callbacks to finish before next phase */ 9765 if (!list_empty(&list)) 9766 rcu_barrier(); 9767 9768 while (!list_empty(&list)) { 9769 struct net_device *dev 9770 = list_first_entry(&list, struct net_device, todo_list); 9771 list_del(&dev->todo_list); 9772 9773 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { 9774 pr_err("network todo '%s' but state %d\n", 9775 dev->name, dev->reg_state); 9776 dump_stack(); 9777 continue; 9778 } 9779 9780 dev->reg_state = NETREG_UNREGISTERED; 9781 9782 netdev_wait_allrefs(dev); 9783 9784 /* paranoia */ 9785 BUG_ON(netdev_refcnt_read(dev)); 9786 BUG_ON(!list_empty(&dev->ptype_all)); 9787 BUG_ON(!list_empty(&dev->ptype_specific)); 9788 WARN_ON(rcu_access_pointer(dev->ip_ptr)); 9789 WARN_ON(rcu_access_pointer(dev->ip6_ptr)); 9790 #if IS_ENABLED(CONFIG_DECNET) 9791 WARN_ON(dev->dn_ptr); 9792 #endif 9793 if (dev->priv_destructor) 9794 dev->priv_destructor(dev); 9795 if (dev->needs_free_netdev) 9796 free_netdev(dev); 9797 9798 /* Report a network device has been unregistered */ 9799 rtnl_lock(); 9800 dev_net(dev)->dev_unreg_count--; 9801 __rtnl_unlock(); 9802 wake_up(&netdev_unregistering_wq); 9803 9804 /* Free network device */ 9805 kobject_put(&dev->dev.kobj); 9806 } 9807 } 9808 9809 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has 9810 * all the same fields in the same order as net_device_stats, with only 9811 * the type differing, but rtnl_link_stats64 may have additional fields 9812 * at the end for newer counters. 9813 */ 9814 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64, 9815 const struct net_device_stats *netdev_stats) 9816 { 9817 #if BITS_PER_LONG == 64 9818 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats)); 9819 memcpy(stats64, netdev_stats, sizeof(*netdev_stats)); 9820 /* zero out counters that only exist in rtnl_link_stats64 */ 9821 memset((char *)stats64 + sizeof(*netdev_stats), 0, 9822 sizeof(*stats64) - sizeof(*netdev_stats)); 9823 #else 9824 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long); 9825 const unsigned long *src = (const unsigned long *)netdev_stats; 9826 u64 *dst = (u64 *)stats64; 9827 9828 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64)); 9829 for (i = 0; i < n; i++) 9830 dst[i] = src[i]; 9831 /* zero out counters that only exist in rtnl_link_stats64 */ 9832 memset((char *)stats64 + n * sizeof(u64), 0, 9833 sizeof(*stats64) - n * sizeof(u64)); 9834 #endif 9835 } 9836 EXPORT_SYMBOL(netdev_stats_to_stats64); 9837 9838 /** 9839 * dev_get_stats - get network device statistics 9840 * @dev: device to get statistics from 9841 * @storage: place to store stats 9842 * 9843 * Get network statistics from device. Return @storage. 9844 * The device driver may provide its own method by setting 9845 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats; 9846 * otherwise the internal statistics structure is used. 9847 */ 9848 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, 9849 struct rtnl_link_stats64 *storage) 9850 { 9851 const struct net_device_ops *ops = dev->netdev_ops; 9852 9853 if (ops->ndo_get_stats64) { 9854 memset(storage, 0, sizeof(*storage)); 9855 ops->ndo_get_stats64(dev, storage); 9856 } else if (ops->ndo_get_stats) { 9857 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); 9858 } else { 9859 netdev_stats_to_stats64(storage, &dev->stats); 9860 } 9861 storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped); 9862 storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped); 9863 storage->rx_nohandler += (unsigned long)atomic_long_read(&dev->rx_nohandler); 9864 return storage; 9865 } 9866 EXPORT_SYMBOL(dev_get_stats); 9867 9868 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) 9869 { 9870 struct netdev_queue *queue = dev_ingress_queue(dev); 9871 9872 #ifdef CONFIG_NET_CLS_ACT 9873 if (queue) 9874 return queue; 9875 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 9876 if (!queue) 9877 return NULL; 9878 netdev_init_one_queue(dev, queue, NULL); 9879 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); 9880 queue->qdisc_sleeping = &noop_qdisc; 9881 rcu_assign_pointer(dev->ingress_queue, queue); 9882 #endif 9883 return queue; 9884 } 9885 9886 static const struct ethtool_ops default_ethtool_ops; 9887 9888 void netdev_set_default_ethtool_ops(struct net_device *dev, 9889 const struct ethtool_ops *ops) 9890 { 9891 if (dev->ethtool_ops == &default_ethtool_ops) 9892 dev->ethtool_ops = ops; 9893 } 9894 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops); 9895 9896 void netdev_freemem(struct net_device *dev) 9897 { 9898 char *addr = (char *)dev - dev->padded; 9899 9900 kvfree(addr); 9901 } 9902 9903 /** 9904 * alloc_netdev_mqs - allocate network device 9905 * @sizeof_priv: size of private data to allocate space for 9906 * @name: device name format string 9907 * @name_assign_type: origin of device name 9908 * @setup: callback to initialize device 9909 * @txqs: the number of TX subqueues to allocate 9910 * @rxqs: the number of RX subqueues to allocate 9911 * 9912 * Allocates a struct net_device with private data area for driver use 9913 * and performs basic initialization. Also allocates subqueue structs 9914 * for each queue on the device. 9915 */ 9916 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, 9917 unsigned char name_assign_type, 9918 void (*setup)(struct net_device *), 9919 unsigned int txqs, unsigned int rxqs) 9920 { 9921 struct net_device *dev; 9922 unsigned int alloc_size; 9923 struct net_device *p; 9924 9925 BUG_ON(strlen(name) >= sizeof(dev->name)); 9926 9927 if (txqs < 1) { 9928 pr_err("alloc_netdev: Unable to allocate device with zero queues\n"); 9929 return NULL; 9930 } 9931 9932 if (rxqs < 1) { 9933 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n"); 9934 return NULL; 9935 } 9936 9937 alloc_size = sizeof(struct net_device); 9938 if (sizeof_priv) { 9939 /* ensure 32-byte alignment of private area */ 9940 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN); 9941 alloc_size += sizeof_priv; 9942 } 9943 /* ensure 32-byte alignment of whole construct */ 9944 alloc_size += NETDEV_ALIGN - 1; 9945 9946 p = kvzalloc(alloc_size, GFP_KERNEL | __GFP_RETRY_MAYFAIL); 9947 if (!p) 9948 return NULL; 9949 9950 dev = PTR_ALIGN(p, NETDEV_ALIGN); 9951 dev->padded = (char *)dev - (char *)p; 9952 9953 dev->pcpu_refcnt = alloc_percpu(int); 9954 if (!dev->pcpu_refcnt) 9955 goto free_dev; 9956 9957 if (dev_addr_init(dev)) 9958 goto free_pcpu; 9959 9960 dev_mc_init(dev); 9961 dev_uc_init(dev); 9962 9963 dev_net_set(dev, &init_net); 9964 9965 dev->gso_max_size = GSO_MAX_SIZE; 9966 dev->gso_max_segs = GSO_MAX_SEGS; 9967 dev->upper_level = 1; 9968 dev->lower_level = 1; 9969 9970 INIT_LIST_HEAD(&dev->napi_list); 9971 INIT_LIST_HEAD(&dev->unreg_list); 9972 INIT_LIST_HEAD(&dev->close_list); 9973 INIT_LIST_HEAD(&dev->link_watch_list); 9974 INIT_LIST_HEAD(&dev->adj_list.upper); 9975 INIT_LIST_HEAD(&dev->adj_list.lower); 9976 INIT_LIST_HEAD(&dev->ptype_all); 9977 INIT_LIST_HEAD(&dev->ptype_specific); 9978 INIT_LIST_HEAD(&dev->net_notifier_list); 9979 #ifdef CONFIG_NET_SCHED 9980 hash_init(dev->qdisc_hash); 9981 #endif 9982 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; 9983 setup(dev); 9984 9985 if (!dev->tx_queue_len) { 9986 dev->priv_flags |= IFF_NO_QUEUE; 9987 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; 9988 } 9989 9990 dev->num_tx_queues = txqs; 9991 dev->real_num_tx_queues = txqs; 9992 if (netif_alloc_netdev_queues(dev)) 9993 goto free_all; 9994 9995 dev->num_rx_queues = rxqs; 9996 dev->real_num_rx_queues = rxqs; 9997 if (netif_alloc_rx_queues(dev)) 9998 goto free_all; 9999 10000 strcpy(dev->name, name); 10001 dev->name_assign_type = name_assign_type; 10002 dev->group = INIT_NETDEV_GROUP; 10003 if (!dev->ethtool_ops) 10004 dev->ethtool_ops = &default_ethtool_ops; 10005 10006 nf_hook_ingress_init(dev); 10007 10008 return dev; 10009 10010 free_all: 10011 free_netdev(dev); 10012 return NULL; 10013 10014 free_pcpu: 10015 free_percpu(dev->pcpu_refcnt); 10016 free_dev: 10017 netdev_freemem(dev); 10018 return NULL; 10019 } 10020 EXPORT_SYMBOL(alloc_netdev_mqs); 10021 10022 /** 10023 * free_netdev - free network device 10024 * @dev: device 10025 * 10026 * This function does the last stage of destroying an allocated device 10027 * interface. The reference to the device object is released. If this 10028 * is the last reference then it will be freed.Must be called in process 10029 * context. 10030 */ 10031 void free_netdev(struct net_device *dev) 10032 { 10033 struct napi_struct *p, *n; 10034 10035 might_sleep(); 10036 netif_free_tx_queues(dev); 10037 netif_free_rx_queues(dev); 10038 10039 kfree(rcu_dereference_protected(dev->ingress_queue, 1)); 10040 10041 /* Flush device addresses */ 10042 dev_addr_flush(dev); 10043 10044 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) 10045 netif_napi_del(p); 10046 10047 free_percpu(dev->pcpu_refcnt); 10048 dev->pcpu_refcnt = NULL; 10049 free_percpu(dev->xdp_bulkq); 10050 dev->xdp_bulkq = NULL; 10051 10052 /* Compatibility with error handling in drivers */ 10053 if (dev->reg_state == NETREG_UNINITIALIZED) { 10054 netdev_freemem(dev); 10055 return; 10056 } 10057 10058 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); 10059 dev->reg_state = NETREG_RELEASED; 10060 10061 /* will free via device release */ 10062 put_device(&dev->dev); 10063 } 10064 EXPORT_SYMBOL(free_netdev); 10065 10066 /** 10067 * synchronize_net - Synchronize with packet receive processing 10068 * 10069 * Wait for packets currently being received to be done. 10070 * Does not block later packets from starting. 10071 */ 10072 void synchronize_net(void) 10073 { 10074 might_sleep(); 10075 if (rtnl_is_locked()) 10076 synchronize_rcu_expedited(); 10077 else 10078 synchronize_rcu(); 10079 } 10080 EXPORT_SYMBOL(synchronize_net); 10081 10082 /** 10083 * unregister_netdevice_queue - remove device from the kernel 10084 * @dev: device 10085 * @head: list 10086 * 10087 * This function shuts down a device interface and removes it 10088 * from the kernel tables. 10089 * If head not NULL, device is queued to be unregistered later. 10090 * 10091 * Callers must hold the rtnl semaphore. You may want 10092 * unregister_netdev() instead of this. 10093 */ 10094 10095 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) 10096 { 10097 ASSERT_RTNL(); 10098 10099 if (head) { 10100 list_move_tail(&dev->unreg_list, head); 10101 } else { 10102 rollback_registered(dev); 10103 /* Finish processing unregister after unlock */ 10104 net_set_todo(dev); 10105 } 10106 } 10107 EXPORT_SYMBOL(unregister_netdevice_queue); 10108 10109 /** 10110 * unregister_netdevice_many - unregister many devices 10111 * @head: list of devices 10112 * 10113 * Note: As most callers use a stack allocated list_head, 10114 * we force a list_del() to make sure stack wont be corrupted later. 10115 */ 10116 void unregister_netdevice_many(struct list_head *head) 10117 { 10118 struct net_device *dev; 10119 10120 if (!list_empty(head)) { 10121 rollback_registered_many(head); 10122 list_for_each_entry(dev, head, unreg_list) 10123 net_set_todo(dev); 10124 list_del(head); 10125 } 10126 } 10127 EXPORT_SYMBOL(unregister_netdevice_many); 10128 10129 /** 10130 * unregister_netdev - remove device from the kernel 10131 * @dev: device 10132 * 10133 * This function shuts down a device interface and removes it 10134 * from the kernel tables. 10135 * 10136 * This is just a wrapper for unregister_netdevice that takes 10137 * the rtnl semaphore. In general you want to use this and not 10138 * unregister_netdevice. 10139 */ 10140 void unregister_netdev(struct net_device *dev) 10141 { 10142 rtnl_lock(); 10143 unregister_netdevice(dev); 10144 rtnl_unlock(); 10145 } 10146 EXPORT_SYMBOL(unregister_netdev); 10147 10148 /** 10149 * dev_change_net_namespace - move device to different nethost namespace 10150 * @dev: device 10151 * @net: network namespace 10152 * @pat: If not NULL name pattern to try if the current device name 10153 * is already taken in the destination network namespace. 10154 * 10155 * This function shuts down a device interface and moves it 10156 * to a new network namespace. On success 0 is returned, on 10157 * a failure a netagive errno code is returned. 10158 * 10159 * Callers must hold the rtnl semaphore. 10160 */ 10161 10162 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat) 10163 { 10164 struct net *net_old = dev_net(dev); 10165 int err, new_nsid, new_ifindex; 10166 10167 ASSERT_RTNL(); 10168 10169 /* Don't allow namespace local devices to be moved. */ 10170 err = -EINVAL; 10171 if (dev->features & NETIF_F_NETNS_LOCAL) 10172 goto out; 10173 10174 /* Ensure the device has been registrered */ 10175 if (dev->reg_state != NETREG_REGISTERED) 10176 goto out; 10177 10178 /* Get out if there is nothing todo */ 10179 err = 0; 10180 if (net_eq(net_old, net)) 10181 goto out; 10182 10183 /* Pick the destination device name, and ensure 10184 * we can use it in the destination network namespace. 10185 */ 10186 err = -EEXIST; 10187 if (__dev_get_by_name(net, dev->name)) { 10188 /* We get here if we can't use the current device name */ 10189 if (!pat) 10190 goto out; 10191 err = dev_get_valid_name(net, dev, pat); 10192 if (err < 0) 10193 goto out; 10194 } 10195 10196 /* 10197 * And now a mini version of register_netdevice unregister_netdevice. 10198 */ 10199 10200 /* If device is running close it first. */ 10201 dev_close(dev); 10202 10203 /* And unlink it from device chain */ 10204 unlist_netdevice(dev); 10205 10206 synchronize_net(); 10207 10208 /* Shutdown queueing discipline. */ 10209 dev_shutdown(dev); 10210 10211 /* Notify protocols, that we are about to destroy 10212 * this device. They should clean all the things. 10213 * 10214 * Note that dev->reg_state stays at NETREG_REGISTERED. 10215 * This is wanted because this way 8021q and macvlan know 10216 * the device is just moving and can keep their slaves up. 10217 */ 10218 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 10219 rcu_barrier(); 10220 10221 new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL); 10222 /* If there is an ifindex conflict assign a new one */ 10223 if (__dev_get_by_index(net, dev->ifindex)) 10224 new_ifindex = dev_new_index(net); 10225 else 10226 new_ifindex = dev->ifindex; 10227 10228 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid, 10229 new_ifindex); 10230 10231 /* 10232 * Flush the unicast and multicast chains 10233 */ 10234 dev_uc_flush(dev); 10235 dev_mc_flush(dev); 10236 10237 /* Send a netdev-removed uevent to the old namespace */ 10238 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); 10239 netdev_adjacent_del_links(dev); 10240 10241 /* Move per-net netdevice notifiers that are following the netdevice */ 10242 move_netdevice_notifiers_dev_net(dev, net); 10243 10244 /* Actually switch the network namespace */ 10245 dev_net_set(dev, net); 10246 dev->ifindex = new_ifindex; 10247 10248 /* Send a netdev-add uevent to the new namespace */ 10249 kobject_uevent(&dev->dev.kobj, KOBJ_ADD); 10250 netdev_adjacent_add_links(dev); 10251 10252 /* Fixup kobjects */ 10253 err = device_rename(&dev->dev, dev->name); 10254 WARN_ON(err); 10255 10256 /* Adapt owner in case owning user namespace of target network 10257 * namespace is different from the original one. 10258 */ 10259 err = netdev_change_owner(dev, net_old, net); 10260 WARN_ON(err); 10261 10262 /* Add the device back in the hashes */ 10263 list_netdevice(dev); 10264 10265 /* Notify protocols, that a new device appeared. */ 10266 call_netdevice_notifiers(NETDEV_REGISTER, dev); 10267 10268 /* 10269 * Prevent userspace races by waiting until the network 10270 * device is fully setup before sending notifications. 10271 */ 10272 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); 10273 10274 synchronize_net(); 10275 err = 0; 10276 out: 10277 return err; 10278 } 10279 EXPORT_SYMBOL_GPL(dev_change_net_namespace); 10280 10281 static int dev_cpu_dead(unsigned int oldcpu) 10282 { 10283 struct sk_buff **list_skb; 10284 struct sk_buff *skb; 10285 unsigned int cpu; 10286 struct softnet_data *sd, *oldsd, *remsd = NULL; 10287 10288 local_irq_disable(); 10289 cpu = smp_processor_id(); 10290 sd = &per_cpu(softnet_data, cpu); 10291 oldsd = &per_cpu(softnet_data, oldcpu); 10292 10293 /* Find end of our completion_queue. */ 10294 list_skb = &sd->completion_queue; 10295 while (*list_skb) 10296 list_skb = &(*list_skb)->next; 10297 /* Append completion queue from offline CPU. */ 10298 *list_skb = oldsd->completion_queue; 10299 oldsd->completion_queue = NULL; 10300 10301 /* Append output queue from offline CPU. */ 10302 if (oldsd->output_queue) { 10303 *sd->output_queue_tailp = oldsd->output_queue; 10304 sd->output_queue_tailp = oldsd->output_queue_tailp; 10305 oldsd->output_queue = NULL; 10306 oldsd->output_queue_tailp = &oldsd->output_queue; 10307 } 10308 /* Append NAPI poll list from offline CPU, with one exception : 10309 * process_backlog() must be called by cpu owning percpu backlog. 10310 * We properly handle process_queue & input_pkt_queue later. 10311 */ 10312 while (!list_empty(&oldsd->poll_list)) { 10313 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, 10314 struct napi_struct, 10315 poll_list); 10316 10317 list_del_init(&napi->poll_list); 10318 if (napi->poll == process_backlog) 10319 napi->state = 0; 10320 else 10321 ____napi_schedule(sd, napi); 10322 } 10323 10324 raise_softirq_irqoff(NET_TX_SOFTIRQ); 10325 local_irq_enable(); 10326 10327 #ifdef CONFIG_RPS 10328 remsd = oldsd->rps_ipi_list; 10329 oldsd->rps_ipi_list = NULL; 10330 #endif 10331 /* send out pending IPI's on offline CPU */ 10332 net_rps_send_ipi(remsd); 10333 10334 /* Process offline CPU's input_pkt_queue */ 10335 while ((skb = __skb_dequeue(&oldsd->process_queue))) { 10336 netif_rx_ni(skb); 10337 input_queue_head_incr(oldsd); 10338 } 10339 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { 10340 netif_rx_ni(skb); 10341 input_queue_head_incr(oldsd); 10342 } 10343 10344 return 0; 10345 } 10346 10347 /** 10348 * netdev_increment_features - increment feature set by one 10349 * @all: current feature set 10350 * @one: new feature set 10351 * @mask: mask feature set 10352 * 10353 * Computes a new feature set after adding a device with feature set 10354 * @one to the master device with current feature set @all. Will not 10355 * enable anything that is off in @mask. Returns the new feature set. 10356 */ 10357 netdev_features_t netdev_increment_features(netdev_features_t all, 10358 netdev_features_t one, netdev_features_t mask) 10359 { 10360 if (mask & NETIF_F_HW_CSUM) 10361 mask |= NETIF_F_CSUM_MASK; 10362 mask |= NETIF_F_VLAN_CHALLENGED; 10363 10364 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask; 10365 all &= one | ~NETIF_F_ALL_FOR_ALL; 10366 10367 /* If one device supports hw checksumming, set for all. */ 10368 if (all & NETIF_F_HW_CSUM) 10369 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM); 10370 10371 return all; 10372 } 10373 EXPORT_SYMBOL(netdev_increment_features); 10374 10375 static struct hlist_head * __net_init netdev_create_hash(void) 10376 { 10377 int i; 10378 struct hlist_head *hash; 10379 10380 hash = kmalloc_array(NETDEV_HASHENTRIES, sizeof(*hash), GFP_KERNEL); 10381 if (hash != NULL) 10382 for (i = 0; i < NETDEV_HASHENTRIES; i++) 10383 INIT_HLIST_HEAD(&hash[i]); 10384 10385 return hash; 10386 } 10387 10388 /* Initialize per network namespace state */ 10389 static int __net_init netdev_init(struct net *net) 10390 { 10391 BUILD_BUG_ON(GRO_HASH_BUCKETS > 10392 8 * sizeof_field(struct napi_struct, gro_bitmask)); 10393 10394 if (net != &init_net) 10395 INIT_LIST_HEAD(&net->dev_base_head); 10396 10397 net->dev_name_head = netdev_create_hash(); 10398 if (net->dev_name_head == NULL) 10399 goto err_name; 10400 10401 net->dev_index_head = netdev_create_hash(); 10402 if (net->dev_index_head == NULL) 10403 goto err_idx; 10404 10405 RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain); 10406 10407 return 0; 10408 10409 err_idx: 10410 kfree(net->dev_name_head); 10411 err_name: 10412 return -ENOMEM; 10413 } 10414 10415 /** 10416 * netdev_drivername - network driver for the device 10417 * @dev: network device 10418 * 10419 * Determine network driver for device. 10420 */ 10421 const char *netdev_drivername(const struct net_device *dev) 10422 { 10423 const struct device_driver *driver; 10424 const struct device *parent; 10425 const char *empty = ""; 10426 10427 parent = dev->dev.parent; 10428 if (!parent) 10429 return empty; 10430 10431 driver = parent->driver; 10432 if (driver && driver->name) 10433 return driver->name; 10434 return empty; 10435 } 10436 10437 static void __netdev_printk(const char *level, const struct net_device *dev, 10438 struct va_format *vaf) 10439 { 10440 if (dev && dev->dev.parent) { 10441 dev_printk_emit(level[1] - '0', 10442 dev->dev.parent, 10443 "%s %s %s%s: %pV", 10444 dev_driver_string(dev->dev.parent), 10445 dev_name(dev->dev.parent), 10446 netdev_name(dev), netdev_reg_state(dev), 10447 vaf); 10448 } else if (dev) { 10449 printk("%s%s%s: %pV", 10450 level, netdev_name(dev), netdev_reg_state(dev), vaf); 10451 } else { 10452 printk("%s(NULL net_device): %pV", level, vaf); 10453 } 10454 } 10455 10456 void netdev_printk(const char *level, const struct net_device *dev, 10457 const char *format, ...) 10458 { 10459 struct va_format vaf; 10460 va_list args; 10461 10462 va_start(args, format); 10463 10464 vaf.fmt = format; 10465 vaf.va = &args; 10466 10467 __netdev_printk(level, dev, &vaf); 10468 10469 va_end(args); 10470 } 10471 EXPORT_SYMBOL(netdev_printk); 10472 10473 #define define_netdev_printk_level(func, level) \ 10474 void func(const struct net_device *dev, const char *fmt, ...) \ 10475 { \ 10476 struct va_format vaf; \ 10477 va_list args; \ 10478 \ 10479 va_start(args, fmt); \ 10480 \ 10481 vaf.fmt = fmt; \ 10482 vaf.va = &args; \ 10483 \ 10484 __netdev_printk(level, dev, &vaf); \ 10485 \ 10486 va_end(args); \ 10487 } \ 10488 EXPORT_SYMBOL(func); 10489 10490 define_netdev_printk_level(netdev_emerg, KERN_EMERG); 10491 define_netdev_printk_level(netdev_alert, KERN_ALERT); 10492 define_netdev_printk_level(netdev_crit, KERN_CRIT); 10493 define_netdev_printk_level(netdev_err, KERN_ERR); 10494 define_netdev_printk_level(netdev_warn, KERN_WARNING); 10495 define_netdev_printk_level(netdev_notice, KERN_NOTICE); 10496 define_netdev_printk_level(netdev_info, KERN_INFO); 10497 10498 static void __net_exit netdev_exit(struct net *net) 10499 { 10500 kfree(net->dev_name_head); 10501 kfree(net->dev_index_head); 10502 if (net != &init_net) 10503 WARN_ON_ONCE(!list_empty(&net->dev_base_head)); 10504 } 10505 10506 static struct pernet_operations __net_initdata netdev_net_ops = { 10507 .init = netdev_init, 10508 .exit = netdev_exit, 10509 }; 10510 10511 static void __net_exit default_device_exit(struct net *net) 10512 { 10513 struct net_device *dev, *aux; 10514 /* 10515 * Push all migratable network devices back to the 10516 * initial network namespace 10517 */ 10518 rtnl_lock(); 10519 for_each_netdev_safe(net, dev, aux) { 10520 int err; 10521 char fb_name[IFNAMSIZ]; 10522 10523 /* Ignore unmoveable devices (i.e. loopback) */ 10524 if (dev->features & NETIF_F_NETNS_LOCAL) 10525 continue; 10526 10527 /* Leave virtual devices for the generic cleanup */ 10528 if (dev->rtnl_link_ops) 10529 continue; 10530 10531 /* Push remaining network devices to init_net */ 10532 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); 10533 if (__dev_get_by_name(&init_net, fb_name)) 10534 snprintf(fb_name, IFNAMSIZ, "dev%%d"); 10535 err = dev_change_net_namespace(dev, &init_net, fb_name); 10536 if (err) { 10537 pr_emerg("%s: failed to move %s to init_net: %d\n", 10538 __func__, dev->name, err); 10539 BUG(); 10540 } 10541 } 10542 rtnl_unlock(); 10543 } 10544 10545 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list) 10546 { 10547 /* Return with the rtnl_lock held when there are no network 10548 * devices unregistering in any network namespace in net_list. 10549 */ 10550 struct net *net; 10551 bool unregistering; 10552 DEFINE_WAIT_FUNC(wait, woken_wake_function); 10553 10554 add_wait_queue(&netdev_unregistering_wq, &wait); 10555 for (;;) { 10556 unregistering = false; 10557 rtnl_lock(); 10558 list_for_each_entry(net, net_list, exit_list) { 10559 if (net->dev_unreg_count > 0) { 10560 unregistering = true; 10561 break; 10562 } 10563 } 10564 if (!unregistering) 10565 break; 10566 __rtnl_unlock(); 10567 10568 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); 10569 } 10570 remove_wait_queue(&netdev_unregistering_wq, &wait); 10571 } 10572 10573 static void __net_exit default_device_exit_batch(struct list_head *net_list) 10574 { 10575 /* At exit all network devices most be removed from a network 10576 * namespace. Do this in the reverse order of registration. 10577 * Do this across as many network namespaces as possible to 10578 * improve batching efficiency. 10579 */ 10580 struct net_device *dev; 10581 struct net *net; 10582 LIST_HEAD(dev_kill_list); 10583 10584 /* To prevent network device cleanup code from dereferencing 10585 * loopback devices or network devices that have been freed 10586 * wait here for all pending unregistrations to complete, 10587 * before unregistring the loopback device and allowing the 10588 * network namespace be freed. 10589 * 10590 * The netdev todo list containing all network devices 10591 * unregistrations that happen in default_device_exit_batch 10592 * will run in the rtnl_unlock() at the end of 10593 * default_device_exit_batch. 10594 */ 10595 rtnl_lock_unregistering(net_list); 10596 list_for_each_entry(net, net_list, exit_list) { 10597 for_each_netdev_reverse(net, dev) { 10598 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) 10599 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); 10600 else 10601 unregister_netdevice_queue(dev, &dev_kill_list); 10602 } 10603 } 10604 unregister_netdevice_many(&dev_kill_list); 10605 rtnl_unlock(); 10606 } 10607 10608 static struct pernet_operations __net_initdata default_device_ops = { 10609 .exit = default_device_exit, 10610 .exit_batch = default_device_exit_batch, 10611 }; 10612 10613 /* 10614 * Initialize the DEV module. At boot time this walks the device list and 10615 * unhooks any devices that fail to initialise (normally hardware not 10616 * present) and leaves us with a valid list of present and active devices. 10617 * 10618 */ 10619 10620 /* 10621 * This is called single threaded during boot, so no need 10622 * to take the rtnl semaphore. 10623 */ 10624 static int __init net_dev_init(void) 10625 { 10626 int i, rc = -ENOMEM; 10627 10628 BUG_ON(!dev_boot_phase); 10629 10630 if (dev_proc_init()) 10631 goto out; 10632 10633 if (netdev_kobject_init()) 10634 goto out; 10635 10636 INIT_LIST_HEAD(&ptype_all); 10637 for (i = 0; i < PTYPE_HASH_SIZE; i++) 10638 INIT_LIST_HEAD(&ptype_base[i]); 10639 10640 INIT_LIST_HEAD(&offload_base); 10641 10642 if (register_pernet_subsys(&netdev_net_ops)) 10643 goto out; 10644 10645 /* 10646 * Initialise the packet receive queues. 10647 */ 10648 10649 for_each_possible_cpu(i) { 10650 struct work_struct *flush = per_cpu_ptr(&flush_works, i); 10651 struct softnet_data *sd = &per_cpu(softnet_data, i); 10652 10653 INIT_WORK(flush, flush_backlog); 10654 10655 skb_queue_head_init(&sd->input_pkt_queue); 10656 skb_queue_head_init(&sd->process_queue); 10657 #ifdef CONFIG_XFRM_OFFLOAD 10658 skb_queue_head_init(&sd->xfrm_backlog); 10659 #endif 10660 INIT_LIST_HEAD(&sd->poll_list); 10661 sd->output_queue_tailp = &sd->output_queue; 10662 #ifdef CONFIG_RPS 10663 sd->csd.func = rps_trigger_softirq; 10664 sd->csd.info = sd; 10665 sd->cpu = i; 10666 #endif 10667 10668 init_gro_hash(&sd->backlog); 10669 sd->backlog.poll = process_backlog; 10670 sd->backlog.weight = weight_p; 10671 } 10672 10673 dev_boot_phase = 0; 10674 10675 /* The loopback device is special if any other network devices 10676 * is present in a network namespace the loopback device must 10677 * be present. Since we now dynamically allocate and free the 10678 * loopback device ensure this invariant is maintained by 10679 * keeping the loopback device as the first device on the 10680 * list of network devices. Ensuring the loopback devices 10681 * is the first device that appears and the last network device 10682 * that disappears. 10683 */ 10684 if (register_pernet_device(&loopback_net_ops)) 10685 goto out; 10686 10687 if (register_pernet_device(&default_device_ops)) 10688 goto out; 10689 10690 open_softirq(NET_TX_SOFTIRQ, net_tx_action); 10691 open_softirq(NET_RX_SOFTIRQ, net_rx_action); 10692 10693 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead", 10694 NULL, dev_cpu_dead); 10695 WARN_ON(rc < 0); 10696 rc = 0; 10697 out: 10698 return rc; 10699 } 10700 10701 subsys_initcall(net_dev_init); 10702