xref: /openbmc/linux/net/core/dev.c (revision 232b0b08)
1 /*
2  *      NET3    Protocol independent device support routines.
3  *
4  *		This program is free software; you can redistribute it and/or
5  *		modify it under the terms of the GNU General Public License
6  *		as published by the Free Software Foundation; either version
7  *		2 of the License, or (at your option) any later version.
8  *
9  *	Derived from the non IP parts of dev.c 1.0.19
10  *              Authors:	Ross Biro
11  *				Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12  *				Mark Evans, <evansmp@uhura.aston.ac.uk>
13  *
14  *	Additional Authors:
15  *		Florian la Roche <rzsfl@rz.uni-sb.de>
16  *		Alan Cox <gw4pts@gw4pts.ampr.org>
17  *		David Hinds <dahinds@users.sourceforge.net>
18  *		Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19  *		Adam Sulmicki <adam@cfar.umd.edu>
20  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
21  *
22  *	Changes:
23  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
24  *                                      to 2 if register_netdev gets called
25  *                                      before net_dev_init & also removed a
26  *                                      few lines of code in the process.
27  *		Alan Cox	:	device private ioctl copies fields back.
28  *		Alan Cox	:	Transmit queue code does relevant
29  *					stunts to keep the queue safe.
30  *		Alan Cox	:	Fixed double lock.
31  *		Alan Cox	:	Fixed promisc NULL pointer trap
32  *		????????	:	Support the full private ioctl range
33  *		Alan Cox	:	Moved ioctl permission check into
34  *					drivers
35  *		Tim Kordas	:	SIOCADDMULTI/SIOCDELMULTI
36  *		Alan Cox	:	100 backlog just doesn't cut it when
37  *					you start doing multicast video 8)
38  *		Alan Cox	:	Rewrote net_bh and list manager.
39  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
40  *		Alan Cox	:	Took out transmit every packet pass
41  *					Saved a few bytes in the ioctl handler
42  *		Alan Cox	:	Network driver sets packet type before
43  *					calling netif_rx. Saves a function
44  *					call a packet.
45  *		Alan Cox	:	Hashed net_bh()
46  *		Richard Kooijman:	Timestamp fixes.
47  *		Alan Cox	:	Wrong field in SIOCGIFDSTADDR
48  *		Alan Cox	:	Device lock protection.
49  *              Alan Cox        :       Fixed nasty side effect of device close
50  *					changes.
51  *		Rudi Cilibrasi	:	Pass the right thing to
52  *					set_mac_address()
53  *		Dave Miller	:	32bit quantity for the device lock to
54  *					make it work out on a Sparc.
55  *		Bjorn Ekwall	:	Added KERNELD hack.
56  *		Alan Cox	:	Cleaned up the backlog initialise.
57  *		Craig Metz	:	SIOCGIFCONF fix if space for under
58  *					1 device.
59  *	    Thomas Bogendoerfer :	Return ENODEV for dev_open, if there
60  *					is no device open function.
61  *		Andi Kleen	:	Fix error reporting for SIOCGIFCONF
62  *	    Michael Chastain	:	Fix signed/unsigned for SIOCGIFCONF
63  *		Cyrus Durgin	:	Cleaned for KMOD
64  *		Adam Sulmicki   :	Bug Fix : Network Device Unload
65  *					A network device unload needs to purge
66  *					the backlog queue.
67  *	Paul Rusty Russell	:	SIOCSIFNAME
68  *              Pekka Riikonen  :	Netdev boot-time settings code
69  *              Andrew Morton   :       Make unregister_netdevice wait
70  *                                      indefinitely on dev->refcnt
71  *              J Hadi Salim    :       - Backlog queue sampling
72  *				        - netif_rx() feedback
73  */
74 
75 #include <linux/uaccess.h>
76 #include <linux/bitops.h>
77 #include <linux/capability.h>
78 #include <linux/cpu.h>
79 #include <linux/types.h>
80 #include <linux/kernel.h>
81 #include <linux/hash.h>
82 #include <linux/slab.h>
83 #include <linux/sched.h>
84 #include <linux/mutex.h>
85 #include <linux/string.h>
86 #include <linux/mm.h>
87 #include <linux/socket.h>
88 #include <linux/sockios.h>
89 #include <linux/errno.h>
90 #include <linux/interrupt.h>
91 #include <linux/if_ether.h>
92 #include <linux/netdevice.h>
93 #include <linux/etherdevice.h>
94 #include <linux/ethtool.h>
95 #include <linux/notifier.h>
96 #include <linux/skbuff.h>
97 #include <linux/bpf.h>
98 #include <net/net_namespace.h>
99 #include <net/sock.h>
100 #include <net/busy_poll.h>
101 #include <linux/rtnetlink.h>
102 #include <linux/stat.h>
103 #include <net/dst.h>
104 #include <net/dst_metadata.h>
105 #include <net/pkt_sched.h>
106 #include <net/checksum.h>
107 #include <net/xfrm.h>
108 #include <linux/highmem.h>
109 #include <linux/init.h>
110 #include <linux/module.h>
111 #include <linux/netpoll.h>
112 #include <linux/rcupdate.h>
113 #include <linux/delay.h>
114 #include <net/iw_handler.h>
115 #include <asm/current.h>
116 #include <linux/audit.h>
117 #include <linux/dmaengine.h>
118 #include <linux/err.h>
119 #include <linux/ctype.h>
120 #include <linux/if_arp.h>
121 #include <linux/if_vlan.h>
122 #include <linux/ip.h>
123 #include <net/ip.h>
124 #include <net/mpls.h>
125 #include <linux/ipv6.h>
126 #include <linux/in.h>
127 #include <linux/jhash.h>
128 #include <linux/random.h>
129 #include <trace/events/napi.h>
130 #include <trace/events/net.h>
131 #include <trace/events/skb.h>
132 #include <linux/pci.h>
133 #include <linux/inetdevice.h>
134 #include <linux/cpu_rmap.h>
135 #include <linux/static_key.h>
136 #include <linux/hashtable.h>
137 #include <linux/vmalloc.h>
138 #include <linux/if_macvlan.h>
139 #include <linux/errqueue.h>
140 #include <linux/hrtimer.h>
141 #include <linux/netfilter_ingress.h>
142 #include <linux/crash_dump.h>
143 
144 #include "net-sysfs.h"
145 
146 /* Instead of increasing this, you should create a hash table. */
147 #define MAX_GRO_SKBS 8
148 
149 /* This should be increased if a protocol with a bigger head is added. */
150 #define GRO_MAX_HEAD (MAX_HEADER + 128)
151 
152 static DEFINE_SPINLOCK(ptype_lock);
153 static DEFINE_SPINLOCK(offload_lock);
154 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
155 struct list_head ptype_all __read_mostly;	/* Taps */
156 static struct list_head offload_base __read_mostly;
157 
158 static int netif_rx_internal(struct sk_buff *skb);
159 static int call_netdevice_notifiers_info(unsigned long val,
160 					 struct net_device *dev,
161 					 struct netdev_notifier_info *info);
162 
163 /*
164  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
165  * semaphore.
166  *
167  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
168  *
169  * Writers must hold the rtnl semaphore while they loop through the
170  * dev_base_head list, and hold dev_base_lock for writing when they do the
171  * actual updates.  This allows pure readers to access the list even
172  * while a writer is preparing to update it.
173  *
174  * To put it another way, dev_base_lock is held for writing only to
175  * protect against pure readers; the rtnl semaphore provides the
176  * protection against other writers.
177  *
178  * See, for example usages, register_netdevice() and
179  * unregister_netdevice(), which must be called with the rtnl
180  * semaphore held.
181  */
182 DEFINE_RWLOCK(dev_base_lock);
183 EXPORT_SYMBOL(dev_base_lock);
184 
185 /* protects napi_hash addition/deletion and napi_gen_id */
186 static DEFINE_SPINLOCK(napi_hash_lock);
187 
188 static unsigned int napi_gen_id = NR_CPUS;
189 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
190 
191 static seqcount_t devnet_rename_seq;
192 
193 static inline void dev_base_seq_inc(struct net *net)
194 {
195 	while (++net->dev_base_seq == 0)
196 		;
197 }
198 
199 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
200 {
201 	unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
202 
203 	return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
204 }
205 
206 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
207 {
208 	return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
209 }
210 
211 static inline void rps_lock(struct softnet_data *sd)
212 {
213 #ifdef CONFIG_RPS
214 	spin_lock(&sd->input_pkt_queue.lock);
215 #endif
216 }
217 
218 static inline void rps_unlock(struct softnet_data *sd)
219 {
220 #ifdef CONFIG_RPS
221 	spin_unlock(&sd->input_pkt_queue.lock);
222 #endif
223 }
224 
225 /* Device list insertion */
226 static void list_netdevice(struct net_device *dev)
227 {
228 	struct net *net = dev_net(dev);
229 
230 	ASSERT_RTNL();
231 
232 	write_lock_bh(&dev_base_lock);
233 	list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
234 	hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
235 	hlist_add_head_rcu(&dev->index_hlist,
236 			   dev_index_hash(net, dev->ifindex));
237 	write_unlock_bh(&dev_base_lock);
238 
239 	dev_base_seq_inc(net);
240 }
241 
242 /* Device list removal
243  * caller must respect a RCU grace period before freeing/reusing dev
244  */
245 static void unlist_netdevice(struct net_device *dev)
246 {
247 	ASSERT_RTNL();
248 
249 	/* Unlink dev from the device chain */
250 	write_lock_bh(&dev_base_lock);
251 	list_del_rcu(&dev->dev_list);
252 	hlist_del_rcu(&dev->name_hlist);
253 	hlist_del_rcu(&dev->index_hlist);
254 	write_unlock_bh(&dev_base_lock);
255 
256 	dev_base_seq_inc(dev_net(dev));
257 }
258 
259 /*
260  *	Our notifier list
261  */
262 
263 static RAW_NOTIFIER_HEAD(netdev_chain);
264 
265 /*
266  *	Device drivers call our routines to queue packets here. We empty the
267  *	queue in the local softnet handler.
268  */
269 
270 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
271 EXPORT_PER_CPU_SYMBOL(softnet_data);
272 
273 #ifdef CONFIG_LOCKDEP
274 /*
275  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
276  * according to dev->type
277  */
278 static const unsigned short netdev_lock_type[] = {
279 	 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
280 	 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
281 	 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
282 	 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
283 	 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
284 	 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
285 	 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
286 	 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
287 	 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
288 	 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
289 	 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
290 	 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
291 	 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
292 	 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
293 	 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
294 
295 static const char *const netdev_lock_name[] = {
296 	"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
297 	"_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
298 	"_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
299 	"_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
300 	"_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
301 	"_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
302 	"_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
303 	"_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
304 	"_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
305 	"_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
306 	"_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
307 	"_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
308 	"_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
309 	"_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
310 	"_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
311 
312 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
313 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
314 
315 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
316 {
317 	int i;
318 
319 	for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
320 		if (netdev_lock_type[i] == dev_type)
321 			return i;
322 	/* the last key is used by default */
323 	return ARRAY_SIZE(netdev_lock_type) - 1;
324 }
325 
326 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
327 						 unsigned short dev_type)
328 {
329 	int i;
330 
331 	i = netdev_lock_pos(dev_type);
332 	lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
333 				   netdev_lock_name[i]);
334 }
335 
336 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
337 {
338 	int i;
339 
340 	i = netdev_lock_pos(dev->type);
341 	lockdep_set_class_and_name(&dev->addr_list_lock,
342 				   &netdev_addr_lock_key[i],
343 				   netdev_lock_name[i]);
344 }
345 #else
346 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
347 						 unsigned short dev_type)
348 {
349 }
350 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
351 {
352 }
353 #endif
354 
355 /*******************************************************************************
356  *
357  *		Protocol management and registration routines
358  *
359  *******************************************************************************/
360 
361 
362 /*
363  *	Add a protocol ID to the list. Now that the input handler is
364  *	smarter we can dispense with all the messy stuff that used to be
365  *	here.
366  *
367  *	BEWARE!!! Protocol handlers, mangling input packets,
368  *	MUST BE last in hash buckets and checking protocol handlers
369  *	MUST start from promiscuous ptype_all chain in net_bh.
370  *	It is true now, do not change it.
371  *	Explanation follows: if protocol handler, mangling packet, will
372  *	be the first on list, it is not able to sense, that packet
373  *	is cloned and should be copied-on-write, so that it will
374  *	change it and subsequent readers will get broken packet.
375  *							--ANK (980803)
376  */
377 
378 static inline struct list_head *ptype_head(const struct packet_type *pt)
379 {
380 	if (pt->type == htons(ETH_P_ALL))
381 		return pt->dev ? &pt->dev->ptype_all : &ptype_all;
382 	else
383 		return pt->dev ? &pt->dev->ptype_specific :
384 				 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
385 }
386 
387 /**
388  *	dev_add_pack - add packet handler
389  *	@pt: packet type declaration
390  *
391  *	Add a protocol handler to the networking stack. The passed &packet_type
392  *	is linked into kernel lists and may not be freed until it has been
393  *	removed from the kernel lists.
394  *
395  *	This call does not sleep therefore it can not
396  *	guarantee all CPU's that are in middle of receiving packets
397  *	will see the new packet type (until the next received packet).
398  */
399 
400 void dev_add_pack(struct packet_type *pt)
401 {
402 	struct list_head *head = ptype_head(pt);
403 
404 	spin_lock(&ptype_lock);
405 	list_add_rcu(&pt->list, head);
406 	spin_unlock(&ptype_lock);
407 }
408 EXPORT_SYMBOL(dev_add_pack);
409 
410 /**
411  *	__dev_remove_pack	 - remove packet handler
412  *	@pt: packet type declaration
413  *
414  *	Remove a protocol handler that was previously added to the kernel
415  *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
416  *	from the kernel lists and can be freed or reused once this function
417  *	returns.
418  *
419  *      The packet type might still be in use by receivers
420  *	and must not be freed until after all the CPU's have gone
421  *	through a quiescent state.
422  */
423 void __dev_remove_pack(struct packet_type *pt)
424 {
425 	struct list_head *head = ptype_head(pt);
426 	struct packet_type *pt1;
427 
428 	spin_lock(&ptype_lock);
429 
430 	list_for_each_entry(pt1, head, list) {
431 		if (pt == pt1) {
432 			list_del_rcu(&pt->list);
433 			goto out;
434 		}
435 	}
436 
437 	pr_warn("dev_remove_pack: %p not found\n", pt);
438 out:
439 	spin_unlock(&ptype_lock);
440 }
441 EXPORT_SYMBOL(__dev_remove_pack);
442 
443 /**
444  *	dev_remove_pack	 - remove packet handler
445  *	@pt: packet type declaration
446  *
447  *	Remove a protocol handler that was previously added to the kernel
448  *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
449  *	from the kernel lists and can be freed or reused once this function
450  *	returns.
451  *
452  *	This call sleeps to guarantee that no CPU is looking at the packet
453  *	type after return.
454  */
455 void dev_remove_pack(struct packet_type *pt)
456 {
457 	__dev_remove_pack(pt);
458 
459 	synchronize_net();
460 }
461 EXPORT_SYMBOL(dev_remove_pack);
462 
463 
464 /**
465  *	dev_add_offload - register offload handlers
466  *	@po: protocol offload declaration
467  *
468  *	Add protocol offload handlers to the networking stack. The passed
469  *	&proto_offload is linked into kernel lists and may not be freed until
470  *	it has been removed from the kernel lists.
471  *
472  *	This call does not sleep therefore it can not
473  *	guarantee all CPU's that are in middle of receiving packets
474  *	will see the new offload handlers (until the next received packet).
475  */
476 void dev_add_offload(struct packet_offload *po)
477 {
478 	struct packet_offload *elem;
479 
480 	spin_lock(&offload_lock);
481 	list_for_each_entry(elem, &offload_base, list) {
482 		if (po->priority < elem->priority)
483 			break;
484 	}
485 	list_add_rcu(&po->list, elem->list.prev);
486 	spin_unlock(&offload_lock);
487 }
488 EXPORT_SYMBOL(dev_add_offload);
489 
490 /**
491  *	__dev_remove_offload	 - remove offload handler
492  *	@po: packet offload declaration
493  *
494  *	Remove a protocol offload handler that was previously added to the
495  *	kernel offload handlers by dev_add_offload(). The passed &offload_type
496  *	is removed from the kernel lists and can be freed or reused once this
497  *	function returns.
498  *
499  *      The packet type might still be in use by receivers
500  *	and must not be freed until after all the CPU's have gone
501  *	through a quiescent state.
502  */
503 static void __dev_remove_offload(struct packet_offload *po)
504 {
505 	struct list_head *head = &offload_base;
506 	struct packet_offload *po1;
507 
508 	spin_lock(&offload_lock);
509 
510 	list_for_each_entry(po1, head, list) {
511 		if (po == po1) {
512 			list_del_rcu(&po->list);
513 			goto out;
514 		}
515 	}
516 
517 	pr_warn("dev_remove_offload: %p not found\n", po);
518 out:
519 	spin_unlock(&offload_lock);
520 }
521 
522 /**
523  *	dev_remove_offload	 - remove packet offload handler
524  *	@po: packet offload declaration
525  *
526  *	Remove a packet offload handler that was previously added to the kernel
527  *	offload handlers by dev_add_offload(). The passed &offload_type is
528  *	removed from the kernel lists and can be freed or reused once this
529  *	function returns.
530  *
531  *	This call sleeps to guarantee that no CPU is looking at the packet
532  *	type after return.
533  */
534 void dev_remove_offload(struct packet_offload *po)
535 {
536 	__dev_remove_offload(po);
537 
538 	synchronize_net();
539 }
540 EXPORT_SYMBOL(dev_remove_offload);
541 
542 /******************************************************************************
543  *
544  *		      Device Boot-time Settings Routines
545  *
546  ******************************************************************************/
547 
548 /* Boot time configuration table */
549 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
550 
551 /**
552  *	netdev_boot_setup_add	- add new setup entry
553  *	@name: name of the device
554  *	@map: configured settings for the device
555  *
556  *	Adds new setup entry to the dev_boot_setup list.  The function
557  *	returns 0 on error and 1 on success.  This is a generic routine to
558  *	all netdevices.
559  */
560 static int netdev_boot_setup_add(char *name, struct ifmap *map)
561 {
562 	struct netdev_boot_setup *s;
563 	int i;
564 
565 	s = dev_boot_setup;
566 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
567 		if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
568 			memset(s[i].name, 0, sizeof(s[i].name));
569 			strlcpy(s[i].name, name, IFNAMSIZ);
570 			memcpy(&s[i].map, map, sizeof(s[i].map));
571 			break;
572 		}
573 	}
574 
575 	return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
576 }
577 
578 /**
579  * netdev_boot_setup_check	- check boot time settings
580  * @dev: the netdevice
581  *
582  * Check boot time settings for the device.
583  * The found settings are set for the device to be used
584  * later in the device probing.
585  * Returns 0 if no settings found, 1 if they are.
586  */
587 int netdev_boot_setup_check(struct net_device *dev)
588 {
589 	struct netdev_boot_setup *s = dev_boot_setup;
590 	int i;
591 
592 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
593 		if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
594 		    !strcmp(dev->name, s[i].name)) {
595 			dev->irq = s[i].map.irq;
596 			dev->base_addr = s[i].map.base_addr;
597 			dev->mem_start = s[i].map.mem_start;
598 			dev->mem_end = s[i].map.mem_end;
599 			return 1;
600 		}
601 	}
602 	return 0;
603 }
604 EXPORT_SYMBOL(netdev_boot_setup_check);
605 
606 
607 /**
608  * netdev_boot_base	- get address from boot time settings
609  * @prefix: prefix for network device
610  * @unit: id for network device
611  *
612  * Check boot time settings for the base address of device.
613  * The found settings are set for the device to be used
614  * later in the device probing.
615  * Returns 0 if no settings found.
616  */
617 unsigned long netdev_boot_base(const char *prefix, int unit)
618 {
619 	const struct netdev_boot_setup *s = dev_boot_setup;
620 	char name[IFNAMSIZ];
621 	int i;
622 
623 	sprintf(name, "%s%d", prefix, unit);
624 
625 	/*
626 	 * If device already registered then return base of 1
627 	 * to indicate not to probe for this interface
628 	 */
629 	if (__dev_get_by_name(&init_net, name))
630 		return 1;
631 
632 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
633 		if (!strcmp(name, s[i].name))
634 			return s[i].map.base_addr;
635 	return 0;
636 }
637 
638 /*
639  * Saves at boot time configured settings for any netdevice.
640  */
641 int __init netdev_boot_setup(char *str)
642 {
643 	int ints[5];
644 	struct ifmap map;
645 
646 	str = get_options(str, ARRAY_SIZE(ints), ints);
647 	if (!str || !*str)
648 		return 0;
649 
650 	/* Save settings */
651 	memset(&map, 0, sizeof(map));
652 	if (ints[0] > 0)
653 		map.irq = ints[1];
654 	if (ints[0] > 1)
655 		map.base_addr = ints[2];
656 	if (ints[0] > 2)
657 		map.mem_start = ints[3];
658 	if (ints[0] > 3)
659 		map.mem_end = ints[4];
660 
661 	/* Add new entry to the list */
662 	return netdev_boot_setup_add(str, &map);
663 }
664 
665 __setup("netdev=", netdev_boot_setup);
666 
667 /*******************************************************************************
668  *
669  *			    Device Interface Subroutines
670  *
671  *******************************************************************************/
672 
673 /**
674  *	dev_get_iflink	- get 'iflink' value of a interface
675  *	@dev: targeted interface
676  *
677  *	Indicates the ifindex the interface is linked to.
678  *	Physical interfaces have the same 'ifindex' and 'iflink' values.
679  */
680 
681 int dev_get_iflink(const struct net_device *dev)
682 {
683 	if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
684 		return dev->netdev_ops->ndo_get_iflink(dev);
685 
686 	return dev->ifindex;
687 }
688 EXPORT_SYMBOL(dev_get_iflink);
689 
690 /**
691  *	dev_fill_metadata_dst - Retrieve tunnel egress information.
692  *	@dev: targeted interface
693  *	@skb: The packet.
694  *
695  *	For better visibility of tunnel traffic OVS needs to retrieve
696  *	egress tunnel information for a packet. Following API allows
697  *	user to get this info.
698  */
699 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
700 {
701 	struct ip_tunnel_info *info;
702 
703 	if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
704 		return -EINVAL;
705 
706 	info = skb_tunnel_info_unclone(skb);
707 	if (!info)
708 		return -ENOMEM;
709 	if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
710 		return -EINVAL;
711 
712 	return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
713 }
714 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
715 
716 /**
717  *	__dev_get_by_name	- find a device by its name
718  *	@net: the applicable net namespace
719  *	@name: name to find
720  *
721  *	Find an interface by name. Must be called under RTNL semaphore
722  *	or @dev_base_lock. If the name is found a pointer to the device
723  *	is returned. If the name is not found then %NULL is returned. The
724  *	reference counters are not incremented so the caller must be
725  *	careful with locks.
726  */
727 
728 struct net_device *__dev_get_by_name(struct net *net, const char *name)
729 {
730 	struct net_device *dev;
731 	struct hlist_head *head = dev_name_hash(net, name);
732 
733 	hlist_for_each_entry(dev, head, name_hlist)
734 		if (!strncmp(dev->name, name, IFNAMSIZ))
735 			return dev;
736 
737 	return NULL;
738 }
739 EXPORT_SYMBOL(__dev_get_by_name);
740 
741 /**
742  * dev_get_by_name_rcu	- find a device by its name
743  * @net: the applicable net namespace
744  * @name: name to find
745  *
746  * Find an interface by name.
747  * If the name is found a pointer to the device is returned.
748  * If the name is not found then %NULL is returned.
749  * The reference counters are not incremented so the caller must be
750  * careful with locks. The caller must hold RCU lock.
751  */
752 
753 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
754 {
755 	struct net_device *dev;
756 	struct hlist_head *head = dev_name_hash(net, name);
757 
758 	hlist_for_each_entry_rcu(dev, head, name_hlist)
759 		if (!strncmp(dev->name, name, IFNAMSIZ))
760 			return dev;
761 
762 	return NULL;
763 }
764 EXPORT_SYMBOL(dev_get_by_name_rcu);
765 
766 /**
767  *	dev_get_by_name		- find a device by its name
768  *	@net: the applicable net namespace
769  *	@name: name to find
770  *
771  *	Find an interface by name. This can be called from any
772  *	context and does its own locking. The returned handle has
773  *	the usage count incremented and the caller must use dev_put() to
774  *	release it when it is no longer needed. %NULL is returned if no
775  *	matching device is found.
776  */
777 
778 struct net_device *dev_get_by_name(struct net *net, const char *name)
779 {
780 	struct net_device *dev;
781 
782 	rcu_read_lock();
783 	dev = dev_get_by_name_rcu(net, name);
784 	if (dev)
785 		dev_hold(dev);
786 	rcu_read_unlock();
787 	return dev;
788 }
789 EXPORT_SYMBOL(dev_get_by_name);
790 
791 /**
792  *	__dev_get_by_index - find a device by its ifindex
793  *	@net: the applicable net namespace
794  *	@ifindex: index of device
795  *
796  *	Search for an interface by index. Returns %NULL if the device
797  *	is not found or a pointer to the device. The device has not
798  *	had its reference counter increased so the caller must be careful
799  *	about locking. The caller must hold either the RTNL semaphore
800  *	or @dev_base_lock.
801  */
802 
803 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
804 {
805 	struct net_device *dev;
806 	struct hlist_head *head = dev_index_hash(net, ifindex);
807 
808 	hlist_for_each_entry(dev, head, index_hlist)
809 		if (dev->ifindex == ifindex)
810 			return dev;
811 
812 	return NULL;
813 }
814 EXPORT_SYMBOL(__dev_get_by_index);
815 
816 /**
817  *	dev_get_by_index_rcu - find a device by its ifindex
818  *	@net: the applicable net namespace
819  *	@ifindex: index of device
820  *
821  *	Search for an interface by index. Returns %NULL if the device
822  *	is not found or a pointer to the device. The device has not
823  *	had its reference counter increased so the caller must be careful
824  *	about locking. The caller must hold RCU lock.
825  */
826 
827 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
828 {
829 	struct net_device *dev;
830 	struct hlist_head *head = dev_index_hash(net, ifindex);
831 
832 	hlist_for_each_entry_rcu(dev, head, index_hlist)
833 		if (dev->ifindex == ifindex)
834 			return dev;
835 
836 	return NULL;
837 }
838 EXPORT_SYMBOL(dev_get_by_index_rcu);
839 
840 
841 /**
842  *	dev_get_by_index - find a device by its ifindex
843  *	@net: the applicable net namespace
844  *	@ifindex: index of device
845  *
846  *	Search for an interface by index. Returns NULL if the device
847  *	is not found or a pointer to the device. The device returned has
848  *	had a reference added and the pointer is safe until the user calls
849  *	dev_put to indicate they have finished with it.
850  */
851 
852 struct net_device *dev_get_by_index(struct net *net, int ifindex)
853 {
854 	struct net_device *dev;
855 
856 	rcu_read_lock();
857 	dev = dev_get_by_index_rcu(net, ifindex);
858 	if (dev)
859 		dev_hold(dev);
860 	rcu_read_unlock();
861 	return dev;
862 }
863 EXPORT_SYMBOL(dev_get_by_index);
864 
865 /**
866  *	netdev_get_name - get a netdevice name, knowing its ifindex.
867  *	@net: network namespace
868  *	@name: a pointer to the buffer where the name will be stored.
869  *	@ifindex: the ifindex of the interface to get the name from.
870  *
871  *	The use of raw_seqcount_begin() and cond_resched() before
872  *	retrying is required as we want to give the writers a chance
873  *	to complete when CONFIG_PREEMPT is not set.
874  */
875 int netdev_get_name(struct net *net, char *name, int ifindex)
876 {
877 	struct net_device *dev;
878 	unsigned int seq;
879 
880 retry:
881 	seq = raw_seqcount_begin(&devnet_rename_seq);
882 	rcu_read_lock();
883 	dev = dev_get_by_index_rcu(net, ifindex);
884 	if (!dev) {
885 		rcu_read_unlock();
886 		return -ENODEV;
887 	}
888 
889 	strcpy(name, dev->name);
890 	rcu_read_unlock();
891 	if (read_seqcount_retry(&devnet_rename_seq, seq)) {
892 		cond_resched();
893 		goto retry;
894 	}
895 
896 	return 0;
897 }
898 
899 /**
900  *	dev_getbyhwaddr_rcu - find a device by its hardware address
901  *	@net: the applicable net namespace
902  *	@type: media type of device
903  *	@ha: hardware address
904  *
905  *	Search for an interface by MAC address. Returns NULL if the device
906  *	is not found or a pointer to the device.
907  *	The caller must hold RCU or RTNL.
908  *	The returned device has not had its ref count increased
909  *	and the caller must therefore be careful about locking
910  *
911  */
912 
913 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
914 				       const char *ha)
915 {
916 	struct net_device *dev;
917 
918 	for_each_netdev_rcu(net, dev)
919 		if (dev->type == type &&
920 		    !memcmp(dev->dev_addr, ha, dev->addr_len))
921 			return dev;
922 
923 	return NULL;
924 }
925 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
926 
927 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
928 {
929 	struct net_device *dev;
930 
931 	ASSERT_RTNL();
932 	for_each_netdev(net, dev)
933 		if (dev->type == type)
934 			return dev;
935 
936 	return NULL;
937 }
938 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
939 
940 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
941 {
942 	struct net_device *dev, *ret = NULL;
943 
944 	rcu_read_lock();
945 	for_each_netdev_rcu(net, dev)
946 		if (dev->type == type) {
947 			dev_hold(dev);
948 			ret = dev;
949 			break;
950 		}
951 	rcu_read_unlock();
952 	return ret;
953 }
954 EXPORT_SYMBOL(dev_getfirstbyhwtype);
955 
956 /**
957  *	__dev_get_by_flags - find any device with given flags
958  *	@net: the applicable net namespace
959  *	@if_flags: IFF_* values
960  *	@mask: bitmask of bits in if_flags to check
961  *
962  *	Search for any interface with the given flags. Returns NULL if a device
963  *	is not found or a pointer to the device. Must be called inside
964  *	rtnl_lock(), and result refcount is unchanged.
965  */
966 
967 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
968 				      unsigned short mask)
969 {
970 	struct net_device *dev, *ret;
971 
972 	ASSERT_RTNL();
973 
974 	ret = NULL;
975 	for_each_netdev(net, dev) {
976 		if (((dev->flags ^ if_flags) & mask) == 0) {
977 			ret = dev;
978 			break;
979 		}
980 	}
981 	return ret;
982 }
983 EXPORT_SYMBOL(__dev_get_by_flags);
984 
985 /**
986  *	dev_valid_name - check if name is okay for network device
987  *	@name: name string
988  *
989  *	Network device names need to be valid file names to
990  *	to allow sysfs to work.  We also disallow any kind of
991  *	whitespace.
992  */
993 bool dev_valid_name(const char *name)
994 {
995 	if (*name == '\0')
996 		return false;
997 	if (strlen(name) >= IFNAMSIZ)
998 		return false;
999 	if (!strcmp(name, ".") || !strcmp(name, ".."))
1000 		return false;
1001 
1002 	while (*name) {
1003 		if (*name == '/' || *name == ':' || isspace(*name))
1004 			return false;
1005 		name++;
1006 	}
1007 	return true;
1008 }
1009 EXPORT_SYMBOL(dev_valid_name);
1010 
1011 /**
1012  *	__dev_alloc_name - allocate a name for a device
1013  *	@net: network namespace to allocate the device name in
1014  *	@name: name format string
1015  *	@buf:  scratch buffer and result name string
1016  *
1017  *	Passed a format string - eg "lt%d" it will try and find a suitable
1018  *	id. It scans list of devices to build up a free map, then chooses
1019  *	the first empty slot. The caller must hold the dev_base or rtnl lock
1020  *	while allocating the name and adding the device in order to avoid
1021  *	duplicates.
1022  *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1023  *	Returns the number of the unit assigned or a negative errno code.
1024  */
1025 
1026 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1027 {
1028 	int i = 0;
1029 	const char *p;
1030 	const int max_netdevices = 8*PAGE_SIZE;
1031 	unsigned long *inuse;
1032 	struct net_device *d;
1033 
1034 	p = strnchr(name, IFNAMSIZ-1, '%');
1035 	if (p) {
1036 		/*
1037 		 * Verify the string as this thing may have come from
1038 		 * the user.  There must be either one "%d" and no other "%"
1039 		 * characters.
1040 		 */
1041 		if (p[1] != 'd' || strchr(p + 2, '%'))
1042 			return -EINVAL;
1043 
1044 		/* Use one page as a bit array of possible slots */
1045 		inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1046 		if (!inuse)
1047 			return -ENOMEM;
1048 
1049 		for_each_netdev(net, d) {
1050 			if (!sscanf(d->name, name, &i))
1051 				continue;
1052 			if (i < 0 || i >= max_netdevices)
1053 				continue;
1054 
1055 			/*  avoid cases where sscanf is not exact inverse of printf */
1056 			snprintf(buf, IFNAMSIZ, name, i);
1057 			if (!strncmp(buf, d->name, IFNAMSIZ))
1058 				set_bit(i, inuse);
1059 		}
1060 
1061 		i = find_first_zero_bit(inuse, max_netdevices);
1062 		free_page((unsigned long) inuse);
1063 	}
1064 
1065 	if (buf != name)
1066 		snprintf(buf, IFNAMSIZ, name, i);
1067 	if (!__dev_get_by_name(net, buf))
1068 		return i;
1069 
1070 	/* It is possible to run out of possible slots
1071 	 * when the name is long and there isn't enough space left
1072 	 * for the digits, or if all bits are used.
1073 	 */
1074 	return -ENFILE;
1075 }
1076 
1077 /**
1078  *	dev_alloc_name - allocate a name for a device
1079  *	@dev: device
1080  *	@name: name format string
1081  *
1082  *	Passed a format string - eg "lt%d" it will try and find a suitable
1083  *	id. It scans list of devices to build up a free map, then chooses
1084  *	the first empty slot. The caller must hold the dev_base or rtnl lock
1085  *	while allocating the name and adding the device in order to avoid
1086  *	duplicates.
1087  *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1088  *	Returns the number of the unit assigned or a negative errno code.
1089  */
1090 
1091 int dev_alloc_name(struct net_device *dev, const char *name)
1092 {
1093 	char buf[IFNAMSIZ];
1094 	struct net *net;
1095 	int ret;
1096 
1097 	BUG_ON(!dev_net(dev));
1098 	net = dev_net(dev);
1099 	ret = __dev_alloc_name(net, name, buf);
1100 	if (ret >= 0)
1101 		strlcpy(dev->name, buf, IFNAMSIZ);
1102 	return ret;
1103 }
1104 EXPORT_SYMBOL(dev_alloc_name);
1105 
1106 static int dev_alloc_name_ns(struct net *net,
1107 			     struct net_device *dev,
1108 			     const char *name)
1109 {
1110 	char buf[IFNAMSIZ];
1111 	int ret;
1112 
1113 	ret = __dev_alloc_name(net, name, buf);
1114 	if (ret >= 0)
1115 		strlcpy(dev->name, buf, IFNAMSIZ);
1116 	return ret;
1117 }
1118 
1119 static int dev_get_valid_name(struct net *net,
1120 			      struct net_device *dev,
1121 			      const char *name)
1122 {
1123 	BUG_ON(!net);
1124 
1125 	if (!dev_valid_name(name))
1126 		return -EINVAL;
1127 
1128 	if (strchr(name, '%'))
1129 		return dev_alloc_name_ns(net, dev, name);
1130 	else if (__dev_get_by_name(net, name))
1131 		return -EEXIST;
1132 	else if (dev->name != name)
1133 		strlcpy(dev->name, name, IFNAMSIZ);
1134 
1135 	return 0;
1136 }
1137 
1138 /**
1139  *	dev_change_name - change name of a device
1140  *	@dev: device
1141  *	@newname: name (or format string) must be at least IFNAMSIZ
1142  *
1143  *	Change name of a device, can pass format strings "eth%d".
1144  *	for wildcarding.
1145  */
1146 int dev_change_name(struct net_device *dev, const char *newname)
1147 {
1148 	unsigned char old_assign_type;
1149 	char oldname[IFNAMSIZ];
1150 	int err = 0;
1151 	int ret;
1152 	struct net *net;
1153 
1154 	ASSERT_RTNL();
1155 	BUG_ON(!dev_net(dev));
1156 
1157 	net = dev_net(dev);
1158 	if (dev->flags & IFF_UP)
1159 		return -EBUSY;
1160 
1161 	write_seqcount_begin(&devnet_rename_seq);
1162 
1163 	if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1164 		write_seqcount_end(&devnet_rename_seq);
1165 		return 0;
1166 	}
1167 
1168 	memcpy(oldname, dev->name, IFNAMSIZ);
1169 
1170 	err = dev_get_valid_name(net, dev, newname);
1171 	if (err < 0) {
1172 		write_seqcount_end(&devnet_rename_seq);
1173 		return err;
1174 	}
1175 
1176 	if (oldname[0] && !strchr(oldname, '%'))
1177 		netdev_info(dev, "renamed from %s\n", oldname);
1178 
1179 	old_assign_type = dev->name_assign_type;
1180 	dev->name_assign_type = NET_NAME_RENAMED;
1181 
1182 rollback:
1183 	ret = device_rename(&dev->dev, dev->name);
1184 	if (ret) {
1185 		memcpy(dev->name, oldname, IFNAMSIZ);
1186 		dev->name_assign_type = old_assign_type;
1187 		write_seqcount_end(&devnet_rename_seq);
1188 		return ret;
1189 	}
1190 
1191 	write_seqcount_end(&devnet_rename_seq);
1192 
1193 	netdev_adjacent_rename_links(dev, oldname);
1194 
1195 	write_lock_bh(&dev_base_lock);
1196 	hlist_del_rcu(&dev->name_hlist);
1197 	write_unlock_bh(&dev_base_lock);
1198 
1199 	synchronize_rcu();
1200 
1201 	write_lock_bh(&dev_base_lock);
1202 	hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1203 	write_unlock_bh(&dev_base_lock);
1204 
1205 	ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1206 	ret = notifier_to_errno(ret);
1207 
1208 	if (ret) {
1209 		/* err >= 0 after dev_alloc_name() or stores the first errno */
1210 		if (err >= 0) {
1211 			err = ret;
1212 			write_seqcount_begin(&devnet_rename_seq);
1213 			memcpy(dev->name, oldname, IFNAMSIZ);
1214 			memcpy(oldname, newname, IFNAMSIZ);
1215 			dev->name_assign_type = old_assign_type;
1216 			old_assign_type = NET_NAME_RENAMED;
1217 			goto rollback;
1218 		} else {
1219 			pr_err("%s: name change rollback failed: %d\n",
1220 			       dev->name, ret);
1221 		}
1222 	}
1223 
1224 	return err;
1225 }
1226 
1227 /**
1228  *	dev_set_alias - change ifalias of a device
1229  *	@dev: device
1230  *	@alias: name up to IFALIASZ
1231  *	@len: limit of bytes to copy from info
1232  *
1233  *	Set ifalias for a device,
1234  */
1235 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1236 {
1237 	char *new_ifalias;
1238 
1239 	ASSERT_RTNL();
1240 
1241 	if (len >= IFALIASZ)
1242 		return -EINVAL;
1243 
1244 	if (!len) {
1245 		kfree(dev->ifalias);
1246 		dev->ifalias = NULL;
1247 		return 0;
1248 	}
1249 
1250 	new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1251 	if (!new_ifalias)
1252 		return -ENOMEM;
1253 	dev->ifalias = new_ifalias;
1254 
1255 	strlcpy(dev->ifalias, alias, len+1);
1256 	return len;
1257 }
1258 
1259 
1260 /**
1261  *	netdev_features_change - device changes features
1262  *	@dev: device to cause notification
1263  *
1264  *	Called to indicate a device has changed features.
1265  */
1266 void netdev_features_change(struct net_device *dev)
1267 {
1268 	call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1269 }
1270 EXPORT_SYMBOL(netdev_features_change);
1271 
1272 /**
1273  *	netdev_state_change - device changes state
1274  *	@dev: device to cause notification
1275  *
1276  *	Called to indicate a device has changed state. This function calls
1277  *	the notifier chains for netdev_chain and sends a NEWLINK message
1278  *	to the routing socket.
1279  */
1280 void netdev_state_change(struct net_device *dev)
1281 {
1282 	if (dev->flags & IFF_UP) {
1283 		struct netdev_notifier_change_info change_info;
1284 
1285 		change_info.flags_changed = 0;
1286 		call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1287 					      &change_info.info);
1288 		rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1289 	}
1290 }
1291 EXPORT_SYMBOL(netdev_state_change);
1292 
1293 /**
1294  * netdev_notify_peers - notify network peers about existence of @dev
1295  * @dev: network device
1296  *
1297  * Generate traffic such that interested network peers are aware of
1298  * @dev, such as by generating a gratuitous ARP. This may be used when
1299  * a device wants to inform the rest of the network about some sort of
1300  * reconfiguration such as a failover event or virtual machine
1301  * migration.
1302  */
1303 void netdev_notify_peers(struct net_device *dev)
1304 {
1305 	rtnl_lock();
1306 	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1307 	call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1308 	rtnl_unlock();
1309 }
1310 EXPORT_SYMBOL(netdev_notify_peers);
1311 
1312 static int __dev_open(struct net_device *dev)
1313 {
1314 	const struct net_device_ops *ops = dev->netdev_ops;
1315 	int ret;
1316 
1317 	ASSERT_RTNL();
1318 
1319 	if (!netif_device_present(dev))
1320 		return -ENODEV;
1321 
1322 	/* Block netpoll from trying to do any rx path servicing.
1323 	 * If we don't do this there is a chance ndo_poll_controller
1324 	 * or ndo_poll may be running while we open the device
1325 	 */
1326 	netpoll_poll_disable(dev);
1327 
1328 	ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1329 	ret = notifier_to_errno(ret);
1330 	if (ret)
1331 		return ret;
1332 
1333 	set_bit(__LINK_STATE_START, &dev->state);
1334 
1335 	if (ops->ndo_validate_addr)
1336 		ret = ops->ndo_validate_addr(dev);
1337 
1338 	if (!ret && ops->ndo_open)
1339 		ret = ops->ndo_open(dev);
1340 
1341 	netpoll_poll_enable(dev);
1342 
1343 	if (ret)
1344 		clear_bit(__LINK_STATE_START, &dev->state);
1345 	else {
1346 		dev->flags |= IFF_UP;
1347 		dev_set_rx_mode(dev);
1348 		dev_activate(dev);
1349 		add_device_randomness(dev->dev_addr, dev->addr_len);
1350 	}
1351 
1352 	return ret;
1353 }
1354 
1355 /**
1356  *	dev_open	- prepare an interface for use.
1357  *	@dev:	device to open
1358  *
1359  *	Takes a device from down to up state. The device's private open
1360  *	function is invoked and then the multicast lists are loaded. Finally
1361  *	the device is moved into the up state and a %NETDEV_UP message is
1362  *	sent to the netdev notifier chain.
1363  *
1364  *	Calling this function on an active interface is a nop. On a failure
1365  *	a negative errno code is returned.
1366  */
1367 int dev_open(struct net_device *dev)
1368 {
1369 	int ret;
1370 
1371 	if (dev->flags & IFF_UP)
1372 		return 0;
1373 
1374 	ret = __dev_open(dev);
1375 	if (ret < 0)
1376 		return ret;
1377 
1378 	rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1379 	call_netdevice_notifiers(NETDEV_UP, dev);
1380 
1381 	return ret;
1382 }
1383 EXPORT_SYMBOL(dev_open);
1384 
1385 static int __dev_close_many(struct list_head *head)
1386 {
1387 	struct net_device *dev;
1388 
1389 	ASSERT_RTNL();
1390 	might_sleep();
1391 
1392 	list_for_each_entry(dev, head, close_list) {
1393 		/* Temporarily disable netpoll until the interface is down */
1394 		netpoll_poll_disable(dev);
1395 
1396 		call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1397 
1398 		clear_bit(__LINK_STATE_START, &dev->state);
1399 
1400 		/* Synchronize to scheduled poll. We cannot touch poll list, it
1401 		 * can be even on different cpu. So just clear netif_running().
1402 		 *
1403 		 * dev->stop() will invoke napi_disable() on all of it's
1404 		 * napi_struct instances on this device.
1405 		 */
1406 		smp_mb__after_atomic(); /* Commit netif_running(). */
1407 	}
1408 
1409 	dev_deactivate_many(head);
1410 
1411 	list_for_each_entry(dev, head, close_list) {
1412 		const struct net_device_ops *ops = dev->netdev_ops;
1413 
1414 		/*
1415 		 *	Call the device specific close. This cannot fail.
1416 		 *	Only if device is UP
1417 		 *
1418 		 *	We allow it to be called even after a DETACH hot-plug
1419 		 *	event.
1420 		 */
1421 		if (ops->ndo_stop)
1422 			ops->ndo_stop(dev);
1423 
1424 		dev->flags &= ~IFF_UP;
1425 		netpoll_poll_enable(dev);
1426 	}
1427 
1428 	return 0;
1429 }
1430 
1431 static int __dev_close(struct net_device *dev)
1432 {
1433 	int retval;
1434 	LIST_HEAD(single);
1435 
1436 	list_add(&dev->close_list, &single);
1437 	retval = __dev_close_many(&single);
1438 	list_del(&single);
1439 
1440 	return retval;
1441 }
1442 
1443 int dev_close_many(struct list_head *head, bool unlink)
1444 {
1445 	struct net_device *dev, *tmp;
1446 
1447 	/* Remove the devices that don't need to be closed */
1448 	list_for_each_entry_safe(dev, tmp, head, close_list)
1449 		if (!(dev->flags & IFF_UP))
1450 			list_del_init(&dev->close_list);
1451 
1452 	__dev_close_many(head);
1453 
1454 	list_for_each_entry_safe(dev, tmp, head, close_list) {
1455 		rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1456 		call_netdevice_notifiers(NETDEV_DOWN, dev);
1457 		if (unlink)
1458 			list_del_init(&dev->close_list);
1459 	}
1460 
1461 	return 0;
1462 }
1463 EXPORT_SYMBOL(dev_close_many);
1464 
1465 /**
1466  *	dev_close - shutdown an interface.
1467  *	@dev: device to shutdown
1468  *
1469  *	This function moves an active device into down state. A
1470  *	%NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1471  *	is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1472  *	chain.
1473  */
1474 int dev_close(struct net_device *dev)
1475 {
1476 	if (dev->flags & IFF_UP) {
1477 		LIST_HEAD(single);
1478 
1479 		list_add(&dev->close_list, &single);
1480 		dev_close_many(&single, true);
1481 		list_del(&single);
1482 	}
1483 	return 0;
1484 }
1485 EXPORT_SYMBOL(dev_close);
1486 
1487 
1488 /**
1489  *	dev_disable_lro - disable Large Receive Offload on a device
1490  *	@dev: device
1491  *
1492  *	Disable Large Receive Offload (LRO) on a net device.  Must be
1493  *	called under RTNL.  This is needed if received packets may be
1494  *	forwarded to another interface.
1495  */
1496 void dev_disable_lro(struct net_device *dev)
1497 {
1498 	struct net_device *lower_dev;
1499 	struct list_head *iter;
1500 
1501 	dev->wanted_features &= ~NETIF_F_LRO;
1502 	netdev_update_features(dev);
1503 
1504 	if (unlikely(dev->features & NETIF_F_LRO))
1505 		netdev_WARN(dev, "failed to disable LRO!\n");
1506 
1507 	netdev_for_each_lower_dev(dev, lower_dev, iter)
1508 		dev_disable_lro(lower_dev);
1509 }
1510 EXPORT_SYMBOL(dev_disable_lro);
1511 
1512 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1513 				   struct net_device *dev)
1514 {
1515 	struct netdev_notifier_info info;
1516 
1517 	netdev_notifier_info_init(&info, dev);
1518 	return nb->notifier_call(nb, val, &info);
1519 }
1520 
1521 static int dev_boot_phase = 1;
1522 
1523 /**
1524  * register_netdevice_notifier - register a network notifier block
1525  * @nb: notifier
1526  *
1527  * Register a notifier to be called when network device events occur.
1528  * The notifier passed is linked into the kernel structures and must
1529  * not be reused until it has been unregistered. A negative errno code
1530  * is returned on a failure.
1531  *
1532  * When registered all registration and up events are replayed
1533  * to the new notifier to allow device to have a race free
1534  * view of the network device list.
1535  */
1536 
1537 int register_netdevice_notifier(struct notifier_block *nb)
1538 {
1539 	struct net_device *dev;
1540 	struct net_device *last;
1541 	struct net *net;
1542 	int err;
1543 
1544 	rtnl_lock();
1545 	err = raw_notifier_chain_register(&netdev_chain, nb);
1546 	if (err)
1547 		goto unlock;
1548 	if (dev_boot_phase)
1549 		goto unlock;
1550 	for_each_net(net) {
1551 		for_each_netdev(net, dev) {
1552 			err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1553 			err = notifier_to_errno(err);
1554 			if (err)
1555 				goto rollback;
1556 
1557 			if (!(dev->flags & IFF_UP))
1558 				continue;
1559 
1560 			call_netdevice_notifier(nb, NETDEV_UP, dev);
1561 		}
1562 	}
1563 
1564 unlock:
1565 	rtnl_unlock();
1566 	return err;
1567 
1568 rollback:
1569 	last = dev;
1570 	for_each_net(net) {
1571 		for_each_netdev(net, dev) {
1572 			if (dev == last)
1573 				goto outroll;
1574 
1575 			if (dev->flags & IFF_UP) {
1576 				call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1577 							dev);
1578 				call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1579 			}
1580 			call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1581 		}
1582 	}
1583 
1584 outroll:
1585 	raw_notifier_chain_unregister(&netdev_chain, nb);
1586 	goto unlock;
1587 }
1588 EXPORT_SYMBOL(register_netdevice_notifier);
1589 
1590 /**
1591  * unregister_netdevice_notifier - unregister a network notifier block
1592  * @nb: notifier
1593  *
1594  * Unregister a notifier previously registered by
1595  * register_netdevice_notifier(). The notifier is unlinked into the
1596  * kernel structures and may then be reused. A negative errno code
1597  * is returned on a failure.
1598  *
1599  * After unregistering unregister and down device events are synthesized
1600  * for all devices on the device list to the removed notifier to remove
1601  * the need for special case cleanup code.
1602  */
1603 
1604 int unregister_netdevice_notifier(struct notifier_block *nb)
1605 {
1606 	struct net_device *dev;
1607 	struct net *net;
1608 	int err;
1609 
1610 	rtnl_lock();
1611 	err = raw_notifier_chain_unregister(&netdev_chain, nb);
1612 	if (err)
1613 		goto unlock;
1614 
1615 	for_each_net(net) {
1616 		for_each_netdev(net, dev) {
1617 			if (dev->flags & IFF_UP) {
1618 				call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1619 							dev);
1620 				call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1621 			}
1622 			call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1623 		}
1624 	}
1625 unlock:
1626 	rtnl_unlock();
1627 	return err;
1628 }
1629 EXPORT_SYMBOL(unregister_netdevice_notifier);
1630 
1631 /**
1632  *	call_netdevice_notifiers_info - call all network notifier blocks
1633  *	@val: value passed unmodified to notifier function
1634  *	@dev: net_device pointer passed unmodified to notifier function
1635  *	@info: notifier information data
1636  *
1637  *	Call all network notifier blocks.  Parameters and return value
1638  *	are as for raw_notifier_call_chain().
1639  */
1640 
1641 static int call_netdevice_notifiers_info(unsigned long val,
1642 					 struct net_device *dev,
1643 					 struct netdev_notifier_info *info)
1644 {
1645 	ASSERT_RTNL();
1646 	netdev_notifier_info_init(info, dev);
1647 	return raw_notifier_call_chain(&netdev_chain, val, info);
1648 }
1649 
1650 /**
1651  *	call_netdevice_notifiers - call all network notifier blocks
1652  *      @val: value passed unmodified to notifier function
1653  *      @dev: net_device pointer passed unmodified to notifier function
1654  *
1655  *	Call all network notifier blocks.  Parameters and return value
1656  *	are as for raw_notifier_call_chain().
1657  */
1658 
1659 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1660 {
1661 	struct netdev_notifier_info info;
1662 
1663 	return call_netdevice_notifiers_info(val, dev, &info);
1664 }
1665 EXPORT_SYMBOL(call_netdevice_notifiers);
1666 
1667 #ifdef CONFIG_NET_INGRESS
1668 static struct static_key ingress_needed __read_mostly;
1669 
1670 void net_inc_ingress_queue(void)
1671 {
1672 	static_key_slow_inc(&ingress_needed);
1673 }
1674 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1675 
1676 void net_dec_ingress_queue(void)
1677 {
1678 	static_key_slow_dec(&ingress_needed);
1679 }
1680 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1681 #endif
1682 
1683 #ifdef CONFIG_NET_EGRESS
1684 static struct static_key egress_needed __read_mostly;
1685 
1686 void net_inc_egress_queue(void)
1687 {
1688 	static_key_slow_inc(&egress_needed);
1689 }
1690 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1691 
1692 void net_dec_egress_queue(void)
1693 {
1694 	static_key_slow_dec(&egress_needed);
1695 }
1696 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1697 #endif
1698 
1699 static struct static_key netstamp_needed __read_mostly;
1700 #ifdef HAVE_JUMP_LABEL
1701 static atomic_t netstamp_needed_deferred;
1702 static atomic_t netstamp_wanted;
1703 static void netstamp_clear(struct work_struct *work)
1704 {
1705 	int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1706 	int wanted;
1707 
1708 	wanted = atomic_add_return(deferred, &netstamp_wanted);
1709 	if (wanted > 0)
1710 		static_key_enable(&netstamp_needed);
1711 	else
1712 		static_key_disable(&netstamp_needed);
1713 }
1714 static DECLARE_WORK(netstamp_work, netstamp_clear);
1715 #endif
1716 
1717 void net_enable_timestamp(void)
1718 {
1719 #ifdef HAVE_JUMP_LABEL
1720 	int wanted;
1721 
1722 	while (1) {
1723 		wanted = atomic_read(&netstamp_wanted);
1724 		if (wanted <= 0)
1725 			break;
1726 		if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
1727 			return;
1728 	}
1729 	atomic_inc(&netstamp_needed_deferred);
1730 	schedule_work(&netstamp_work);
1731 #else
1732 	static_key_slow_inc(&netstamp_needed);
1733 #endif
1734 }
1735 EXPORT_SYMBOL(net_enable_timestamp);
1736 
1737 void net_disable_timestamp(void)
1738 {
1739 #ifdef HAVE_JUMP_LABEL
1740 	int wanted;
1741 
1742 	while (1) {
1743 		wanted = atomic_read(&netstamp_wanted);
1744 		if (wanted <= 1)
1745 			break;
1746 		if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
1747 			return;
1748 	}
1749 	atomic_dec(&netstamp_needed_deferred);
1750 	schedule_work(&netstamp_work);
1751 #else
1752 	static_key_slow_dec(&netstamp_needed);
1753 #endif
1754 }
1755 EXPORT_SYMBOL(net_disable_timestamp);
1756 
1757 static inline void net_timestamp_set(struct sk_buff *skb)
1758 {
1759 	skb->tstamp = 0;
1760 	if (static_key_false(&netstamp_needed))
1761 		__net_timestamp(skb);
1762 }
1763 
1764 #define net_timestamp_check(COND, SKB)			\
1765 	if (static_key_false(&netstamp_needed)) {		\
1766 		if ((COND) && !(SKB)->tstamp)	\
1767 			__net_timestamp(SKB);		\
1768 	}						\
1769 
1770 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
1771 {
1772 	unsigned int len;
1773 
1774 	if (!(dev->flags & IFF_UP))
1775 		return false;
1776 
1777 	len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1778 	if (skb->len <= len)
1779 		return true;
1780 
1781 	/* if TSO is enabled, we don't care about the length as the packet
1782 	 * could be forwarded without being segmented before
1783 	 */
1784 	if (skb_is_gso(skb))
1785 		return true;
1786 
1787 	return false;
1788 }
1789 EXPORT_SYMBOL_GPL(is_skb_forwardable);
1790 
1791 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1792 {
1793 	int ret = ____dev_forward_skb(dev, skb);
1794 
1795 	if (likely(!ret)) {
1796 		skb->protocol = eth_type_trans(skb, dev);
1797 		skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1798 	}
1799 
1800 	return ret;
1801 }
1802 EXPORT_SYMBOL_GPL(__dev_forward_skb);
1803 
1804 /**
1805  * dev_forward_skb - loopback an skb to another netif
1806  *
1807  * @dev: destination network device
1808  * @skb: buffer to forward
1809  *
1810  * return values:
1811  *	NET_RX_SUCCESS	(no congestion)
1812  *	NET_RX_DROP     (packet was dropped, but freed)
1813  *
1814  * dev_forward_skb can be used for injecting an skb from the
1815  * start_xmit function of one device into the receive queue
1816  * of another device.
1817  *
1818  * The receiving device may be in another namespace, so
1819  * we have to clear all information in the skb that could
1820  * impact namespace isolation.
1821  */
1822 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1823 {
1824 	return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1825 }
1826 EXPORT_SYMBOL_GPL(dev_forward_skb);
1827 
1828 static inline int deliver_skb(struct sk_buff *skb,
1829 			      struct packet_type *pt_prev,
1830 			      struct net_device *orig_dev)
1831 {
1832 	if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1833 		return -ENOMEM;
1834 	atomic_inc(&skb->users);
1835 	return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1836 }
1837 
1838 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1839 					  struct packet_type **pt,
1840 					  struct net_device *orig_dev,
1841 					  __be16 type,
1842 					  struct list_head *ptype_list)
1843 {
1844 	struct packet_type *ptype, *pt_prev = *pt;
1845 
1846 	list_for_each_entry_rcu(ptype, ptype_list, list) {
1847 		if (ptype->type != type)
1848 			continue;
1849 		if (pt_prev)
1850 			deliver_skb(skb, pt_prev, orig_dev);
1851 		pt_prev = ptype;
1852 	}
1853 	*pt = pt_prev;
1854 }
1855 
1856 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1857 {
1858 	if (!ptype->af_packet_priv || !skb->sk)
1859 		return false;
1860 
1861 	if (ptype->id_match)
1862 		return ptype->id_match(ptype, skb->sk);
1863 	else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1864 		return true;
1865 
1866 	return false;
1867 }
1868 
1869 /*
1870  *	Support routine. Sends outgoing frames to any network
1871  *	taps currently in use.
1872  */
1873 
1874 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1875 {
1876 	struct packet_type *ptype;
1877 	struct sk_buff *skb2 = NULL;
1878 	struct packet_type *pt_prev = NULL;
1879 	struct list_head *ptype_list = &ptype_all;
1880 
1881 	rcu_read_lock();
1882 again:
1883 	list_for_each_entry_rcu(ptype, ptype_list, list) {
1884 		/* Never send packets back to the socket
1885 		 * they originated from - MvS (miquels@drinkel.ow.org)
1886 		 */
1887 		if (skb_loop_sk(ptype, skb))
1888 			continue;
1889 
1890 		if (pt_prev) {
1891 			deliver_skb(skb2, pt_prev, skb->dev);
1892 			pt_prev = ptype;
1893 			continue;
1894 		}
1895 
1896 		/* need to clone skb, done only once */
1897 		skb2 = skb_clone(skb, GFP_ATOMIC);
1898 		if (!skb2)
1899 			goto out_unlock;
1900 
1901 		net_timestamp_set(skb2);
1902 
1903 		/* skb->nh should be correctly
1904 		 * set by sender, so that the second statement is
1905 		 * just protection against buggy protocols.
1906 		 */
1907 		skb_reset_mac_header(skb2);
1908 
1909 		if (skb_network_header(skb2) < skb2->data ||
1910 		    skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1911 			net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1912 					     ntohs(skb2->protocol),
1913 					     dev->name);
1914 			skb_reset_network_header(skb2);
1915 		}
1916 
1917 		skb2->transport_header = skb2->network_header;
1918 		skb2->pkt_type = PACKET_OUTGOING;
1919 		pt_prev = ptype;
1920 	}
1921 
1922 	if (ptype_list == &ptype_all) {
1923 		ptype_list = &dev->ptype_all;
1924 		goto again;
1925 	}
1926 out_unlock:
1927 	if (pt_prev)
1928 		pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1929 	rcu_read_unlock();
1930 }
1931 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
1932 
1933 /**
1934  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1935  * @dev: Network device
1936  * @txq: number of queues available
1937  *
1938  * If real_num_tx_queues is changed the tc mappings may no longer be
1939  * valid. To resolve this verify the tc mapping remains valid and if
1940  * not NULL the mapping. With no priorities mapping to this
1941  * offset/count pair it will no longer be used. In the worst case TC0
1942  * is invalid nothing can be done so disable priority mappings. If is
1943  * expected that drivers will fix this mapping if they can before
1944  * calling netif_set_real_num_tx_queues.
1945  */
1946 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1947 {
1948 	int i;
1949 	struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1950 
1951 	/* If TC0 is invalidated disable TC mapping */
1952 	if (tc->offset + tc->count > txq) {
1953 		pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1954 		dev->num_tc = 0;
1955 		return;
1956 	}
1957 
1958 	/* Invalidated prio to tc mappings set to TC0 */
1959 	for (i = 1; i < TC_BITMASK + 1; i++) {
1960 		int q = netdev_get_prio_tc_map(dev, i);
1961 
1962 		tc = &dev->tc_to_txq[q];
1963 		if (tc->offset + tc->count > txq) {
1964 			pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1965 				i, q);
1966 			netdev_set_prio_tc_map(dev, i, 0);
1967 		}
1968 	}
1969 }
1970 
1971 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
1972 {
1973 	if (dev->num_tc) {
1974 		struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1975 		int i;
1976 
1977 		for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
1978 			if ((txq - tc->offset) < tc->count)
1979 				return i;
1980 		}
1981 
1982 		return -1;
1983 	}
1984 
1985 	return 0;
1986 }
1987 
1988 #ifdef CONFIG_XPS
1989 static DEFINE_MUTEX(xps_map_mutex);
1990 #define xmap_dereference(P)		\
1991 	rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1992 
1993 static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
1994 			     int tci, u16 index)
1995 {
1996 	struct xps_map *map = NULL;
1997 	int pos;
1998 
1999 	if (dev_maps)
2000 		map = xmap_dereference(dev_maps->cpu_map[tci]);
2001 	if (!map)
2002 		return false;
2003 
2004 	for (pos = map->len; pos--;) {
2005 		if (map->queues[pos] != index)
2006 			continue;
2007 
2008 		if (map->len > 1) {
2009 			map->queues[pos] = map->queues[--map->len];
2010 			break;
2011 		}
2012 
2013 		RCU_INIT_POINTER(dev_maps->cpu_map[tci], NULL);
2014 		kfree_rcu(map, rcu);
2015 		return false;
2016 	}
2017 
2018 	return true;
2019 }
2020 
2021 static bool remove_xps_queue_cpu(struct net_device *dev,
2022 				 struct xps_dev_maps *dev_maps,
2023 				 int cpu, u16 offset, u16 count)
2024 {
2025 	int num_tc = dev->num_tc ? : 1;
2026 	bool active = false;
2027 	int tci;
2028 
2029 	for (tci = cpu * num_tc; num_tc--; tci++) {
2030 		int i, j;
2031 
2032 		for (i = count, j = offset; i--; j++) {
2033 			if (!remove_xps_queue(dev_maps, cpu, j))
2034 				break;
2035 		}
2036 
2037 		active |= i < 0;
2038 	}
2039 
2040 	return active;
2041 }
2042 
2043 static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2044 				   u16 count)
2045 {
2046 	struct xps_dev_maps *dev_maps;
2047 	int cpu, i;
2048 	bool active = false;
2049 
2050 	mutex_lock(&xps_map_mutex);
2051 	dev_maps = xmap_dereference(dev->xps_maps);
2052 
2053 	if (!dev_maps)
2054 		goto out_no_maps;
2055 
2056 	for_each_possible_cpu(cpu)
2057 		active |= remove_xps_queue_cpu(dev, dev_maps, cpu,
2058 					       offset, count);
2059 
2060 	if (!active) {
2061 		RCU_INIT_POINTER(dev->xps_maps, NULL);
2062 		kfree_rcu(dev_maps, rcu);
2063 	}
2064 
2065 	for (i = offset + (count - 1); count--; i--)
2066 		netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2067 					     NUMA_NO_NODE);
2068 
2069 out_no_maps:
2070 	mutex_unlock(&xps_map_mutex);
2071 }
2072 
2073 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2074 {
2075 	netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2076 }
2077 
2078 static struct xps_map *expand_xps_map(struct xps_map *map,
2079 				      int cpu, u16 index)
2080 {
2081 	struct xps_map *new_map;
2082 	int alloc_len = XPS_MIN_MAP_ALLOC;
2083 	int i, pos;
2084 
2085 	for (pos = 0; map && pos < map->len; pos++) {
2086 		if (map->queues[pos] != index)
2087 			continue;
2088 		return map;
2089 	}
2090 
2091 	/* Need to add queue to this CPU's existing map */
2092 	if (map) {
2093 		if (pos < map->alloc_len)
2094 			return map;
2095 
2096 		alloc_len = map->alloc_len * 2;
2097 	}
2098 
2099 	/* Need to allocate new map to store queue on this CPU's map */
2100 	new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2101 			       cpu_to_node(cpu));
2102 	if (!new_map)
2103 		return NULL;
2104 
2105 	for (i = 0; i < pos; i++)
2106 		new_map->queues[i] = map->queues[i];
2107 	new_map->alloc_len = alloc_len;
2108 	new_map->len = pos;
2109 
2110 	return new_map;
2111 }
2112 
2113 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2114 			u16 index)
2115 {
2116 	struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2117 	int i, cpu, tci, numa_node_id = -2;
2118 	int maps_sz, num_tc = 1, tc = 0;
2119 	struct xps_map *map, *new_map;
2120 	bool active = false;
2121 
2122 	if (dev->num_tc) {
2123 		num_tc = dev->num_tc;
2124 		tc = netdev_txq_to_tc(dev, index);
2125 		if (tc < 0)
2126 			return -EINVAL;
2127 	}
2128 
2129 	maps_sz = XPS_DEV_MAPS_SIZE(num_tc);
2130 	if (maps_sz < L1_CACHE_BYTES)
2131 		maps_sz = L1_CACHE_BYTES;
2132 
2133 	mutex_lock(&xps_map_mutex);
2134 
2135 	dev_maps = xmap_dereference(dev->xps_maps);
2136 
2137 	/* allocate memory for queue storage */
2138 	for_each_cpu_and(cpu, cpu_online_mask, mask) {
2139 		if (!new_dev_maps)
2140 			new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2141 		if (!new_dev_maps) {
2142 			mutex_unlock(&xps_map_mutex);
2143 			return -ENOMEM;
2144 		}
2145 
2146 		tci = cpu * num_tc + tc;
2147 		map = dev_maps ? xmap_dereference(dev_maps->cpu_map[tci]) :
2148 				 NULL;
2149 
2150 		map = expand_xps_map(map, cpu, index);
2151 		if (!map)
2152 			goto error;
2153 
2154 		RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2155 	}
2156 
2157 	if (!new_dev_maps)
2158 		goto out_no_new_maps;
2159 
2160 	for_each_possible_cpu(cpu) {
2161 		/* copy maps belonging to foreign traffic classes */
2162 		for (i = tc, tci = cpu * num_tc; dev_maps && i--; tci++) {
2163 			/* fill in the new device map from the old device map */
2164 			map = xmap_dereference(dev_maps->cpu_map[tci]);
2165 			RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2166 		}
2167 
2168 		/* We need to explicitly update tci as prevous loop
2169 		 * could break out early if dev_maps is NULL.
2170 		 */
2171 		tci = cpu * num_tc + tc;
2172 
2173 		if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2174 			/* add queue to CPU maps */
2175 			int pos = 0;
2176 
2177 			map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2178 			while ((pos < map->len) && (map->queues[pos] != index))
2179 				pos++;
2180 
2181 			if (pos == map->len)
2182 				map->queues[map->len++] = index;
2183 #ifdef CONFIG_NUMA
2184 			if (numa_node_id == -2)
2185 				numa_node_id = cpu_to_node(cpu);
2186 			else if (numa_node_id != cpu_to_node(cpu))
2187 				numa_node_id = -1;
2188 #endif
2189 		} else if (dev_maps) {
2190 			/* fill in the new device map from the old device map */
2191 			map = xmap_dereference(dev_maps->cpu_map[tci]);
2192 			RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2193 		}
2194 
2195 		/* copy maps belonging to foreign traffic classes */
2196 		for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2197 			/* fill in the new device map from the old device map */
2198 			map = xmap_dereference(dev_maps->cpu_map[tci]);
2199 			RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2200 		}
2201 	}
2202 
2203 	rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2204 
2205 	/* Cleanup old maps */
2206 	if (!dev_maps)
2207 		goto out_no_old_maps;
2208 
2209 	for_each_possible_cpu(cpu) {
2210 		for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2211 			new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2212 			map = xmap_dereference(dev_maps->cpu_map[tci]);
2213 			if (map && map != new_map)
2214 				kfree_rcu(map, rcu);
2215 		}
2216 	}
2217 
2218 	kfree_rcu(dev_maps, rcu);
2219 
2220 out_no_old_maps:
2221 	dev_maps = new_dev_maps;
2222 	active = true;
2223 
2224 out_no_new_maps:
2225 	/* update Tx queue numa node */
2226 	netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2227 				     (numa_node_id >= 0) ? numa_node_id :
2228 				     NUMA_NO_NODE);
2229 
2230 	if (!dev_maps)
2231 		goto out_no_maps;
2232 
2233 	/* removes queue from unused CPUs */
2234 	for_each_possible_cpu(cpu) {
2235 		for (i = tc, tci = cpu * num_tc; i--; tci++)
2236 			active |= remove_xps_queue(dev_maps, tci, index);
2237 		if (!cpumask_test_cpu(cpu, mask) || !cpu_online(cpu))
2238 			active |= remove_xps_queue(dev_maps, tci, index);
2239 		for (i = num_tc - tc, tci++; --i; tci++)
2240 			active |= remove_xps_queue(dev_maps, tci, index);
2241 	}
2242 
2243 	/* free map if not active */
2244 	if (!active) {
2245 		RCU_INIT_POINTER(dev->xps_maps, NULL);
2246 		kfree_rcu(dev_maps, rcu);
2247 	}
2248 
2249 out_no_maps:
2250 	mutex_unlock(&xps_map_mutex);
2251 
2252 	return 0;
2253 error:
2254 	/* remove any maps that we added */
2255 	for_each_possible_cpu(cpu) {
2256 		for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2257 			new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2258 			map = dev_maps ?
2259 			      xmap_dereference(dev_maps->cpu_map[tci]) :
2260 			      NULL;
2261 			if (new_map && new_map != map)
2262 				kfree(new_map);
2263 		}
2264 	}
2265 
2266 	mutex_unlock(&xps_map_mutex);
2267 
2268 	kfree(new_dev_maps);
2269 	return -ENOMEM;
2270 }
2271 EXPORT_SYMBOL(netif_set_xps_queue);
2272 
2273 #endif
2274 void netdev_reset_tc(struct net_device *dev)
2275 {
2276 #ifdef CONFIG_XPS
2277 	netif_reset_xps_queues_gt(dev, 0);
2278 #endif
2279 	dev->num_tc = 0;
2280 	memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2281 	memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2282 }
2283 EXPORT_SYMBOL(netdev_reset_tc);
2284 
2285 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2286 {
2287 	if (tc >= dev->num_tc)
2288 		return -EINVAL;
2289 
2290 #ifdef CONFIG_XPS
2291 	netif_reset_xps_queues(dev, offset, count);
2292 #endif
2293 	dev->tc_to_txq[tc].count = count;
2294 	dev->tc_to_txq[tc].offset = offset;
2295 	return 0;
2296 }
2297 EXPORT_SYMBOL(netdev_set_tc_queue);
2298 
2299 int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2300 {
2301 	if (num_tc > TC_MAX_QUEUE)
2302 		return -EINVAL;
2303 
2304 #ifdef CONFIG_XPS
2305 	netif_reset_xps_queues_gt(dev, 0);
2306 #endif
2307 	dev->num_tc = num_tc;
2308 	return 0;
2309 }
2310 EXPORT_SYMBOL(netdev_set_num_tc);
2311 
2312 /*
2313  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2314  * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2315  */
2316 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2317 {
2318 	int rc;
2319 
2320 	if (txq < 1 || txq > dev->num_tx_queues)
2321 		return -EINVAL;
2322 
2323 	if (dev->reg_state == NETREG_REGISTERED ||
2324 	    dev->reg_state == NETREG_UNREGISTERING) {
2325 		ASSERT_RTNL();
2326 
2327 		rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2328 						  txq);
2329 		if (rc)
2330 			return rc;
2331 
2332 		if (dev->num_tc)
2333 			netif_setup_tc(dev, txq);
2334 
2335 		if (txq < dev->real_num_tx_queues) {
2336 			qdisc_reset_all_tx_gt(dev, txq);
2337 #ifdef CONFIG_XPS
2338 			netif_reset_xps_queues_gt(dev, txq);
2339 #endif
2340 		}
2341 	}
2342 
2343 	dev->real_num_tx_queues = txq;
2344 	return 0;
2345 }
2346 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2347 
2348 #ifdef CONFIG_SYSFS
2349 /**
2350  *	netif_set_real_num_rx_queues - set actual number of RX queues used
2351  *	@dev: Network device
2352  *	@rxq: Actual number of RX queues
2353  *
2354  *	This must be called either with the rtnl_lock held or before
2355  *	registration of the net device.  Returns 0 on success, or a
2356  *	negative error code.  If called before registration, it always
2357  *	succeeds.
2358  */
2359 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2360 {
2361 	int rc;
2362 
2363 	if (rxq < 1 || rxq > dev->num_rx_queues)
2364 		return -EINVAL;
2365 
2366 	if (dev->reg_state == NETREG_REGISTERED) {
2367 		ASSERT_RTNL();
2368 
2369 		rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2370 						  rxq);
2371 		if (rc)
2372 			return rc;
2373 	}
2374 
2375 	dev->real_num_rx_queues = rxq;
2376 	return 0;
2377 }
2378 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2379 #endif
2380 
2381 /**
2382  * netif_get_num_default_rss_queues - default number of RSS queues
2383  *
2384  * This routine should set an upper limit on the number of RSS queues
2385  * used by default by multiqueue devices.
2386  */
2387 int netif_get_num_default_rss_queues(void)
2388 {
2389 	return is_kdump_kernel() ?
2390 		1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2391 }
2392 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2393 
2394 static void __netif_reschedule(struct Qdisc *q)
2395 {
2396 	struct softnet_data *sd;
2397 	unsigned long flags;
2398 
2399 	local_irq_save(flags);
2400 	sd = this_cpu_ptr(&softnet_data);
2401 	q->next_sched = NULL;
2402 	*sd->output_queue_tailp = q;
2403 	sd->output_queue_tailp = &q->next_sched;
2404 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
2405 	local_irq_restore(flags);
2406 }
2407 
2408 void __netif_schedule(struct Qdisc *q)
2409 {
2410 	if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2411 		__netif_reschedule(q);
2412 }
2413 EXPORT_SYMBOL(__netif_schedule);
2414 
2415 struct dev_kfree_skb_cb {
2416 	enum skb_free_reason reason;
2417 };
2418 
2419 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2420 {
2421 	return (struct dev_kfree_skb_cb *)skb->cb;
2422 }
2423 
2424 void netif_schedule_queue(struct netdev_queue *txq)
2425 {
2426 	rcu_read_lock();
2427 	if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2428 		struct Qdisc *q = rcu_dereference(txq->qdisc);
2429 
2430 		__netif_schedule(q);
2431 	}
2432 	rcu_read_unlock();
2433 }
2434 EXPORT_SYMBOL(netif_schedule_queue);
2435 
2436 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2437 {
2438 	if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2439 		struct Qdisc *q;
2440 
2441 		rcu_read_lock();
2442 		q = rcu_dereference(dev_queue->qdisc);
2443 		__netif_schedule(q);
2444 		rcu_read_unlock();
2445 	}
2446 }
2447 EXPORT_SYMBOL(netif_tx_wake_queue);
2448 
2449 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2450 {
2451 	unsigned long flags;
2452 
2453 	if (likely(atomic_read(&skb->users) == 1)) {
2454 		smp_rmb();
2455 		atomic_set(&skb->users, 0);
2456 	} else if (likely(!atomic_dec_and_test(&skb->users))) {
2457 		return;
2458 	}
2459 	get_kfree_skb_cb(skb)->reason = reason;
2460 	local_irq_save(flags);
2461 	skb->next = __this_cpu_read(softnet_data.completion_queue);
2462 	__this_cpu_write(softnet_data.completion_queue, skb);
2463 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
2464 	local_irq_restore(flags);
2465 }
2466 EXPORT_SYMBOL(__dev_kfree_skb_irq);
2467 
2468 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2469 {
2470 	if (in_irq() || irqs_disabled())
2471 		__dev_kfree_skb_irq(skb, reason);
2472 	else
2473 		dev_kfree_skb(skb);
2474 }
2475 EXPORT_SYMBOL(__dev_kfree_skb_any);
2476 
2477 
2478 /**
2479  * netif_device_detach - mark device as removed
2480  * @dev: network device
2481  *
2482  * Mark device as removed from system and therefore no longer available.
2483  */
2484 void netif_device_detach(struct net_device *dev)
2485 {
2486 	if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2487 	    netif_running(dev)) {
2488 		netif_tx_stop_all_queues(dev);
2489 	}
2490 }
2491 EXPORT_SYMBOL(netif_device_detach);
2492 
2493 /**
2494  * netif_device_attach - mark device as attached
2495  * @dev: network device
2496  *
2497  * Mark device as attached from system and restart if needed.
2498  */
2499 void netif_device_attach(struct net_device *dev)
2500 {
2501 	if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2502 	    netif_running(dev)) {
2503 		netif_tx_wake_all_queues(dev);
2504 		__netdev_watchdog_up(dev);
2505 	}
2506 }
2507 EXPORT_SYMBOL(netif_device_attach);
2508 
2509 /*
2510  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2511  * to be used as a distribution range.
2512  */
2513 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2514 		  unsigned int num_tx_queues)
2515 {
2516 	u32 hash;
2517 	u16 qoffset = 0;
2518 	u16 qcount = num_tx_queues;
2519 
2520 	if (skb_rx_queue_recorded(skb)) {
2521 		hash = skb_get_rx_queue(skb);
2522 		while (unlikely(hash >= num_tx_queues))
2523 			hash -= num_tx_queues;
2524 		return hash;
2525 	}
2526 
2527 	if (dev->num_tc) {
2528 		u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2529 
2530 		qoffset = dev->tc_to_txq[tc].offset;
2531 		qcount = dev->tc_to_txq[tc].count;
2532 	}
2533 
2534 	return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2535 }
2536 EXPORT_SYMBOL(__skb_tx_hash);
2537 
2538 static void skb_warn_bad_offload(const struct sk_buff *skb)
2539 {
2540 	static const netdev_features_t null_features;
2541 	struct net_device *dev = skb->dev;
2542 	const char *name = "";
2543 
2544 	if (!net_ratelimit())
2545 		return;
2546 
2547 	if (dev) {
2548 		if (dev->dev.parent)
2549 			name = dev_driver_string(dev->dev.parent);
2550 		else
2551 			name = netdev_name(dev);
2552 	}
2553 	WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2554 	     "gso_type=%d ip_summed=%d\n",
2555 	     name, dev ? &dev->features : &null_features,
2556 	     skb->sk ? &skb->sk->sk_route_caps : &null_features,
2557 	     skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2558 	     skb_shinfo(skb)->gso_type, skb->ip_summed);
2559 }
2560 
2561 /*
2562  * Invalidate hardware checksum when packet is to be mangled, and
2563  * complete checksum manually on outgoing path.
2564  */
2565 int skb_checksum_help(struct sk_buff *skb)
2566 {
2567 	__wsum csum;
2568 	int ret = 0, offset;
2569 
2570 	if (skb->ip_summed == CHECKSUM_COMPLETE)
2571 		goto out_set_summed;
2572 
2573 	if (unlikely(skb_shinfo(skb)->gso_size)) {
2574 		skb_warn_bad_offload(skb);
2575 		return -EINVAL;
2576 	}
2577 
2578 	/* Before computing a checksum, we should make sure no frag could
2579 	 * be modified by an external entity : checksum could be wrong.
2580 	 */
2581 	if (skb_has_shared_frag(skb)) {
2582 		ret = __skb_linearize(skb);
2583 		if (ret)
2584 			goto out;
2585 	}
2586 
2587 	offset = skb_checksum_start_offset(skb);
2588 	BUG_ON(offset >= skb_headlen(skb));
2589 	csum = skb_checksum(skb, offset, skb->len - offset, 0);
2590 
2591 	offset += skb->csum_offset;
2592 	BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2593 
2594 	if (skb_cloned(skb) &&
2595 	    !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2596 		ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2597 		if (ret)
2598 			goto out;
2599 	}
2600 
2601 	*(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2602 out_set_summed:
2603 	skb->ip_summed = CHECKSUM_NONE;
2604 out:
2605 	return ret;
2606 }
2607 EXPORT_SYMBOL(skb_checksum_help);
2608 
2609 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2610 {
2611 	__be16 type = skb->protocol;
2612 
2613 	/* Tunnel gso handlers can set protocol to ethernet. */
2614 	if (type == htons(ETH_P_TEB)) {
2615 		struct ethhdr *eth;
2616 
2617 		if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2618 			return 0;
2619 
2620 		eth = (struct ethhdr *)skb_mac_header(skb);
2621 		type = eth->h_proto;
2622 	}
2623 
2624 	return __vlan_get_protocol(skb, type, depth);
2625 }
2626 
2627 /**
2628  *	skb_mac_gso_segment - mac layer segmentation handler.
2629  *	@skb: buffer to segment
2630  *	@features: features for the output path (see dev->features)
2631  */
2632 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2633 				    netdev_features_t features)
2634 {
2635 	struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2636 	struct packet_offload *ptype;
2637 	int vlan_depth = skb->mac_len;
2638 	__be16 type = skb_network_protocol(skb, &vlan_depth);
2639 
2640 	if (unlikely(!type))
2641 		return ERR_PTR(-EINVAL);
2642 
2643 	__skb_pull(skb, vlan_depth);
2644 
2645 	rcu_read_lock();
2646 	list_for_each_entry_rcu(ptype, &offload_base, list) {
2647 		if (ptype->type == type && ptype->callbacks.gso_segment) {
2648 			segs = ptype->callbacks.gso_segment(skb, features);
2649 			break;
2650 		}
2651 	}
2652 	rcu_read_unlock();
2653 
2654 	__skb_push(skb, skb->data - skb_mac_header(skb));
2655 
2656 	return segs;
2657 }
2658 EXPORT_SYMBOL(skb_mac_gso_segment);
2659 
2660 
2661 /* openvswitch calls this on rx path, so we need a different check.
2662  */
2663 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2664 {
2665 	if (tx_path)
2666 		return skb->ip_summed != CHECKSUM_PARTIAL &&
2667 		       skb->ip_summed != CHECKSUM_NONE;
2668 
2669 	return skb->ip_summed == CHECKSUM_NONE;
2670 }
2671 
2672 /**
2673  *	__skb_gso_segment - Perform segmentation on skb.
2674  *	@skb: buffer to segment
2675  *	@features: features for the output path (see dev->features)
2676  *	@tx_path: whether it is called in TX path
2677  *
2678  *	This function segments the given skb and returns a list of segments.
2679  *
2680  *	It may return NULL if the skb requires no segmentation.  This is
2681  *	only possible when GSO is used for verifying header integrity.
2682  *
2683  *	Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
2684  */
2685 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2686 				  netdev_features_t features, bool tx_path)
2687 {
2688 	struct sk_buff *segs;
2689 
2690 	if (unlikely(skb_needs_check(skb, tx_path))) {
2691 		int err;
2692 
2693 		/* We're going to init ->check field in TCP or UDP header */
2694 		err = skb_cow_head(skb, 0);
2695 		if (err < 0)
2696 			return ERR_PTR(err);
2697 	}
2698 
2699 	/* Only report GSO partial support if it will enable us to
2700 	 * support segmentation on this frame without needing additional
2701 	 * work.
2702 	 */
2703 	if (features & NETIF_F_GSO_PARTIAL) {
2704 		netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
2705 		struct net_device *dev = skb->dev;
2706 
2707 		partial_features |= dev->features & dev->gso_partial_features;
2708 		if (!skb_gso_ok(skb, features | partial_features))
2709 			features &= ~NETIF_F_GSO_PARTIAL;
2710 	}
2711 
2712 	BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2713 		     sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2714 
2715 	SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2716 	SKB_GSO_CB(skb)->encap_level = 0;
2717 
2718 	skb_reset_mac_header(skb);
2719 	skb_reset_mac_len(skb);
2720 
2721 	segs = skb_mac_gso_segment(skb, features);
2722 
2723 	if (unlikely(skb_needs_check(skb, tx_path)))
2724 		skb_warn_bad_offload(skb);
2725 
2726 	return segs;
2727 }
2728 EXPORT_SYMBOL(__skb_gso_segment);
2729 
2730 /* Take action when hardware reception checksum errors are detected. */
2731 #ifdef CONFIG_BUG
2732 void netdev_rx_csum_fault(struct net_device *dev)
2733 {
2734 	if (net_ratelimit()) {
2735 		pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2736 		dump_stack();
2737 	}
2738 }
2739 EXPORT_SYMBOL(netdev_rx_csum_fault);
2740 #endif
2741 
2742 /* Actually, we should eliminate this check as soon as we know, that:
2743  * 1. IOMMU is present and allows to map all the memory.
2744  * 2. No high memory really exists on this machine.
2745  */
2746 
2747 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2748 {
2749 #ifdef CONFIG_HIGHMEM
2750 	int i;
2751 
2752 	if (!(dev->features & NETIF_F_HIGHDMA)) {
2753 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2754 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2755 
2756 			if (PageHighMem(skb_frag_page(frag)))
2757 				return 1;
2758 		}
2759 	}
2760 
2761 	if (PCI_DMA_BUS_IS_PHYS) {
2762 		struct device *pdev = dev->dev.parent;
2763 
2764 		if (!pdev)
2765 			return 0;
2766 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2767 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2768 			dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2769 
2770 			if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2771 				return 1;
2772 		}
2773 	}
2774 #endif
2775 	return 0;
2776 }
2777 
2778 /* If MPLS offload request, verify we are testing hardware MPLS features
2779  * instead of standard features for the netdev.
2780  */
2781 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
2782 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2783 					   netdev_features_t features,
2784 					   __be16 type)
2785 {
2786 	if (eth_p_mpls(type))
2787 		features &= skb->dev->mpls_features;
2788 
2789 	return features;
2790 }
2791 #else
2792 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2793 					   netdev_features_t features,
2794 					   __be16 type)
2795 {
2796 	return features;
2797 }
2798 #endif
2799 
2800 static netdev_features_t harmonize_features(struct sk_buff *skb,
2801 	netdev_features_t features)
2802 {
2803 	int tmp;
2804 	__be16 type;
2805 
2806 	type = skb_network_protocol(skb, &tmp);
2807 	features = net_mpls_features(skb, features, type);
2808 
2809 	if (skb->ip_summed != CHECKSUM_NONE &&
2810 	    !can_checksum_protocol(features, type)) {
2811 		features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2812 	}
2813 	if (illegal_highdma(skb->dev, skb))
2814 		features &= ~NETIF_F_SG;
2815 
2816 	return features;
2817 }
2818 
2819 netdev_features_t passthru_features_check(struct sk_buff *skb,
2820 					  struct net_device *dev,
2821 					  netdev_features_t features)
2822 {
2823 	return features;
2824 }
2825 EXPORT_SYMBOL(passthru_features_check);
2826 
2827 static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2828 					     struct net_device *dev,
2829 					     netdev_features_t features)
2830 {
2831 	return vlan_features_check(skb, features);
2832 }
2833 
2834 static netdev_features_t gso_features_check(const struct sk_buff *skb,
2835 					    struct net_device *dev,
2836 					    netdev_features_t features)
2837 {
2838 	u16 gso_segs = skb_shinfo(skb)->gso_segs;
2839 
2840 	if (gso_segs > dev->gso_max_segs)
2841 		return features & ~NETIF_F_GSO_MASK;
2842 
2843 	/* Support for GSO partial features requires software
2844 	 * intervention before we can actually process the packets
2845 	 * so we need to strip support for any partial features now
2846 	 * and we can pull them back in after we have partially
2847 	 * segmented the frame.
2848 	 */
2849 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
2850 		features &= ~dev->gso_partial_features;
2851 
2852 	/* Make sure to clear the IPv4 ID mangling feature if the
2853 	 * IPv4 header has the potential to be fragmented.
2854 	 */
2855 	if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
2856 		struct iphdr *iph = skb->encapsulation ?
2857 				    inner_ip_hdr(skb) : ip_hdr(skb);
2858 
2859 		if (!(iph->frag_off & htons(IP_DF)))
2860 			features &= ~NETIF_F_TSO_MANGLEID;
2861 	}
2862 
2863 	return features;
2864 }
2865 
2866 netdev_features_t netif_skb_features(struct sk_buff *skb)
2867 {
2868 	struct net_device *dev = skb->dev;
2869 	netdev_features_t features = dev->features;
2870 
2871 	if (skb_is_gso(skb))
2872 		features = gso_features_check(skb, dev, features);
2873 
2874 	/* If encapsulation offload request, verify we are testing
2875 	 * hardware encapsulation features instead of standard
2876 	 * features for the netdev
2877 	 */
2878 	if (skb->encapsulation)
2879 		features &= dev->hw_enc_features;
2880 
2881 	if (skb_vlan_tagged(skb))
2882 		features = netdev_intersect_features(features,
2883 						     dev->vlan_features |
2884 						     NETIF_F_HW_VLAN_CTAG_TX |
2885 						     NETIF_F_HW_VLAN_STAG_TX);
2886 
2887 	if (dev->netdev_ops->ndo_features_check)
2888 		features &= dev->netdev_ops->ndo_features_check(skb, dev,
2889 								features);
2890 	else
2891 		features &= dflt_features_check(skb, dev, features);
2892 
2893 	return harmonize_features(skb, features);
2894 }
2895 EXPORT_SYMBOL(netif_skb_features);
2896 
2897 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
2898 		    struct netdev_queue *txq, bool more)
2899 {
2900 	unsigned int len;
2901 	int rc;
2902 
2903 	if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
2904 		dev_queue_xmit_nit(skb, dev);
2905 
2906 	len = skb->len;
2907 	trace_net_dev_start_xmit(skb, dev);
2908 	rc = netdev_start_xmit(skb, dev, txq, more);
2909 	trace_net_dev_xmit(skb, rc, dev, len);
2910 
2911 	return rc;
2912 }
2913 
2914 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2915 				    struct netdev_queue *txq, int *ret)
2916 {
2917 	struct sk_buff *skb = first;
2918 	int rc = NETDEV_TX_OK;
2919 
2920 	while (skb) {
2921 		struct sk_buff *next = skb->next;
2922 
2923 		skb->next = NULL;
2924 		rc = xmit_one(skb, dev, txq, next != NULL);
2925 		if (unlikely(!dev_xmit_complete(rc))) {
2926 			skb->next = next;
2927 			goto out;
2928 		}
2929 
2930 		skb = next;
2931 		if (netif_xmit_stopped(txq) && skb) {
2932 			rc = NETDEV_TX_BUSY;
2933 			break;
2934 		}
2935 	}
2936 
2937 out:
2938 	*ret = rc;
2939 	return skb;
2940 }
2941 
2942 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2943 					  netdev_features_t features)
2944 {
2945 	if (skb_vlan_tag_present(skb) &&
2946 	    !vlan_hw_offload_capable(features, skb->vlan_proto))
2947 		skb = __vlan_hwaccel_push_inside(skb);
2948 	return skb;
2949 }
2950 
2951 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
2952 {
2953 	netdev_features_t features;
2954 
2955 	features = netif_skb_features(skb);
2956 	skb = validate_xmit_vlan(skb, features);
2957 	if (unlikely(!skb))
2958 		goto out_null;
2959 
2960 	if (netif_needs_gso(skb, features)) {
2961 		struct sk_buff *segs;
2962 
2963 		segs = skb_gso_segment(skb, features);
2964 		if (IS_ERR(segs)) {
2965 			goto out_kfree_skb;
2966 		} else if (segs) {
2967 			consume_skb(skb);
2968 			skb = segs;
2969 		}
2970 	} else {
2971 		if (skb_needs_linearize(skb, features) &&
2972 		    __skb_linearize(skb))
2973 			goto out_kfree_skb;
2974 
2975 		/* If packet is not checksummed and device does not
2976 		 * support checksumming for this protocol, complete
2977 		 * checksumming here.
2978 		 */
2979 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
2980 			if (skb->encapsulation)
2981 				skb_set_inner_transport_header(skb,
2982 							       skb_checksum_start_offset(skb));
2983 			else
2984 				skb_set_transport_header(skb,
2985 							 skb_checksum_start_offset(skb));
2986 			if (!(features & NETIF_F_CSUM_MASK) &&
2987 			    skb_checksum_help(skb))
2988 				goto out_kfree_skb;
2989 		}
2990 	}
2991 
2992 	return skb;
2993 
2994 out_kfree_skb:
2995 	kfree_skb(skb);
2996 out_null:
2997 	atomic_long_inc(&dev->tx_dropped);
2998 	return NULL;
2999 }
3000 
3001 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
3002 {
3003 	struct sk_buff *next, *head = NULL, *tail;
3004 
3005 	for (; skb != NULL; skb = next) {
3006 		next = skb->next;
3007 		skb->next = NULL;
3008 
3009 		/* in case skb wont be segmented, point to itself */
3010 		skb->prev = skb;
3011 
3012 		skb = validate_xmit_skb(skb, dev);
3013 		if (!skb)
3014 			continue;
3015 
3016 		if (!head)
3017 			head = skb;
3018 		else
3019 			tail->next = skb;
3020 		/* If skb was segmented, skb->prev points to
3021 		 * the last segment. If not, it still contains skb.
3022 		 */
3023 		tail = skb->prev;
3024 	}
3025 	return head;
3026 }
3027 EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3028 
3029 static void qdisc_pkt_len_init(struct sk_buff *skb)
3030 {
3031 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
3032 
3033 	qdisc_skb_cb(skb)->pkt_len = skb->len;
3034 
3035 	/* To get more precise estimation of bytes sent on wire,
3036 	 * we add to pkt_len the headers size of all segments
3037 	 */
3038 	if (shinfo->gso_size)  {
3039 		unsigned int hdr_len;
3040 		u16 gso_segs = shinfo->gso_segs;
3041 
3042 		/* mac layer + network layer */
3043 		hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3044 
3045 		/* + transport layer */
3046 		if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
3047 			hdr_len += tcp_hdrlen(skb);
3048 		else
3049 			hdr_len += sizeof(struct udphdr);
3050 
3051 		if (shinfo->gso_type & SKB_GSO_DODGY)
3052 			gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3053 						shinfo->gso_size);
3054 
3055 		qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3056 	}
3057 }
3058 
3059 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3060 				 struct net_device *dev,
3061 				 struct netdev_queue *txq)
3062 {
3063 	spinlock_t *root_lock = qdisc_lock(q);
3064 	struct sk_buff *to_free = NULL;
3065 	bool contended;
3066 	int rc;
3067 
3068 	qdisc_calculate_pkt_len(skb, q);
3069 	/*
3070 	 * Heuristic to force contended enqueues to serialize on a
3071 	 * separate lock before trying to get qdisc main lock.
3072 	 * This permits qdisc->running owner to get the lock more
3073 	 * often and dequeue packets faster.
3074 	 */
3075 	contended = qdisc_is_running(q);
3076 	if (unlikely(contended))
3077 		spin_lock(&q->busylock);
3078 
3079 	spin_lock(root_lock);
3080 	if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3081 		__qdisc_drop(skb, &to_free);
3082 		rc = NET_XMIT_DROP;
3083 	} else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3084 		   qdisc_run_begin(q)) {
3085 		/*
3086 		 * This is a work-conserving queue; there are no old skbs
3087 		 * waiting to be sent out; and the qdisc is not running -
3088 		 * xmit the skb directly.
3089 		 */
3090 
3091 		qdisc_bstats_update(q, skb);
3092 
3093 		if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3094 			if (unlikely(contended)) {
3095 				spin_unlock(&q->busylock);
3096 				contended = false;
3097 			}
3098 			__qdisc_run(q);
3099 		} else
3100 			qdisc_run_end(q);
3101 
3102 		rc = NET_XMIT_SUCCESS;
3103 	} else {
3104 		rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3105 		if (qdisc_run_begin(q)) {
3106 			if (unlikely(contended)) {
3107 				spin_unlock(&q->busylock);
3108 				contended = false;
3109 			}
3110 			__qdisc_run(q);
3111 		}
3112 	}
3113 	spin_unlock(root_lock);
3114 	if (unlikely(to_free))
3115 		kfree_skb_list(to_free);
3116 	if (unlikely(contended))
3117 		spin_unlock(&q->busylock);
3118 	return rc;
3119 }
3120 
3121 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3122 static void skb_update_prio(struct sk_buff *skb)
3123 {
3124 	struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
3125 
3126 	if (!skb->priority && skb->sk && map) {
3127 		unsigned int prioidx =
3128 			sock_cgroup_prioidx(&skb->sk->sk_cgrp_data);
3129 
3130 		if (prioidx < map->priomap_len)
3131 			skb->priority = map->priomap[prioidx];
3132 	}
3133 }
3134 #else
3135 #define skb_update_prio(skb)
3136 #endif
3137 
3138 DEFINE_PER_CPU(int, xmit_recursion);
3139 EXPORT_SYMBOL(xmit_recursion);
3140 
3141 /**
3142  *	dev_loopback_xmit - loop back @skb
3143  *	@net: network namespace this loopback is happening in
3144  *	@sk:  sk needed to be a netfilter okfn
3145  *	@skb: buffer to transmit
3146  */
3147 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3148 {
3149 	skb_reset_mac_header(skb);
3150 	__skb_pull(skb, skb_network_offset(skb));
3151 	skb->pkt_type = PACKET_LOOPBACK;
3152 	skb->ip_summed = CHECKSUM_UNNECESSARY;
3153 	WARN_ON(!skb_dst(skb));
3154 	skb_dst_force(skb);
3155 	netif_rx_ni(skb);
3156 	return 0;
3157 }
3158 EXPORT_SYMBOL(dev_loopback_xmit);
3159 
3160 #ifdef CONFIG_NET_EGRESS
3161 static struct sk_buff *
3162 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3163 {
3164 	struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3165 	struct tcf_result cl_res;
3166 
3167 	if (!cl)
3168 		return skb;
3169 
3170 	/* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3171 	qdisc_bstats_cpu_update(cl->q, skb);
3172 
3173 	switch (tc_classify(skb, cl, &cl_res, false)) {
3174 	case TC_ACT_OK:
3175 	case TC_ACT_RECLASSIFY:
3176 		skb->tc_index = TC_H_MIN(cl_res.classid);
3177 		break;
3178 	case TC_ACT_SHOT:
3179 		qdisc_qstats_cpu_drop(cl->q);
3180 		*ret = NET_XMIT_DROP;
3181 		kfree_skb(skb);
3182 		return NULL;
3183 	case TC_ACT_STOLEN:
3184 	case TC_ACT_QUEUED:
3185 		*ret = NET_XMIT_SUCCESS;
3186 		consume_skb(skb);
3187 		return NULL;
3188 	case TC_ACT_REDIRECT:
3189 		/* No need to push/pop skb's mac_header here on egress! */
3190 		skb_do_redirect(skb);
3191 		*ret = NET_XMIT_SUCCESS;
3192 		return NULL;
3193 	default:
3194 		break;
3195 	}
3196 
3197 	return skb;
3198 }
3199 #endif /* CONFIG_NET_EGRESS */
3200 
3201 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3202 {
3203 #ifdef CONFIG_XPS
3204 	struct xps_dev_maps *dev_maps;
3205 	struct xps_map *map;
3206 	int queue_index = -1;
3207 
3208 	rcu_read_lock();
3209 	dev_maps = rcu_dereference(dev->xps_maps);
3210 	if (dev_maps) {
3211 		unsigned int tci = skb->sender_cpu - 1;
3212 
3213 		if (dev->num_tc) {
3214 			tci *= dev->num_tc;
3215 			tci += netdev_get_prio_tc_map(dev, skb->priority);
3216 		}
3217 
3218 		map = rcu_dereference(dev_maps->cpu_map[tci]);
3219 		if (map) {
3220 			if (map->len == 1)
3221 				queue_index = map->queues[0];
3222 			else
3223 				queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3224 									   map->len)];
3225 			if (unlikely(queue_index >= dev->real_num_tx_queues))
3226 				queue_index = -1;
3227 		}
3228 	}
3229 	rcu_read_unlock();
3230 
3231 	return queue_index;
3232 #else
3233 	return -1;
3234 #endif
3235 }
3236 
3237 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3238 {
3239 	struct sock *sk = skb->sk;
3240 	int queue_index = sk_tx_queue_get(sk);
3241 
3242 	if (queue_index < 0 || skb->ooo_okay ||
3243 	    queue_index >= dev->real_num_tx_queues) {
3244 		int new_index = get_xps_queue(dev, skb);
3245 
3246 		if (new_index < 0)
3247 			new_index = skb_tx_hash(dev, skb);
3248 
3249 		if (queue_index != new_index && sk &&
3250 		    sk_fullsock(sk) &&
3251 		    rcu_access_pointer(sk->sk_dst_cache))
3252 			sk_tx_queue_set(sk, new_index);
3253 
3254 		queue_index = new_index;
3255 	}
3256 
3257 	return queue_index;
3258 }
3259 
3260 struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3261 				    struct sk_buff *skb,
3262 				    void *accel_priv)
3263 {
3264 	int queue_index = 0;
3265 
3266 #ifdef CONFIG_XPS
3267 	u32 sender_cpu = skb->sender_cpu - 1;
3268 
3269 	if (sender_cpu >= (u32)NR_CPUS)
3270 		skb->sender_cpu = raw_smp_processor_id() + 1;
3271 #endif
3272 
3273 	if (dev->real_num_tx_queues != 1) {
3274 		const struct net_device_ops *ops = dev->netdev_ops;
3275 
3276 		if (ops->ndo_select_queue)
3277 			queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3278 							    __netdev_pick_tx);
3279 		else
3280 			queue_index = __netdev_pick_tx(dev, skb);
3281 
3282 		if (!accel_priv)
3283 			queue_index = netdev_cap_txqueue(dev, queue_index);
3284 	}
3285 
3286 	skb_set_queue_mapping(skb, queue_index);
3287 	return netdev_get_tx_queue(dev, queue_index);
3288 }
3289 
3290 /**
3291  *	__dev_queue_xmit - transmit a buffer
3292  *	@skb: buffer to transmit
3293  *	@accel_priv: private data used for L2 forwarding offload
3294  *
3295  *	Queue a buffer for transmission to a network device. The caller must
3296  *	have set the device and priority and built the buffer before calling
3297  *	this function. The function can be called from an interrupt.
3298  *
3299  *	A negative errno code is returned on a failure. A success does not
3300  *	guarantee the frame will be transmitted as it may be dropped due
3301  *	to congestion or traffic shaping.
3302  *
3303  * -----------------------------------------------------------------------------------
3304  *      I notice this method can also return errors from the queue disciplines,
3305  *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
3306  *      be positive.
3307  *
3308  *      Regardless of the return value, the skb is consumed, so it is currently
3309  *      difficult to retry a send to this method.  (You can bump the ref count
3310  *      before sending to hold a reference for retry if you are careful.)
3311  *
3312  *      When calling this method, interrupts MUST be enabled.  This is because
3313  *      the BH enable code must have IRQs enabled so that it will not deadlock.
3314  *          --BLG
3315  */
3316 static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
3317 {
3318 	struct net_device *dev = skb->dev;
3319 	struct netdev_queue *txq;
3320 	struct Qdisc *q;
3321 	int rc = -ENOMEM;
3322 
3323 	skb_reset_mac_header(skb);
3324 
3325 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3326 		__skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3327 
3328 	/* Disable soft irqs for various locks below. Also
3329 	 * stops preemption for RCU.
3330 	 */
3331 	rcu_read_lock_bh();
3332 
3333 	skb_update_prio(skb);
3334 
3335 	qdisc_pkt_len_init(skb);
3336 #ifdef CONFIG_NET_CLS_ACT
3337 	skb->tc_at_ingress = 0;
3338 # ifdef CONFIG_NET_EGRESS
3339 	if (static_key_false(&egress_needed)) {
3340 		skb = sch_handle_egress(skb, &rc, dev);
3341 		if (!skb)
3342 			goto out;
3343 	}
3344 # endif
3345 #endif
3346 	/* If device/qdisc don't need skb->dst, release it right now while
3347 	 * its hot in this cpu cache.
3348 	 */
3349 	if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3350 		skb_dst_drop(skb);
3351 	else
3352 		skb_dst_force(skb);
3353 
3354 	txq = netdev_pick_tx(dev, skb, accel_priv);
3355 	q = rcu_dereference_bh(txq->qdisc);
3356 
3357 	trace_net_dev_queue(skb);
3358 	if (q->enqueue) {
3359 		rc = __dev_xmit_skb(skb, q, dev, txq);
3360 		goto out;
3361 	}
3362 
3363 	/* The device has no queue. Common case for software devices:
3364 	 * loopback, all the sorts of tunnels...
3365 
3366 	 * Really, it is unlikely that netif_tx_lock protection is necessary
3367 	 * here.  (f.e. loopback and IP tunnels are clean ignoring statistics
3368 	 * counters.)
3369 	 * However, it is possible, that they rely on protection
3370 	 * made by us here.
3371 
3372 	 * Check this and shot the lock. It is not prone from deadlocks.
3373 	 *Either shot noqueue qdisc, it is even simpler 8)
3374 	 */
3375 	if (dev->flags & IFF_UP) {
3376 		int cpu = smp_processor_id(); /* ok because BHs are off */
3377 
3378 		if (txq->xmit_lock_owner != cpu) {
3379 			if (unlikely(__this_cpu_read(xmit_recursion) >
3380 				     XMIT_RECURSION_LIMIT))
3381 				goto recursion_alert;
3382 
3383 			skb = validate_xmit_skb(skb, dev);
3384 			if (!skb)
3385 				goto out;
3386 
3387 			HARD_TX_LOCK(dev, txq, cpu);
3388 
3389 			if (!netif_xmit_stopped(txq)) {
3390 				__this_cpu_inc(xmit_recursion);
3391 				skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3392 				__this_cpu_dec(xmit_recursion);
3393 				if (dev_xmit_complete(rc)) {
3394 					HARD_TX_UNLOCK(dev, txq);
3395 					goto out;
3396 				}
3397 			}
3398 			HARD_TX_UNLOCK(dev, txq);
3399 			net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3400 					     dev->name);
3401 		} else {
3402 			/* Recursion is detected! It is possible,
3403 			 * unfortunately
3404 			 */
3405 recursion_alert:
3406 			net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3407 					     dev->name);
3408 		}
3409 	}
3410 
3411 	rc = -ENETDOWN;
3412 	rcu_read_unlock_bh();
3413 
3414 	atomic_long_inc(&dev->tx_dropped);
3415 	kfree_skb_list(skb);
3416 	return rc;
3417 out:
3418 	rcu_read_unlock_bh();
3419 	return rc;
3420 }
3421 
3422 int dev_queue_xmit(struct sk_buff *skb)
3423 {
3424 	return __dev_queue_xmit(skb, NULL);
3425 }
3426 EXPORT_SYMBOL(dev_queue_xmit);
3427 
3428 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3429 {
3430 	return __dev_queue_xmit(skb, accel_priv);
3431 }
3432 EXPORT_SYMBOL(dev_queue_xmit_accel);
3433 
3434 
3435 /*************************************************************************
3436  *			Receiver routines
3437  *************************************************************************/
3438 
3439 int netdev_max_backlog __read_mostly = 1000;
3440 EXPORT_SYMBOL(netdev_max_backlog);
3441 
3442 int netdev_tstamp_prequeue __read_mostly = 1;
3443 int netdev_budget __read_mostly = 300;
3444 int weight_p __read_mostly = 64;           /* old backlog weight */
3445 int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
3446 int dev_weight_tx_bias __read_mostly = 1;  /* bias for output_queue quota */
3447 int dev_rx_weight __read_mostly = 64;
3448 int dev_tx_weight __read_mostly = 64;
3449 
3450 /* Called with irq disabled */
3451 static inline void ____napi_schedule(struct softnet_data *sd,
3452 				     struct napi_struct *napi)
3453 {
3454 	list_add_tail(&napi->poll_list, &sd->poll_list);
3455 	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
3456 }
3457 
3458 #ifdef CONFIG_RPS
3459 
3460 /* One global table that all flow-based protocols share. */
3461 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3462 EXPORT_SYMBOL(rps_sock_flow_table);
3463 u32 rps_cpu_mask __read_mostly;
3464 EXPORT_SYMBOL(rps_cpu_mask);
3465 
3466 struct static_key rps_needed __read_mostly;
3467 EXPORT_SYMBOL(rps_needed);
3468 struct static_key rfs_needed __read_mostly;
3469 EXPORT_SYMBOL(rfs_needed);
3470 
3471 static struct rps_dev_flow *
3472 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3473 	    struct rps_dev_flow *rflow, u16 next_cpu)
3474 {
3475 	if (next_cpu < nr_cpu_ids) {
3476 #ifdef CONFIG_RFS_ACCEL
3477 		struct netdev_rx_queue *rxqueue;
3478 		struct rps_dev_flow_table *flow_table;
3479 		struct rps_dev_flow *old_rflow;
3480 		u32 flow_id;
3481 		u16 rxq_index;
3482 		int rc;
3483 
3484 		/* Should we steer this flow to a different hardware queue? */
3485 		if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3486 		    !(dev->features & NETIF_F_NTUPLE))
3487 			goto out;
3488 		rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3489 		if (rxq_index == skb_get_rx_queue(skb))
3490 			goto out;
3491 
3492 		rxqueue = dev->_rx + rxq_index;
3493 		flow_table = rcu_dereference(rxqueue->rps_flow_table);
3494 		if (!flow_table)
3495 			goto out;
3496 		flow_id = skb_get_hash(skb) & flow_table->mask;
3497 		rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3498 							rxq_index, flow_id);
3499 		if (rc < 0)
3500 			goto out;
3501 		old_rflow = rflow;
3502 		rflow = &flow_table->flows[flow_id];
3503 		rflow->filter = rc;
3504 		if (old_rflow->filter == rflow->filter)
3505 			old_rflow->filter = RPS_NO_FILTER;
3506 	out:
3507 #endif
3508 		rflow->last_qtail =
3509 			per_cpu(softnet_data, next_cpu).input_queue_head;
3510 	}
3511 
3512 	rflow->cpu = next_cpu;
3513 	return rflow;
3514 }
3515 
3516 /*
3517  * get_rps_cpu is called from netif_receive_skb and returns the target
3518  * CPU from the RPS map of the receiving queue for a given skb.
3519  * rcu_read_lock must be held on entry.
3520  */
3521 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3522 		       struct rps_dev_flow **rflowp)
3523 {
3524 	const struct rps_sock_flow_table *sock_flow_table;
3525 	struct netdev_rx_queue *rxqueue = dev->_rx;
3526 	struct rps_dev_flow_table *flow_table;
3527 	struct rps_map *map;
3528 	int cpu = -1;
3529 	u32 tcpu;
3530 	u32 hash;
3531 
3532 	if (skb_rx_queue_recorded(skb)) {
3533 		u16 index = skb_get_rx_queue(skb);
3534 
3535 		if (unlikely(index >= dev->real_num_rx_queues)) {
3536 			WARN_ONCE(dev->real_num_rx_queues > 1,
3537 				  "%s received packet on queue %u, but number "
3538 				  "of RX queues is %u\n",
3539 				  dev->name, index, dev->real_num_rx_queues);
3540 			goto done;
3541 		}
3542 		rxqueue += index;
3543 	}
3544 
3545 	/* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3546 
3547 	flow_table = rcu_dereference(rxqueue->rps_flow_table);
3548 	map = rcu_dereference(rxqueue->rps_map);
3549 	if (!flow_table && !map)
3550 		goto done;
3551 
3552 	skb_reset_network_header(skb);
3553 	hash = skb_get_hash(skb);
3554 	if (!hash)
3555 		goto done;
3556 
3557 	sock_flow_table = rcu_dereference(rps_sock_flow_table);
3558 	if (flow_table && sock_flow_table) {
3559 		struct rps_dev_flow *rflow;
3560 		u32 next_cpu;
3561 		u32 ident;
3562 
3563 		/* First check into global flow table if there is a match */
3564 		ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3565 		if ((ident ^ hash) & ~rps_cpu_mask)
3566 			goto try_rps;
3567 
3568 		next_cpu = ident & rps_cpu_mask;
3569 
3570 		/* OK, now we know there is a match,
3571 		 * we can look at the local (per receive queue) flow table
3572 		 */
3573 		rflow = &flow_table->flows[hash & flow_table->mask];
3574 		tcpu = rflow->cpu;
3575 
3576 		/*
3577 		 * If the desired CPU (where last recvmsg was done) is
3578 		 * different from current CPU (one in the rx-queue flow
3579 		 * table entry), switch if one of the following holds:
3580 		 *   - Current CPU is unset (>= nr_cpu_ids).
3581 		 *   - Current CPU is offline.
3582 		 *   - The current CPU's queue tail has advanced beyond the
3583 		 *     last packet that was enqueued using this table entry.
3584 		 *     This guarantees that all previous packets for the flow
3585 		 *     have been dequeued, thus preserving in order delivery.
3586 		 */
3587 		if (unlikely(tcpu != next_cpu) &&
3588 		    (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
3589 		     ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3590 		      rflow->last_qtail)) >= 0)) {
3591 			tcpu = next_cpu;
3592 			rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3593 		}
3594 
3595 		if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
3596 			*rflowp = rflow;
3597 			cpu = tcpu;
3598 			goto done;
3599 		}
3600 	}
3601 
3602 try_rps:
3603 
3604 	if (map) {
3605 		tcpu = map->cpus[reciprocal_scale(hash, map->len)];
3606 		if (cpu_online(tcpu)) {
3607 			cpu = tcpu;
3608 			goto done;
3609 		}
3610 	}
3611 
3612 done:
3613 	return cpu;
3614 }
3615 
3616 #ifdef CONFIG_RFS_ACCEL
3617 
3618 /**
3619  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3620  * @dev: Device on which the filter was set
3621  * @rxq_index: RX queue index
3622  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3623  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3624  *
3625  * Drivers that implement ndo_rx_flow_steer() should periodically call
3626  * this function for each installed filter and remove the filters for
3627  * which it returns %true.
3628  */
3629 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3630 			 u32 flow_id, u16 filter_id)
3631 {
3632 	struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3633 	struct rps_dev_flow_table *flow_table;
3634 	struct rps_dev_flow *rflow;
3635 	bool expire = true;
3636 	unsigned int cpu;
3637 
3638 	rcu_read_lock();
3639 	flow_table = rcu_dereference(rxqueue->rps_flow_table);
3640 	if (flow_table && flow_id <= flow_table->mask) {
3641 		rflow = &flow_table->flows[flow_id];
3642 		cpu = ACCESS_ONCE(rflow->cpu);
3643 		if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
3644 		    ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3645 			   rflow->last_qtail) <
3646 		     (int)(10 * flow_table->mask)))
3647 			expire = false;
3648 	}
3649 	rcu_read_unlock();
3650 	return expire;
3651 }
3652 EXPORT_SYMBOL(rps_may_expire_flow);
3653 
3654 #endif /* CONFIG_RFS_ACCEL */
3655 
3656 /* Called from hardirq (IPI) context */
3657 static void rps_trigger_softirq(void *data)
3658 {
3659 	struct softnet_data *sd = data;
3660 
3661 	____napi_schedule(sd, &sd->backlog);
3662 	sd->received_rps++;
3663 }
3664 
3665 #endif /* CONFIG_RPS */
3666 
3667 /*
3668  * Check if this softnet_data structure is another cpu one
3669  * If yes, queue it to our IPI list and return 1
3670  * If no, return 0
3671  */
3672 static int rps_ipi_queued(struct softnet_data *sd)
3673 {
3674 #ifdef CONFIG_RPS
3675 	struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
3676 
3677 	if (sd != mysd) {
3678 		sd->rps_ipi_next = mysd->rps_ipi_list;
3679 		mysd->rps_ipi_list = sd;
3680 
3681 		__raise_softirq_irqoff(NET_RX_SOFTIRQ);
3682 		return 1;
3683 	}
3684 #endif /* CONFIG_RPS */
3685 	return 0;
3686 }
3687 
3688 #ifdef CONFIG_NET_FLOW_LIMIT
3689 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3690 #endif
3691 
3692 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3693 {
3694 #ifdef CONFIG_NET_FLOW_LIMIT
3695 	struct sd_flow_limit *fl;
3696 	struct softnet_data *sd;
3697 	unsigned int old_flow, new_flow;
3698 
3699 	if (qlen < (netdev_max_backlog >> 1))
3700 		return false;
3701 
3702 	sd = this_cpu_ptr(&softnet_data);
3703 
3704 	rcu_read_lock();
3705 	fl = rcu_dereference(sd->flow_limit);
3706 	if (fl) {
3707 		new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3708 		old_flow = fl->history[fl->history_head];
3709 		fl->history[fl->history_head] = new_flow;
3710 
3711 		fl->history_head++;
3712 		fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3713 
3714 		if (likely(fl->buckets[old_flow]))
3715 			fl->buckets[old_flow]--;
3716 
3717 		if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3718 			fl->count++;
3719 			rcu_read_unlock();
3720 			return true;
3721 		}
3722 	}
3723 	rcu_read_unlock();
3724 #endif
3725 	return false;
3726 }
3727 
3728 /*
3729  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3730  * queue (may be a remote CPU queue).
3731  */
3732 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3733 			      unsigned int *qtail)
3734 {
3735 	struct softnet_data *sd;
3736 	unsigned long flags;
3737 	unsigned int qlen;
3738 
3739 	sd = &per_cpu(softnet_data, cpu);
3740 
3741 	local_irq_save(flags);
3742 
3743 	rps_lock(sd);
3744 	if (!netif_running(skb->dev))
3745 		goto drop;
3746 	qlen = skb_queue_len(&sd->input_pkt_queue);
3747 	if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3748 		if (qlen) {
3749 enqueue:
3750 			__skb_queue_tail(&sd->input_pkt_queue, skb);
3751 			input_queue_tail_incr_save(sd, qtail);
3752 			rps_unlock(sd);
3753 			local_irq_restore(flags);
3754 			return NET_RX_SUCCESS;
3755 		}
3756 
3757 		/* Schedule NAPI for backlog device
3758 		 * We can use non atomic operation since we own the queue lock
3759 		 */
3760 		if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3761 			if (!rps_ipi_queued(sd))
3762 				____napi_schedule(sd, &sd->backlog);
3763 		}
3764 		goto enqueue;
3765 	}
3766 
3767 drop:
3768 	sd->dropped++;
3769 	rps_unlock(sd);
3770 
3771 	local_irq_restore(flags);
3772 
3773 	atomic_long_inc(&skb->dev->rx_dropped);
3774 	kfree_skb(skb);
3775 	return NET_RX_DROP;
3776 }
3777 
3778 static int netif_rx_internal(struct sk_buff *skb)
3779 {
3780 	int ret;
3781 
3782 	net_timestamp_check(netdev_tstamp_prequeue, skb);
3783 
3784 	trace_netif_rx(skb);
3785 #ifdef CONFIG_RPS
3786 	if (static_key_false(&rps_needed)) {
3787 		struct rps_dev_flow voidflow, *rflow = &voidflow;
3788 		int cpu;
3789 
3790 		preempt_disable();
3791 		rcu_read_lock();
3792 
3793 		cpu = get_rps_cpu(skb->dev, skb, &rflow);
3794 		if (cpu < 0)
3795 			cpu = smp_processor_id();
3796 
3797 		ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3798 
3799 		rcu_read_unlock();
3800 		preempt_enable();
3801 	} else
3802 #endif
3803 	{
3804 		unsigned int qtail;
3805 
3806 		ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3807 		put_cpu();
3808 	}
3809 	return ret;
3810 }
3811 
3812 /**
3813  *	netif_rx	-	post buffer to the network code
3814  *	@skb: buffer to post
3815  *
3816  *	This function receives a packet from a device driver and queues it for
3817  *	the upper (protocol) levels to process.  It always succeeds. The buffer
3818  *	may be dropped during processing for congestion control or by the
3819  *	protocol layers.
3820  *
3821  *	return values:
3822  *	NET_RX_SUCCESS	(no congestion)
3823  *	NET_RX_DROP     (packet was dropped)
3824  *
3825  */
3826 
3827 int netif_rx(struct sk_buff *skb)
3828 {
3829 	trace_netif_rx_entry(skb);
3830 
3831 	return netif_rx_internal(skb);
3832 }
3833 EXPORT_SYMBOL(netif_rx);
3834 
3835 int netif_rx_ni(struct sk_buff *skb)
3836 {
3837 	int err;
3838 
3839 	trace_netif_rx_ni_entry(skb);
3840 
3841 	preempt_disable();
3842 	err = netif_rx_internal(skb);
3843 	if (local_softirq_pending())
3844 		do_softirq();
3845 	preempt_enable();
3846 
3847 	return err;
3848 }
3849 EXPORT_SYMBOL(netif_rx_ni);
3850 
3851 static __latent_entropy void net_tx_action(struct softirq_action *h)
3852 {
3853 	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
3854 
3855 	if (sd->completion_queue) {
3856 		struct sk_buff *clist;
3857 
3858 		local_irq_disable();
3859 		clist = sd->completion_queue;
3860 		sd->completion_queue = NULL;
3861 		local_irq_enable();
3862 
3863 		while (clist) {
3864 			struct sk_buff *skb = clist;
3865 
3866 			clist = clist->next;
3867 
3868 			WARN_ON(atomic_read(&skb->users));
3869 			if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3870 				trace_consume_skb(skb);
3871 			else
3872 				trace_kfree_skb(skb, net_tx_action);
3873 
3874 			if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
3875 				__kfree_skb(skb);
3876 			else
3877 				__kfree_skb_defer(skb);
3878 		}
3879 
3880 		__kfree_skb_flush();
3881 	}
3882 
3883 	if (sd->output_queue) {
3884 		struct Qdisc *head;
3885 
3886 		local_irq_disable();
3887 		head = sd->output_queue;
3888 		sd->output_queue = NULL;
3889 		sd->output_queue_tailp = &sd->output_queue;
3890 		local_irq_enable();
3891 
3892 		while (head) {
3893 			struct Qdisc *q = head;
3894 			spinlock_t *root_lock;
3895 
3896 			head = head->next_sched;
3897 
3898 			root_lock = qdisc_lock(q);
3899 			spin_lock(root_lock);
3900 			/* We need to make sure head->next_sched is read
3901 			 * before clearing __QDISC_STATE_SCHED
3902 			 */
3903 			smp_mb__before_atomic();
3904 			clear_bit(__QDISC_STATE_SCHED, &q->state);
3905 			qdisc_run(q);
3906 			spin_unlock(root_lock);
3907 		}
3908 	}
3909 }
3910 
3911 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
3912 /* This hook is defined here for ATM LANE */
3913 int (*br_fdb_test_addr_hook)(struct net_device *dev,
3914 			     unsigned char *addr) __read_mostly;
3915 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
3916 #endif
3917 
3918 static inline struct sk_buff *
3919 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
3920 		   struct net_device *orig_dev)
3921 {
3922 #ifdef CONFIG_NET_CLS_ACT
3923 	struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3924 	struct tcf_result cl_res;
3925 
3926 	/* If there's at least one ingress present somewhere (so
3927 	 * we get here via enabled static key), remaining devices
3928 	 * that are not configured with an ingress qdisc will bail
3929 	 * out here.
3930 	 */
3931 	if (!cl)
3932 		return skb;
3933 	if (*pt_prev) {
3934 		*ret = deliver_skb(skb, *pt_prev, orig_dev);
3935 		*pt_prev = NULL;
3936 	}
3937 
3938 	qdisc_skb_cb(skb)->pkt_len = skb->len;
3939 	skb->tc_at_ingress = 1;
3940 	qdisc_bstats_cpu_update(cl->q, skb);
3941 
3942 	switch (tc_classify(skb, cl, &cl_res, false)) {
3943 	case TC_ACT_OK:
3944 	case TC_ACT_RECLASSIFY:
3945 		skb->tc_index = TC_H_MIN(cl_res.classid);
3946 		break;
3947 	case TC_ACT_SHOT:
3948 		qdisc_qstats_cpu_drop(cl->q);
3949 		kfree_skb(skb);
3950 		return NULL;
3951 	case TC_ACT_STOLEN:
3952 	case TC_ACT_QUEUED:
3953 		consume_skb(skb);
3954 		return NULL;
3955 	case TC_ACT_REDIRECT:
3956 		/* skb_mac_header check was done by cls/act_bpf, so
3957 		 * we can safely push the L2 header back before
3958 		 * redirecting to another netdev
3959 		 */
3960 		__skb_push(skb, skb->mac_len);
3961 		skb_do_redirect(skb);
3962 		return NULL;
3963 	default:
3964 		break;
3965 	}
3966 #endif /* CONFIG_NET_CLS_ACT */
3967 	return skb;
3968 }
3969 
3970 /**
3971  *	netdev_is_rx_handler_busy - check if receive handler is registered
3972  *	@dev: device to check
3973  *
3974  *	Check if a receive handler is already registered for a given device.
3975  *	Return true if there one.
3976  *
3977  *	The caller must hold the rtnl_mutex.
3978  */
3979 bool netdev_is_rx_handler_busy(struct net_device *dev)
3980 {
3981 	ASSERT_RTNL();
3982 	return dev && rtnl_dereference(dev->rx_handler);
3983 }
3984 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
3985 
3986 /**
3987  *	netdev_rx_handler_register - register receive handler
3988  *	@dev: device to register a handler for
3989  *	@rx_handler: receive handler to register
3990  *	@rx_handler_data: data pointer that is used by rx handler
3991  *
3992  *	Register a receive handler for a device. This handler will then be
3993  *	called from __netif_receive_skb. A negative errno code is returned
3994  *	on a failure.
3995  *
3996  *	The caller must hold the rtnl_mutex.
3997  *
3998  *	For a general description of rx_handler, see enum rx_handler_result.
3999  */
4000 int netdev_rx_handler_register(struct net_device *dev,
4001 			       rx_handler_func_t *rx_handler,
4002 			       void *rx_handler_data)
4003 {
4004 	if (netdev_is_rx_handler_busy(dev))
4005 		return -EBUSY;
4006 
4007 	/* Note: rx_handler_data must be set before rx_handler */
4008 	rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
4009 	rcu_assign_pointer(dev->rx_handler, rx_handler);
4010 
4011 	return 0;
4012 }
4013 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
4014 
4015 /**
4016  *	netdev_rx_handler_unregister - unregister receive handler
4017  *	@dev: device to unregister a handler from
4018  *
4019  *	Unregister a receive handler from a device.
4020  *
4021  *	The caller must hold the rtnl_mutex.
4022  */
4023 void netdev_rx_handler_unregister(struct net_device *dev)
4024 {
4025 
4026 	ASSERT_RTNL();
4027 	RCU_INIT_POINTER(dev->rx_handler, NULL);
4028 	/* a reader seeing a non NULL rx_handler in a rcu_read_lock()
4029 	 * section has a guarantee to see a non NULL rx_handler_data
4030 	 * as well.
4031 	 */
4032 	synchronize_net();
4033 	RCU_INIT_POINTER(dev->rx_handler_data, NULL);
4034 }
4035 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
4036 
4037 /*
4038  * Limit the use of PFMEMALLOC reserves to those protocols that implement
4039  * the special handling of PFMEMALLOC skbs.
4040  */
4041 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
4042 {
4043 	switch (skb->protocol) {
4044 	case htons(ETH_P_ARP):
4045 	case htons(ETH_P_IP):
4046 	case htons(ETH_P_IPV6):
4047 	case htons(ETH_P_8021Q):
4048 	case htons(ETH_P_8021AD):
4049 		return true;
4050 	default:
4051 		return false;
4052 	}
4053 }
4054 
4055 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4056 			     int *ret, struct net_device *orig_dev)
4057 {
4058 #ifdef CONFIG_NETFILTER_INGRESS
4059 	if (nf_hook_ingress_active(skb)) {
4060 		int ingress_retval;
4061 
4062 		if (*pt_prev) {
4063 			*ret = deliver_skb(skb, *pt_prev, orig_dev);
4064 			*pt_prev = NULL;
4065 		}
4066 
4067 		rcu_read_lock();
4068 		ingress_retval = nf_hook_ingress(skb);
4069 		rcu_read_unlock();
4070 		return ingress_retval;
4071 	}
4072 #endif /* CONFIG_NETFILTER_INGRESS */
4073 	return 0;
4074 }
4075 
4076 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
4077 {
4078 	struct packet_type *ptype, *pt_prev;
4079 	rx_handler_func_t *rx_handler;
4080 	struct net_device *orig_dev;
4081 	bool deliver_exact = false;
4082 	int ret = NET_RX_DROP;
4083 	__be16 type;
4084 
4085 	net_timestamp_check(!netdev_tstamp_prequeue, skb);
4086 
4087 	trace_netif_receive_skb(skb);
4088 
4089 	orig_dev = skb->dev;
4090 
4091 	skb_reset_network_header(skb);
4092 	if (!skb_transport_header_was_set(skb))
4093 		skb_reset_transport_header(skb);
4094 	skb_reset_mac_len(skb);
4095 
4096 	pt_prev = NULL;
4097 
4098 another_round:
4099 	skb->skb_iif = skb->dev->ifindex;
4100 
4101 	__this_cpu_inc(softnet_data.processed);
4102 
4103 	if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4104 	    skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
4105 		skb = skb_vlan_untag(skb);
4106 		if (unlikely(!skb))
4107 			goto out;
4108 	}
4109 
4110 	if (skb_skip_tc_classify(skb))
4111 		goto skip_classify;
4112 
4113 	if (pfmemalloc)
4114 		goto skip_taps;
4115 
4116 	list_for_each_entry_rcu(ptype, &ptype_all, list) {
4117 		if (pt_prev)
4118 			ret = deliver_skb(skb, pt_prev, orig_dev);
4119 		pt_prev = ptype;
4120 	}
4121 
4122 	list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4123 		if (pt_prev)
4124 			ret = deliver_skb(skb, pt_prev, orig_dev);
4125 		pt_prev = ptype;
4126 	}
4127 
4128 skip_taps:
4129 #ifdef CONFIG_NET_INGRESS
4130 	if (static_key_false(&ingress_needed)) {
4131 		skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
4132 		if (!skb)
4133 			goto out;
4134 
4135 		if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
4136 			goto out;
4137 	}
4138 #endif
4139 	skb_reset_tc(skb);
4140 skip_classify:
4141 	if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
4142 		goto drop;
4143 
4144 	if (skb_vlan_tag_present(skb)) {
4145 		if (pt_prev) {
4146 			ret = deliver_skb(skb, pt_prev, orig_dev);
4147 			pt_prev = NULL;
4148 		}
4149 		if (vlan_do_receive(&skb))
4150 			goto another_round;
4151 		else if (unlikely(!skb))
4152 			goto out;
4153 	}
4154 
4155 	rx_handler = rcu_dereference(skb->dev->rx_handler);
4156 	if (rx_handler) {
4157 		if (pt_prev) {
4158 			ret = deliver_skb(skb, pt_prev, orig_dev);
4159 			pt_prev = NULL;
4160 		}
4161 		switch (rx_handler(&skb)) {
4162 		case RX_HANDLER_CONSUMED:
4163 			ret = NET_RX_SUCCESS;
4164 			goto out;
4165 		case RX_HANDLER_ANOTHER:
4166 			goto another_round;
4167 		case RX_HANDLER_EXACT:
4168 			deliver_exact = true;
4169 		case RX_HANDLER_PASS:
4170 			break;
4171 		default:
4172 			BUG();
4173 		}
4174 	}
4175 
4176 	if (unlikely(skb_vlan_tag_present(skb))) {
4177 		if (skb_vlan_tag_get_id(skb))
4178 			skb->pkt_type = PACKET_OTHERHOST;
4179 		/* Note: we might in the future use prio bits
4180 		 * and set skb->priority like in vlan_do_receive()
4181 		 * For the time being, just ignore Priority Code Point
4182 		 */
4183 		skb->vlan_tci = 0;
4184 	}
4185 
4186 	type = skb->protocol;
4187 
4188 	/* deliver only exact match when indicated */
4189 	if (likely(!deliver_exact)) {
4190 		deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4191 				       &ptype_base[ntohs(type) &
4192 						   PTYPE_HASH_MASK]);
4193 	}
4194 
4195 	deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4196 			       &orig_dev->ptype_specific);
4197 
4198 	if (unlikely(skb->dev != orig_dev)) {
4199 		deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4200 				       &skb->dev->ptype_specific);
4201 	}
4202 
4203 	if (pt_prev) {
4204 		if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
4205 			goto drop;
4206 		else
4207 			ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
4208 	} else {
4209 drop:
4210 		if (!deliver_exact)
4211 			atomic_long_inc(&skb->dev->rx_dropped);
4212 		else
4213 			atomic_long_inc(&skb->dev->rx_nohandler);
4214 		kfree_skb(skb);
4215 		/* Jamal, now you will not able to escape explaining
4216 		 * me how you were going to use this. :-)
4217 		 */
4218 		ret = NET_RX_DROP;
4219 	}
4220 
4221 out:
4222 	return ret;
4223 }
4224 
4225 static int __netif_receive_skb(struct sk_buff *skb)
4226 {
4227 	int ret;
4228 
4229 	if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
4230 		unsigned long pflags = current->flags;
4231 
4232 		/*
4233 		 * PFMEMALLOC skbs are special, they should
4234 		 * - be delivered to SOCK_MEMALLOC sockets only
4235 		 * - stay away from userspace
4236 		 * - have bounded memory usage
4237 		 *
4238 		 * Use PF_MEMALLOC as this saves us from propagating the allocation
4239 		 * context down to all allocation sites.
4240 		 */
4241 		current->flags |= PF_MEMALLOC;
4242 		ret = __netif_receive_skb_core(skb, true);
4243 		tsk_restore_flags(current, pflags, PF_MEMALLOC);
4244 	} else
4245 		ret = __netif_receive_skb_core(skb, false);
4246 
4247 	return ret;
4248 }
4249 
4250 static int netif_receive_skb_internal(struct sk_buff *skb)
4251 {
4252 	int ret;
4253 
4254 	net_timestamp_check(netdev_tstamp_prequeue, skb);
4255 
4256 	if (skb_defer_rx_timestamp(skb))
4257 		return NET_RX_SUCCESS;
4258 
4259 	rcu_read_lock();
4260 
4261 #ifdef CONFIG_RPS
4262 	if (static_key_false(&rps_needed)) {
4263 		struct rps_dev_flow voidflow, *rflow = &voidflow;
4264 		int cpu = get_rps_cpu(skb->dev, skb, &rflow);
4265 
4266 		if (cpu >= 0) {
4267 			ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4268 			rcu_read_unlock();
4269 			return ret;
4270 		}
4271 	}
4272 #endif
4273 	ret = __netif_receive_skb(skb);
4274 	rcu_read_unlock();
4275 	return ret;
4276 }
4277 
4278 /**
4279  *	netif_receive_skb - process receive buffer from network
4280  *	@skb: buffer to process
4281  *
4282  *	netif_receive_skb() is the main receive data processing function.
4283  *	It always succeeds. The buffer may be dropped during processing
4284  *	for congestion control or by the protocol layers.
4285  *
4286  *	This function may only be called from softirq context and interrupts
4287  *	should be enabled.
4288  *
4289  *	Return values (usually ignored):
4290  *	NET_RX_SUCCESS: no congestion
4291  *	NET_RX_DROP: packet was dropped
4292  */
4293 int netif_receive_skb(struct sk_buff *skb)
4294 {
4295 	trace_netif_receive_skb_entry(skb);
4296 
4297 	return netif_receive_skb_internal(skb);
4298 }
4299 EXPORT_SYMBOL(netif_receive_skb);
4300 
4301 DEFINE_PER_CPU(struct work_struct, flush_works);
4302 
4303 /* Network device is going away, flush any packets still pending */
4304 static void flush_backlog(struct work_struct *work)
4305 {
4306 	struct sk_buff *skb, *tmp;
4307 	struct softnet_data *sd;
4308 
4309 	local_bh_disable();
4310 	sd = this_cpu_ptr(&softnet_data);
4311 
4312 	local_irq_disable();
4313 	rps_lock(sd);
4314 	skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
4315 		if (skb->dev->reg_state == NETREG_UNREGISTERING) {
4316 			__skb_unlink(skb, &sd->input_pkt_queue);
4317 			kfree_skb(skb);
4318 			input_queue_head_incr(sd);
4319 		}
4320 	}
4321 	rps_unlock(sd);
4322 	local_irq_enable();
4323 
4324 	skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4325 		if (skb->dev->reg_state == NETREG_UNREGISTERING) {
4326 			__skb_unlink(skb, &sd->process_queue);
4327 			kfree_skb(skb);
4328 			input_queue_head_incr(sd);
4329 		}
4330 	}
4331 	local_bh_enable();
4332 }
4333 
4334 static void flush_all_backlogs(void)
4335 {
4336 	unsigned int cpu;
4337 
4338 	get_online_cpus();
4339 
4340 	for_each_online_cpu(cpu)
4341 		queue_work_on(cpu, system_highpri_wq,
4342 			      per_cpu_ptr(&flush_works, cpu));
4343 
4344 	for_each_online_cpu(cpu)
4345 		flush_work(per_cpu_ptr(&flush_works, cpu));
4346 
4347 	put_online_cpus();
4348 }
4349 
4350 static int napi_gro_complete(struct sk_buff *skb)
4351 {
4352 	struct packet_offload *ptype;
4353 	__be16 type = skb->protocol;
4354 	struct list_head *head = &offload_base;
4355 	int err = -ENOENT;
4356 
4357 	BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4358 
4359 	if (NAPI_GRO_CB(skb)->count == 1) {
4360 		skb_shinfo(skb)->gso_size = 0;
4361 		goto out;
4362 	}
4363 
4364 	rcu_read_lock();
4365 	list_for_each_entry_rcu(ptype, head, list) {
4366 		if (ptype->type != type || !ptype->callbacks.gro_complete)
4367 			continue;
4368 
4369 		err = ptype->callbacks.gro_complete(skb, 0);
4370 		break;
4371 	}
4372 	rcu_read_unlock();
4373 
4374 	if (err) {
4375 		WARN_ON(&ptype->list == head);
4376 		kfree_skb(skb);
4377 		return NET_RX_SUCCESS;
4378 	}
4379 
4380 out:
4381 	return netif_receive_skb_internal(skb);
4382 }
4383 
4384 /* napi->gro_list contains packets ordered by age.
4385  * youngest packets at the head of it.
4386  * Complete skbs in reverse order to reduce latencies.
4387  */
4388 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
4389 {
4390 	struct sk_buff *skb, *prev = NULL;
4391 
4392 	/* scan list and build reverse chain */
4393 	for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4394 		skb->prev = prev;
4395 		prev = skb;
4396 	}
4397 
4398 	for (skb = prev; skb; skb = prev) {
4399 		skb->next = NULL;
4400 
4401 		if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4402 			return;
4403 
4404 		prev = skb->prev;
4405 		napi_gro_complete(skb);
4406 		napi->gro_count--;
4407 	}
4408 
4409 	napi->gro_list = NULL;
4410 }
4411 EXPORT_SYMBOL(napi_gro_flush);
4412 
4413 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4414 {
4415 	struct sk_buff *p;
4416 	unsigned int maclen = skb->dev->hard_header_len;
4417 	u32 hash = skb_get_hash_raw(skb);
4418 
4419 	for (p = napi->gro_list; p; p = p->next) {
4420 		unsigned long diffs;
4421 
4422 		NAPI_GRO_CB(p)->flush = 0;
4423 
4424 		if (hash != skb_get_hash_raw(p)) {
4425 			NAPI_GRO_CB(p)->same_flow = 0;
4426 			continue;
4427 		}
4428 
4429 		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4430 		diffs |= p->vlan_tci ^ skb->vlan_tci;
4431 		diffs |= skb_metadata_dst_cmp(p, skb);
4432 		if (maclen == ETH_HLEN)
4433 			diffs |= compare_ether_header(skb_mac_header(p),
4434 						      skb_mac_header(skb));
4435 		else if (!diffs)
4436 			diffs = memcmp(skb_mac_header(p),
4437 				       skb_mac_header(skb),
4438 				       maclen);
4439 		NAPI_GRO_CB(p)->same_flow = !diffs;
4440 	}
4441 }
4442 
4443 static void skb_gro_reset_offset(struct sk_buff *skb)
4444 {
4445 	const struct skb_shared_info *pinfo = skb_shinfo(skb);
4446 	const skb_frag_t *frag0 = &pinfo->frags[0];
4447 
4448 	NAPI_GRO_CB(skb)->data_offset = 0;
4449 	NAPI_GRO_CB(skb)->frag0 = NULL;
4450 	NAPI_GRO_CB(skb)->frag0_len = 0;
4451 
4452 	if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4453 	    pinfo->nr_frags &&
4454 	    !PageHighMem(skb_frag_page(frag0))) {
4455 		NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4456 		NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
4457 						    skb_frag_size(frag0),
4458 						    skb->end - skb->tail);
4459 	}
4460 }
4461 
4462 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4463 {
4464 	struct skb_shared_info *pinfo = skb_shinfo(skb);
4465 
4466 	BUG_ON(skb->end - skb->tail < grow);
4467 
4468 	memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4469 
4470 	skb->data_len -= grow;
4471 	skb->tail += grow;
4472 
4473 	pinfo->frags[0].page_offset += grow;
4474 	skb_frag_size_sub(&pinfo->frags[0], grow);
4475 
4476 	if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4477 		skb_frag_unref(skb, 0);
4478 		memmove(pinfo->frags, pinfo->frags + 1,
4479 			--pinfo->nr_frags * sizeof(pinfo->frags[0]));
4480 	}
4481 }
4482 
4483 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4484 {
4485 	struct sk_buff **pp = NULL;
4486 	struct packet_offload *ptype;
4487 	__be16 type = skb->protocol;
4488 	struct list_head *head = &offload_base;
4489 	int same_flow;
4490 	enum gro_result ret;
4491 	int grow;
4492 
4493 	if (!(skb->dev->features & NETIF_F_GRO))
4494 		goto normal;
4495 
4496 	if (skb->csum_bad)
4497 		goto normal;
4498 
4499 	gro_list_prepare(napi, skb);
4500 
4501 	rcu_read_lock();
4502 	list_for_each_entry_rcu(ptype, head, list) {
4503 		if (ptype->type != type || !ptype->callbacks.gro_receive)
4504 			continue;
4505 
4506 		skb_set_network_header(skb, skb_gro_offset(skb));
4507 		skb_reset_mac_len(skb);
4508 		NAPI_GRO_CB(skb)->same_flow = 0;
4509 		NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
4510 		NAPI_GRO_CB(skb)->free = 0;
4511 		NAPI_GRO_CB(skb)->encap_mark = 0;
4512 		NAPI_GRO_CB(skb)->recursion_counter = 0;
4513 		NAPI_GRO_CB(skb)->is_fou = 0;
4514 		NAPI_GRO_CB(skb)->is_atomic = 1;
4515 		NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
4516 
4517 		/* Setup for GRO checksum validation */
4518 		switch (skb->ip_summed) {
4519 		case CHECKSUM_COMPLETE:
4520 			NAPI_GRO_CB(skb)->csum = skb->csum;
4521 			NAPI_GRO_CB(skb)->csum_valid = 1;
4522 			NAPI_GRO_CB(skb)->csum_cnt = 0;
4523 			break;
4524 		case CHECKSUM_UNNECESSARY:
4525 			NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4526 			NAPI_GRO_CB(skb)->csum_valid = 0;
4527 			break;
4528 		default:
4529 			NAPI_GRO_CB(skb)->csum_cnt = 0;
4530 			NAPI_GRO_CB(skb)->csum_valid = 0;
4531 		}
4532 
4533 		pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
4534 		break;
4535 	}
4536 	rcu_read_unlock();
4537 
4538 	if (&ptype->list == head)
4539 		goto normal;
4540 
4541 	if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) {
4542 		ret = GRO_CONSUMED;
4543 		goto ok;
4544 	}
4545 
4546 	same_flow = NAPI_GRO_CB(skb)->same_flow;
4547 	ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
4548 
4549 	if (pp) {
4550 		struct sk_buff *nskb = *pp;
4551 
4552 		*pp = nskb->next;
4553 		nskb->next = NULL;
4554 		napi_gro_complete(nskb);
4555 		napi->gro_count--;
4556 	}
4557 
4558 	if (same_flow)
4559 		goto ok;
4560 
4561 	if (NAPI_GRO_CB(skb)->flush)
4562 		goto normal;
4563 
4564 	if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4565 		struct sk_buff *nskb = napi->gro_list;
4566 
4567 		/* locate the end of the list to select the 'oldest' flow */
4568 		while (nskb->next) {
4569 			pp = &nskb->next;
4570 			nskb = *pp;
4571 		}
4572 		*pp = NULL;
4573 		nskb->next = NULL;
4574 		napi_gro_complete(nskb);
4575 	} else {
4576 		napi->gro_count++;
4577 	}
4578 	NAPI_GRO_CB(skb)->count = 1;
4579 	NAPI_GRO_CB(skb)->age = jiffies;
4580 	NAPI_GRO_CB(skb)->last = skb;
4581 	skb_shinfo(skb)->gso_size = skb_gro_len(skb);
4582 	skb->next = napi->gro_list;
4583 	napi->gro_list = skb;
4584 	ret = GRO_HELD;
4585 
4586 pull:
4587 	grow = skb_gro_offset(skb) - skb_headlen(skb);
4588 	if (grow > 0)
4589 		gro_pull_from_frag0(skb, grow);
4590 ok:
4591 	return ret;
4592 
4593 normal:
4594 	ret = GRO_NORMAL;
4595 	goto pull;
4596 }
4597 
4598 struct packet_offload *gro_find_receive_by_type(__be16 type)
4599 {
4600 	struct list_head *offload_head = &offload_base;
4601 	struct packet_offload *ptype;
4602 
4603 	list_for_each_entry_rcu(ptype, offload_head, list) {
4604 		if (ptype->type != type || !ptype->callbacks.gro_receive)
4605 			continue;
4606 		return ptype;
4607 	}
4608 	return NULL;
4609 }
4610 EXPORT_SYMBOL(gro_find_receive_by_type);
4611 
4612 struct packet_offload *gro_find_complete_by_type(__be16 type)
4613 {
4614 	struct list_head *offload_head = &offload_base;
4615 	struct packet_offload *ptype;
4616 
4617 	list_for_each_entry_rcu(ptype, offload_head, list) {
4618 		if (ptype->type != type || !ptype->callbacks.gro_complete)
4619 			continue;
4620 		return ptype;
4621 	}
4622 	return NULL;
4623 }
4624 EXPORT_SYMBOL(gro_find_complete_by_type);
4625 
4626 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
4627 {
4628 	switch (ret) {
4629 	case GRO_NORMAL:
4630 		if (netif_receive_skb_internal(skb))
4631 			ret = GRO_DROP;
4632 		break;
4633 
4634 	case GRO_DROP:
4635 		kfree_skb(skb);
4636 		break;
4637 
4638 	case GRO_MERGED_FREE:
4639 		if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) {
4640 			skb_dst_drop(skb);
4641 			secpath_reset(skb);
4642 			kmem_cache_free(skbuff_head_cache, skb);
4643 		} else {
4644 			__kfree_skb(skb);
4645 		}
4646 		break;
4647 
4648 	case GRO_HELD:
4649 	case GRO_MERGED:
4650 	case GRO_CONSUMED:
4651 		break;
4652 	}
4653 
4654 	return ret;
4655 }
4656 
4657 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4658 {
4659 	skb_mark_napi_id(skb, napi);
4660 	trace_napi_gro_receive_entry(skb);
4661 
4662 	skb_gro_reset_offset(skb);
4663 
4664 	return napi_skb_finish(dev_gro_receive(napi, skb), skb);
4665 }
4666 EXPORT_SYMBOL(napi_gro_receive);
4667 
4668 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
4669 {
4670 	if (unlikely(skb->pfmemalloc)) {
4671 		consume_skb(skb);
4672 		return;
4673 	}
4674 	__skb_pull(skb, skb_headlen(skb));
4675 	/* restore the reserve we had after netdev_alloc_skb_ip_align() */
4676 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
4677 	skb->vlan_tci = 0;
4678 	skb->dev = napi->dev;
4679 	skb->skb_iif = 0;
4680 	skb->encapsulation = 0;
4681 	skb_shinfo(skb)->gso_type = 0;
4682 	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
4683 	secpath_reset(skb);
4684 
4685 	napi->skb = skb;
4686 }
4687 
4688 struct sk_buff *napi_get_frags(struct napi_struct *napi)
4689 {
4690 	struct sk_buff *skb = napi->skb;
4691 
4692 	if (!skb) {
4693 		skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
4694 		if (skb) {
4695 			napi->skb = skb;
4696 			skb_mark_napi_id(skb, napi);
4697 		}
4698 	}
4699 	return skb;
4700 }
4701 EXPORT_SYMBOL(napi_get_frags);
4702 
4703 static gro_result_t napi_frags_finish(struct napi_struct *napi,
4704 				      struct sk_buff *skb,
4705 				      gro_result_t ret)
4706 {
4707 	switch (ret) {
4708 	case GRO_NORMAL:
4709 	case GRO_HELD:
4710 		__skb_push(skb, ETH_HLEN);
4711 		skb->protocol = eth_type_trans(skb, skb->dev);
4712 		if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
4713 			ret = GRO_DROP;
4714 		break;
4715 
4716 	case GRO_DROP:
4717 	case GRO_MERGED_FREE:
4718 		napi_reuse_skb(napi, skb);
4719 		break;
4720 
4721 	case GRO_MERGED:
4722 	case GRO_CONSUMED:
4723 		break;
4724 	}
4725 
4726 	return ret;
4727 }
4728 
4729 /* Upper GRO stack assumes network header starts at gro_offset=0
4730  * Drivers could call both napi_gro_frags() and napi_gro_receive()
4731  * We copy ethernet header into skb->data to have a common layout.
4732  */
4733 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
4734 {
4735 	struct sk_buff *skb = napi->skb;
4736 	const struct ethhdr *eth;
4737 	unsigned int hlen = sizeof(*eth);
4738 
4739 	napi->skb = NULL;
4740 
4741 	skb_reset_mac_header(skb);
4742 	skb_gro_reset_offset(skb);
4743 
4744 	eth = skb_gro_header_fast(skb, 0);
4745 	if (unlikely(skb_gro_header_hard(skb, hlen))) {
4746 		eth = skb_gro_header_slow(skb, hlen, 0);
4747 		if (unlikely(!eth)) {
4748 			net_warn_ratelimited("%s: dropping impossible skb from %s\n",
4749 					     __func__, napi->dev->name);
4750 			napi_reuse_skb(napi, skb);
4751 			return NULL;
4752 		}
4753 	} else {
4754 		gro_pull_from_frag0(skb, hlen);
4755 		NAPI_GRO_CB(skb)->frag0 += hlen;
4756 		NAPI_GRO_CB(skb)->frag0_len -= hlen;
4757 	}
4758 	__skb_pull(skb, hlen);
4759 
4760 	/*
4761 	 * This works because the only protocols we care about don't require
4762 	 * special handling.
4763 	 * We'll fix it up properly in napi_frags_finish()
4764 	 */
4765 	skb->protocol = eth->h_proto;
4766 
4767 	return skb;
4768 }
4769 
4770 gro_result_t napi_gro_frags(struct napi_struct *napi)
4771 {
4772 	struct sk_buff *skb = napi_frags_skb(napi);
4773 
4774 	if (!skb)
4775 		return GRO_DROP;
4776 
4777 	trace_napi_gro_frags_entry(skb);
4778 
4779 	return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
4780 }
4781 EXPORT_SYMBOL(napi_gro_frags);
4782 
4783 /* Compute the checksum from gro_offset and return the folded value
4784  * after adding in any pseudo checksum.
4785  */
4786 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
4787 {
4788 	__wsum wsum;
4789 	__sum16 sum;
4790 
4791 	wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
4792 
4793 	/* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4794 	sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
4795 	if (likely(!sum)) {
4796 		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
4797 		    !skb->csum_complete_sw)
4798 			netdev_rx_csum_fault(skb->dev);
4799 	}
4800 
4801 	NAPI_GRO_CB(skb)->csum = wsum;
4802 	NAPI_GRO_CB(skb)->csum_valid = 1;
4803 
4804 	return sum;
4805 }
4806 EXPORT_SYMBOL(__skb_gro_checksum_complete);
4807 
4808 /*
4809  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
4810  * Note: called with local irq disabled, but exits with local irq enabled.
4811  */
4812 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4813 {
4814 #ifdef CONFIG_RPS
4815 	struct softnet_data *remsd = sd->rps_ipi_list;
4816 
4817 	if (remsd) {
4818 		sd->rps_ipi_list = NULL;
4819 
4820 		local_irq_enable();
4821 
4822 		/* Send pending IPI's to kick RPS processing on remote cpus. */
4823 		while (remsd) {
4824 			struct softnet_data *next = remsd->rps_ipi_next;
4825 
4826 			if (cpu_online(remsd->cpu))
4827 				smp_call_function_single_async(remsd->cpu,
4828 							   &remsd->csd);
4829 			remsd = next;
4830 		}
4831 	} else
4832 #endif
4833 		local_irq_enable();
4834 }
4835 
4836 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
4837 {
4838 #ifdef CONFIG_RPS
4839 	return sd->rps_ipi_list != NULL;
4840 #else
4841 	return false;
4842 #endif
4843 }
4844 
4845 static int process_backlog(struct napi_struct *napi, int quota)
4846 {
4847 	struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
4848 	bool again = true;
4849 	int work = 0;
4850 
4851 	/* Check if we have pending ipi, its better to send them now,
4852 	 * not waiting net_rx_action() end.
4853 	 */
4854 	if (sd_has_rps_ipi_waiting(sd)) {
4855 		local_irq_disable();
4856 		net_rps_action_and_irq_enable(sd);
4857 	}
4858 
4859 	napi->weight = dev_rx_weight;
4860 	while (again) {
4861 		struct sk_buff *skb;
4862 
4863 		while ((skb = __skb_dequeue(&sd->process_queue))) {
4864 			rcu_read_lock();
4865 			__netif_receive_skb(skb);
4866 			rcu_read_unlock();
4867 			input_queue_head_incr(sd);
4868 			if (++work >= quota)
4869 				return work;
4870 
4871 		}
4872 
4873 		local_irq_disable();
4874 		rps_lock(sd);
4875 		if (skb_queue_empty(&sd->input_pkt_queue)) {
4876 			/*
4877 			 * Inline a custom version of __napi_complete().
4878 			 * only current cpu owns and manipulates this napi,
4879 			 * and NAPI_STATE_SCHED is the only possible flag set
4880 			 * on backlog.
4881 			 * We can use a plain write instead of clear_bit(),
4882 			 * and we dont need an smp_mb() memory barrier.
4883 			 */
4884 			napi->state = 0;
4885 			again = false;
4886 		} else {
4887 			skb_queue_splice_tail_init(&sd->input_pkt_queue,
4888 						   &sd->process_queue);
4889 		}
4890 		rps_unlock(sd);
4891 		local_irq_enable();
4892 	}
4893 
4894 	return work;
4895 }
4896 
4897 /**
4898  * __napi_schedule - schedule for receive
4899  * @n: entry to schedule
4900  *
4901  * The entry's receive function will be scheduled to run.
4902  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
4903  */
4904 void __napi_schedule(struct napi_struct *n)
4905 {
4906 	unsigned long flags;
4907 
4908 	local_irq_save(flags);
4909 	____napi_schedule(this_cpu_ptr(&softnet_data), n);
4910 	local_irq_restore(flags);
4911 }
4912 EXPORT_SYMBOL(__napi_schedule);
4913 
4914 /**
4915  *	napi_schedule_prep - check if napi can be scheduled
4916  *	@n: napi context
4917  *
4918  * Test if NAPI routine is already running, and if not mark
4919  * it as running.  This is used as a condition variable
4920  * insure only one NAPI poll instance runs.  We also make
4921  * sure there is no pending NAPI disable.
4922  */
4923 bool napi_schedule_prep(struct napi_struct *n)
4924 {
4925 	unsigned long val, new;
4926 
4927 	do {
4928 		val = READ_ONCE(n->state);
4929 		if (unlikely(val & NAPIF_STATE_DISABLE))
4930 			return false;
4931 		new = val | NAPIF_STATE_SCHED;
4932 
4933 		/* Sets STATE_MISSED bit if STATE_SCHED was already set
4934 		 * This was suggested by Alexander Duyck, as compiler
4935 		 * emits better code than :
4936 		 * if (val & NAPIF_STATE_SCHED)
4937 		 *     new |= NAPIF_STATE_MISSED;
4938 		 */
4939 		new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
4940 						   NAPIF_STATE_MISSED;
4941 	} while (cmpxchg(&n->state, val, new) != val);
4942 
4943 	return !(val & NAPIF_STATE_SCHED);
4944 }
4945 EXPORT_SYMBOL(napi_schedule_prep);
4946 
4947 /**
4948  * __napi_schedule_irqoff - schedule for receive
4949  * @n: entry to schedule
4950  *
4951  * Variant of __napi_schedule() assuming hard irqs are masked
4952  */
4953 void __napi_schedule_irqoff(struct napi_struct *n)
4954 {
4955 	____napi_schedule(this_cpu_ptr(&softnet_data), n);
4956 }
4957 EXPORT_SYMBOL(__napi_schedule_irqoff);
4958 
4959 bool napi_complete_done(struct napi_struct *n, int work_done)
4960 {
4961 	unsigned long flags, val, new;
4962 
4963 	/*
4964 	 * 1) Don't let napi dequeue from the cpu poll list
4965 	 *    just in case its running on a different cpu.
4966 	 * 2) If we are busy polling, do nothing here, we have
4967 	 *    the guarantee we will be called later.
4968 	 */
4969 	if (unlikely(n->state & (NAPIF_STATE_NPSVC |
4970 				 NAPIF_STATE_IN_BUSY_POLL)))
4971 		return false;
4972 
4973 	if (n->gro_list) {
4974 		unsigned long timeout = 0;
4975 
4976 		if (work_done)
4977 			timeout = n->dev->gro_flush_timeout;
4978 
4979 		if (timeout)
4980 			hrtimer_start(&n->timer, ns_to_ktime(timeout),
4981 				      HRTIMER_MODE_REL_PINNED);
4982 		else
4983 			napi_gro_flush(n, false);
4984 	}
4985 	if (unlikely(!list_empty(&n->poll_list))) {
4986 		/* If n->poll_list is not empty, we need to mask irqs */
4987 		local_irq_save(flags);
4988 		list_del_init(&n->poll_list);
4989 		local_irq_restore(flags);
4990 	}
4991 
4992 	do {
4993 		val = READ_ONCE(n->state);
4994 
4995 		WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
4996 
4997 		new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED);
4998 
4999 		/* If STATE_MISSED was set, leave STATE_SCHED set,
5000 		 * because we will call napi->poll() one more time.
5001 		 * This C code was suggested by Alexander Duyck to help gcc.
5002 		 */
5003 		new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
5004 						    NAPIF_STATE_SCHED;
5005 	} while (cmpxchg(&n->state, val, new) != val);
5006 
5007 	if (unlikely(val & NAPIF_STATE_MISSED)) {
5008 		__napi_schedule(n);
5009 		return false;
5010 	}
5011 
5012 	return true;
5013 }
5014 EXPORT_SYMBOL(napi_complete_done);
5015 
5016 /* must be called under rcu_read_lock(), as we dont take a reference */
5017 static struct napi_struct *napi_by_id(unsigned int napi_id)
5018 {
5019 	unsigned int hash = napi_id % HASH_SIZE(napi_hash);
5020 	struct napi_struct *napi;
5021 
5022 	hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
5023 		if (napi->napi_id == napi_id)
5024 			return napi;
5025 
5026 	return NULL;
5027 }
5028 
5029 #if defined(CONFIG_NET_RX_BUSY_POLL)
5030 
5031 #define BUSY_POLL_BUDGET 8
5032 
5033 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)
5034 {
5035 	int rc;
5036 
5037 	/* Busy polling means there is a high chance device driver hard irq
5038 	 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
5039 	 * set in napi_schedule_prep().
5040 	 * Since we are about to call napi->poll() once more, we can safely
5041 	 * clear NAPI_STATE_MISSED.
5042 	 *
5043 	 * Note: x86 could use a single "lock and ..." instruction
5044 	 * to perform these two clear_bit()
5045 	 */
5046 	clear_bit(NAPI_STATE_MISSED, &napi->state);
5047 	clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
5048 
5049 	local_bh_disable();
5050 
5051 	/* All we really want here is to re-enable device interrupts.
5052 	 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
5053 	 */
5054 	rc = napi->poll(napi, BUSY_POLL_BUDGET);
5055 	netpoll_poll_unlock(have_poll_lock);
5056 	if (rc == BUSY_POLL_BUDGET)
5057 		__napi_schedule(napi);
5058 	local_bh_enable();
5059 	if (local_softirq_pending())
5060 		do_softirq();
5061 }
5062 
5063 bool sk_busy_loop(struct sock *sk, int nonblock)
5064 {
5065 	unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0;
5066 	int (*napi_poll)(struct napi_struct *napi, int budget);
5067 	void *have_poll_lock = NULL;
5068 	struct napi_struct *napi;
5069 	int rc;
5070 
5071 restart:
5072 	rc = false;
5073 	napi_poll = NULL;
5074 
5075 	rcu_read_lock();
5076 
5077 	napi = napi_by_id(sk->sk_napi_id);
5078 	if (!napi)
5079 		goto out;
5080 
5081 	preempt_disable();
5082 	for (;;) {
5083 		rc = 0;
5084 		local_bh_disable();
5085 		if (!napi_poll) {
5086 			unsigned long val = READ_ONCE(napi->state);
5087 
5088 			/* If multiple threads are competing for this napi,
5089 			 * we avoid dirtying napi->state as much as we can.
5090 			 */
5091 			if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
5092 				   NAPIF_STATE_IN_BUSY_POLL))
5093 				goto count;
5094 			if (cmpxchg(&napi->state, val,
5095 				    val | NAPIF_STATE_IN_BUSY_POLL |
5096 					  NAPIF_STATE_SCHED) != val)
5097 				goto count;
5098 			have_poll_lock = netpoll_poll_lock(napi);
5099 			napi_poll = napi->poll;
5100 		}
5101 		rc = napi_poll(napi, BUSY_POLL_BUDGET);
5102 		trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);
5103 count:
5104 		if (rc > 0)
5105 			__NET_ADD_STATS(sock_net(sk),
5106 					LINUX_MIB_BUSYPOLLRXPACKETS, rc);
5107 		local_bh_enable();
5108 
5109 		if (nonblock || !skb_queue_empty(&sk->sk_receive_queue) ||
5110 		    busy_loop_timeout(end_time))
5111 			break;
5112 
5113 		if (unlikely(need_resched())) {
5114 			if (napi_poll)
5115 				busy_poll_stop(napi, have_poll_lock);
5116 			preempt_enable();
5117 			rcu_read_unlock();
5118 			cond_resched();
5119 			rc = !skb_queue_empty(&sk->sk_receive_queue);
5120 			if (rc || busy_loop_timeout(end_time))
5121 				return rc;
5122 			goto restart;
5123 		}
5124 		cpu_relax();
5125 	}
5126 	if (napi_poll)
5127 		busy_poll_stop(napi, have_poll_lock);
5128 	preempt_enable();
5129 	rc = !skb_queue_empty(&sk->sk_receive_queue);
5130 out:
5131 	rcu_read_unlock();
5132 	return rc;
5133 }
5134 EXPORT_SYMBOL(sk_busy_loop);
5135 
5136 #endif /* CONFIG_NET_RX_BUSY_POLL */
5137 
5138 static void napi_hash_add(struct napi_struct *napi)
5139 {
5140 	if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
5141 	    test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
5142 		return;
5143 
5144 	spin_lock(&napi_hash_lock);
5145 
5146 	/* 0..NR_CPUS+1 range is reserved for sender_cpu use */
5147 	do {
5148 		if (unlikely(++napi_gen_id < NR_CPUS + 1))
5149 			napi_gen_id = NR_CPUS + 1;
5150 	} while (napi_by_id(napi_gen_id));
5151 	napi->napi_id = napi_gen_id;
5152 
5153 	hlist_add_head_rcu(&napi->napi_hash_node,
5154 			   &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
5155 
5156 	spin_unlock(&napi_hash_lock);
5157 }
5158 
5159 /* Warning : caller is responsible to make sure rcu grace period
5160  * is respected before freeing memory containing @napi
5161  */
5162 bool napi_hash_del(struct napi_struct *napi)
5163 {
5164 	bool rcu_sync_needed = false;
5165 
5166 	spin_lock(&napi_hash_lock);
5167 
5168 	if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
5169 		rcu_sync_needed = true;
5170 		hlist_del_rcu(&napi->napi_hash_node);
5171 	}
5172 	spin_unlock(&napi_hash_lock);
5173 	return rcu_sync_needed;
5174 }
5175 EXPORT_SYMBOL_GPL(napi_hash_del);
5176 
5177 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
5178 {
5179 	struct napi_struct *napi;
5180 
5181 	napi = container_of(timer, struct napi_struct, timer);
5182 
5183 	/* Note : we use a relaxed variant of napi_schedule_prep() not setting
5184 	 * NAPI_STATE_MISSED, since we do not react to a device IRQ.
5185 	 */
5186 	if (napi->gro_list && !napi_disable_pending(napi) &&
5187 	    !test_and_set_bit(NAPI_STATE_SCHED, &napi->state))
5188 		__napi_schedule_irqoff(napi);
5189 
5190 	return HRTIMER_NORESTART;
5191 }
5192 
5193 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
5194 		    int (*poll)(struct napi_struct *, int), int weight)
5195 {
5196 	INIT_LIST_HEAD(&napi->poll_list);
5197 	hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
5198 	napi->timer.function = napi_watchdog;
5199 	napi->gro_count = 0;
5200 	napi->gro_list = NULL;
5201 	napi->skb = NULL;
5202 	napi->poll = poll;
5203 	if (weight > NAPI_POLL_WEIGHT)
5204 		pr_err_once("netif_napi_add() called with weight %d on device %s\n",
5205 			    weight, dev->name);
5206 	napi->weight = weight;
5207 	list_add(&napi->dev_list, &dev->napi_list);
5208 	napi->dev = dev;
5209 #ifdef CONFIG_NETPOLL
5210 	napi->poll_owner = -1;
5211 #endif
5212 	set_bit(NAPI_STATE_SCHED, &napi->state);
5213 	napi_hash_add(napi);
5214 }
5215 EXPORT_SYMBOL(netif_napi_add);
5216 
5217 void napi_disable(struct napi_struct *n)
5218 {
5219 	might_sleep();
5220 	set_bit(NAPI_STATE_DISABLE, &n->state);
5221 
5222 	while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
5223 		msleep(1);
5224 	while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
5225 		msleep(1);
5226 
5227 	hrtimer_cancel(&n->timer);
5228 
5229 	clear_bit(NAPI_STATE_DISABLE, &n->state);
5230 }
5231 EXPORT_SYMBOL(napi_disable);
5232 
5233 /* Must be called in process context */
5234 void netif_napi_del(struct napi_struct *napi)
5235 {
5236 	might_sleep();
5237 	if (napi_hash_del(napi))
5238 		synchronize_net();
5239 	list_del_init(&napi->dev_list);
5240 	napi_free_frags(napi);
5241 
5242 	kfree_skb_list(napi->gro_list);
5243 	napi->gro_list = NULL;
5244 	napi->gro_count = 0;
5245 }
5246 EXPORT_SYMBOL(netif_napi_del);
5247 
5248 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
5249 {
5250 	void *have;
5251 	int work, weight;
5252 
5253 	list_del_init(&n->poll_list);
5254 
5255 	have = netpoll_poll_lock(n);
5256 
5257 	weight = n->weight;
5258 
5259 	/* This NAPI_STATE_SCHED test is for avoiding a race
5260 	 * with netpoll's poll_napi().  Only the entity which
5261 	 * obtains the lock and sees NAPI_STATE_SCHED set will
5262 	 * actually make the ->poll() call.  Therefore we avoid
5263 	 * accidentally calling ->poll() when NAPI is not scheduled.
5264 	 */
5265 	work = 0;
5266 	if (test_bit(NAPI_STATE_SCHED, &n->state)) {
5267 		work = n->poll(n, weight);
5268 		trace_napi_poll(n, work, weight);
5269 	}
5270 
5271 	WARN_ON_ONCE(work > weight);
5272 
5273 	if (likely(work < weight))
5274 		goto out_unlock;
5275 
5276 	/* Drivers must not modify the NAPI state if they
5277 	 * consume the entire weight.  In such cases this code
5278 	 * still "owns" the NAPI instance and therefore can
5279 	 * move the instance around on the list at-will.
5280 	 */
5281 	if (unlikely(napi_disable_pending(n))) {
5282 		napi_complete(n);
5283 		goto out_unlock;
5284 	}
5285 
5286 	if (n->gro_list) {
5287 		/* flush too old packets
5288 		 * If HZ < 1000, flush all packets.
5289 		 */
5290 		napi_gro_flush(n, HZ >= 1000);
5291 	}
5292 
5293 	/* Some drivers may have called napi_schedule
5294 	 * prior to exhausting their budget.
5295 	 */
5296 	if (unlikely(!list_empty(&n->poll_list))) {
5297 		pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
5298 			     n->dev ? n->dev->name : "backlog");
5299 		goto out_unlock;
5300 	}
5301 
5302 	list_add_tail(&n->poll_list, repoll);
5303 
5304 out_unlock:
5305 	netpoll_poll_unlock(have);
5306 
5307 	return work;
5308 }
5309 
5310 static __latent_entropy void net_rx_action(struct softirq_action *h)
5311 {
5312 	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5313 	unsigned long time_limit = jiffies + 2;
5314 	int budget = netdev_budget;
5315 	LIST_HEAD(list);
5316 	LIST_HEAD(repoll);
5317 
5318 	local_irq_disable();
5319 	list_splice_init(&sd->poll_list, &list);
5320 	local_irq_enable();
5321 
5322 	for (;;) {
5323 		struct napi_struct *n;
5324 
5325 		if (list_empty(&list)) {
5326 			if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
5327 				goto out;
5328 			break;
5329 		}
5330 
5331 		n = list_first_entry(&list, struct napi_struct, poll_list);
5332 		budget -= napi_poll(n, &repoll);
5333 
5334 		/* If softirq window is exhausted then punt.
5335 		 * Allow this to run for 2 jiffies since which will allow
5336 		 * an average latency of 1.5/HZ.
5337 		 */
5338 		if (unlikely(budget <= 0 ||
5339 			     time_after_eq(jiffies, time_limit))) {
5340 			sd->time_squeeze++;
5341 			break;
5342 		}
5343 	}
5344 
5345 	local_irq_disable();
5346 
5347 	list_splice_tail_init(&sd->poll_list, &list);
5348 	list_splice_tail(&repoll, &list);
5349 	list_splice(&list, &sd->poll_list);
5350 	if (!list_empty(&sd->poll_list))
5351 		__raise_softirq_irqoff(NET_RX_SOFTIRQ);
5352 
5353 	net_rps_action_and_irq_enable(sd);
5354 out:
5355 	__kfree_skb_flush();
5356 }
5357 
5358 struct netdev_adjacent {
5359 	struct net_device *dev;
5360 
5361 	/* upper master flag, there can only be one master device per list */
5362 	bool master;
5363 
5364 	/* counter for the number of times this device was added to us */
5365 	u16 ref_nr;
5366 
5367 	/* private field for the users */
5368 	void *private;
5369 
5370 	struct list_head list;
5371 	struct rcu_head rcu;
5372 };
5373 
5374 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
5375 						 struct list_head *adj_list)
5376 {
5377 	struct netdev_adjacent *adj;
5378 
5379 	list_for_each_entry(adj, adj_list, list) {
5380 		if (adj->dev == adj_dev)
5381 			return adj;
5382 	}
5383 	return NULL;
5384 }
5385 
5386 static int __netdev_has_upper_dev(struct net_device *upper_dev, void *data)
5387 {
5388 	struct net_device *dev = data;
5389 
5390 	return upper_dev == dev;
5391 }
5392 
5393 /**
5394  * netdev_has_upper_dev - Check if device is linked to an upper device
5395  * @dev: device
5396  * @upper_dev: upper device to check
5397  *
5398  * Find out if a device is linked to specified upper device and return true
5399  * in case it is. Note that this checks only immediate upper device,
5400  * not through a complete stack of devices. The caller must hold the RTNL lock.
5401  */
5402 bool netdev_has_upper_dev(struct net_device *dev,
5403 			  struct net_device *upper_dev)
5404 {
5405 	ASSERT_RTNL();
5406 
5407 	return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5408 					     upper_dev);
5409 }
5410 EXPORT_SYMBOL(netdev_has_upper_dev);
5411 
5412 /**
5413  * netdev_has_upper_dev_all - Check if device is linked to an upper device
5414  * @dev: device
5415  * @upper_dev: upper device to check
5416  *
5417  * Find out if a device is linked to specified upper device and return true
5418  * in case it is. Note that this checks the entire upper device chain.
5419  * The caller must hold rcu lock.
5420  */
5421 
5422 bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
5423 				  struct net_device *upper_dev)
5424 {
5425 	return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5426 					       upper_dev);
5427 }
5428 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
5429 
5430 /**
5431  * netdev_has_any_upper_dev - Check if device is linked to some device
5432  * @dev: device
5433  *
5434  * Find out if a device is linked to an upper device and return true in case
5435  * it is. The caller must hold the RTNL lock.
5436  */
5437 static bool netdev_has_any_upper_dev(struct net_device *dev)
5438 {
5439 	ASSERT_RTNL();
5440 
5441 	return !list_empty(&dev->adj_list.upper);
5442 }
5443 
5444 /**
5445  * netdev_master_upper_dev_get - Get master upper device
5446  * @dev: device
5447  *
5448  * Find a master upper device and return pointer to it or NULL in case
5449  * it's not there. The caller must hold the RTNL lock.
5450  */
5451 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
5452 {
5453 	struct netdev_adjacent *upper;
5454 
5455 	ASSERT_RTNL();
5456 
5457 	if (list_empty(&dev->adj_list.upper))
5458 		return NULL;
5459 
5460 	upper = list_first_entry(&dev->adj_list.upper,
5461 				 struct netdev_adjacent, list);
5462 	if (likely(upper->master))
5463 		return upper->dev;
5464 	return NULL;
5465 }
5466 EXPORT_SYMBOL(netdev_master_upper_dev_get);
5467 
5468 /**
5469  * netdev_has_any_lower_dev - Check if device is linked to some device
5470  * @dev: device
5471  *
5472  * Find out if a device is linked to a lower device and return true in case
5473  * it is. The caller must hold the RTNL lock.
5474  */
5475 static bool netdev_has_any_lower_dev(struct net_device *dev)
5476 {
5477 	ASSERT_RTNL();
5478 
5479 	return !list_empty(&dev->adj_list.lower);
5480 }
5481 
5482 void *netdev_adjacent_get_private(struct list_head *adj_list)
5483 {
5484 	struct netdev_adjacent *adj;
5485 
5486 	adj = list_entry(adj_list, struct netdev_adjacent, list);
5487 
5488 	return adj->private;
5489 }
5490 EXPORT_SYMBOL(netdev_adjacent_get_private);
5491 
5492 /**
5493  * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
5494  * @dev: device
5495  * @iter: list_head ** of the current position
5496  *
5497  * Gets the next device from the dev's upper list, starting from iter
5498  * position. The caller must hold RCU read lock.
5499  */
5500 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5501 						 struct list_head **iter)
5502 {
5503 	struct netdev_adjacent *upper;
5504 
5505 	WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5506 
5507 	upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5508 
5509 	if (&upper->list == &dev->adj_list.upper)
5510 		return NULL;
5511 
5512 	*iter = &upper->list;
5513 
5514 	return upper->dev;
5515 }
5516 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5517 
5518 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
5519 						    struct list_head **iter)
5520 {
5521 	struct netdev_adjacent *upper;
5522 
5523 	WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5524 
5525 	upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5526 
5527 	if (&upper->list == &dev->adj_list.upper)
5528 		return NULL;
5529 
5530 	*iter = &upper->list;
5531 
5532 	return upper->dev;
5533 }
5534 
5535 int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
5536 				  int (*fn)(struct net_device *dev,
5537 					    void *data),
5538 				  void *data)
5539 {
5540 	struct net_device *udev;
5541 	struct list_head *iter;
5542 	int ret;
5543 
5544 	for (iter = &dev->adj_list.upper,
5545 	     udev = netdev_next_upper_dev_rcu(dev, &iter);
5546 	     udev;
5547 	     udev = netdev_next_upper_dev_rcu(dev, &iter)) {
5548 		/* first is the upper device itself */
5549 		ret = fn(udev, data);
5550 		if (ret)
5551 			return ret;
5552 
5553 		/* then look at all of its upper devices */
5554 		ret = netdev_walk_all_upper_dev_rcu(udev, fn, data);
5555 		if (ret)
5556 			return ret;
5557 	}
5558 
5559 	return 0;
5560 }
5561 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
5562 
5563 /**
5564  * netdev_lower_get_next_private - Get the next ->private from the
5565  *				   lower neighbour list
5566  * @dev: device
5567  * @iter: list_head ** of the current position
5568  *
5569  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5570  * list, starting from iter position. The caller must hold either hold the
5571  * RTNL lock or its own locking that guarantees that the neighbour lower
5572  * list will remain unchanged.
5573  */
5574 void *netdev_lower_get_next_private(struct net_device *dev,
5575 				    struct list_head **iter)
5576 {
5577 	struct netdev_adjacent *lower;
5578 
5579 	lower = list_entry(*iter, struct netdev_adjacent, list);
5580 
5581 	if (&lower->list == &dev->adj_list.lower)
5582 		return NULL;
5583 
5584 	*iter = lower->list.next;
5585 
5586 	return lower->private;
5587 }
5588 EXPORT_SYMBOL(netdev_lower_get_next_private);
5589 
5590 /**
5591  * netdev_lower_get_next_private_rcu - Get the next ->private from the
5592  *				       lower neighbour list, RCU
5593  *				       variant
5594  * @dev: device
5595  * @iter: list_head ** of the current position
5596  *
5597  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5598  * list, starting from iter position. The caller must hold RCU read lock.
5599  */
5600 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5601 					struct list_head **iter)
5602 {
5603 	struct netdev_adjacent *lower;
5604 
5605 	WARN_ON_ONCE(!rcu_read_lock_held());
5606 
5607 	lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5608 
5609 	if (&lower->list == &dev->adj_list.lower)
5610 		return NULL;
5611 
5612 	*iter = &lower->list;
5613 
5614 	return lower->private;
5615 }
5616 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5617 
5618 /**
5619  * netdev_lower_get_next - Get the next device from the lower neighbour
5620  *                         list
5621  * @dev: device
5622  * @iter: list_head ** of the current position
5623  *
5624  * Gets the next netdev_adjacent from the dev's lower neighbour
5625  * list, starting from iter position. The caller must hold RTNL lock or
5626  * its own locking that guarantees that the neighbour lower
5627  * list will remain unchanged.
5628  */
5629 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5630 {
5631 	struct netdev_adjacent *lower;
5632 
5633 	lower = list_entry(*iter, struct netdev_adjacent, list);
5634 
5635 	if (&lower->list == &dev->adj_list.lower)
5636 		return NULL;
5637 
5638 	*iter = lower->list.next;
5639 
5640 	return lower->dev;
5641 }
5642 EXPORT_SYMBOL(netdev_lower_get_next);
5643 
5644 static struct net_device *netdev_next_lower_dev(struct net_device *dev,
5645 						struct list_head **iter)
5646 {
5647 	struct netdev_adjacent *lower;
5648 
5649 	lower = list_entry((*iter)->next, struct netdev_adjacent, list);
5650 
5651 	if (&lower->list == &dev->adj_list.lower)
5652 		return NULL;
5653 
5654 	*iter = &lower->list;
5655 
5656 	return lower->dev;
5657 }
5658 
5659 int netdev_walk_all_lower_dev(struct net_device *dev,
5660 			      int (*fn)(struct net_device *dev,
5661 					void *data),
5662 			      void *data)
5663 {
5664 	struct net_device *ldev;
5665 	struct list_head *iter;
5666 	int ret;
5667 
5668 	for (iter = &dev->adj_list.lower,
5669 	     ldev = netdev_next_lower_dev(dev, &iter);
5670 	     ldev;
5671 	     ldev = netdev_next_lower_dev(dev, &iter)) {
5672 		/* first is the lower device itself */
5673 		ret = fn(ldev, data);
5674 		if (ret)
5675 			return ret;
5676 
5677 		/* then look at all of its lower devices */
5678 		ret = netdev_walk_all_lower_dev(ldev, fn, data);
5679 		if (ret)
5680 			return ret;
5681 	}
5682 
5683 	return 0;
5684 }
5685 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
5686 
5687 static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
5688 						    struct list_head **iter)
5689 {
5690 	struct netdev_adjacent *lower;
5691 
5692 	lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5693 	if (&lower->list == &dev->adj_list.lower)
5694 		return NULL;
5695 
5696 	*iter = &lower->list;
5697 
5698 	return lower->dev;
5699 }
5700 
5701 int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
5702 				  int (*fn)(struct net_device *dev,
5703 					    void *data),
5704 				  void *data)
5705 {
5706 	struct net_device *ldev;
5707 	struct list_head *iter;
5708 	int ret;
5709 
5710 	for (iter = &dev->adj_list.lower,
5711 	     ldev = netdev_next_lower_dev_rcu(dev, &iter);
5712 	     ldev;
5713 	     ldev = netdev_next_lower_dev_rcu(dev, &iter)) {
5714 		/* first is the lower device itself */
5715 		ret = fn(ldev, data);
5716 		if (ret)
5717 			return ret;
5718 
5719 		/* then look at all of its lower devices */
5720 		ret = netdev_walk_all_lower_dev_rcu(ldev, fn, data);
5721 		if (ret)
5722 			return ret;
5723 	}
5724 
5725 	return 0;
5726 }
5727 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
5728 
5729 /**
5730  * netdev_lower_get_first_private_rcu - Get the first ->private from the
5731  *				       lower neighbour list, RCU
5732  *				       variant
5733  * @dev: device
5734  *
5735  * Gets the first netdev_adjacent->private from the dev's lower neighbour
5736  * list. The caller must hold RCU read lock.
5737  */
5738 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
5739 {
5740 	struct netdev_adjacent *lower;
5741 
5742 	lower = list_first_or_null_rcu(&dev->adj_list.lower,
5743 			struct netdev_adjacent, list);
5744 	if (lower)
5745 		return lower->private;
5746 	return NULL;
5747 }
5748 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
5749 
5750 /**
5751  * netdev_master_upper_dev_get_rcu - Get master upper device
5752  * @dev: device
5753  *
5754  * Find a master upper device and return pointer to it or NULL in case
5755  * it's not there. The caller must hold the RCU read lock.
5756  */
5757 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
5758 {
5759 	struct netdev_adjacent *upper;
5760 
5761 	upper = list_first_or_null_rcu(&dev->adj_list.upper,
5762 				       struct netdev_adjacent, list);
5763 	if (upper && likely(upper->master))
5764 		return upper->dev;
5765 	return NULL;
5766 }
5767 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
5768 
5769 static int netdev_adjacent_sysfs_add(struct net_device *dev,
5770 			      struct net_device *adj_dev,
5771 			      struct list_head *dev_list)
5772 {
5773 	char linkname[IFNAMSIZ+7];
5774 
5775 	sprintf(linkname, dev_list == &dev->adj_list.upper ?
5776 		"upper_%s" : "lower_%s", adj_dev->name);
5777 	return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
5778 				 linkname);
5779 }
5780 static void netdev_adjacent_sysfs_del(struct net_device *dev,
5781 			       char *name,
5782 			       struct list_head *dev_list)
5783 {
5784 	char linkname[IFNAMSIZ+7];
5785 
5786 	sprintf(linkname, dev_list == &dev->adj_list.upper ?
5787 		"upper_%s" : "lower_%s", name);
5788 	sysfs_remove_link(&(dev->dev.kobj), linkname);
5789 }
5790 
5791 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
5792 						 struct net_device *adj_dev,
5793 						 struct list_head *dev_list)
5794 {
5795 	return (dev_list == &dev->adj_list.upper ||
5796 		dev_list == &dev->adj_list.lower) &&
5797 		net_eq(dev_net(dev), dev_net(adj_dev));
5798 }
5799 
5800 static int __netdev_adjacent_dev_insert(struct net_device *dev,
5801 					struct net_device *adj_dev,
5802 					struct list_head *dev_list,
5803 					void *private, bool master)
5804 {
5805 	struct netdev_adjacent *adj;
5806 	int ret;
5807 
5808 	adj = __netdev_find_adj(adj_dev, dev_list);
5809 
5810 	if (adj) {
5811 		adj->ref_nr += 1;
5812 		pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
5813 			 dev->name, adj_dev->name, adj->ref_nr);
5814 
5815 		return 0;
5816 	}
5817 
5818 	adj = kmalloc(sizeof(*adj), GFP_KERNEL);
5819 	if (!adj)
5820 		return -ENOMEM;
5821 
5822 	adj->dev = adj_dev;
5823 	adj->master = master;
5824 	adj->ref_nr = 1;
5825 	adj->private = private;
5826 	dev_hold(adj_dev);
5827 
5828 	pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
5829 		 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
5830 
5831 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
5832 		ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
5833 		if (ret)
5834 			goto free_adj;
5835 	}
5836 
5837 	/* Ensure that master link is always the first item in list. */
5838 	if (master) {
5839 		ret = sysfs_create_link(&(dev->dev.kobj),
5840 					&(adj_dev->dev.kobj), "master");
5841 		if (ret)
5842 			goto remove_symlinks;
5843 
5844 		list_add_rcu(&adj->list, dev_list);
5845 	} else {
5846 		list_add_tail_rcu(&adj->list, dev_list);
5847 	}
5848 
5849 	return 0;
5850 
5851 remove_symlinks:
5852 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5853 		netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5854 free_adj:
5855 	kfree(adj);
5856 	dev_put(adj_dev);
5857 
5858 	return ret;
5859 }
5860 
5861 static void __netdev_adjacent_dev_remove(struct net_device *dev,
5862 					 struct net_device *adj_dev,
5863 					 u16 ref_nr,
5864 					 struct list_head *dev_list)
5865 {
5866 	struct netdev_adjacent *adj;
5867 
5868 	pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
5869 		 dev->name, adj_dev->name, ref_nr);
5870 
5871 	adj = __netdev_find_adj(adj_dev, dev_list);
5872 
5873 	if (!adj) {
5874 		pr_err("Adjacency does not exist for device %s from %s\n",
5875 		       dev->name, adj_dev->name);
5876 		WARN_ON(1);
5877 		return;
5878 	}
5879 
5880 	if (adj->ref_nr > ref_nr) {
5881 		pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
5882 			 dev->name, adj_dev->name, ref_nr,
5883 			 adj->ref_nr - ref_nr);
5884 		adj->ref_nr -= ref_nr;
5885 		return;
5886 	}
5887 
5888 	if (adj->master)
5889 		sysfs_remove_link(&(dev->dev.kobj), "master");
5890 
5891 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5892 		netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5893 
5894 	list_del_rcu(&adj->list);
5895 	pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
5896 		 adj_dev->name, dev->name, adj_dev->name);
5897 	dev_put(adj_dev);
5898 	kfree_rcu(adj, rcu);
5899 }
5900 
5901 static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
5902 					    struct net_device *upper_dev,
5903 					    struct list_head *up_list,
5904 					    struct list_head *down_list,
5905 					    void *private, bool master)
5906 {
5907 	int ret;
5908 
5909 	ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
5910 					   private, master);
5911 	if (ret)
5912 		return ret;
5913 
5914 	ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
5915 					   private, false);
5916 	if (ret) {
5917 		__netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
5918 		return ret;
5919 	}
5920 
5921 	return 0;
5922 }
5923 
5924 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
5925 					       struct net_device *upper_dev,
5926 					       u16 ref_nr,
5927 					       struct list_head *up_list,
5928 					       struct list_head *down_list)
5929 {
5930 	__netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
5931 	__netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
5932 }
5933 
5934 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
5935 						struct net_device *upper_dev,
5936 						void *private, bool master)
5937 {
5938 	return __netdev_adjacent_dev_link_lists(dev, upper_dev,
5939 						&dev->adj_list.upper,
5940 						&upper_dev->adj_list.lower,
5941 						private, master);
5942 }
5943 
5944 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
5945 						   struct net_device *upper_dev)
5946 {
5947 	__netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
5948 					   &dev->adj_list.upper,
5949 					   &upper_dev->adj_list.lower);
5950 }
5951 
5952 static int __netdev_upper_dev_link(struct net_device *dev,
5953 				   struct net_device *upper_dev, bool master,
5954 				   void *upper_priv, void *upper_info)
5955 {
5956 	struct netdev_notifier_changeupper_info changeupper_info;
5957 	int ret = 0;
5958 
5959 	ASSERT_RTNL();
5960 
5961 	if (dev == upper_dev)
5962 		return -EBUSY;
5963 
5964 	/* To prevent loops, check if dev is not upper device to upper_dev. */
5965 	if (netdev_has_upper_dev(upper_dev, dev))
5966 		return -EBUSY;
5967 
5968 	if (netdev_has_upper_dev(dev, upper_dev))
5969 		return -EEXIST;
5970 
5971 	if (master && netdev_master_upper_dev_get(dev))
5972 		return -EBUSY;
5973 
5974 	changeupper_info.upper_dev = upper_dev;
5975 	changeupper_info.master = master;
5976 	changeupper_info.linking = true;
5977 	changeupper_info.upper_info = upper_info;
5978 
5979 	ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5980 					    &changeupper_info.info);
5981 	ret = notifier_to_errno(ret);
5982 	if (ret)
5983 		return ret;
5984 
5985 	ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
5986 						   master);
5987 	if (ret)
5988 		return ret;
5989 
5990 	ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5991 					    &changeupper_info.info);
5992 	ret = notifier_to_errno(ret);
5993 	if (ret)
5994 		goto rollback;
5995 
5996 	return 0;
5997 
5998 rollback:
5999 	__netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
6000 
6001 	return ret;
6002 }
6003 
6004 /**
6005  * netdev_upper_dev_link - Add a link to the upper device
6006  * @dev: device
6007  * @upper_dev: new upper device
6008  *
6009  * Adds a link to device which is upper to this one. The caller must hold
6010  * the RTNL lock. On a failure a negative errno code is returned.
6011  * On success the reference counts are adjusted and the function
6012  * returns zero.
6013  */
6014 int netdev_upper_dev_link(struct net_device *dev,
6015 			  struct net_device *upper_dev)
6016 {
6017 	return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL);
6018 }
6019 EXPORT_SYMBOL(netdev_upper_dev_link);
6020 
6021 /**
6022  * netdev_master_upper_dev_link - Add a master link to the upper device
6023  * @dev: device
6024  * @upper_dev: new upper device
6025  * @upper_priv: upper device private
6026  * @upper_info: upper info to be passed down via notifier
6027  *
6028  * Adds a link to device which is upper to this one. In this case, only
6029  * one master upper device can be linked, although other non-master devices
6030  * might be linked as well. The caller must hold the RTNL lock.
6031  * On a failure a negative errno code is returned. On success the reference
6032  * counts are adjusted and the function returns zero.
6033  */
6034 int netdev_master_upper_dev_link(struct net_device *dev,
6035 				 struct net_device *upper_dev,
6036 				 void *upper_priv, void *upper_info)
6037 {
6038 	return __netdev_upper_dev_link(dev, upper_dev, true,
6039 				       upper_priv, upper_info);
6040 }
6041 EXPORT_SYMBOL(netdev_master_upper_dev_link);
6042 
6043 /**
6044  * netdev_upper_dev_unlink - Removes a link to upper device
6045  * @dev: device
6046  * @upper_dev: new upper device
6047  *
6048  * Removes a link to device which is upper to this one. The caller must hold
6049  * the RTNL lock.
6050  */
6051 void netdev_upper_dev_unlink(struct net_device *dev,
6052 			     struct net_device *upper_dev)
6053 {
6054 	struct netdev_notifier_changeupper_info changeupper_info;
6055 
6056 	ASSERT_RTNL();
6057 
6058 	changeupper_info.upper_dev = upper_dev;
6059 	changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
6060 	changeupper_info.linking = false;
6061 
6062 	call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
6063 				      &changeupper_info.info);
6064 
6065 	__netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
6066 
6067 	call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
6068 				      &changeupper_info.info);
6069 }
6070 EXPORT_SYMBOL(netdev_upper_dev_unlink);
6071 
6072 /**
6073  * netdev_bonding_info_change - Dispatch event about slave change
6074  * @dev: device
6075  * @bonding_info: info to dispatch
6076  *
6077  * Send NETDEV_BONDING_INFO to netdev notifiers with info.
6078  * The caller must hold the RTNL lock.
6079  */
6080 void netdev_bonding_info_change(struct net_device *dev,
6081 				struct netdev_bonding_info *bonding_info)
6082 {
6083 	struct netdev_notifier_bonding_info	info;
6084 
6085 	memcpy(&info.bonding_info, bonding_info,
6086 	       sizeof(struct netdev_bonding_info));
6087 	call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
6088 				      &info.info);
6089 }
6090 EXPORT_SYMBOL(netdev_bonding_info_change);
6091 
6092 static void netdev_adjacent_add_links(struct net_device *dev)
6093 {
6094 	struct netdev_adjacent *iter;
6095 
6096 	struct net *net = dev_net(dev);
6097 
6098 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
6099 		if (!net_eq(net, dev_net(iter->dev)))
6100 			continue;
6101 		netdev_adjacent_sysfs_add(iter->dev, dev,
6102 					  &iter->dev->adj_list.lower);
6103 		netdev_adjacent_sysfs_add(dev, iter->dev,
6104 					  &dev->adj_list.upper);
6105 	}
6106 
6107 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
6108 		if (!net_eq(net, dev_net(iter->dev)))
6109 			continue;
6110 		netdev_adjacent_sysfs_add(iter->dev, dev,
6111 					  &iter->dev->adj_list.upper);
6112 		netdev_adjacent_sysfs_add(dev, iter->dev,
6113 					  &dev->adj_list.lower);
6114 	}
6115 }
6116 
6117 static void netdev_adjacent_del_links(struct net_device *dev)
6118 {
6119 	struct netdev_adjacent *iter;
6120 
6121 	struct net *net = dev_net(dev);
6122 
6123 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
6124 		if (!net_eq(net, dev_net(iter->dev)))
6125 			continue;
6126 		netdev_adjacent_sysfs_del(iter->dev, dev->name,
6127 					  &iter->dev->adj_list.lower);
6128 		netdev_adjacent_sysfs_del(dev, iter->dev->name,
6129 					  &dev->adj_list.upper);
6130 	}
6131 
6132 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
6133 		if (!net_eq(net, dev_net(iter->dev)))
6134 			continue;
6135 		netdev_adjacent_sysfs_del(iter->dev, dev->name,
6136 					  &iter->dev->adj_list.upper);
6137 		netdev_adjacent_sysfs_del(dev, iter->dev->name,
6138 					  &dev->adj_list.lower);
6139 	}
6140 }
6141 
6142 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
6143 {
6144 	struct netdev_adjacent *iter;
6145 
6146 	struct net *net = dev_net(dev);
6147 
6148 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
6149 		if (!net_eq(net, dev_net(iter->dev)))
6150 			continue;
6151 		netdev_adjacent_sysfs_del(iter->dev, oldname,
6152 					  &iter->dev->adj_list.lower);
6153 		netdev_adjacent_sysfs_add(iter->dev, dev,
6154 					  &iter->dev->adj_list.lower);
6155 	}
6156 
6157 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
6158 		if (!net_eq(net, dev_net(iter->dev)))
6159 			continue;
6160 		netdev_adjacent_sysfs_del(iter->dev, oldname,
6161 					  &iter->dev->adj_list.upper);
6162 		netdev_adjacent_sysfs_add(iter->dev, dev,
6163 					  &iter->dev->adj_list.upper);
6164 	}
6165 }
6166 
6167 void *netdev_lower_dev_get_private(struct net_device *dev,
6168 				   struct net_device *lower_dev)
6169 {
6170 	struct netdev_adjacent *lower;
6171 
6172 	if (!lower_dev)
6173 		return NULL;
6174 	lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
6175 	if (!lower)
6176 		return NULL;
6177 
6178 	return lower->private;
6179 }
6180 EXPORT_SYMBOL(netdev_lower_dev_get_private);
6181 
6182 
6183 int dev_get_nest_level(struct net_device *dev)
6184 {
6185 	struct net_device *lower = NULL;
6186 	struct list_head *iter;
6187 	int max_nest = -1;
6188 	int nest;
6189 
6190 	ASSERT_RTNL();
6191 
6192 	netdev_for_each_lower_dev(dev, lower, iter) {
6193 		nest = dev_get_nest_level(lower);
6194 		if (max_nest < nest)
6195 			max_nest = nest;
6196 	}
6197 
6198 	return max_nest + 1;
6199 }
6200 EXPORT_SYMBOL(dev_get_nest_level);
6201 
6202 /**
6203  * netdev_lower_change - Dispatch event about lower device state change
6204  * @lower_dev: device
6205  * @lower_state_info: state to dispatch
6206  *
6207  * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
6208  * The caller must hold the RTNL lock.
6209  */
6210 void netdev_lower_state_changed(struct net_device *lower_dev,
6211 				void *lower_state_info)
6212 {
6213 	struct netdev_notifier_changelowerstate_info changelowerstate_info;
6214 
6215 	ASSERT_RTNL();
6216 	changelowerstate_info.lower_state_info = lower_state_info;
6217 	call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev,
6218 				      &changelowerstate_info.info);
6219 }
6220 EXPORT_SYMBOL(netdev_lower_state_changed);
6221 
6222 static void dev_change_rx_flags(struct net_device *dev, int flags)
6223 {
6224 	const struct net_device_ops *ops = dev->netdev_ops;
6225 
6226 	if (ops->ndo_change_rx_flags)
6227 		ops->ndo_change_rx_flags(dev, flags);
6228 }
6229 
6230 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
6231 {
6232 	unsigned int old_flags = dev->flags;
6233 	kuid_t uid;
6234 	kgid_t gid;
6235 
6236 	ASSERT_RTNL();
6237 
6238 	dev->flags |= IFF_PROMISC;
6239 	dev->promiscuity += inc;
6240 	if (dev->promiscuity == 0) {
6241 		/*
6242 		 * Avoid overflow.
6243 		 * If inc causes overflow, untouch promisc and return error.
6244 		 */
6245 		if (inc < 0)
6246 			dev->flags &= ~IFF_PROMISC;
6247 		else {
6248 			dev->promiscuity -= inc;
6249 			pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
6250 				dev->name);
6251 			return -EOVERFLOW;
6252 		}
6253 	}
6254 	if (dev->flags != old_flags) {
6255 		pr_info("device %s %s promiscuous mode\n",
6256 			dev->name,
6257 			dev->flags & IFF_PROMISC ? "entered" : "left");
6258 		if (audit_enabled) {
6259 			current_uid_gid(&uid, &gid);
6260 			audit_log(current->audit_context, GFP_ATOMIC,
6261 				AUDIT_ANOM_PROMISCUOUS,
6262 				"dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
6263 				dev->name, (dev->flags & IFF_PROMISC),
6264 				(old_flags & IFF_PROMISC),
6265 				from_kuid(&init_user_ns, audit_get_loginuid(current)),
6266 				from_kuid(&init_user_ns, uid),
6267 				from_kgid(&init_user_ns, gid),
6268 				audit_get_sessionid(current));
6269 		}
6270 
6271 		dev_change_rx_flags(dev, IFF_PROMISC);
6272 	}
6273 	if (notify)
6274 		__dev_notify_flags(dev, old_flags, IFF_PROMISC);
6275 	return 0;
6276 }
6277 
6278 /**
6279  *	dev_set_promiscuity	- update promiscuity count on a device
6280  *	@dev: device
6281  *	@inc: modifier
6282  *
6283  *	Add or remove promiscuity from a device. While the count in the device
6284  *	remains above zero the interface remains promiscuous. Once it hits zero
6285  *	the device reverts back to normal filtering operation. A negative inc
6286  *	value is used to drop promiscuity on the device.
6287  *	Return 0 if successful or a negative errno code on error.
6288  */
6289 int dev_set_promiscuity(struct net_device *dev, int inc)
6290 {
6291 	unsigned int old_flags = dev->flags;
6292 	int err;
6293 
6294 	err = __dev_set_promiscuity(dev, inc, true);
6295 	if (err < 0)
6296 		return err;
6297 	if (dev->flags != old_flags)
6298 		dev_set_rx_mode(dev);
6299 	return err;
6300 }
6301 EXPORT_SYMBOL(dev_set_promiscuity);
6302 
6303 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
6304 {
6305 	unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
6306 
6307 	ASSERT_RTNL();
6308 
6309 	dev->flags |= IFF_ALLMULTI;
6310 	dev->allmulti += inc;
6311 	if (dev->allmulti == 0) {
6312 		/*
6313 		 * Avoid overflow.
6314 		 * If inc causes overflow, untouch allmulti and return error.
6315 		 */
6316 		if (inc < 0)
6317 			dev->flags &= ~IFF_ALLMULTI;
6318 		else {
6319 			dev->allmulti -= inc;
6320 			pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
6321 				dev->name);
6322 			return -EOVERFLOW;
6323 		}
6324 	}
6325 	if (dev->flags ^ old_flags) {
6326 		dev_change_rx_flags(dev, IFF_ALLMULTI);
6327 		dev_set_rx_mode(dev);
6328 		if (notify)
6329 			__dev_notify_flags(dev, old_flags,
6330 					   dev->gflags ^ old_gflags);
6331 	}
6332 	return 0;
6333 }
6334 
6335 /**
6336  *	dev_set_allmulti	- update allmulti count on a device
6337  *	@dev: device
6338  *	@inc: modifier
6339  *
6340  *	Add or remove reception of all multicast frames to a device. While the
6341  *	count in the device remains above zero the interface remains listening
6342  *	to all interfaces. Once it hits zero the device reverts back to normal
6343  *	filtering operation. A negative @inc value is used to drop the counter
6344  *	when releasing a resource needing all multicasts.
6345  *	Return 0 if successful or a negative errno code on error.
6346  */
6347 
6348 int dev_set_allmulti(struct net_device *dev, int inc)
6349 {
6350 	return __dev_set_allmulti(dev, inc, true);
6351 }
6352 EXPORT_SYMBOL(dev_set_allmulti);
6353 
6354 /*
6355  *	Upload unicast and multicast address lists to device and
6356  *	configure RX filtering. When the device doesn't support unicast
6357  *	filtering it is put in promiscuous mode while unicast addresses
6358  *	are present.
6359  */
6360 void __dev_set_rx_mode(struct net_device *dev)
6361 {
6362 	const struct net_device_ops *ops = dev->netdev_ops;
6363 
6364 	/* dev_open will call this function so the list will stay sane. */
6365 	if (!(dev->flags&IFF_UP))
6366 		return;
6367 
6368 	if (!netif_device_present(dev))
6369 		return;
6370 
6371 	if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
6372 		/* Unicast addresses changes may only happen under the rtnl,
6373 		 * therefore calling __dev_set_promiscuity here is safe.
6374 		 */
6375 		if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
6376 			__dev_set_promiscuity(dev, 1, false);
6377 			dev->uc_promisc = true;
6378 		} else if (netdev_uc_empty(dev) && dev->uc_promisc) {
6379 			__dev_set_promiscuity(dev, -1, false);
6380 			dev->uc_promisc = false;
6381 		}
6382 	}
6383 
6384 	if (ops->ndo_set_rx_mode)
6385 		ops->ndo_set_rx_mode(dev);
6386 }
6387 
6388 void dev_set_rx_mode(struct net_device *dev)
6389 {
6390 	netif_addr_lock_bh(dev);
6391 	__dev_set_rx_mode(dev);
6392 	netif_addr_unlock_bh(dev);
6393 }
6394 
6395 /**
6396  *	dev_get_flags - get flags reported to userspace
6397  *	@dev: device
6398  *
6399  *	Get the combination of flag bits exported through APIs to userspace.
6400  */
6401 unsigned int dev_get_flags(const struct net_device *dev)
6402 {
6403 	unsigned int flags;
6404 
6405 	flags = (dev->flags & ~(IFF_PROMISC |
6406 				IFF_ALLMULTI |
6407 				IFF_RUNNING |
6408 				IFF_LOWER_UP |
6409 				IFF_DORMANT)) |
6410 		(dev->gflags & (IFF_PROMISC |
6411 				IFF_ALLMULTI));
6412 
6413 	if (netif_running(dev)) {
6414 		if (netif_oper_up(dev))
6415 			flags |= IFF_RUNNING;
6416 		if (netif_carrier_ok(dev))
6417 			flags |= IFF_LOWER_UP;
6418 		if (netif_dormant(dev))
6419 			flags |= IFF_DORMANT;
6420 	}
6421 
6422 	return flags;
6423 }
6424 EXPORT_SYMBOL(dev_get_flags);
6425 
6426 int __dev_change_flags(struct net_device *dev, unsigned int flags)
6427 {
6428 	unsigned int old_flags = dev->flags;
6429 	int ret;
6430 
6431 	ASSERT_RTNL();
6432 
6433 	/*
6434 	 *	Set the flags on our device.
6435 	 */
6436 
6437 	dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
6438 			       IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
6439 			       IFF_AUTOMEDIA)) |
6440 		     (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
6441 				    IFF_ALLMULTI));
6442 
6443 	/*
6444 	 *	Load in the correct multicast list now the flags have changed.
6445 	 */
6446 
6447 	if ((old_flags ^ flags) & IFF_MULTICAST)
6448 		dev_change_rx_flags(dev, IFF_MULTICAST);
6449 
6450 	dev_set_rx_mode(dev);
6451 
6452 	/*
6453 	 *	Have we downed the interface. We handle IFF_UP ourselves
6454 	 *	according to user attempts to set it, rather than blindly
6455 	 *	setting it.
6456 	 */
6457 
6458 	ret = 0;
6459 	if ((old_flags ^ flags) & IFF_UP)
6460 		ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
6461 
6462 	if ((flags ^ dev->gflags) & IFF_PROMISC) {
6463 		int inc = (flags & IFF_PROMISC) ? 1 : -1;
6464 		unsigned int old_flags = dev->flags;
6465 
6466 		dev->gflags ^= IFF_PROMISC;
6467 
6468 		if (__dev_set_promiscuity(dev, inc, false) >= 0)
6469 			if (dev->flags != old_flags)
6470 				dev_set_rx_mode(dev);
6471 	}
6472 
6473 	/* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
6474 	 * is important. Some (broken) drivers set IFF_PROMISC, when
6475 	 * IFF_ALLMULTI is requested not asking us and not reporting.
6476 	 */
6477 	if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
6478 		int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
6479 
6480 		dev->gflags ^= IFF_ALLMULTI;
6481 		__dev_set_allmulti(dev, inc, false);
6482 	}
6483 
6484 	return ret;
6485 }
6486 
6487 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
6488 			unsigned int gchanges)
6489 {
6490 	unsigned int changes = dev->flags ^ old_flags;
6491 
6492 	if (gchanges)
6493 		rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
6494 
6495 	if (changes & IFF_UP) {
6496 		if (dev->flags & IFF_UP)
6497 			call_netdevice_notifiers(NETDEV_UP, dev);
6498 		else
6499 			call_netdevice_notifiers(NETDEV_DOWN, dev);
6500 	}
6501 
6502 	if (dev->flags & IFF_UP &&
6503 	    (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
6504 		struct netdev_notifier_change_info change_info;
6505 
6506 		change_info.flags_changed = changes;
6507 		call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
6508 					      &change_info.info);
6509 	}
6510 }
6511 
6512 /**
6513  *	dev_change_flags - change device settings
6514  *	@dev: device
6515  *	@flags: device state flags
6516  *
6517  *	Change settings on device based state flags. The flags are
6518  *	in the userspace exported format.
6519  */
6520 int dev_change_flags(struct net_device *dev, unsigned int flags)
6521 {
6522 	int ret;
6523 	unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
6524 
6525 	ret = __dev_change_flags(dev, flags);
6526 	if (ret < 0)
6527 		return ret;
6528 
6529 	changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
6530 	__dev_notify_flags(dev, old_flags, changes);
6531 	return ret;
6532 }
6533 EXPORT_SYMBOL(dev_change_flags);
6534 
6535 static int __dev_set_mtu(struct net_device *dev, int new_mtu)
6536 {
6537 	const struct net_device_ops *ops = dev->netdev_ops;
6538 
6539 	if (ops->ndo_change_mtu)
6540 		return ops->ndo_change_mtu(dev, new_mtu);
6541 
6542 	dev->mtu = new_mtu;
6543 	return 0;
6544 }
6545 
6546 /**
6547  *	dev_set_mtu - Change maximum transfer unit
6548  *	@dev: device
6549  *	@new_mtu: new transfer unit
6550  *
6551  *	Change the maximum transfer size of the network device.
6552  */
6553 int dev_set_mtu(struct net_device *dev, int new_mtu)
6554 {
6555 	int err, orig_mtu;
6556 
6557 	if (new_mtu == dev->mtu)
6558 		return 0;
6559 
6560 	/* MTU must be positive, and in range */
6561 	if (new_mtu < 0 || new_mtu < dev->min_mtu) {
6562 		net_err_ratelimited("%s: Invalid MTU %d requested, hw min %d\n",
6563 				    dev->name, new_mtu, dev->min_mtu);
6564 		return -EINVAL;
6565 	}
6566 
6567 	if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
6568 		net_err_ratelimited("%s: Invalid MTU %d requested, hw max %d\n",
6569 				    dev->name, new_mtu, dev->max_mtu);
6570 		return -EINVAL;
6571 	}
6572 
6573 	if (!netif_device_present(dev))
6574 		return -ENODEV;
6575 
6576 	err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
6577 	err = notifier_to_errno(err);
6578 	if (err)
6579 		return err;
6580 
6581 	orig_mtu = dev->mtu;
6582 	err = __dev_set_mtu(dev, new_mtu);
6583 
6584 	if (!err) {
6585 		err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6586 		err = notifier_to_errno(err);
6587 		if (err) {
6588 			/* setting mtu back and notifying everyone again,
6589 			 * so that they have a chance to revert changes.
6590 			 */
6591 			__dev_set_mtu(dev, orig_mtu);
6592 			call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6593 		}
6594 	}
6595 	return err;
6596 }
6597 EXPORT_SYMBOL(dev_set_mtu);
6598 
6599 /**
6600  *	dev_set_group - Change group this device belongs to
6601  *	@dev: device
6602  *	@new_group: group this device should belong to
6603  */
6604 void dev_set_group(struct net_device *dev, int new_group)
6605 {
6606 	dev->group = new_group;
6607 }
6608 EXPORT_SYMBOL(dev_set_group);
6609 
6610 /**
6611  *	dev_set_mac_address - Change Media Access Control Address
6612  *	@dev: device
6613  *	@sa: new address
6614  *
6615  *	Change the hardware (MAC) address of the device
6616  */
6617 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
6618 {
6619 	const struct net_device_ops *ops = dev->netdev_ops;
6620 	int err;
6621 
6622 	if (!ops->ndo_set_mac_address)
6623 		return -EOPNOTSUPP;
6624 	if (sa->sa_family != dev->type)
6625 		return -EINVAL;
6626 	if (!netif_device_present(dev))
6627 		return -ENODEV;
6628 	err = ops->ndo_set_mac_address(dev, sa);
6629 	if (err)
6630 		return err;
6631 	dev->addr_assign_type = NET_ADDR_SET;
6632 	call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
6633 	add_device_randomness(dev->dev_addr, dev->addr_len);
6634 	return 0;
6635 }
6636 EXPORT_SYMBOL(dev_set_mac_address);
6637 
6638 /**
6639  *	dev_change_carrier - Change device carrier
6640  *	@dev: device
6641  *	@new_carrier: new value
6642  *
6643  *	Change device carrier
6644  */
6645 int dev_change_carrier(struct net_device *dev, bool new_carrier)
6646 {
6647 	const struct net_device_ops *ops = dev->netdev_ops;
6648 
6649 	if (!ops->ndo_change_carrier)
6650 		return -EOPNOTSUPP;
6651 	if (!netif_device_present(dev))
6652 		return -ENODEV;
6653 	return ops->ndo_change_carrier(dev, new_carrier);
6654 }
6655 EXPORT_SYMBOL(dev_change_carrier);
6656 
6657 /**
6658  *	dev_get_phys_port_id - Get device physical port ID
6659  *	@dev: device
6660  *	@ppid: port ID
6661  *
6662  *	Get device physical port ID
6663  */
6664 int dev_get_phys_port_id(struct net_device *dev,
6665 			 struct netdev_phys_item_id *ppid)
6666 {
6667 	const struct net_device_ops *ops = dev->netdev_ops;
6668 
6669 	if (!ops->ndo_get_phys_port_id)
6670 		return -EOPNOTSUPP;
6671 	return ops->ndo_get_phys_port_id(dev, ppid);
6672 }
6673 EXPORT_SYMBOL(dev_get_phys_port_id);
6674 
6675 /**
6676  *	dev_get_phys_port_name - Get device physical port name
6677  *	@dev: device
6678  *	@name: port name
6679  *	@len: limit of bytes to copy to name
6680  *
6681  *	Get device physical port name
6682  */
6683 int dev_get_phys_port_name(struct net_device *dev,
6684 			   char *name, size_t len)
6685 {
6686 	const struct net_device_ops *ops = dev->netdev_ops;
6687 
6688 	if (!ops->ndo_get_phys_port_name)
6689 		return -EOPNOTSUPP;
6690 	return ops->ndo_get_phys_port_name(dev, name, len);
6691 }
6692 EXPORT_SYMBOL(dev_get_phys_port_name);
6693 
6694 /**
6695  *	dev_change_proto_down - update protocol port state information
6696  *	@dev: device
6697  *	@proto_down: new value
6698  *
6699  *	This info can be used by switch drivers to set the phys state of the
6700  *	port.
6701  */
6702 int dev_change_proto_down(struct net_device *dev, bool proto_down)
6703 {
6704 	const struct net_device_ops *ops = dev->netdev_ops;
6705 
6706 	if (!ops->ndo_change_proto_down)
6707 		return -EOPNOTSUPP;
6708 	if (!netif_device_present(dev))
6709 		return -ENODEV;
6710 	return ops->ndo_change_proto_down(dev, proto_down);
6711 }
6712 EXPORT_SYMBOL(dev_change_proto_down);
6713 
6714 /**
6715  *	dev_change_xdp_fd - set or clear a bpf program for a device rx path
6716  *	@dev: device
6717  *	@fd: new program fd or negative value to clear
6718  *	@flags: xdp-related flags
6719  *
6720  *	Set or clear a bpf program for a device
6721  */
6722 int dev_change_xdp_fd(struct net_device *dev, int fd, u32 flags)
6723 {
6724 	const struct net_device_ops *ops = dev->netdev_ops;
6725 	struct bpf_prog *prog = NULL;
6726 	struct netdev_xdp xdp;
6727 	int err;
6728 
6729 	ASSERT_RTNL();
6730 
6731 	if (!ops->ndo_xdp)
6732 		return -EOPNOTSUPP;
6733 	if (fd >= 0) {
6734 		if (flags & XDP_FLAGS_UPDATE_IF_NOEXIST) {
6735 			memset(&xdp, 0, sizeof(xdp));
6736 			xdp.command = XDP_QUERY_PROG;
6737 
6738 			err = ops->ndo_xdp(dev, &xdp);
6739 			if (err < 0)
6740 				return err;
6741 			if (xdp.prog_attached)
6742 				return -EBUSY;
6743 		}
6744 
6745 		prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_XDP);
6746 		if (IS_ERR(prog))
6747 			return PTR_ERR(prog);
6748 	}
6749 
6750 	memset(&xdp, 0, sizeof(xdp));
6751 	xdp.command = XDP_SETUP_PROG;
6752 	xdp.prog = prog;
6753 
6754 	err = ops->ndo_xdp(dev, &xdp);
6755 	if (err < 0 && prog)
6756 		bpf_prog_put(prog);
6757 
6758 	return err;
6759 }
6760 EXPORT_SYMBOL(dev_change_xdp_fd);
6761 
6762 /**
6763  *	dev_new_index	-	allocate an ifindex
6764  *	@net: the applicable net namespace
6765  *
6766  *	Returns a suitable unique value for a new device interface
6767  *	number.  The caller must hold the rtnl semaphore or the
6768  *	dev_base_lock to be sure it remains unique.
6769  */
6770 static int dev_new_index(struct net *net)
6771 {
6772 	int ifindex = net->ifindex;
6773 
6774 	for (;;) {
6775 		if (++ifindex <= 0)
6776 			ifindex = 1;
6777 		if (!__dev_get_by_index(net, ifindex))
6778 			return net->ifindex = ifindex;
6779 	}
6780 }
6781 
6782 /* Delayed registration/unregisteration */
6783 static LIST_HEAD(net_todo_list);
6784 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
6785 
6786 static void net_set_todo(struct net_device *dev)
6787 {
6788 	list_add_tail(&dev->todo_list, &net_todo_list);
6789 	dev_net(dev)->dev_unreg_count++;
6790 }
6791 
6792 static void rollback_registered_many(struct list_head *head)
6793 {
6794 	struct net_device *dev, *tmp;
6795 	LIST_HEAD(close_head);
6796 
6797 	BUG_ON(dev_boot_phase);
6798 	ASSERT_RTNL();
6799 
6800 	list_for_each_entry_safe(dev, tmp, head, unreg_list) {
6801 		/* Some devices call without registering
6802 		 * for initialization unwind. Remove those
6803 		 * devices and proceed with the remaining.
6804 		 */
6805 		if (dev->reg_state == NETREG_UNINITIALIZED) {
6806 			pr_debug("unregister_netdevice: device %s/%p never was registered\n",
6807 				 dev->name, dev);
6808 
6809 			WARN_ON(1);
6810 			list_del(&dev->unreg_list);
6811 			continue;
6812 		}
6813 		dev->dismantle = true;
6814 		BUG_ON(dev->reg_state != NETREG_REGISTERED);
6815 	}
6816 
6817 	/* If device is running, close it first. */
6818 	list_for_each_entry(dev, head, unreg_list)
6819 		list_add_tail(&dev->close_list, &close_head);
6820 	dev_close_many(&close_head, true);
6821 
6822 	list_for_each_entry(dev, head, unreg_list) {
6823 		/* And unlink it from device chain. */
6824 		unlist_netdevice(dev);
6825 
6826 		dev->reg_state = NETREG_UNREGISTERING;
6827 	}
6828 	flush_all_backlogs();
6829 
6830 	synchronize_net();
6831 
6832 	list_for_each_entry(dev, head, unreg_list) {
6833 		struct sk_buff *skb = NULL;
6834 
6835 		/* Shutdown queueing discipline. */
6836 		dev_shutdown(dev);
6837 
6838 
6839 		/* Notify protocols, that we are about to destroy
6840 		 * this device. They should clean all the things.
6841 		 */
6842 		call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6843 
6844 		if (!dev->rtnl_link_ops ||
6845 		    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6846 			skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
6847 						     GFP_KERNEL);
6848 
6849 		/*
6850 		 *	Flush the unicast and multicast chains
6851 		 */
6852 		dev_uc_flush(dev);
6853 		dev_mc_flush(dev);
6854 
6855 		if (dev->netdev_ops->ndo_uninit)
6856 			dev->netdev_ops->ndo_uninit(dev);
6857 
6858 		if (skb)
6859 			rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
6860 
6861 		/* Notifier chain MUST detach us all upper devices. */
6862 		WARN_ON(netdev_has_any_upper_dev(dev));
6863 		WARN_ON(netdev_has_any_lower_dev(dev));
6864 
6865 		/* Remove entries from kobject tree */
6866 		netdev_unregister_kobject(dev);
6867 #ifdef CONFIG_XPS
6868 		/* Remove XPS queueing entries */
6869 		netif_reset_xps_queues_gt(dev, 0);
6870 #endif
6871 	}
6872 
6873 	synchronize_net();
6874 
6875 	list_for_each_entry(dev, head, unreg_list)
6876 		dev_put(dev);
6877 }
6878 
6879 static void rollback_registered(struct net_device *dev)
6880 {
6881 	LIST_HEAD(single);
6882 
6883 	list_add(&dev->unreg_list, &single);
6884 	rollback_registered_many(&single);
6885 	list_del(&single);
6886 }
6887 
6888 static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
6889 	struct net_device *upper, netdev_features_t features)
6890 {
6891 	netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6892 	netdev_features_t feature;
6893 	int feature_bit;
6894 
6895 	for_each_netdev_feature(&upper_disables, feature_bit) {
6896 		feature = __NETIF_F_BIT(feature_bit);
6897 		if (!(upper->wanted_features & feature)
6898 		    && (features & feature)) {
6899 			netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
6900 				   &feature, upper->name);
6901 			features &= ~feature;
6902 		}
6903 	}
6904 
6905 	return features;
6906 }
6907 
6908 static void netdev_sync_lower_features(struct net_device *upper,
6909 	struct net_device *lower, netdev_features_t features)
6910 {
6911 	netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6912 	netdev_features_t feature;
6913 	int feature_bit;
6914 
6915 	for_each_netdev_feature(&upper_disables, feature_bit) {
6916 		feature = __NETIF_F_BIT(feature_bit);
6917 		if (!(features & feature) && (lower->features & feature)) {
6918 			netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
6919 				   &feature, lower->name);
6920 			lower->wanted_features &= ~feature;
6921 			netdev_update_features(lower);
6922 
6923 			if (unlikely(lower->features & feature))
6924 				netdev_WARN(upper, "failed to disable %pNF on %s!\n",
6925 					    &feature, lower->name);
6926 		}
6927 	}
6928 }
6929 
6930 static netdev_features_t netdev_fix_features(struct net_device *dev,
6931 	netdev_features_t features)
6932 {
6933 	/* Fix illegal checksum combinations */
6934 	if ((features & NETIF_F_HW_CSUM) &&
6935 	    (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
6936 		netdev_warn(dev, "mixed HW and IP checksum settings.\n");
6937 		features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
6938 	}
6939 
6940 	/* TSO requires that SG is present as well. */
6941 	if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
6942 		netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
6943 		features &= ~NETIF_F_ALL_TSO;
6944 	}
6945 
6946 	if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
6947 					!(features & NETIF_F_IP_CSUM)) {
6948 		netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
6949 		features &= ~NETIF_F_TSO;
6950 		features &= ~NETIF_F_TSO_ECN;
6951 	}
6952 
6953 	if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
6954 					 !(features & NETIF_F_IPV6_CSUM)) {
6955 		netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
6956 		features &= ~NETIF_F_TSO6;
6957 	}
6958 
6959 	/* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
6960 	if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
6961 		features &= ~NETIF_F_TSO_MANGLEID;
6962 
6963 	/* TSO ECN requires that TSO is present as well. */
6964 	if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
6965 		features &= ~NETIF_F_TSO_ECN;
6966 
6967 	/* Software GSO depends on SG. */
6968 	if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
6969 		netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
6970 		features &= ~NETIF_F_GSO;
6971 	}
6972 
6973 	/* UFO needs SG and checksumming */
6974 	if (features & NETIF_F_UFO) {
6975 		/* maybe split UFO into V4 and V6? */
6976 		if (!(features & NETIF_F_HW_CSUM) &&
6977 		    ((features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) !=
6978 		     (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) {
6979 			netdev_dbg(dev,
6980 				"Dropping NETIF_F_UFO since no checksum offload features.\n");
6981 			features &= ~NETIF_F_UFO;
6982 		}
6983 
6984 		if (!(features & NETIF_F_SG)) {
6985 			netdev_dbg(dev,
6986 				"Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
6987 			features &= ~NETIF_F_UFO;
6988 		}
6989 	}
6990 
6991 	/* GSO partial features require GSO partial be set */
6992 	if ((features & dev->gso_partial_features) &&
6993 	    !(features & NETIF_F_GSO_PARTIAL)) {
6994 		netdev_dbg(dev,
6995 			   "Dropping partially supported GSO features since no GSO partial.\n");
6996 		features &= ~dev->gso_partial_features;
6997 	}
6998 
6999 	return features;
7000 }
7001 
7002 int __netdev_update_features(struct net_device *dev)
7003 {
7004 	struct net_device *upper, *lower;
7005 	netdev_features_t features;
7006 	struct list_head *iter;
7007 	int err = -1;
7008 
7009 	ASSERT_RTNL();
7010 
7011 	features = netdev_get_wanted_features(dev);
7012 
7013 	if (dev->netdev_ops->ndo_fix_features)
7014 		features = dev->netdev_ops->ndo_fix_features(dev, features);
7015 
7016 	/* driver might be less strict about feature dependencies */
7017 	features = netdev_fix_features(dev, features);
7018 
7019 	/* some features can't be enabled if they're off an an upper device */
7020 	netdev_for_each_upper_dev_rcu(dev, upper, iter)
7021 		features = netdev_sync_upper_features(dev, upper, features);
7022 
7023 	if (dev->features == features)
7024 		goto sync_lower;
7025 
7026 	netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
7027 		&dev->features, &features);
7028 
7029 	if (dev->netdev_ops->ndo_set_features)
7030 		err = dev->netdev_ops->ndo_set_features(dev, features);
7031 	else
7032 		err = 0;
7033 
7034 	if (unlikely(err < 0)) {
7035 		netdev_err(dev,
7036 			"set_features() failed (%d); wanted %pNF, left %pNF\n",
7037 			err, &features, &dev->features);
7038 		/* return non-0 since some features might have changed and
7039 		 * it's better to fire a spurious notification than miss it
7040 		 */
7041 		return -1;
7042 	}
7043 
7044 sync_lower:
7045 	/* some features must be disabled on lower devices when disabled
7046 	 * on an upper device (think: bonding master or bridge)
7047 	 */
7048 	netdev_for_each_lower_dev(dev, lower, iter)
7049 		netdev_sync_lower_features(dev, lower, features);
7050 
7051 	if (!err)
7052 		dev->features = features;
7053 
7054 	return err < 0 ? 0 : 1;
7055 }
7056 
7057 /**
7058  *	netdev_update_features - recalculate device features
7059  *	@dev: the device to check
7060  *
7061  *	Recalculate dev->features set and send notifications if it
7062  *	has changed. Should be called after driver or hardware dependent
7063  *	conditions might have changed that influence the features.
7064  */
7065 void netdev_update_features(struct net_device *dev)
7066 {
7067 	if (__netdev_update_features(dev))
7068 		netdev_features_change(dev);
7069 }
7070 EXPORT_SYMBOL(netdev_update_features);
7071 
7072 /**
7073  *	netdev_change_features - recalculate device features
7074  *	@dev: the device to check
7075  *
7076  *	Recalculate dev->features set and send notifications even
7077  *	if they have not changed. Should be called instead of
7078  *	netdev_update_features() if also dev->vlan_features might
7079  *	have changed to allow the changes to be propagated to stacked
7080  *	VLAN devices.
7081  */
7082 void netdev_change_features(struct net_device *dev)
7083 {
7084 	__netdev_update_features(dev);
7085 	netdev_features_change(dev);
7086 }
7087 EXPORT_SYMBOL(netdev_change_features);
7088 
7089 /**
7090  *	netif_stacked_transfer_operstate -	transfer operstate
7091  *	@rootdev: the root or lower level device to transfer state from
7092  *	@dev: the device to transfer operstate to
7093  *
7094  *	Transfer operational state from root to device. This is normally
7095  *	called when a stacking relationship exists between the root
7096  *	device and the device(a leaf device).
7097  */
7098 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
7099 					struct net_device *dev)
7100 {
7101 	if (rootdev->operstate == IF_OPER_DORMANT)
7102 		netif_dormant_on(dev);
7103 	else
7104 		netif_dormant_off(dev);
7105 
7106 	if (netif_carrier_ok(rootdev)) {
7107 		if (!netif_carrier_ok(dev))
7108 			netif_carrier_on(dev);
7109 	} else {
7110 		if (netif_carrier_ok(dev))
7111 			netif_carrier_off(dev);
7112 	}
7113 }
7114 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
7115 
7116 #ifdef CONFIG_SYSFS
7117 static int netif_alloc_rx_queues(struct net_device *dev)
7118 {
7119 	unsigned int i, count = dev->num_rx_queues;
7120 	struct netdev_rx_queue *rx;
7121 	size_t sz = count * sizeof(*rx);
7122 
7123 	BUG_ON(count < 1);
7124 
7125 	rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7126 	if (!rx) {
7127 		rx = vzalloc(sz);
7128 		if (!rx)
7129 			return -ENOMEM;
7130 	}
7131 	dev->_rx = rx;
7132 
7133 	for (i = 0; i < count; i++)
7134 		rx[i].dev = dev;
7135 	return 0;
7136 }
7137 #endif
7138 
7139 static void netdev_init_one_queue(struct net_device *dev,
7140 				  struct netdev_queue *queue, void *_unused)
7141 {
7142 	/* Initialize queue lock */
7143 	spin_lock_init(&queue->_xmit_lock);
7144 	netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
7145 	queue->xmit_lock_owner = -1;
7146 	netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
7147 	queue->dev = dev;
7148 #ifdef CONFIG_BQL
7149 	dql_init(&queue->dql, HZ);
7150 #endif
7151 }
7152 
7153 static void netif_free_tx_queues(struct net_device *dev)
7154 {
7155 	kvfree(dev->_tx);
7156 }
7157 
7158 static int netif_alloc_netdev_queues(struct net_device *dev)
7159 {
7160 	unsigned int count = dev->num_tx_queues;
7161 	struct netdev_queue *tx;
7162 	size_t sz = count * sizeof(*tx);
7163 
7164 	if (count < 1 || count > 0xffff)
7165 		return -EINVAL;
7166 
7167 	tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7168 	if (!tx) {
7169 		tx = vzalloc(sz);
7170 		if (!tx)
7171 			return -ENOMEM;
7172 	}
7173 	dev->_tx = tx;
7174 
7175 	netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
7176 	spin_lock_init(&dev->tx_global_lock);
7177 
7178 	return 0;
7179 }
7180 
7181 void netif_tx_stop_all_queues(struct net_device *dev)
7182 {
7183 	unsigned int i;
7184 
7185 	for (i = 0; i < dev->num_tx_queues; i++) {
7186 		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
7187 
7188 		netif_tx_stop_queue(txq);
7189 	}
7190 }
7191 EXPORT_SYMBOL(netif_tx_stop_all_queues);
7192 
7193 /**
7194  *	register_netdevice	- register a network device
7195  *	@dev: device to register
7196  *
7197  *	Take a completed network device structure and add it to the kernel
7198  *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7199  *	chain. 0 is returned on success. A negative errno code is returned
7200  *	on a failure to set up the device, or if the name is a duplicate.
7201  *
7202  *	Callers must hold the rtnl semaphore. You may want
7203  *	register_netdev() instead of this.
7204  *
7205  *	BUGS:
7206  *	The locking appears insufficient to guarantee two parallel registers
7207  *	will not get the same name.
7208  */
7209 
7210 int register_netdevice(struct net_device *dev)
7211 {
7212 	int ret;
7213 	struct net *net = dev_net(dev);
7214 
7215 	BUG_ON(dev_boot_phase);
7216 	ASSERT_RTNL();
7217 
7218 	might_sleep();
7219 
7220 	/* When net_device's are persistent, this will be fatal. */
7221 	BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
7222 	BUG_ON(!net);
7223 
7224 	spin_lock_init(&dev->addr_list_lock);
7225 	netdev_set_addr_lockdep_class(dev);
7226 
7227 	ret = dev_get_valid_name(net, dev, dev->name);
7228 	if (ret < 0)
7229 		goto out;
7230 
7231 	/* Init, if this function is available */
7232 	if (dev->netdev_ops->ndo_init) {
7233 		ret = dev->netdev_ops->ndo_init(dev);
7234 		if (ret) {
7235 			if (ret > 0)
7236 				ret = -EIO;
7237 			goto out;
7238 		}
7239 	}
7240 
7241 	if (((dev->hw_features | dev->features) &
7242 	     NETIF_F_HW_VLAN_CTAG_FILTER) &&
7243 	    (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
7244 	     !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
7245 		netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
7246 		ret = -EINVAL;
7247 		goto err_uninit;
7248 	}
7249 
7250 	ret = -EBUSY;
7251 	if (!dev->ifindex)
7252 		dev->ifindex = dev_new_index(net);
7253 	else if (__dev_get_by_index(net, dev->ifindex))
7254 		goto err_uninit;
7255 
7256 	/* Transfer changeable features to wanted_features and enable
7257 	 * software offloads (GSO and GRO).
7258 	 */
7259 	dev->hw_features |= NETIF_F_SOFT_FEATURES;
7260 	dev->features |= NETIF_F_SOFT_FEATURES;
7261 	dev->wanted_features = dev->features & dev->hw_features;
7262 
7263 	if (!(dev->flags & IFF_LOOPBACK))
7264 		dev->hw_features |= NETIF_F_NOCACHE_COPY;
7265 
7266 	/* If IPv4 TCP segmentation offload is supported we should also
7267 	 * allow the device to enable segmenting the frame with the option
7268 	 * of ignoring a static IP ID value.  This doesn't enable the
7269 	 * feature itself but allows the user to enable it later.
7270 	 */
7271 	if (dev->hw_features & NETIF_F_TSO)
7272 		dev->hw_features |= NETIF_F_TSO_MANGLEID;
7273 	if (dev->vlan_features & NETIF_F_TSO)
7274 		dev->vlan_features |= NETIF_F_TSO_MANGLEID;
7275 	if (dev->mpls_features & NETIF_F_TSO)
7276 		dev->mpls_features |= NETIF_F_TSO_MANGLEID;
7277 	if (dev->hw_enc_features & NETIF_F_TSO)
7278 		dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
7279 
7280 	/* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
7281 	 */
7282 	dev->vlan_features |= NETIF_F_HIGHDMA;
7283 
7284 	/* Make NETIF_F_SG inheritable to tunnel devices.
7285 	 */
7286 	dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
7287 
7288 	/* Make NETIF_F_SG inheritable to MPLS.
7289 	 */
7290 	dev->mpls_features |= NETIF_F_SG;
7291 
7292 	ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
7293 	ret = notifier_to_errno(ret);
7294 	if (ret)
7295 		goto err_uninit;
7296 
7297 	ret = netdev_register_kobject(dev);
7298 	if (ret)
7299 		goto err_uninit;
7300 	dev->reg_state = NETREG_REGISTERED;
7301 
7302 	__netdev_update_features(dev);
7303 
7304 	/*
7305 	 *	Default initial state at registry is that the
7306 	 *	device is present.
7307 	 */
7308 
7309 	set_bit(__LINK_STATE_PRESENT, &dev->state);
7310 
7311 	linkwatch_init_dev(dev);
7312 
7313 	dev_init_scheduler(dev);
7314 	dev_hold(dev);
7315 	list_netdevice(dev);
7316 	add_device_randomness(dev->dev_addr, dev->addr_len);
7317 
7318 	/* If the device has permanent device address, driver should
7319 	 * set dev_addr and also addr_assign_type should be set to
7320 	 * NET_ADDR_PERM (default value).
7321 	 */
7322 	if (dev->addr_assign_type == NET_ADDR_PERM)
7323 		memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
7324 
7325 	/* Notify protocols, that a new device appeared. */
7326 	ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
7327 	ret = notifier_to_errno(ret);
7328 	if (ret) {
7329 		rollback_registered(dev);
7330 		dev->reg_state = NETREG_UNREGISTERED;
7331 	}
7332 	/*
7333 	 *	Prevent userspace races by waiting until the network
7334 	 *	device is fully setup before sending notifications.
7335 	 */
7336 	if (!dev->rtnl_link_ops ||
7337 	    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
7338 		rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
7339 
7340 out:
7341 	return ret;
7342 
7343 err_uninit:
7344 	if (dev->netdev_ops->ndo_uninit)
7345 		dev->netdev_ops->ndo_uninit(dev);
7346 	goto out;
7347 }
7348 EXPORT_SYMBOL(register_netdevice);
7349 
7350 /**
7351  *	init_dummy_netdev	- init a dummy network device for NAPI
7352  *	@dev: device to init
7353  *
7354  *	This takes a network device structure and initialize the minimum
7355  *	amount of fields so it can be used to schedule NAPI polls without
7356  *	registering a full blown interface. This is to be used by drivers
7357  *	that need to tie several hardware interfaces to a single NAPI
7358  *	poll scheduler due to HW limitations.
7359  */
7360 int init_dummy_netdev(struct net_device *dev)
7361 {
7362 	/* Clear everything. Note we don't initialize spinlocks
7363 	 * are they aren't supposed to be taken by any of the
7364 	 * NAPI code and this dummy netdev is supposed to be
7365 	 * only ever used for NAPI polls
7366 	 */
7367 	memset(dev, 0, sizeof(struct net_device));
7368 
7369 	/* make sure we BUG if trying to hit standard
7370 	 * register/unregister code path
7371 	 */
7372 	dev->reg_state = NETREG_DUMMY;
7373 
7374 	/* NAPI wants this */
7375 	INIT_LIST_HEAD(&dev->napi_list);
7376 
7377 	/* a dummy interface is started by default */
7378 	set_bit(__LINK_STATE_PRESENT, &dev->state);
7379 	set_bit(__LINK_STATE_START, &dev->state);
7380 
7381 	/* Note : We dont allocate pcpu_refcnt for dummy devices,
7382 	 * because users of this 'device' dont need to change
7383 	 * its refcount.
7384 	 */
7385 
7386 	return 0;
7387 }
7388 EXPORT_SYMBOL_GPL(init_dummy_netdev);
7389 
7390 
7391 /**
7392  *	register_netdev	- register a network device
7393  *	@dev: device to register
7394  *
7395  *	Take a completed network device structure and add it to the kernel
7396  *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7397  *	chain. 0 is returned on success. A negative errno code is returned
7398  *	on a failure to set up the device, or if the name is a duplicate.
7399  *
7400  *	This is a wrapper around register_netdevice that takes the rtnl semaphore
7401  *	and expands the device name if you passed a format string to
7402  *	alloc_netdev.
7403  */
7404 int register_netdev(struct net_device *dev)
7405 {
7406 	int err;
7407 
7408 	rtnl_lock();
7409 	err = register_netdevice(dev);
7410 	rtnl_unlock();
7411 	return err;
7412 }
7413 EXPORT_SYMBOL(register_netdev);
7414 
7415 int netdev_refcnt_read(const struct net_device *dev)
7416 {
7417 	int i, refcnt = 0;
7418 
7419 	for_each_possible_cpu(i)
7420 		refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
7421 	return refcnt;
7422 }
7423 EXPORT_SYMBOL(netdev_refcnt_read);
7424 
7425 /**
7426  * netdev_wait_allrefs - wait until all references are gone.
7427  * @dev: target net_device
7428  *
7429  * This is called when unregistering network devices.
7430  *
7431  * Any protocol or device that holds a reference should register
7432  * for netdevice notification, and cleanup and put back the
7433  * reference if they receive an UNREGISTER event.
7434  * We can get stuck here if buggy protocols don't correctly
7435  * call dev_put.
7436  */
7437 static void netdev_wait_allrefs(struct net_device *dev)
7438 {
7439 	unsigned long rebroadcast_time, warning_time;
7440 	int refcnt;
7441 
7442 	linkwatch_forget_dev(dev);
7443 
7444 	rebroadcast_time = warning_time = jiffies;
7445 	refcnt = netdev_refcnt_read(dev);
7446 
7447 	while (refcnt != 0) {
7448 		if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
7449 			rtnl_lock();
7450 
7451 			/* Rebroadcast unregister notification */
7452 			call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7453 
7454 			__rtnl_unlock();
7455 			rcu_barrier();
7456 			rtnl_lock();
7457 
7458 			call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7459 			if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
7460 				     &dev->state)) {
7461 				/* We must not have linkwatch events
7462 				 * pending on unregister. If this
7463 				 * happens, we simply run the queue
7464 				 * unscheduled, resulting in a noop
7465 				 * for this device.
7466 				 */
7467 				linkwatch_run_queue();
7468 			}
7469 
7470 			__rtnl_unlock();
7471 
7472 			rebroadcast_time = jiffies;
7473 		}
7474 
7475 		msleep(250);
7476 
7477 		refcnt = netdev_refcnt_read(dev);
7478 
7479 		if (time_after(jiffies, warning_time + 10 * HZ)) {
7480 			pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
7481 				 dev->name, refcnt);
7482 			warning_time = jiffies;
7483 		}
7484 	}
7485 }
7486 
7487 /* The sequence is:
7488  *
7489  *	rtnl_lock();
7490  *	...
7491  *	register_netdevice(x1);
7492  *	register_netdevice(x2);
7493  *	...
7494  *	unregister_netdevice(y1);
7495  *	unregister_netdevice(y2);
7496  *      ...
7497  *	rtnl_unlock();
7498  *	free_netdev(y1);
7499  *	free_netdev(y2);
7500  *
7501  * We are invoked by rtnl_unlock().
7502  * This allows us to deal with problems:
7503  * 1) We can delete sysfs objects which invoke hotplug
7504  *    without deadlocking with linkwatch via keventd.
7505  * 2) Since we run with the RTNL semaphore not held, we can sleep
7506  *    safely in order to wait for the netdev refcnt to drop to zero.
7507  *
7508  * We must not return until all unregister events added during
7509  * the interval the lock was held have been completed.
7510  */
7511 void netdev_run_todo(void)
7512 {
7513 	struct list_head list;
7514 
7515 	/* Snapshot list, allow later requests */
7516 	list_replace_init(&net_todo_list, &list);
7517 
7518 	__rtnl_unlock();
7519 
7520 
7521 	/* Wait for rcu callbacks to finish before next phase */
7522 	if (!list_empty(&list))
7523 		rcu_barrier();
7524 
7525 	while (!list_empty(&list)) {
7526 		struct net_device *dev
7527 			= list_first_entry(&list, struct net_device, todo_list);
7528 		list_del(&dev->todo_list);
7529 
7530 		rtnl_lock();
7531 		call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7532 		__rtnl_unlock();
7533 
7534 		if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
7535 			pr_err("network todo '%s' but state %d\n",
7536 			       dev->name, dev->reg_state);
7537 			dump_stack();
7538 			continue;
7539 		}
7540 
7541 		dev->reg_state = NETREG_UNREGISTERED;
7542 
7543 		netdev_wait_allrefs(dev);
7544 
7545 		/* paranoia */
7546 		BUG_ON(netdev_refcnt_read(dev));
7547 		BUG_ON(!list_empty(&dev->ptype_all));
7548 		BUG_ON(!list_empty(&dev->ptype_specific));
7549 		WARN_ON(rcu_access_pointer(dev->ip_ptr));
7550 		WARN_ON(rcu_access_pointer(dev->ip6_ptr));
7551 		WARN_ON(dev->dn_ptr);
7552 
7553 		if (dev->destructor)
7554 			dev->destructor(dev);
7555 
7556 		/* Report a network device has been unregistered */
7557 		rtnl_lock();
7558 		dev_net(dev)->dev_unreg_count--;
7559 		__rtnl_unlock();
7560 		wake_up(&netdev_unregistering_wq);
7561 
7562 		/* Free network device */
7563 		kobject_put(&dev->dev.kobj);
7564 	}
7565 }
7566 
7567 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
7568  * all the same fields in the same order as net_device_stats, with only
7569  * the type differing, but rtnl_link_stats64 may have additional fields
7570  * at the end for newer counters.
7571  */
7572 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
7573 			     const struct net_device_stats *netdev_stats)
7574 {
7575 #if BITS_PER_LONG == 64
7576 	BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
7577 	memcpy(stats64, netdev_stats, sizeof(*stats64));
7578 	/* zero out counters that only exist in rtnl_link_stats64 */
7579 	memset((char *)stats64 + sizeof(*netdev_stats), 0,
7580 	       sizeof(*stats64) - sizeof(*netdev_stats));
7581 #else
7582 	size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
7583 	const unsigned long *src = (const unsigned long *)netdev_stats;
7584 	u64 *dst = (u64 *)stats64;
7585 
7586 	BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
7587 	for (i = 0; i < n; i++)
7588 		dst[i] = src[i];
7589 	/* zero out counters that only exist in rtnl_link_stats64 */
7590 	memset((char *)stats64 + n * sizeof(u64), 0,
7591 	       sizeof(*stats64) - n * sizeof(u64));
7592 #endif
7593 }
7594 EXPORT_SYMBOL(netdev_stats_to_stats64);
7595 
7596 /**
7597  *	dev_get_stats	- get network device statistics
7598  *	@dev: device to get statistics from
7599  *	@storage: place to store stats
7600  *
7601  *	Get network statistics from device. Return @storage.
7602  *	The device driver may provide its own method by setting
7603  *	dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
7604  *	otherwise the internal statistics structure is used.
7605  */
7606 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
7607 					struct rtnl_link_stats64 *storage)
7608 {
7609 	const struct net_device_ops *ops = dev->netdev_ops;
7610 
7611 	if (ops->ndo_get_stats64) {
7612 		memset(storage, 0, sizeof(*storage));
7613 		ops->ndo_get_stats64(dev, storage);
7614 	} else if (ops->ndo_get_stats) {
7615 		netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
7616 	} else {
7617 		netdev_stats_to_stats64(storage, &dev->stats);
7618 	}
7619 	storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
7620 	storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
7621 	storage->rx_nohandler += atomic_long_read(&dev->rx_nohandler);
7622 	return storage;
7623 }
7624 EXPORT_SYMBOL(dev_get_stats);
7625 
7626 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
7627 {
7628 	struct netdev_queue *queue = dev_ingress_queue(dev);
7629 
7630 #ifdef CONFIG_NET_CLS_ACT
7631 	if (queue)
7632 		return queue;
7633 	queue = kzalloc(sizeof(*queue), GFP_KERNEL);
7634 	if (!queue)
7635 		return NULL;
7636 	netdev_init_one_queue(dev, queue, NULL);
7637 	RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
7638 	queue->qdisc_sleeping = &noop_qdisc;
7639 	rcu_assign_pointer(dev->ingress_queue, queue);
7640 #endif
7641 	return queue;
7642 }
7643 
7644 static const struct ethtool_ops default_ethtool_ops;
7645 
7646 void netdev_set_default_ethtool_ops(struct net_device *dev,
7647 				    const struct ethtool_ops *ops)
7648 {
7649 	if (dev->ethtool_ops == &default_ethtool_ops)
7650 		dev->ethtool_ops = ops;
7651 }
7652 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
7653 
7654 void netdev_freemem(struct net_device *dev)
7655 {
7656 	char *addr = (char *)dev - dev->padded;
7657 
7658 	kvfree(addr);
7659 }
7660 
7661 /**
7662  * alloc_netdev_mqs - allocate network device
7663  * @sizeof_priv: size of private data to allocate space for
7664  * @name: device name format string
7665  * @name_assign_type: origin of device name
7666  * @setup: callback to initialize device
7667  * @txqs: the number of TX subqueues to allocate
7668  * @rxqs: the number of RX subqueues to allocate
7669  *
7670  * Allocates a struct net_device with private data area for driver use
7671  * and performs basic initialization.  Also allocates subqueue structs
7672  * for each queue on the device.
7673  */
7674 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
7675 		unsigned char name_assign_type,
7676 		void (*setup)(struct net_device *),
7677 		unsigned int txqs, unsigned int rxqs)
7678 {
7679 	struct net_device *dev;
7680 	size_t alloc_size;
7681 	struct net_device *p;
7682 
7683 	BUG_ON(strlen(name) >= sizeof(dev->name));
7684 
7685 	if (txqs < 1) {
7686 		pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
7687 		return NULL;
7688 	}
7689 
7690 #ifdef CONFIG_SYSFS
7691 	if (rxqs < 1) {
7692 		pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
7693 		return NULL;
7694 	}
7695 #endif
7696 
7697 	alloc_size = sizeof(struct net_device);
7698 	if (sizeof_priv) {
7699 		/* ensure 32-byte alignment of private area */
7700 		alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
7701 		alloc_size += sizeof_priv;
7702 	}
7703 	/* ensure 32-byte alignment of whole construct */
7704 	alloc_size += NETDEV_ALIGN - 1;
7705 
7706 	p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7707 	if (!p)
7708 		p = vzalloc(alloc_size);
7709 	if (!p)
7710 		return NULL;
7711 
7712 	dev = PTR_ALIGN(p, NETDEV_ALIGN);
7713 	dev->padded = (char *)dev - (char *)p;
7714 
7715 	dev->pcpu_refcnt = alloc_percpu(int);
7716 	if (!dev->pcpu_refcnt)
7717 		goto free_dev;
7718 
7719 	if (dev_addr_init(dev))
7720 		goto free_pcpu;
7721 
7722 	dev_mc_init(dev);
7723 	dev_uc_init(dev);
7724 
7725 	dev_net_set(dev, &init_net);
7726 
7727 	dev->gso_max_size = GSO_MAX_SIZE;
7728 	dev->gso_max_segs = GSO_MAX_SEGS;
7729 
7730 	INIT_LIST_HEAD(&dev->napi_list);
7731 	INIT_LIST_HEAD(&dev->unreg_list);
7732 	INIT_LIST_HEAD(&dev->close_list);
7733 	INIT_LIST_HEAD(&dev->link_watch_list);
7734 	INIT_LIST_HEAD(&dev->adj_list.upper);
7735 	INIT_LIST_HEAD(&dev->adj_list.lower);
7736 	INIT_LIST_HEAD(&dev->ptype_all);
7737 	INIT_LIST_HEAD(&dev->ptype_specific);
7738 #ifdef CONFIG_NET_SCHED
7739 	hash_init(dev->qdisc_hash);
7740 #endif
7741 	dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
7742 	setup(dev);
7743 
7744 	if (!dev->tx_queue_len) {
7745 		dev->priv_flags |= IFF_NO_QUEUE;
7746 		dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
7747 	}
7748 
7749 	dev->num_tx_queues = txqs;
7750 	dev->real_num_tx_queues = txqs;
7751 	if (netif_alloc_netdev_queues(dev))
7752 		goto free_all;
7753 
7754 #ifdef CONFIG_SYSFS
7755 	dev->num_rx_queues = rxqs;
7756 	dev->real_num_rx_queues = rxqs;
7757 	if (netif_alloc_rx_queues(dev))
7758 		goto free_all;
7759 #endif
7760 
7761 	strcpy(dev->name, name);
7762 	dev->name_assign_type = name_assign_type;
7763 	dev->group = INIT_NETDEV_GROUP;
7764 	if (!dev->ethtool_ops)
7765 		dev->ethtool_ops = &default_ethtool_ops;
7766 
7767 	nf_hook_ingress_init(dev);
7768 
7769 	return dev;
7770 
7771 free_all:
7772 	free_netdev(dev);
7773 	return NULL;
7774 
7775 free_pcpu:
7776 	free_percpu(dev->pcpu_refcnt);
7777 free_dev:
7778 	netdev_freemem(dev);
7779 	return NULL;
7780 }
7781 EXPORT_SYMBOL(alloc_netdev_mqs);
7782 
7783 /**
7784  * free_netdev - free network device
7785  * @dev: device
7786  *
7787  * This function does the last stage of destroying an allocated device
7788  * interface. The reference to the device object is released. If this
7789  * is the last reference then it will be freed.Must be called in process
7790  * context.
7791  */
7792 void free_netdev(struct net_device *dev)
7793 {
7794 	struct napi_struct *p, *n;
7795 
7796 	might_sleep();
7797 	netif_free_tx_queues(dev);
7798 #ifdef CONFIG_SYSFS
7799 	kvfree(dev->_rx);
7800 #endif
7801 
7802 	kfree(rcu_dereference_protected(dev->ingress_queue, 1));
7803 
7804 	/* Flush device addresses */
7805 	dev_addr_flush(dev);
7806 
7807 	list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
7808 		netif_napi_del(p);
7809 
7810 	free_percpu(dev->pcpu_refcnt);
7811 	dev->pcpu_refcnt = NULL;
7812 
7813 	/*  Compatibility with error handling in drivers */
7814 	if (dev->reg_state == NETREG_UNINITIALIZED) {
7815 		netdev_freemem(dev);
7816 		return;
7817 	}
7818 
7819 	BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
7820 	dev->reg_state = NETREG_RELEASED;
7821 
7822 	/* will free via device release */
7823 	put_device(&dev->dev);
7824 }
7825 EXPORT_SYMBOL(free_netdev);
7826 
7827 /**
7828  *	synchronize_net -  Synchronize with packet receive processing
7829  *
7830  *	Wait for packets currently being received to be done.
7831  *	Does not block later packets from starting.
7832  */
7833 void synchronize_net(void)
7834 {
7835 	might_sleep();
7836 	if (rtnl_is_locked())
7837 		synchronize_rcu_expedited();
7838 	else
7839 		synchronize_rcu();
7840 }
7841 EXPORT_SYMBOL(synchronize_net);
7842 
7843 /**
7844  *	unregister_netdevice_queue - remove device from the kernel
7845  *	@dev: device
7846  *	@head: list
7847  *
7848  *	This function shuts down a device interface and removes it
7849  *	from the kernel tables.
7850  *	If head not NULL, device is queued to be unregistered later.
7851  *
7852  *	Callers must hold the rtnl semaphore.  You may want
7853  *	unregister_netdev() instead of this.
7854  */
7855 
7856 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
7857 {
7858 	ASSERT_RTNL();
7859 
7860 	if (head) {
7861 		list_move_tail(&dev->unreg_list, head);
7862 	} else {
7863 		rollback_registered(dev);
7864 		/* Finish processing unregister after unlock */
7865 		net_set_todo(dev);
7866 	}
7867 }
7868 EXPORT_SYMBOL(unregister_netdevice_queue);
7869 
7870 /**
7871  *	unregister_netdevice_many - unregister many devices
7872  *	@head: list of devices
7873  *
7874  *  Note: As most callers use a stack allocated list_head,
7875  *  we force a list_del() to make sure stack wont be corrupted later.
7876  */
7877 void unregister_netdevice_many(struct list_head *head)
7878 {
7879 	struct net_device *dev;
7880 
7881 	if (!list_empty(head)) {
7882 		rollback_registered_many(head);
7883 		list_for_each_entry(dev, head, unreg_list)
7884 			net_set_todo(dev);
7885 		list_del(head);
7886 	}
7887 }
7888 EXPORT_SYMBOL(unregister_netdevice_many);
7889 
7890 /**
7891  *	unregister_netdev - remove device from the kernel
7892  *	@dev: device
7893  *
7894  *	This function shuts down a device interface and removes it
7895  *	from the kernel tables.
7896  *
7897  *	This is just a wrapper for unregister_netdevice that takes
7898  *	the rtnl semaphore.  In general you want to use this and not
7899  *	unregister_netdevice.
7900  */
7901 void unregister_netdev(struct net_device *dev)
7902 {
7903 	rtnl_lock();
7904 	unregister_netdevice(dev);
7905 	rtnl_unlock();
7906 }
7907 EXPORT_SYMBOL(unregister_netdev);
7908 
7909 /**
7910  *	dev_change_net_namespace - move device to different nethost namespace
7911  *	@dev: device
7912  *	@net: network namespace
7913  *	@pat: If not NULL name pattern to try if the current device name
7914  *	      is already taken in the destination network namespace.
7915  *
7916  *	This function shuts down a device interface and moves it
7917  *	to a new network namespace. On success 0 is returned, on
7918  *	a failure a netagive errno code is returned.
7919  *
7920  *	Callers must hold the rtnl semaphore.
7921  */
7922 
7923 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
7924 {
7925 	int err;
7926 
7927 	ASSERT_RTNL();
7928 
7929 	/* Don't allow namespace local devices to be moved. */
7930 	err = -EINVAL;
7931 	if (dev->features & NETIF_F_NETNS_LOCAL)
7932 		goto out;
7933 
7934 	/* Ensure the device has been registrered */
7935 	if (dev->reg_state != NETREG_REGISTERED)
7936 		goto out;
7937 
7938 	/* Get out if there is nothing todo */
7939 	err = 0;
7940 	if (net_eq(dev_net(dev), net))
7941 		goto out;
7942 
7943 	/* Pick the destination device name, and ensure
7944 	 * we can use it in the destination network namespace.
7945 	 */
7946 	err = -EEXIST;
7947 	if (__dev_get_by_name(net, dev->name)) {
7948 		/* We get here if we can't use the current device name */
7949 		if (!pat)
7950 			goto out;
7951 		if (dev_get_valid_name(net, dev, pat) < 0)
7952 			goto out;
7953 	}
7954 
7955 	/*
7956 	 * And now a mini version of register_netdevice unregister_netdevice.
7957 	 */
7958 
7959 	/* If device is running close it first. */
7960 	dev_close(dev);
7961 
7962 	/* And unlink it from device chain */
7963 	err = -ENODEV;
7964 	unlist_netdevice(dev);
7965 
7966 	synchronize_net();
7967 
7968 	/* Shutdown queueing discipline. */
7969 	dev_shutdown(dev);
7970 
7971 	/* Notify protocols, that we are about to destroy
7972 	 * this device. They should clean all the things.
7973 	 *
7974 	 * Note that dev->reg_state stays at NETREG_REGISTERED.
7975 	 * This is wanted because this way 8021q and macvlan know
7976 	 * the device is just moving and can keep their slaves up.
7977 	 */
7978 	call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7979 	rcu_barrier();
7980 	call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7981 	rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
7982 
7983 	/*
7984 	 *	Flush the unicast and multicast chains
7985 	 */
7986 	dev_uc_flush(dev);
7987 	dev_mc_flush(dev);
7988 
7989 	/* Send a netdev-removed uevent to the old namespace */
7990 	kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
7991 	netdev_adjacent_del_links(dev);
7992 
7993 	/* Actually switch the network namespace */
7994 	dev_net_set(dev, net);
7995 
7996 	/* If there is an ifindex conflict assign a new one */
7997 	if (__dev_get_by_index(net, dev->ifindex))
7998 		dev->ifindex = dev_new_index(net);
7999 
8000 	/* Send a netdev-add uevent to the new namespace */
8001 	kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
8002 	netdev_adjacent_add_links(dev);
8003 
8004 	/* Fixup kobjects */
8005 	err = device_rename(&dev->dev, dev->name);
8006 	WARN_ON(err);
8007 
8008 	/* Add the device back in the hashes */
8009 	list_netdevice(dev);
8010 
8011 	/* Notify protocols, that a new device appeared. */
8012 	call_netdevice_notifiers(NETDEV_REGISTER, dev);
8013 
8014 	/*
8015 	 *	Prevent userspace races by waiting until the network
8016 	 *	device is fully setup before sending notifications.
8017 	 */
8018 	rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
8019 
8020 	synchronize_net();
8021 	err = 0;
8022 out:
8023 	return err;
8024 }
8025 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
8026 
8027 static int dev_cpu_dead(unsigned int oldcpu)
8028 {
8029 	struct sk_buff **list_skb;
8030 	struct sk_buff *skb;
8031 	unsigned int cpu;
8032 	struct softnet_data *sd, *oldsd;
8033 
8034 	local_irq_disable();
8035 	cpu = smp_processor_id();
8036 	sd = &per_cpu(softnet_data, cpu);
8037 	oldsd = &per_cpu(softnet_data, oldcpu);
8038 
8039 	/* Find end of our completion_queue. */
8040 	list_skb = &sd->completion_queue;
8041 	while (*list_skb)
8042 		list_skb = &(*list_skb)->next;
8043 	/* Append completion queue from offline CPU. */
8044 	*list_skb = oldsd->completion_queue;
8045 	oldsd->completion_queue = NULL;
8046 
8047 	/* Append output queue from offline CPU. */
8048 	if (oldsd->output_queue) {
8049 		*sd->output_queue_tailp = oldsd->output_queue;
8050 		sd->output_queue_tailp = oldsd->output_queue_tailp;
8051 		oldsd->output_queue = NULL;
8052 		oldsd->output_queue_tailp = &oldsd->output_queue;
8053 	}
8054 	/* Append NAPI poll list from offline CPU, with one exception :
8055 	 * process_backlog() must be called by cpu owning percpu backlog.
8056 	 * We properly handle process_queue & input_pkt_queue later.
8057 	 */
8058 	while (!list_empty(&oldsd->poll_list)) {
8059 		struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
8060 							    struct napi_struct,
8061 							    poll_list);
8062 
8063 		list_del_init(&napi->poll_list);
8064 		if (napi->poll == process_backlog)
8065 			napi->state = 0;
8066 		else
8067 			____napi_schedule(sd, napi);
8068 	}
8069 
8070 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
8071 	local_irq_enable();
8072 
8073 	/* Process offline CPU's input_pkt_queue */
8074 	while ((skb = __skb_dequeue(&oldsd->process_queue))) {
8075 		netif_rx_ni(skb);
8076 		input_queue_head_incr(oldsd);
8077 	}
8078 	while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
8079 		netif_rx_ni(skb);
8080 		input_queue_head_incr(oldsd);
8081 	}
8082 
8083 	return 0;
8084 }
8085 
8086 /**
8087  *	netdev_increment_features - increment feature set by one
8088  *	@all: current feature set
8089  *	@one: new feature set
8090  *	@mask: mask feature set
8091  *
8092  *	Computes a new feature set after adding a device with feature set
8093  *	@one to the master device with current feature set @all.  Will not
8094  *	enable anything that is off in @mask. Returns the new feature set.
8095  */
8096 netdev_features_t netdev_increment_features(netdev_features_t all,
8097 	netdev_features_t one, netdev_features_t mask)
8098 {
8099 	if (mask & NETIF_F_HW_CSUM)
8100 		mask |= NETIF_F_CSUM_MASK;
8101 	mask |= NETIF_F_VLAN_CHALLENGED;
8102 
8103 	all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
8104 	all &= one | ~NETIF_F_ALL_FOR_ALL;
8105 
8106 	/* If one device supports hw checksumming, set for all. */
8107 	if (all & NETIF_F_HW_CSUM)
8108 		all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
8109 
8110 	return all;
8111 }
8112 EXPORT_SYMBOL(netdev_increment_features);
8113 
8114 static struct hlist_head * __net_init netdev_create_hash(void)
8115 {
8116 	int i;
8117 	struct hlist_head *hash;
8118 
8119 	hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
8120 	if (hash != NULL)
8121 		for (i = 0; i < NETDEV_HASHENTRIES; i++)
8122 			INIT_HLIST_HEAD(&hash[i]);
8123 
8124 	return hash;
8125 }
8126 
8127 /* Initialize per network namespace state */
8128 static int __net_init netdev_init(struct net *net)
8129 {
8130 	if (net != &init_net)
8131 		INIT_LIST_HEAD(&net->dev_base_head);
8132 
8133 	net->dev_name_head = netdev_create_hash();
8134 	if (net->dev_name_head == NULL)
8135 		goto err_name;
8136 
8137 	net->dev_index_head = netdev_create_hash();
8138 	if (net->dev_index_head == NULL)
8139 		goto err_idx;
8140 
8141 	return 0;
8142 
8143 err_idx:
8144 	kfree(net->dev_name_head);
8145 err_name:
8146 	return -ENOMEM;
8147 }
8148 
8149 /**
8150  *	netdev_drivername - network driver for the device
8151  *	@dev: network device
8152  *
8153  *	Determine network driver for device.
8154  */
8155 const char *netdev_drivername(const struct net_device *dev)
8156 {
8157 	const struct device_driver *driver;
8158 	const struct device *parent;
8159 	const char *empty = "";
8160 
8161 	parent = dev->dev.parent;
8162 	if (!parent)
8163 		return empty;
8164 
8165 	driver = parent->driver;
8166 	if (driver && driver->name)
8167 		return driver->name;
8168 	return empty;
8169 }
8170 
8171 static void __netdev_printk(const char *level, const struct net_device *dev,
8172 			    struct va_format *vaf)
8173 {
8174 	if (dev && dev->dev.parent) {
8175 		dev_printk_emit(level[1] - '0',
8176 				dev->dev.parent,
8177 				"%s %s %s%s: %pV",
8178 				dev_driver_string(dev->dev.parent),
8179 				dev_name(dev->dev.parent),
8180 				netdev_name(dev), netdev_reg_state(dev),
8181 				vaf);
8182 	} else if (dev) {
8183 		printk("%s%s%s: %pV",
8184 		       level, netdev_name(dev), netdev_reg_state(dev), vaf);
8185 	} else {
8186 		printk("%s(NULL net_device): %pV", level, vaf);
8187 	}
8188 }
8189 
8190 void netdev_printk(const char *level, const struct net_device *dev,
8191 		   const char *format, ...)
8192 {
8193 	struct va_format vaf;
8194 	va_list args;
8195 
8196 	va_start(args, format);
8197 
8198 	vaf.fmt = format;
8199 	vaf.va = &args;
8200 
8201 	__netdev_printk(level, dev, &vaf);
8202 
8203 	va_end(args);
8204 }
8205 EXPORT_SYMBOL(netdev_printk);
8206 
8207 #define define_netdev_printk_level(func, level)			\
8208 void func(const struct net_device *dev, const char *fmt, ...)	\
8209 {								\
8210 	struct va_format vaf;					\
8211 	va_list args;						\
8212 								\
8213 	va_start(args, fmt);					\
8214 								\
8215 	vaf.fmt = fmt;						\
8216 	vaf.va = &args;						\
8217 								\
8218 	__netdev_printk(level, dev, &vaf);			\
8219 								\
8220 	va_end(args);						\
8221 }								\
8222 EXPORT_SYMBOL(func);
8223 
8224 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
8225 define_netdev_printk_level(netdev_alert, KERN_ALERT);
8226 define_netdev_printk_level(netdev_crit, KERN_CRIT);
8227 define_netdev_printk_level(netdev_err, KERN_ERR);
8228 define_netdev_printk_level(netdev_warn, KERN_WARNING);
8229 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
8230 define_netdev_printk_level(netdev_info, KERN_INFO);
8231 
8232 static void __net_exit netdev_exit(struct net *net)
8233 {
8234 	kfree(net->dev_name_head);
8235 	kfree(net->dev_index_head);
8236 }
8237 
8238 static struct pernet_operations __net_initdata netdev_net_ops = {
8239 	.init = netdev_init,
8240 	.exit = netdev_exit,
8241 };
8242 
8243 static void __net_exit default_device_exit(struct net *net)
8244 {
8245 	struct net_device *dev, *aux;
8246 	/*
8247 	 * Push all migratable network devices back to the
8248 	 * initial network namespace
8249 	 */
8250 	rtnl_lock();
8251 	for_each_netdev_safe(net, dev, aux) {
8252 		int err;
8253 		char fb_name[IFNAMSIZ];
8254 
8255 		/* Ignore unmoveable devices (i.e. loopback) */
8256 		if (dev->features & NETIF_F_NETNS_LOCAL)
8257 			continue;
8258 
8259 		/* Leave virtual devices for the generic cleanup */
8260 		if (dev->rtnl_link_ops)
8261 			continue;
8262 
8263 		/* Push remaining network devices to init_net */
8264 		snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
8265 		err = dev_change_net_namespace(dev, &init_net, fb_name);
8266 		if (err) {
8267 			pr_emerg("%s: failed to move %s to init_net: %d\n",
8268 				 __func__, dev->name, err);
8269 			BUG();
8270 		}
8271 	}
8272 	rtnl_unlock();
8273 }
8274 
8275 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
8276 {
8277 	/* Return with the rtnl_lock held when there are no network
8278 	 * devices unregistering in any network namespace in net_list.
8279 	 */
8280 	struct net *net;
8281 	bool unregistering;
8282 	DEFINE_WAIT_FUNC(wait, woken_wake_function);
8283 
8284 	add_wait_queue(&netdev_unregistering_wq, &wait);
8285 	for (;;) {
8286 		unregistering = false;
8287 		rtnl_lock();
8288 		list_for_each_entry(net, net_list, exit_list) {
8289 			if (net->dev_unreg_count > 0) {
8290 				unregistering = true;
8291 				break;
8292 			}
8293 		}
8294 		if (!unregistering)
8295 			break;
8296 		__rtnl_unlock();
8297 
8298 		wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
8299 	}
8300 	remove_wait_queue(&netdev_unregistering_wq, &wait);
8301 }
8302 
8303 static void __net_exit default_device_exit_batch(struct list_head *net_list)
8304 {
8305 	/* At exit all network devices most be removed from a network
8306 	 * namespace.  Do this in the reverse order of registration.
8307 	 * Do this across as many network namespaces as possible to
8308 	 * improve batching efficiency.
8309 	 */
8310 	struct net_device *dev;
8311 	struct net *net;
8312 	LIST_HEAD(dev_kill_list);
8313 
8314 	/* To prevent network device cleanup code from dereferencing
8315 	 * loopback devices or network devices that have been freed
8316 	 * wait here for all pending unregistrations to complete,
8317 	 * before unregistring the loopback device and allowing the
8318 	 * network namespace be freed.
8319 	 *
8320 	 * The netdev todo list containing all network devices
8321 	 * unregistrations that happen in default_device_exit_batch
8322 	 * will run in the rtnl_unlock() at the end of
8323 	 * default_device_exit_batch.
8324 	 */
8325 	rtnl_lock_unregistering(net_list);
8326 	list_for_each_entry(net, net_list, exit_list) {
8327 		for_each_netdev_reverse(net, dev) {
8328 			if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
8329 				dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
8330 			else
8331 				unregister_netdevice_queue(dev, &dev_kill_list);
8332 		}
8333 	}
8334 	unregister_netdevice_many(&dev_kill_list);
8335 	rtnl_unlock();
8336 }
8337 
8338 static struct pernet_operations __net_initdata default_device_ops = {
8339 	.exit = default_device_exit,
8340 	.exit_batch = default_device_exit_batch,
8341 };
8342 
8343 /*
8344  *	Initialize the DEV module. At boot time this walks the device list and
8345  *	unhooks any devices that fail to initialise (normally hardware not
8346  *	present) and leaves us with a valid list of present and active devices.
8347  *
8348  */
8349 
8350 /*
8351  *       This is called single threaded during boot, so no need
8352  *       to take the rtnl semaphore.
8353  */
8354 static int __init net_dev_init(void)
8355 {
8356 	int i, rc = -ENOMEM;
8357 
8358 	BUG_ON(!dev_boot_phase);
8359 
8360 	if (dev_proc_init())
8361 		goto out;
8362 
8363 	if (netdev_kobject_init())
8364 		goto out;
8365 
8366 	INIT_LIST_HEAD(&ptype_all);
8367 	for (i = 0; i < PTYPE_HASH_SIZE; i++)
8368 		INIT_LIST_HEAD(&ptype_base[i]);
8369 
8370 	INIT_LIST_HEAD(&offload_base);
8371 
8372 	if (register_pernet_subsys(&netdev_net_ops))
8373 		goto out;
8374 
8375 	/*
8376 	 *	Initialise the packet receive queues.
8377 	 */
8378 
8379 	for_each_possible_cpu(i) {
8380 		struct work_struct *flush = per_cpu_ptr(&flush_works, i);
8381 		struct softnet_data *sd = &per_cpu(softnet_data, i);
8382 
8383 		INIT_WORK(flush, flush_backlog);
8384 
8385 		skb_queue_head_init(&sd->input_pkt_queue);
8386 		skb_queue_head_init(&sd->process_queue);
8387 		INIT_LIST_HEAD(&sd->poll_list);
8388 		sd->output_queue_tailp = &sd->output_queue;
8389 #ifdef CONFIG_RPS
8390 		sd->csd.func = rps_trigger_softirq;
8391 		sd->csd.info = sd;
8392 		sd->cpu = i;
8393 #endif
8394 
8395 		sd->backlog.poll = process_backlog;
8396 		sd->backlog.weight = weight_p;
8397 	}
8398 
8399 	dev_boot_phase = 0;
8400 
8401 	/* The loopback device is special if any other network devices
8402 	 * is present in a network namespace the loopback device must
8403 	 * be present. Since we now dynamically allocate and free the
8404 	 * loopback device ensure this invariant is maintained by
8405 	 * keeping the loopback device as the first device on the
8406 	 * list of network devices.  Ensuring the loopback devices
8407 	 * is the first device that appears and the last network device
8408 	 * that disappears.
8409 	 */
8410 	if (register_pernet_device(&loopback_net_ops))
8411 		goto out;
8412 
8413 	if (register_pernet_device(&default_device_ops))
8414 		goto out;
8415 
8416 	open_softirq(NET_TX_SOFTIRQ, net_tx_action);
8417 	open_softirq(NET_RX_SOFTIRQ, net_rx_action);
8418 
8419 	rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
8420 				       NULL, dev_cpu_dead);
8421 	WARN_ON(rc < 0);
8422 	dst_subsys_init();
8423 	rc = 0;
8424 out:
8425 	return rc;
8426 }
8427 
8428 subsys_initcall(net_dev_init);
8429