xref: /openbmc/linux/net/core/dev.c (revision 4e1a33b1)
1 /*
2  *      NET3    Protocol independent device support routines.
3  *
4  *		This program is free software; you can redistribute it and/or
5  *		modify it under the terms of the GNU General Public License
6  *		as published by the Free Software Foundation; either version
7  *		2 of the License, or (at your option) any later version.
8  *
9  *	Derived from the non IP parts of dev.c 1.0.19
10  *              Authors:	Ross Biro
11  *				Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12  *				Mark Evans, <evansmp@uhura.aston.ac.uk>
13  *
14  *	Additional Authors:
15  *		Florian la Roche <rzsfl@rz.uni-sb.de>
16  *		Alan Cox <gw4pts@gw4pts.ampr.org>
17  *		David Hinds <dahinds@users.sourceforge.net>
18  *		Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19  *		Adam Sulmicki <adam@cfar.umd.edu>
20  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
21  *
22  *	Changes:
23  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
24  *                                      to 2 if register_netdev gets called
25  *                                      before net_dev_init & also removed a
26  *                                      few lines of code in the process.
27  *		Alan Cox	:	device private ioctl copies fields back.
28  *		Alan Cox	:	Transmit queue code does relevant
29  *					stunts to keep the queue safe.
30  *		Alan Cox	:	Fixed double lock.
31  *		Alan Cox	:	Fixed promisc NULL pointer trap
32  *		????????	:	Support the full private ioctl range
33  *		Alan Cox	:	Moved ioctl permission check into
34  *					drivers
35  *		Tim Kordas	:	SIOCADDMULTI/SIOCDELMULTI
36  *		Alan Cox	:	100 backlog just doesn't cut it when
37  *					you start doing multicast video 8)
38  *		Alan Cox	:	Rewrote net_bh and list manager.
39  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
40  *		Alan Cox	:	Took out transmit every packet pass
41  *					Saved a few bytes in the ioctl handler
42  *		Alan Cox	:	Network driver sets packet type before
43  *					calling netif_rx. Saves a function
44  *					call a packet.
45  *		Alan Cox	:	Hashed net_bh()
46  *		Richard Kooijman:	Timestamp fixes.
47  *		Alan Cox	:	Wrong field in SIOCGIFDSTADDR
48  *		Alan Cox	:	Device lock protection.
49  *              Alan Cox        :       Fixed nasty side effect of device close
50  *					changes.
51  *		Rudi Cilibrasi	:	Pass the right thing to
52  *					set_mac_address()
53  *		Dave Miller	:	32bit quantity for the device lock to
54  *					make it work out on a Sparc.
55  *		Bjorn Ekwall	:	Added KERNELD hack.
56  *		Alan Cox	:	Cleaned up the backlog initialise.
57  *		Craig Metz	:	SIOCGIFCONF fix if space for under
58  *					1 device.
59  *	    Thomas Bogendoerfer :	Return ENODEV for dev_open, if there
60  *					is no device open function.
61  *		Andi Kleen	:	Fix error reporting for SIOCGIFCONF
62  *	    Michael Chastain	:	Fix signed/unsigned for SIOCGIFCONF
63  *		Cyrus Durgin	:	Cleaned for KMOD
64  *		Adam Sulmicki   :	Bug Fix : Network Device Unload
65  *					A network device unload needs to purge
66  *					the backlog queue.
67  *	Paul Rusty Russell	:	SIOCSIFNAME
68  *              Pekka Riikonen  :	Netdev boot-time settings code
69  *              Andrew Morton   :       Make unregister_netdevice wait
70  *                                      indefinitely on dev->refcnt
71  *              J Hadi Salim    :       - Backlog queue sampling
72  *				        - netif_rx() feedback
73  */
74 
75 #include <linux/uaccess.h>
76 #include <linux/bitops.h>
77 #include <linux/capability.h>
78 #include <linux/cpu.h>
79 #include <linux/types.h>
80 #include <linux/kernel.h>
81 #include <linux/hash.h>
82 #include <linux/slab.h>
83 #include <linux/sched.h>
84 #include <linux/mutex.h>
85 #include <linux/string.h>
86 #include <linux/mm.h>
87 #include <linux/socket.h>
88 #include <linux/sockios.h>
89 #include <linux/errno.h>
90 #include <linux/interrupt.h>
91 #include <linux/if_ether.h>
92 #include <linux/netdevice.h>
93 #include <linux/etherdevice.h>
94 #include <linux/ethtool.h>
95 #include <linux/notifier.h>
96 #include <linux/skbuff.h>
97 #include <linux/bpf.h>
98 #include <net/net_namespace.h>
99 #include <net/sock.h>
100 #include <net/busy_poll.h>
101 #include <linux/rtnetlink.h>
102 #include <linux/stat.h>
103 #include <net/dst.h>
104 #include <net/dst_metadata.h>
105 #include <net/pkt_sched.h>
106 #include <net/checksum.h>
107 #include <net/xfrm.h>
108 #include <linux/highmem.h>
109 #include <linux/init.h>
110 #include <linux/module.h>
111 #include <linux/netpoll.h>
112 #include <linux/rcupdate.h>
113 #include <linux/delay.h>
114 #include <net/iw_handler.h>
115 #include <asm/current.h>
116 #include <linux/audit.h>
117 #include <linux/dmaengine.h>
118 #include <linux/err.h>
119 #include <linux/ctype.h>
120 #include <linux/if_arp.h>
121 #include <linux/if_vlan.h>
122 #include <linux/ip.h>
123 #include <net/ip.h>
124 #include <net/mpls.h>
125 #include <linux/ipv6.h>
126 #include <linux/in.h>
127 #include <linux/jhash.h>
128 #include <linux/random.h>
129 #include <trace/events/napi.h>
130 #include <trace/events/net.h>
131 #include <trace/events/skb.h>
132 #include <linux/pci.h>
133 #include <linux/inetdevice.h>
134 #include <linux/cpu_rmap.h>
135 #include <linux/static_key.h>
136 #include <linux/hashtable.h>
137 #include <linux/vmalloc.h>
138 #include <linux/if_macvlan.h>
139 #include <linux/errqueue.h>
140 #include <linux/hrtimer.h>
141 #include <linux/netfilter_ingress.h>
142 #include <linux/crash_dump.h>
143 
144 #include "net-sysfs.h"
145 
146 /* Instead of increasing this, you should create a hash table. */
147 #define MAX_GRO_SKBS 8
148 
149 /* This should be increased if a protocol with a bigger head is added. */
150 #define GRO_MAX_HEAD (MAX_HEADER + 128)
151 
152 static DEFINE_SPINLOCK(ptype_lock);
153 static DEFINE_SPINLOCK(offload_lock);
154 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
155 struct list_head ptype_all __read_mostly;	/* Taps */
156 static struct list_head offload_base __read_mostly;
157 
158 static int netif_rx_internal(struct sk_buff *skb);
159 static int call_netdevice_notifiers_info(unsigned long val,
160 					 struct net_device *dev,
161 					 struct netdev_notifier_info *info);
162 
163 /*
164  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
165  * semaphore.
166  *
167  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
168  *
169  * Writers must hold the rtnl semaphore while they loop through the
170  * dev_base_head list, and hold dev_base_lock for writing when they do the
171  * actual updates.  This allows pure readers to access the list even
172  * while a writer is preparing to update it.
173  *
174  * To put it another way, dev_base_lock is held for writing only to
175  * protect against pure readers; the rtnl semaphore provides the
176  * protection against other writers.
177  *
178  * See, for example usages, register_netdevice() and
179  * unregister_netdevice(), which must be called with the rtnl
180  * semaphore held.
181  */
182 DEFINE_RWLOCK(dev_base_lock);
183 EXPORT_SYMBOL(dev_base_lock);
184 
185 /* protects napi_hash addition/deletion and napi_gen_id */
186 static DEFINE_SPINLOCK(napi_hash_lock);
187 
188 static unsigned int napi_gen_id = NR_CPUS;
189 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
190 
191 static seqcount_t devnet_rename_seq;
192 
193 static inline void dev_base_seq_inc(struct net *net)
194 {
195 	while (++net->dev_base_seq == 0)
196 		;
197 }
198 
199 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
200 {
201 	unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
202 
203 	return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
204 }
205 
206 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
207 {
208 	return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
209 }
210 
211 static inline void rps_lock(struct softnet_data *sd)
212 {
213 #ifdef CONFIG_RPS
214 	spin_lock(&sd->input_pkt_queue.lock);
215 #endif
216 }
217 
218 static inline void rps_unlock(struct softnet_data *sd)
219 {
220 #ifdef CONFIG_RPS
221 	spin_unlock(&sd->input_pkt_queue.lock);
222 #endif
223 }
224 
225 /* Device list insertion */
226 static void list_netdevice(struct net_device *dev)
227 {
228 	struct net *net = dev_net(dev);
229 
230 	ASSERT_RTNL();
231 
232 	write_lock_bh(&dev_base_lock);
233 	list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
234 	hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
235 	hlist_add_head_rcu(&dev->index_hlist,
236 			   dev_index_hash(net, dev->ifindex));
237 	write_unlock_bh(&dev_base_lock);
238 
239 	dev_base_seq_inc(net);
240 }
241 
242 /* Device list removal
243  * caller must respect a RCU grace period before freeing/reusing dev
244  */
245 static void unlist_netdevice(struct net_device *dev)
246 {
247 	ASSERT_RTNL();
248 
249 	/* Unlink dev from the device chain */
250 	write_lock_bh(&dev_base_lock);
251 	list_del_rcu(&dev->dev_list);
252 	hlist_del_rcu(&dev->name_hlist);
253 	hlist_del_rcu(&dev->index_hlist);
254 	write_unlock_bh(&dev_base_lock);
255 
256 	dev_base_seq_inc(dev_net(dev));
257 }
258 
259 /*
260  *	Our notifier list
261  */
262 
263 static RAW_NOTIFIER_HEAD(netdev_chain);
264 
265 /*
266  *	Device drivers call our routines to queue packets here. We empty the
267  *	queue in the local softnet handler.
268  */
269 
270 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
271 EXPORT_PER_CPU_SYMBOL(softnet_data);
272 
273 #ifdef CONFIG_LOCKDEP
274 /*
275  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
276  * according to dev->type
277  */
278 static const unsigned short netdev_lock_type[] = {
279 	 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
280 	 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
281 	 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
282 	 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
283 	 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
284 	 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
285 	 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
286 	 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
287 	 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
288 	 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
289 	 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
290 	 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
291 	 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
292 	 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
293 	 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
294 
295 static const char *const netdev_lock_name[] = {
296 	"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
297 	"_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
298 	"_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
299 	"_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
300 	"_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
301 	"_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
302 	"_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
303 	"_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
304 	"_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
305 	"_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
306 	"_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
307 	"_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
308 	"_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
309 	"_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
310 	"_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
311 
312 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
313 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
314 
315 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
316 {
317 	int i;
318 
319 	for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
320 		if (netdev_lock_type[i] == dev_type)
321 			return i;
322 	/* the last key is used by default */
323 	return ARRAY_SIZE(netdev_lock_type) - 1;
324 }
325 
326 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
327 						 unsigned short dev_type)
328 {
329 	int i;
330 
331 	i = netdev_lock_pos(dev_type);
332 	lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
333 				   netdev_lock_name[i]);
334 }
335 
336 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
337 {
338 	int i;
339 
340 	i = netdev_lock_pos(dev->type);
341 	lockdep_set_class_and_name(&dev->addr_list_lock,
342 				   &netdev_addr_lock_key[i],
343 				   netdev_lock_name[i]);
344 }
345 #else
346 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
347 						 unsigned short dev_type)
348 {
349 }
350 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
351 {
352 }
353 #endif
354 
355 /*******************************************************************************
356  *
357  *		Protocol management and registration routines
358  *
359  *******************************************************************************/
360 
361 
362 /*
363  *	Add a protocol ID to the list. Now that the input handler is
364  *	smarter we can dispense with all the messy stuff that used to be
365  *	here.
366  *
367  *	BEWARE!!! Protocol handlers, mangling input packets,
368  *	MUST BE last in hash buckets and checking protocol handlers
369  *	MUST start from promiscuous ptype_all chain in net_bh.
370  *	It is true now, do not change it.
371  *	Explanation follows: if protocol handler, mangling packet, will
372  *	be the first on list, it is not able to sense, that packet
373  *	is cloned and should be copied-on-write, so that it will
374  *	change it and subsequent readers will get broken packet.
375  *							--ANK (980803)
376  */
377 
378 static inline struct list_head *ptype_head(const struct packet_type *pt)
379 {
380 	if (pt->type == htons(ETH_P_ALL))
381 		return pt->dev ? &pt->dev->ptype_all : &ptype_all;
382 	else
383 		return pt->dev ? &pt->dev->ptype_specific :
384 				 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
385 }
386 
387 /**
388  *	dev_add_pack - add packet handler
389  *	@pt: packet type declaration
390  *
391  *	Add a protocol handler to the networking stack. The passed &packet_type
392  *	is linked into kernel lists and may not be freed until it has been
393  *	removed from the kernel lists.
394  *
395  *	This call does not sleep therefore it can not
396  *	guarantee all CPU's that are in middle of receiving packets
397  *	will see the new packet type (until the next received packet).
398  */
399 
400 void dev_add_pack(struct packet_type *pt)
401 {
402 	struct list_head *head = ptype_head(pt);
403 
404 	spin_lock(&ptype_lock);
405 	list_add_rcu(&pt->list, head);
406 	spin_unlock(&ptype_lock);
407 }
408 EXPORT_SYMBOL(dev_add_pack);
409 
410 /**
411  *	__dev_remove_pack	 - remove packet handler
412  *	@pt: packet type declaration
413  *
414  *	Remove a protocol handler that was previously added to the kernel
415  *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
416  *	from the kernel lists and can be freed or reused once this function
417  *	returns.
418  *
419  *      The packet type might still be in use by receivers
420  *	and must not be freed until after all the CPU's have gone
421  *	through a quiescent state.
422  */
423 void __dev_remove_pack(struct packet_type *pt)
424 {
425 	struct list_head *head = ptype_head(pt);
426 	struct packet_type *pt1;
427 
428 	spin_lock(&ptype_lock);
429 
430 	list_for_each_entry(pt1, head, list) {
431 		if (pt == pt1) {
432 			list_del_rcu(&pt->list);
433 			goto out;
434 		}
435 	}
436 
437 	pr_warn("dev_remove_pack: %p not found\n", pt);
438 out:
439 	spin_unlock(&ptype_lock);
440 }
441 EXPORT_SYMBOL(__dev_remove_pack);
442 
443 /**
444  *	dev_remove_pack	 - remove packet handler
445  *	@pt: packet type declaration
446  *
447  *	Remove a protocol handler that was previously added to the kernel
448  *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
449  *	from the kernel lists and can be freed or reused once this function
450  *	returns.
451  *
452  *	This call sleeps to guarantee that no CPU is looking at the packet
453  *	type after return.
454  */
455 void dev_remove_pack(struct packet_type *pt)
456 {
457 	__dev_remove_pack(pt);
458 
459 	synchronize_net();
460 }
461 EXPORT_SYMBOL(dev_remove_pack);
462 
463 
464 /**
465  *	dev_add_offload - register offload handlers
466  *	@po: protocol offload declaration
467  *
468  *	Add protocol offload handlers to the networking stack. The passed
469  *	&proto_offload is linked into kernel lists and may not be freed until
470  *	it has been removed from the kernel lists.
471  *
472  *	This call does not sleep therefore it can not
473  *	guarantee all CPU's that are in middle of receiving packets
474  *	will see the new offload handlers (until the next received packet).
475  */
476 void dev_add_offload(struct packet_offload *po)
477 {
478 	struct packet_offload *elem;
479 
480 	spin_lock(&offload_lock);
481 	list_for_each_entry(elem, &offload_base, list) {
482 		if (po->priority < elem->priority)
483 			break;
484 	}
485 	list_add_rcu(&po->list, elem->list.prev);
486 	spin_unlock(&offload_lock);
487 }
488 EXPORT_SYMBOL(dev_add_offload);
489 
490 /**
491  *	__dev_remove_offload	 - remove offload handler
492  *	@po: packet offload declaration
493  *
494  *	Remove a protocol offload handler that was previously added to the
495  *	kernel offload handlers by dev_add_offload(). The passed &offload_type
496  *	is removed from the kernel lists and can be freed or reused once this
497  *	function returns.
498  *
499  *      The packet type might still be in use by receivers
500  *	and must not be freed until after all the CPU's have gone
501  *	through a quiescent state.
502  */
503 static void __dev_remove_offload(struct packet_offload *po)
504 {
505 	struct list_head *head = &offload_base;
506 	struct packet_offload *po1;
507 
508 	spin_lock(&offload_lock);
509 
510 	list_for_each_entry(po1, head, list) {
511 		if (po == po1) {
512 			list_del_rcu(&po->list);
513 			goto out;
514 		}
515 	}
516 
517 	pr_warn("dev_remove_offload: %p not found\n", po);
518 out:
519 	spin_unlock(&offload_lock);
520 }
521 
522 /**
523  *	dev_remove_offload	 - remove packet offload handler
524  *	@po: packet offload declaration
525  *
526  *	Remove a packet offload handler that was previously added to the kernel
527  *	offload handlers by dev_add_offload(). The passed &offload_type is
528  *	removed from the kernel lists and can be freed or reused once this
529  *	function returns.
530  *
531  *	This call sleeps to guarantee that no CPU is looking at the packet
532  *	type after return.
533  */
534 void dev_remove_offload(struct packet_offload *po)
535 {
536 	__dev_remove_offload(po);
537 
538 	synchronize_net();
539 }
540 EXPORT_SYMBOL(dev_remove_offload);
541 
542 /******************************************************************************
543  *
544  *		      Device Boot-time Settings Routines
545  *
546  ******************************************************************************/
547 
548 /* Boot time configuration table */
549 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
550 
551 /**
552  *	netdev_boot_setup_add	- add new setup entry
553  *	@name: name of the device
554  *	@map: configured settings for the device
555  *
556  *	Adds new setup entry to the dev_boot_setup list.  The function
557  *	returns 0 on error and 1 on success.  This is a generic routine to
558  *	all netdevices.
559  */
560 static int netdev_boot_setup_add(char *name, struct ifmap *map)
561 {
562 	struct netdev_boot_setup *s;
563 	int i;
564 
565 	s = dev_boot_setup;
566 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
567 		if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
568 			memset(s[i].name, 0, sizeof(s[i].name));
569 			strlcpy(s[i].name, name, IFNAMSIZ);
570 			memcpy(&s[i].map, map, sizeof(s[i].map));
571 			break;
572 		}
573 	}
574 
575 	return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
576 }
577 
578 /**
579  * netdev_boot_setup_check	- check boot time settings
580  * @dev: the netdevice
581  *
582  * Check boot time settings for the device.
583  * The found settings are set for the device to be used
584  * later in the device probing.
585  * Returns 0 if no settings found, 1 if they are.
586  */
587 int netdev_boot_setup_check(struct net_device *dev)
588 {
589 	struct netdev_boot_setup *s = dev_boot_setup;
590 	int i;
591 
592 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
593 		if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
594 		    !strcmp(dev->name, s[i].name)) {
595 			dev->irq = s[i].map.irq;
596 			dev->base_addr = s[i].map.base_addr;
597 			dev->mem_start = s[i].map.mem_start;
598 			dev->mem_end = s[i].map.mem_end;
599 			return 1;
600 		}
601 	}
602 	return 0;
603 }
604 EXPORT_SYMBOL(netdev_boot_setup_check);
605 
606 
607 /**
608  * netdev_boot_base	- get address from boot time settings
609  * @prefix: prefix for network device
610  * @unit: id for network device
611  *
612  * Check boot time settings for the base address of device.
613  * The found settings are set for the device to be used
614  * later in the device probing.
615  * Returns 0 if no settings found.
616  */
617 unsigned long netdev_boot_base(const char *prefix, int unit)
618 {
619 	const struct netdev_boot_setup *s = dev_boot_setup;
620 	char name[IFNAMSIZ];
621 	int i;
622 
623 	sprintf(name, "%s%d", prefix, unit);
624 
625 	/*
626 	 * If device already registered then return base of 1
627 	 * to indicate not to probe for this interface
628 	 */
629 	if (__dev_get_by_name(&init_net, name))
630 		return 1;
631 
632 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
633 		if (!strcmp(name, s[i].name))
634 			return s[i].map.base_addr;
635 	return 0;
636 }
637 
638 /*
639  * Saves at boot time configured settings for any netdevice.
640  */
641 int __init netdev_boot_setup(char *str)
642 {
643 	int ints[5];
644 	struct ifmap map;
645 
646 	str = get_options(str, ARRAY_SIZE(ints), ints);
647 	if (!str || !*str)
648 		return 0;
649 
650 	/* Save settings */
651 	memset(&map, 0, sizeof(map));
652 	if (ints[0] > 0)
653 		map.irq = ints[1];
654 	if (ints[0] > 1)
655 		map.base_addr = ints[2];
656 	if (ints[0] > 2)
657 		map.mem_start = ints[3];
658 	if (ints[0] > 3)
659 		map.mem_end = ints[4];
660 
661 	/* Add new entry to the list */
662 	return netdev_boot_setup_add(str, &map);
663 }
664 
665 __setup("netdev=", netdev_boot_setup);
666 
667 /*******************************************************************************
668  *
669  *			    Device Interface Subroutines
670  *
671  *******************************************************************************/
672 
673 /**
674  *	dev_get_iflink	- get 'iflink' value of a interface
675  *	@dev: targeted interface
676  *
677  *	Indicates the ifindex the interface is linked to.
678  *	Physical interfaces have the same 'ifindex' and 'iflink' values.
679  */
680 
681 int dev_get_iflink(const struct net_device *dev)
682 {
683 	if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
684 		return dev->netdev_ops->ndo_get_iflink(dev);
685 
686 	return dev->ifindex;
687 }
688 EXPORT_SYMBOL(dev_get_iflink);
689 
690 /**
691  *	dev_fill_metadata_dst - Retrieve tunnel egress information.
692  *	@dev: targeted interface
693  *	@skb: The packet.
694  *
695  *	For better visibility of tunnel traffic OVS needs to retrieve
696  *	egress tunnel information for a packet. Following API allows
697  *	user to get this info.
698  */
699 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
700 {
701 	struct ip_tunnel_info *info;
702 
703 	if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
704 		return -EINVAL;
705 
706 	info = skb_tunnel_info_unclone(skb);
707 	if (!info)
708 		return -ENOMEM;
709 	if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
710 		return -EINVAL;
711 
712 	return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
713 }
714 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
715 
716 /**
717  *	__dev_get_by_name	- find a device by its name
718  *	@net: the applicable net namespace
719  *	@name: name to find
720  *
721  *	Find an interface by name. Must be called under RTNL semaphore
722  *	or @dev_base_lock. If the name is found a pointer to the device
723  *	is returned. If the name is not found then %NULL is returned. The
724  *	reference counters are not incremented so the caller must be
725  *	careful with locks.
726  */
727 
728 struct net_device *__dev_get_by_name(struct net *net, const char *name)
729 {
730 	struct net_device *dev;
731 	struct hlist_head *head = dev_name_hash(net, name);
732 
733 	hlist_for_each_entry(dev, head, name_hlist)
734 		if (!strncmp(dev->name, name, IFNAMSIZ))
735 			return dev;
736 
737 	return NULL;
738 }
739 EXPORT_SYMBOL(__dev_get_by_name);
740 
741 /**
742  * dev_get_by_name_rcu	- find a device by its name
743  * @net: the applicable net namespace
744  * @name: name to find
745  *
746  * Find an interface by name.
747  * If the name is found a pointer to the device is returned.
748  * If the name is not found then %NULL is returned.
749  * The reference counters are not incremented so the caller must be
750  * careful with locks. The caller must hold RCU lock.
751  */
752 
753 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
754 {
755 	struct net_device *dev;
756 	struct hlist_head *head = dev_name_hash(net, name);
757 
758 	hlist_for_each_entry_rcu(dev, head, name_hlist)
759 		if (!strncmp(dev->name, name, IFNAMSIZ))
760 			return dev;
761 
762 	return NULL;
763 }
764 EXPORT_SYMBOL(dev_get_by_name_rcu);
765 
766 /**
767  *	dev_get_by_name		- find a device by its name
768  *	@net: the applicable net namespace
769  *	@name: name to find
770  *
771  *	Find an interface by name. This can be called from any
772  *	context and does its own locking. The returned handle has
773  *	the usage count incremented and the caller must use dev_put() to
774  *	release it when it is no longer needed. %NULL is returned if no
775  *	matching device is found.
776  */
777 
778 struct net_device *dev_get_by_name(struct net *net, const char *name)
779 {
780 	struct net_device *dev;
781 
782 	rcu_read_lock();
783 	dev = dev_get_by_name_rcu(net, name);
784 	if (dev)
785 		dev_hold(dev);
786 	rcu_read_unlock();
787 	return dev;
788 }
789 EXPORT_SYMBOL(dev_get_by_name);
790 
791 /**
792  *	__dev_get_by_index - find a device by its ifindex
793  *	@net: the applicable net namespace
794  *	@ifindex: index of device
795  *
796  *	Search for an interface by index. Returns %NULL if the device
797  *	is not found or a pointer to the device. The device has not
798  *	had its reference counter increased so the caller must be careful
799  *	about locking. The caller must hold either the RTNL semaphore
800  *	or @dev_base_lock.
801  */
802 
803 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
804 {
805 	struct net_device *dev;
806 	struct hlist_head *head = dev_index_hash(net, ifindex);
807 
808 	hlist_for_each_entry(dev, head, index_hlist)
809 		if (dev->ifindex == ifindex)
810 			return dev;
811 
812 	return NULL;
813 }
814 EXPORT_SYMBOL(__dev_get_by_index);
815 
816 /**
817  *	dev_get_by_index_rcu - find a device by its ifindex
818  *	@net: the applicable net namespace
819  *	@ifindex: index of device
820  *
821  *	Search for an interface by index. Returns %NULL if the device
822  *	is not found or a pointer to the device. The device has not
823  *	had its reference counter increased so the caller must be careful
824  *	about locking. The caller must hold RCU lock.
825  */
826 
827 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
828 {
829 	struct net_device *dev;
830 	struct hlist_head *head = dev_index_hash(net, ifindex);
831 
832 	hlist_for_each_entry_rcu(dev, head, index_hlist)
833 		if (dev->ifindex == ifindex)
834 			return dev;
835 
836 	return NULL;
837 }
838 EXPORT_SYMBOL(dev_get_by_index_rcu);
839 
840 
841 /**
842  *	dev_get_by_index - find a device by its ifindex
843  *	@net: the applicable net namespace
844  *	@ifindex: index of device
845  *
846  *	Search for an interface by index. Returns NULL if the device
847  *	is not found or a pointer to the device. The device returned has
848  *	had a reference added and the pointer is safe until the user calls
849  *	dev_put to indicate they have finished with it.
850  */
851 
852 struct net_device *dev_get_by_index(struct net *net, int ifindex)
853 {
854 	struct net_device *dev;
855 
856 	rcu_read_lock();
857 	dev = dev_get_by_index_rcu(net, ifindex);
858 	if (dev)
859 		dev_hold(dev);
860 	rcu_read_unlock();
861 	return dev;
862 }
863 EXPORT_SYMBOL(dev_get_by_index);
864 
865 /**
866  *	netdev_get_name - get a netdevice name, knowing its ifindex.
867  *	@net: network namespace
868  *	@name: a pointer to the buffer where the name will be stored.
869  *	@ifindex: the ifindex of the interface to get the name from.
870  *
871  *	The use of raw_seqcount_begin() and cond_resched() before
872  *	retrying is required as we want to give the writers a chance
873  *	to complete when CONFIG_PREEMPT is not set.
874  */
875 int netdev_get_name(struct net *net, char *name, int ifindex)
876 {
877 	struct net_device *dev;
878 	unsigned int seq;
879 
880 retry:
881 	seq = raw_seqcount_begin(&devnet_rename_seq);
882 	rcu_read_lock();
883 	dev = dev_get_by_index_rcu(net, ifindex);
884 	if (!dev) {
885 		rcu_read_unlock();
886 		return -ENODEV;
887 	}
888 
889 	strcpy(name, dev->name);
890 	rcu_read_unlock();
891 	if (read_seqcount_retry(&devnet_rename_seq, seq)) {
892 		cond_resched();
893 		goto retry;
894 	}
895 
896 	return 0;
897 }
898 
899 /**
900  *	dev_getbyhwaddr_rcu - find a device by its hardware address
901  *	@net: the applicable net namespace
902  *	@type: media type of device
903  *	@ha: hardware address
904  *
905  *	Search for an interface by MAC address. Returns NULL if the device
906  *	is not found or a pointer to the device.
907  *	The caller must hold RCU or RTNL.
908  *	The returned device has not had its ref count increased
909  *	and the caller must therefore be careful about locking
910  *
911  */
912 
913 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
914 				       const char *ha)
915 {
916 	struct net_device *dev;
917 
918 	for_each_netdev_rcu(net, dev)
919 		if (dev->type == type &&
920 		    !memcmp(dev->dev_addr, ha, dev->addr_len))
921 			return dev;
922 
923 	return NULL;
924 }
925 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
926 
927 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
928 {
929 	struct net_device *dev;
930 
931 	ASSERT_RTNL();
932 	for_each_netdev(net, dev)
933 		if (dev->type == type)
934 			return dev;
935 
936 	return NULL;
937 }
938 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
939 
940 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
941 {
942 	struct net_device *dev, *ret = NULL;
943 
944 	rcu_read_lock();
945 	for_each_netdev_rcu(net, dev)
946 		if (dev->type == type) {
947 			dev_hold(dev);
948 			ret = dev;
949 			break;
950 		}
951 	rcu_read_unlock();
952 	return ret;
953 }
954 EXPORT_SYMBOL(dev_getfirstbyhwtype);
955 
956 /**
957  *	__dev_get_by_flags - find any device with given flags
958  *	@net: the applicable net namespace
959  *	@if_flags: IFF_* values
960  *	@mask: bitmask of bits in if_flags to check
961  *
962  *	Search for any interface with the given flags. Returns NULL if a device
963  *	is not found or a pointer to the device. Must be called inside
964  *	rtnl_lock(), and result refcount is unchanged.
965  */
966 
967 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
968 				      unsigned short mask)
969 {
970 	struct net_device *dev, *ret;
971 
972 	ASSERT_RTNL();
973 
974 	ret = NULL;
975 	for_each_netdev(net, dev) {
976 		if (((dev->flags ^ if_flags) & mask) == 0) {
977 			ret = dev;
978 			break;
979 		}
980 	}
981 	return ret;
982 }
983 EXPORT_SYMBOL(__dev_get_by_flags);
984 
985 /**
986  *	dev_valid_name - check if name is okay for network device
987  *	@name: name string
988  *
989  *	Network device names need to be valid file names to
990  *	to allow sysfs to work.  We also disallow any kind of
991  *	whitespace.
992  */
993 bool dev_valid_name(const char *name)
994 {
995 	if (*name == '\0')
996 		return false;
997 	if (strlen(name) >= IFNAMSIZ)
998 		return false;
999 	if (!strcmp(name, ".") || !strcmp(name, ".."))
1000 		return false;
1001 
1002 	while (*name) {
1003 		if (*name == '/' || *name == ':' || isspace(*name))
1004 			return false;
1005 		name++;
1006 	}
1007 	return true;
1008 }
1009 EXPORT_SYMBOL(dev_valid_name);
1010 
1011 /**
1012  *	__dev_alloc_name - allocate a name for a device
1013  *	@net: network namespace to allocate the device name in
1014  *	@name: name format string
1015  *	@buf:  scratch buffer and result name string
1016  *
1017  *	Passed a format string - eg "lt%d" it will try and find a suitable
1018  *	id. It scans list of devices to build up a free map, then chooses
1019  *	the first empty slot. The caller must hold the dev_base or rtnl lock
1020  *	while allocating the name and adding the device in order to avoid
1021  *	duplicates.
1022  *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1023  *	Returns the number of the unit assigned or a negative errno code.
1024  */
1025 
1026 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1027 {
1028 	int i = 0;
1029 	const char *p;
1030 	const int max_netdevices = 8*PAGE_SIZE;
1031 	unsigned long *inuse;
1032 	struct net_device *d;
1033 
1034 	p = strnchr(name, IFNAMSIZ-1, '%');
1035 	if (p) {
1036 		/*
1037 		 * Verify the string as this thing may have come from
1038 		 * the user.  There must be either one "%d" and no other "%"
1039 		 * characters.
1040 		 */
1041 		if (p[1] != 'd' || strchr(p + 2, '%'))
1042 			return -EINVAL;
1043 
1044 		/* Use one page as a bit array of possible slots */
1045 		inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1046 		if (!inuse)
1047 			return -ENOMEM;
1048 
1049 		for_each_netdev(net, d) {
1050 			if (!sscanf(d->name, name, &i))
1051 				continue;
1052 			if (i < 0 || i >= max_netdevices)
1053 				continue;
1054 
1055 			/*  avoid cases where sscanf is not exact inverse of printf */
1056 			snprintf(buf, IFNAMSIZ, name, i);
1057 			if (!strncmp(buf, d->name, IFNAMSIZ))
1058 				set_bit(i, inuse);
1059 		}
1060 
1061 		i = find_first_zero_bit(inuse, max_netdevices);
1062 		free_page((unsigned long) inuse);
1063 	}
1064 
1065 	if (buf != name)
1066 		snprintf(buf, IFNAMSIZ, name, i);
1067 	if (!__dev_get_by_name(net, buf))
1068 		return i;
1069 
1070 	/* It is possible to run out of possible slots
1071 	 * when the name is long and there isn't enough space left
1072 	 * for the digits, or if all bits are used.
1073 	 */
1074 	return -ENFILE;
1075 }
1076 
1077 /**
1078  *	dev_alloc_name - allocate a name for a device
1079  *	@dev: device
1080  *	@name: name format string
1081  *
1082  *	Passed a format string - eg "lt%d" it will try and find a suitable
1083  *	id. It scans list of devices to build up a free map, then chooses
1084  *	the first empty slot. The caller must hold the dev_base or rtnl lock
1085  *	while allocating the name and adding the device in order to avoid
1086  *	duplicates.
1087  *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1088  *	Returns the number of the unit assigned or a negative errno code.
1089  */
1090 
1091 int dev_alloc_name(struct net_device *dev, const char *name)
1092 {
1093 	char buf[IFNAMSIZ];
1094 	struct net *net;
1095 	int ret;
1096 
1097 	BUG_ON(!dev_net(dev));
1098 	net = dev_net(dev);
1099 	ret = __dev_alloc_name(net, name, buf);
1100 	if (ret >= 0)
1101 		strlcpy(dev->name, buf, IFNAMSIZ);
1102 	return ret;
1103 }
1104 EXPORT_SYMBOL(dev_alloc_name);
1105 
1106 static int dev_alloc_name_ns(struct net *net,
1107 			     struct net_device *dev,
1108 			     const char *name)
1109 {
1110 	char buf[IFNAMSIZ];
1111 	int ret;
1112 
1113 	ret = __dev_alloc_name(net, name, buf);
1114 	if (ret >= 0)
1115 		strlcpy(dev->name, buf, IFNAMSIZ);
1116 	return ret;
1117 }
1118 
1119 static int dev_get_valid_name(struct net *net,
1120 			      struct net_device *dev,
1121 			      const char *name)
1122 {
1123 	BUG_ON(!net);
1124 
1125 	if (!dev_valid_name(name))
1126 		return -EINVAL;
1127 
1128 	if (strchr(name, '%'))
1129 		return dev_alloc_name_ns(net, dev, name);
1130 	else if (__dev_get_by_name(net, name))
1131 		return -EEXIST;
1132 	else if (dev->name != name)
1133 		strlcpy(dev->name, name, IFNAMSIZ);
1134 
1135 	return 0;
1136 }
1137 
1138 /**
1139  *	dev_change_name - change name of a device
1140  *	@dev: device
1141  *	@newname: name (or format string) must be at least IFNAMSIZ
1142  *
1143  *	Change name of a device, can pass format strings "eth%d".
1144  *	for wildcarding.
1145  */
1146 int dev_change_name(struct net_device *dev, const char *newname)
1147 {
1148 	unsigned char old_assign_type;
1149 	char oldname[IFNAMSIZ];
1150 	int err = 0;
1151 	int ret;
1152 	struct net *net;
1153 
1154 	ASSERT_RTNL();
1155 	BUG_ON(!dev_net(dev));
1156 
1157 	net = dev_net(dev);
1158 	if (dev->flags & IFF_UP)
1159 		return -EBUSY;
1160 
1161 	write_seqcount_begin(&devnet_rename_seq);
1162 
1163 	if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1164 		write_seqcount_end(&devnet_rename_seq);
1165 		return 0;
1166 	}
1167 
1168 	memcpy(oldname, dev->name, IFNAMSIZ);
1169 
1170 	err = dev_get_valid_name(net, dev, newname);
1171 	if (err < 0) {
1172 		write_seqcount_end(&devnet_rename_seq);
1173 		return err;
1174 	}
1175 
1176 	if (oldname[0] && !strchr(oldname, '%'))
1177 		netdev_info(dev, "renamed from %s\n", oldname);
1178 
1179 	old_assign_type = dev->name_assign_type;
1180 	dev->name_assign_type = NET_NAME_RENAMED;
1181 
1182 rollback:
1183 	ret = device_rename(&dev->dev, dev->name);
1184 	if (ret) {
1185 		memcpy(dev->name, oldname, IFNAMSIZ);
1186 		dev->name_assign_type = old_assign_type;
1187 		write_seqcount_end(&devnet_rename_seq);
1188 		return ret;
1189 	}
1190 
1191 	write_seqcount_end(&devnet_rename_seq);
1192 
1193 	netdev_adjacent_rename_links(dev, oldname);
1194 
1195 	write_lock_bh(&dev_base_lock);
1196 	hlist_del_rcu(&dev->name_hlist);
1197 	write_unlock_bh(&dev_base_lock);
1198 
1199 	synchronize_rcu();
1200 
1201 	write_lock_bh(&dev_base_lock);
1202 	hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1203 	write_unlock_bh(&dev_base_lock);
1204 
1205 	ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1206 	ret = notifier_to_errno(ret);
1207 
1208 	if (ret) {
1209 		/* err >= 0 after dev_alloc_name() or stores the first errno */
1210 		if (err >= 0) {
1211 			err = ret;
1212 			write_seqcount_begin(&devnet_rename_seq);
1213 			memcpy(dev->name, oldname, IFNAMSIZ);
1214 			memcpy(oldname, newname, IFNAMSIZ);
1215 			dev->name_assign_type = old_assign_type;
1216 			old_assign_type = NET_NAME_RENAMED;
1217 			goto rollback;
1218 		} else {
1219 			pr_err("%s: name change rollback failed: %d\n",
1220 			       dev->name, ret);
1221 		}
1222 	}
1223 
1224 	return err;
1225 }
1226 
1227 /**
1228  *	dev_set_alias - change ifalias of a device
1229  *	@dev: device
1230  *	@alias: name up to IFALIASZ
1231  *	@len: limit of bytes to copy from info
1232  *
1233  *	Set ifalias for a device,
1234  */
1235 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1236 {
1237 	char *new_ifalias;
1238 
1239 	ASSERT_RTNL();
1240 
1241 	if (len >= IFALIASZ)
1242 		return -EINVAL;
1243 
1244 	if (!len) {
1245 		kfree(dev->ifalias);
1246 		dev->ifalias = NULL;
1247 		return 0;
1248 	}
1249 
1250 	new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1251 	if (!new_ifalias)
1252 		return -ENOMEM;
1253 	dev->ifalias = new_ifalias;
1254 
1255 	strlcpy(dev->ifalias, alias, len+1);
1256 	return len;
1257 }
1258 
1259 
1260 /**
1261  *	netdev_features_change - device changes features
1262  *	@dev: device to cause notification
1263  *
1264  *	Called to indicate a device has changed features.
1265  */
1266 void netdev_features_change(struct net_device *dev)
1267 {
1268 	call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1269 }
1270 EXPORT_SYMBOL(netdev_features_change);
1271 
1272 /**
1273  *	netdev_state_change - device changes state
1274  *	@dev: device to cause notification
1275  *
1276  *	Called to indicate a device has changed state. This function calls
1277  *	the notifier chains for netdev_chain and sends a NEWLINK message
1278  *	to the routing socket.
1279  */
1280 void netdev_state_change(struct net_device *dev)
1281 {
1282 	if (dev->flags & IFF_UP) {
1283 		struct netdev_notifier_change_info change_info;
1284 
1285 		change_info.flags_changed = 0;
1286 		call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1287 					      &change_info.info);
1288 		rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1289 	}
1290 }
1291 EXPORT_SYMBOL(netdev_state_change);
1292 
1293 /**
1294  * netdev_notify_peers - notify network peers about existence of @dev
1295  * @dev: network device
1296  *
1297  * Generate traffic such that interested network peers are aware of
1298  * @dev, such as by generating a gratuitous ARP. This may be used when
1299  * a device wants to inform the rest of the network about some sort of
1300  * reconfiguration such as a failover event or virtual machine
1301  * migration.
1302  */
1303 void netdev_notify_peers(struct net_device *dev)
1304 {
1305 	rtnl_lock();
1306 	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1307 	rtnl_unlock();
1308 }
1309 EXPORT_SYMBOL(netdev_notify_peers);
1310 
1311 static int __dev_open(struct net_device *dev)
1312 {
1313 	const struct net_device_ops *ops = dev->netdev_ops;
1314 	int ret;
1315 
1316 	ASSERT_RTNL();
1317 
1318 	if (!netif_device_present(dev))
1319 		return -ENODEV;
1320 
1321 	/* Block netpoll from trying to do any rx path servicing.
1322 	 * If we don't do this there is a chance ndo_poll_controller
1323 	 * or ndo_poll may be running while we open the device
1324 	 */
1325 	netpoll_poll_disable(dev);
1326 
1327 	ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1328 	ret = notifier_to_errno(ret);
1329 	if (ret)
1330 		return ret;
1331 
1332 	set_bit(__LINK_STATE_START, &dev->state);
1333 
1334 	if (ops->ndo_validate_addr)
1335 		ret = ops->ndo_validate_addr(dev);
1336 
1337 	if (!ret && ops->ndo_open)
1338 		ret = ops->ndo_open(dev);
1339 
1340 	netpoll_poll_enable(dev);
1341 
1342 	if (ret)
1343 		clear_bit(__LINK_STATE_START, &dev->state);
1344 	else {
1345 		dev->flags |= IFF_UP;
1346 		dev_set_rx_mode(dev);
1347 		dev_activate(dev);
1348 		add_device_randomness(dev->dev_addr, dev->addr_len);
1349 	}
1350 
1351 	return ret;
1352 }
1353 
1354 /**
1355  *	dev_open	- prepare an interface for use.
1356  *	@dev:	device to open
1357  *
1358  *	Takes a device from down to up state. The device's private open
1359  *	function is invoked and then the multicast lists are loaded. Finally
1360  *	the device is moved into the up state and a %NETDEV_UP message is
1361  *	sent to the netdev notifier chain.
1362  *
1363  *	Calling this function on an active interface is a nop. On a failure
1364  *	a negative errno code is returned.
1365  */
1366 int dev_open(struct net_device *dev)
1367 {
1368 	int ret;
1369 
1370 	if (dev->flags & IFF_UP)
1371 		return 0;
1372 
1373 	ret = __dev_open(dev);
1374 	if (ret < 0)
1375 		return ret;
1376 
1377 	rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1378 	call_netdevice_notifiers(NETDEV_UP, dev);
1379 
1380 	return ret;
1381 }
1382 EXPORT_SYMBOL(dev_open);
1383 
1384 static int __dev_close_many(struct list_head *head)
1385 {
1386 	struct net_device *dev;
1387 
1388 	ASSERT_RTNL();
1389 	might_sleep();
1390 
1391 	list_for_each_entry(dev, head, close_list) {
1392 		/* Temporarily disable netpoll until the interface is down */
1393 		netpoll_poll_disable(dev);
1394 
1395 		call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1396 
1397 		clear_bit(__LINK_STATE_START, &dev->state);
1398 
1399 		/* Synchronize to scheduled poll. We cannot touch poll list, it
1400 		 * can be even on different cpu. So just clear netif_running().
1401 		 *
1402 		 * dev->stop() will invoke napi_disable() on all of it's
1403 		 * napi_struct instances on this device.
1404 		 */
1405 		smp_mb__after_atomic(); /* Commit netif_running(). */
1406 	}
1407 
1408 	dev_deactivate_many(head);
1409 
1410 	list_for_each_entry(dev, head, close_list) {
1411 		const struct net_device_ops *ops = dev->netdev_ops;
1412 
1413 		/*
1414 		 *	Call the device specific close. This cannot fail.
1415 		 *	Only if device is UP
1416 		 *
1417 		 *	We allow it to be called even after a DETACH hot-plug
1418 		 *	event.
1419 		 */
1420 		if (ops->ndo_stop)
1421 			ops->ndo_stop(dev);
1422 
1423 		dev->flags &= ~IFF_UP;
1424 		netpoll_poll_enable(dev);
1425 	}
1426 
1427 	return 0;
1428 }
1429 
1430 static int __dev_close(struct net_device *dev)
1431 {
1432 	int retval;
1433 	LIST_HEAD(single);
1434 
1435 	list_add(&dev->close_list, &single);
1436 	retval = __dev_close_many(&single);
1437 	list_del(&single);
1438 
1439 	return retval;
1440 }
1441 
1442 int dev_close_many(struct list_head *head, bool unlink)
1443 {
1444 	struct net_device *dev, *tmp;
1445 
1446 	/* Remove the devices that don't need to be closed */
1447 	list_for_each_entry_safe(dev, tmp, head, close_list)
1448 		if (!(dev->flags & IFF_UP))
1449 			list_del_init(&dev->close_list);
1450 
1451 	__dev_close_many(head);
1452 
1453 	list_for_each_entry_safe(dev, tmp, head, close_list) {
1454 		rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1455 		call_netdevice_notifiers(NETDEV_DOWN, dev);
1456 		if (unlink)
1457 			list_del_init(&dev->close_list);
1458 	}
1459 
1460 	return 0;
1461 }
1462 EXPORT_SYMBOL(dev_close_many);
1463 
1464 /**
1465  *	dev_close - shutdown an interface.
1466  *	@dev: device to shutdown
1467  *
1468  *	This function moves an active device into down state. A
1469  *	%NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1470  *	is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1471  *	chain.
1472  */
1473 int dev_close(struct net_device *dev)
1474 {
1475 	if (dev->flags & IFF_UP) {
1476 		LIST_HEAD(single);
1477 
1478 		list_add(&dev->close_list, &single);
1479 		dev_close_many(&single, true);
1480 		list_del(&single);
1481 	}
1482 	return 0;
1483 }
1484 EXPORT_SYMBOL(dev_close);
1485 
1486 
1487 /**
1488  *	dev_disable_lro - disable Large Receive Offload on a device
1489  *	@dev: device
1490  *
1491  *	Disable Large Receive Offload (LRO) on a net device.  Must be
1492  *	called under RTNL.  This is needed if received packets may be
1493  *	forwarded to another interface.
1494  */
1495 void dev_disable_lro(struct net_device *dev)
1496 {
1497 	struct net_device *lower_dev;
1498 	struct list_head *iter;
1499 
1500 	dev->wanted_features &= ~NETIF_F_LRO;
1501 	netdev_update_features(dev);
1502 
1503 	if (unlikely(dev->features & NETIF_F_LRO))
1504 		netdev_WARN(dev, "failed to disable LRO!\n");
1505 
1506 	netdev_for_each_lower_dev(dev, lower_dev, iter)
1507 		dev_disable_lro(lower_dev);
1508 }
1509 EXPORT_SYMBOL(dev_disable_lro);
1510 
1511 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1512 				   struct net_device *dev)
1513 {
1514 	struct netdev_notifier_info info;
1515 
1516 	netdev_notifier_info_init(&info, dev);
1517 	return nb->notifier_call(nb, val, &info);
1518 }
1519 
1520 static int dev_boot_phase = 1;
1521 
1522 /**
1523  * register_netdevice_notifier - register a network notifier block
1524  * @nb: notifier
1525  *
1526  * Register a notifier to be called when network device events occur.
1527  * The notifier passed is linked into the kernel structures and must
1528  * not be reused until it has been unregistered. A negative errno code
1529  * is returned on a failure.
1530  *
1531  * When registered all registration and up events are replayed
1532  * to the new notifier to allow device to have a race free
1533  * view of the network device list.
1534  */
1535 
1536 int register_netdevice_notifier(struct notifier_block *nb)
1537 {
1538 	struct net_device *dev;
1539 	struct net_device *last;
1540 	struct net *net;
1541 	int err;
1542 
1543 	rtnl_lock();
1544 	err = raw_notifier_chain_register(&netdev_chain, nb);
1545 	if (err)
1546 		goto unlock;
1547 	if (dev_boot_phase)
1548 		goto unlock;
1549 	for_each_net(net) {
1550 		for_each_netdev(net, dev) {
1551 			err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1552 			err = notifier_to_errno(err);
1553 			if (err)
1554 				goto rollback;
1555 
1556 			if (!(dev->flags & IFF_UP))
1557 				continue;
1558 
1559 			call_netdevice_notifier(nb, NETDEV_UP, dev);
1560 		}
1561 	}
1562 
1563 unlock:
1564 	rtnl_unlock();
1565 	return err;
1566 
1567 rollback:
1568 	last = dev;
1569 	for_each_net(net) {
1570 		for_each_netdev(net, dev) {
1571 			if (dev == last)
1572 				goto outroll;
1573 
1574 			if (dev->flags & IFF_UP) {
1575 				call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1576 							dev);
1577 				call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1578 			}
1579 			call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1580 		}
1581 	}
1582 
1583 outroll:
1584 	raw_notifier_chain_unregister(&netdev_chain, nb);
1585 	goto unlock;
1586 }
1587 EXPORT_SYMBOL(register_netdevice_notifier);
1588 
1589 /**
1590  * unregister_netdevice_notifier - unregister a network notifier block
1591  * @nb: notifier
1592  *
1593  * Unregister a notifier previously registered by
1594  * register_netdevice_notifier(). The notifier is unlinked into the
1595  * kernel structures and may then be reused. A negative errno code
1596  * is returned on a failure.
1597  *
1598  * After unregistering unregister and down device events are synthesized
1599  * for all devices on the device list to the removed notifier to remove
1600  * the need for special case cleanup code.
1601  */
1602 
1603 int unregister_netdevice_notifier(struct notifier_block *nb)
1604 {
1605 	struct net_device *dev;
1606 	struct net *net;
1607 	int err;
1608 
1609 	rtnl_lock();
1610 	err = raw_notifier_chain_unregister(&netdev_chain, nb);
1611 	if (err)
1612 		goto unlock;
1613 
1614 	for_each_net(net) {
1615 		for_each_netdev(net, dev) {
1616 			if (dev->flags & IFF_UP) {
1617 				call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1618 							dev);
1619 				call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1620 			}
1621 			call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1622 		}
1623 	}
1624 unlock:
1625 	rtnl_unlock();
1626 	return err;
1627 }
1628 EXPORT_SYMBOL(unregister_netdevice_notifier);
1629 
1630 /**
1631  *	call_netdevice_notifiers_info - call all network notifier blocks
1632  *	@val: value passed unmodified to notifier function
1633  *	@dev: net_device pointer passed unmodified to notifier function
1634  *	@info: notifier information data
1635  *
1636  *	Call all network notifier blocks.  Parameters and return value
1637  *	are as for raw_notifier_call_chain().
1638  */
1639 
1640 static int call_netdevice_notifiers_info(unsigned long val,
1641 					 struct net_device *dev,
1642 					 struct netdev_notifier_info *info)
1643 {
1644 	ASSERT_RTNL();
1645 	netdev_notifier_info_init(info, dev);
1646 	return raw_notifier_call_chain(&netdev_chain, val, info);
1647 }
1648 
1649 /**
1650  *	call_netdevice_notifiers - call all network notifier blocks
1651  *      @val: value passed unmodified to notifier function
1652  *      @dev: net_device pointer passed unmodified to notifier function
1653  *
1654  *	Call all network notifier blocks.  Parameters and return value
1655  *	are as for raw_notifier_call_chain().
1656  */
1657 
1658 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1659 {
1660 	struct netdev_notifier_info info;
1661 
1662 	return call_netdevice_notifiers_info(val, dev, &info);
1663 }
1664 EXPORT_SYMBOL(call_netdevice_notifiers);
1665 
1666 #ifdef CONFIG_NET_INGRESS
1667 static struct static_key ingress_needed __read_mostly;
1668 
1669 void net_inc_ingress_queue(void)
1670 {
1671 	static_key_slow_inc(&ingress_needed);
1672 }
1673 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1674 
1675 void net_dec_ingress_queue(void)
1676 {
1677 	static_key_slow_dec(&ingress_needed);
1678 }
1679 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1680 #endif
1681 
1682 #ifdef CONFIG_NET_EGRESS
1683 static struct static_key egress_needed __read_mostly;
1684 
1685 void net_inc_egress_queue(void)
1686 {
1687 	static_key_slow_inc(&egress_needed);
1688 }
1689 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1690 
1691 void net_dec_egress_queue(void)
1692 {
1693 	static_key_slow_dec(&egress_needed);
1694 }
1695 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1696 #endif
1697 
1698 static struct static_key netstamp_needed __read_mostly;
1699 #ifdef HAVE_JUMP_LABEL
1700 static atomic_t netstamp_needed_deferred;
1701 static void netstamp_clear(struct work_struct *work)
1702 {
1703 	int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1704 
1705 	while (deferred--)
1706 		static_key_slow_dec(&netstamp_needed);
1707 }
1708 static DECLARE_WORK(netstamp_work, netstamp_clear);
1709 #endif
1710 
1711 void net_enable_timestamp(void)
1712 {
1713 	static_key_slow_inc(&netstamp_needed);
1714 }
1715 EXPORT_SYMBOL(net_enable_timestamp);
1716 
1717 void net_disable_timestamp(void)
1718 {
1719 #ifdef HAVE_JUMP_LABEL
1720 	/* net_disable_timestamp() can be called from non process context */
1721 	atomic_inc(&netstamp_needed_deferred);
1722 	schedule_work(&netstamp_work);
1723 #else
1724 	static_key_slow_dec(&netstamp_needed);
1725 #endif
1726 }
1727 EXPORT_SYMBOL(net_disable_timestamp);
1728 
1729 static inline void net_timestamp_set(struct sk_buff *skb)
1730 {
1731 	skb->tstamp = 0;
1732 	if (static_key_false(&netstamp_needed))
1733 		__net_timestamp(skb);
1734 }
1735 
1736 #define net_timestamp_check(COND, SKB)			\
1737 	if (static_key_false(&netstamp_needed)) {		\
1738 		if ((COND) && !(SKB)->tstamp)	\
1739 			__net_timestamp(SKB);		\
1740 	}						\
1741 
1742 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
1743 {
1744 	unsigned int len;
1745 
1746 	if (!(dev->flags & IFF_UP))
1747 		return false;
1748 
1749 	len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1750 	if (skb->len <= len)
1751 		return true;
1752 
1753 	/* if TSO is enabled, we don't care about the length as the packet
1754 	 * could be forwarded without being segmented before
1755 	 */
1756 	if (skb_is_gso(skb))
1757 		return true;
1758 
1759 	return false;
1760 }
1761 EXPORT_SYMBOL_GPL(is_skb_forwardable);
1762 
1763 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1764 {
1765 	int ret = ____dev_forward_skb(dev, skb);
1766 
1767 	if (likely(!ret)) {
1768 		skb->protocol = eth_type_trans(skb, dev);
1769 		skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1770 	}
1771 
1772 	return ret;
1773 }
1774 EXPORT_SYMBOL_GPL(__dev_forward_skb);
1775 
1776 /**
1777  * dev_forward_skb - loopback an skb to another netif
1778  *
1779  * @dev: destination network device
1780  * @skb: buffer to forward
1781  *
1782  * return values:
1783  *	NET_RX_SUCCESS	(no congestion)
1784  *	NET_RX_DROP     (packet was dropped, but freed)
1785  *
1786  * dev_forward_skb can be used for injecting an skb from the
1787  * start_xmit function of one device into the receive queue
1788  * of another device.
1789  *
1790  * The receiving device may be in another namespace, so
1791  * we have to clear all information in the skb that could
1792  * impact namespace isolation.
1793  */
1794 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1795 {
1796 	return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1797 }
1798 EXPORT_SYMBOL_GPL(dev_forward_skb);
1799 
1800 static inline int deliver_skb(struct sk_buff *skb,
1801 			      struct packet_type *pt_prev,
1802 			      struct net_device *orig_dev)
1803 {
1804 	if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1805 		return -ENOMEM;
1806 	atomic_inc(&skb->users);
1807 	return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1808 }
1809 
1810 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1811 					  struct packet_type **pt,
1812 					  struct net_device *orig_dev,
1813 					  __be16 type,
1814 					  struct list_head *ptype_list)
1815 {
1816 	struct packet_type *ptype, *pt_prev = *pt;
1817 
1818 	list_for_each_entry_rcu(ptype, ptype_list, list) {
1819 		if (ptype->type != type)
1820 			continue;
1821 		if (pt_prev)
1822 			deliver_skb(skb, pt_prev, orig_dev);
1823 		pt_prev = ptype;
1824 	}
1825 	*pt = pt_prev;
1826 }
1827 
1828 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1829 {
1830 	if (!ptype->af_packet_priv || !skb->sk)
1831 		return false;
1832 
1833 	if (ptype->id_match)
1834 		return ptype->id_match(ptype, skb->sk);
1835 	else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1836 		return true;
1837 
1838 	return false;
1839 }
1840 
1841 /*
1842  *	Support routine. Sends outgoing frames to any network
1843  *	taps currently in use.
1844  */
1845 
1846 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1847 {
1848 	struct packet_type *ptype;
1849 	struct sk_buff *skb2 = NULL;
1850 	struct packet_type *pt_prev = NULL;
1851 	struct list_head *ptype_list = &ptype_all;
1852 
1853 	rcu_read_lock();
1854 again:
1855 	list_for_each_entry_rcu(ptype, ptype_list, list) {
1856 		/* Never send packets back to the socket
1857 		 * they originated from - MvS (miquels@drinkel.ow.org)
1858 		 */
1859 		if (skb_loop_sk(ptype, skb))
1860 			continue;
1861 
1862 		if (pt_prev) {
1863 			deliver_skb(skb2, pt_prev, skb->dev);
1864 			pt_prev = ptype;
1865 			continue;
1866 		}
1867 
1868 		/* need to clone skb, done only once */
1869 		skb2 = skb_clone(skb, GFP_ATOMIC);
1870 		if (!skb2)
1871 			goto out_unlock;
1872 
1873 		net_timestamp_set(skb2);
1874 
1875 		/* skb->nh should be correctly
1876 		 * set by sender, so that the second statement is
1877 		 * just protection against buggy protocols.
1878 		 */
1879 		skb_reset_mac_header(skb2);
1880 
1881 		if (skb_network_header(skb2) < skb2->data ||
1882 		    skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1883 			net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1884 					     ntohs(skb2->protocol),
1885 					     dev->name);
1886 			skb_reset_network_header(skb2);
1887 		}
1888 
1889 		skb2->transport_header = skb2->network_header;
1890 		skb2->pkt_type = PACKET_OUTGOING;
1891 		pt_prev = ptype;
1892 	}
1893 
1894 	if (ptype_list == &ptype_all) {
1895 		ptype_list = &dev->ptype_all;
1896 		goto again;
1897 	}
1898 out_unlock:
1899 	if (pt_prev)
1900 		pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1901 	rcu_read_unlock();
1902 }
1903 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
1904 
1905 /**
1906  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1907  * @dev: Network device
1908  * @txq: number of queues available
1909  *
1910  * If real_num_tx_queues is changed the tc mappings may no longer be
1911  * valid. To resolve this verify the tc mapping remains valid and if
1912  * not NULL the mapping. With no priorities mapping to this
1913  * offset/count pair it will no longer be used. In the worst case TC0
1914  * is invalid nothing can be done so disable priority mappings. If is
1915  * expected that drivers will fix this mapping if they can before
1916  * calling netif_set_real_num_tx_queues.
1917  */
1918 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1919 {
1920 	int i;
1921 	struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1922 
1923 	/* If TC0 is invalidated disable TC mapping */
1924 	if (tc->offset + tc->count > txq) {
1925 		pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1926 		dev->num_tc = 0;
1927 		return;
1928 	}
1929 
1930 	/* Invalidated prio to tc mappings set to TC0 */
1931 	for (i = 1; i < TC_BITMASK + 1; i++) {
1932 		int q = netdev_get_prio_tc_map(dev, i);
1933 
1934 		tc = &dev->tc_to_txq[q];
1935 		if (tc->offset + tc->count > txq) {
1936 			pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1937 				i, q);
1938 			netdev_set_prio_tc_map(dev, i, 0);
1939 		}
1940 	}
1941 }
1942 
1943 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
1944 {
1945 	if (dev->num_tc) {
1946 		struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1947 		int i;
1948 
1949 		for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
1950 			if ((txq - tc->offset) < tc->count)
1951 				return i;
1952 		}
1953 
1954 		return -1;
1955 	}
1956 
1957 	return 0;
1958 }
1959 
1960 #ifdef CONFIG_XPS
1961 static DEFINE_MUTEX(xps_map_mutex);
1962 #define xmap_dereference(P)		\
1963 	rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1964 
1965 static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
1966 			     int tci, u16 index)
1967 {
1968 	struct xps_map *map = NULL;
1969 	int pos;
1970 
1971 	if (dev_maps)
1972 		map = xmap_dereference(dev_maps->cpu_map[tci]);
1973 	if (!map)
1974 		return false;
1975 
1976 	for (pos = map->len; pos--;) {
1977 		if (map->queues[pos] != index)
1978 			continue;
1979 
1980 		if (map->len > 1) {
1981 			map->queues[pos] = map->queues[--map->len];
1982 			break;
1983 		}
1984 
1985 		RCU_INIT_POINTER(dev_maps->cpu_map[tci], NULL);
1986 		kfree_rcu(map, rcu);
1987 		return false;
1988 	}
1989 
1990 	return true;
1991 }
1992 
1993 static bool remove_xps_queue_cpu(struct net_device *dev,
1994 				 struct xps_dev_maps *dev_maps,
1995 				 int cpu, u16 offset, u16 count)
1996 {
1997 	int num_tc = dev->num_tc ? : 1;
1998 	bool active = false;
1999 	int tci;
2000 
2001 	for (tci = cpu * num_tc; num_tc--; tci++) {
2002 		int i, j;
2003 
2004 		for (i = count, j = offset; i--; j++) {
2005 			if (!remove_xps_queue(dev_maps, cpu, j))
2006 				break;
2007 		}
2008 
2009 		active |= i < 0;
2010 	}
2011 
2012 	return active;
2013 }
2014 
2015 static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2016 				   u16 count)
2017 {
2018 	struct xps_dev_maps *dev_maps;
2019 	int cpu, i;
2020 	bool active = false;
2021 
2022 	mutex_lock(&xps_map_mutex);
2023 	dev_maps = xmap_dereference(dev->xps_maps);
2024 
2025 	if (!dev_maps)
2026 		goto out_no_maps;
2027 
2028 	for_each_possible_cpu(cpu)
2029 		active |= remove_xps_queue_cpu(dev, dev_maps, cpu,
2030 					       offset, count);
2031 
2032 	if (!active) {
2033 		RCU_INIT_POINTER(dev->xps_maps, NULL);
2034 		kfree_rcu(dev_maps, rcu);
2035 	}
2036 
2037 	for (i = offset + (count - 1); count--; i--)
2038 		netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2039 					     NUMA_NO_NODE);
2040 
2041 out_no_maps:
2042 	mutex_unlock(&xps_map_mutex);
2043 }
2044 
2045 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2046 {
2047 	netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2048 }
2049 
2050 static struct xps_map *expand_xps_map(struct xps_map *map,
2051 				      int cpu, u16 index)
2052 {
2053 	struct xps_map *new_map;
2054 	int alloc_len = XPS_MIN_MAP_ALLOC;
2055 	int i, pos;
2056 
2057 	for (pos = 0; map && pos < map->len; pos++) {
2058 		if (map->queues[pos] != index)
2059 			continue;
2060 		return map;
2061 	}
2062 
2063 	/* Need to add queue to this CPU's existing map */
2064 	if (map) {
2065 		if (pos < map->alloc_len)
2066 			return map;
2067 
2068 		alloc_len = map->alloc_len * 2;
2069 	}
2070 
2071 	/* Need to allocate new map to store queue on this CPU's map */
2072 	new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2073 			       cpu_to_node(cpu));
2074 	if (!new_map)
2075 		return NULL;
2076 
2077 	for (i = 0; i < pos; i++)
2078 		new_map->queues[i] = map->queues[i];
2079 	new_map->alloc_len = alloc_len;
2080 	new_map->len = pos;
2081 
2082 	return new_map;
2083 }
2084 
2085 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2086 			u16 index)
2087 {
2088 	struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2089 	int i, cpu, tci, numa_node_id = -2;
2090 	int maps_sz, num_tc = 1, tc = 0;
2091 	struct xps_map *map, *new_map;
2092 	bool active = false;
2093 
2094 	if (dev->num_tc) {
2095 		num_tc = dev->num_tc;
2096 		tc = netdev_txq_to_tc(dev, index);
2097 		if (tc < 0)
2098 			return -EINVAL;
2099 	}
2100 
2101 	maps_sz = XPS_DEV_MAPS_SIZE(num_tc);
2102 	if (maps_sz < L1_CACHE_BYTES)
2103 		maps_sz = L1_CACHE_BYTES;
2104 
2105 	mutex_lock(&xps_map_mutex);
2106 
2107 	dev_maps = xmap_dereference(dev->xps_maps);
2108 
2109 	/* allocate memory for queue storage */
2110 	for_each_cpu_and(cpu, cpu_online_mask, mask) {
2111 		if (!new_dev_maps)
2112 			new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2113 		if (!new_dev_maps) {
2114 			mutex_unlock(&xps_map_mutex);
2115 			return -ENOMEM;
2116 		}
2117 
2118 		tci = cpu * num_tc + tc;
2119 		map = dev_maps ? xmap_dereference(dev_maps->cpu_map[tci]) :
2120 				 NULL;
2121 
2122 		map = expand_xps_map(map, cpu, index);
2123 		if (!map)
2124 			goto error;
2125 
2126 		RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2127 	}
2128 
2129 	if (!new_dev_maps)
2130 		goto out_no_new_maps;
2131 
2132 	for_each_possible_cpu(cpu) {
2133 		/* copy maps belonging to foreign traffic classes */
2134 		for (i = tc, tci = cpu * num_tc; dev_maps && i--; tci++) {
2135 			/* fill in the new device map from the old device map */
2136 			map = xmap_dereference(dev_maps->cpu_map[tci]);
2137 			RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2138 		}
2139 
2140 		/* We need to explicitly update tci as prevous loop
2141 		 * could break out early if dev_maps is NULL.
2142 		 */
2143 		tci = cpu * num_tc + tc;
2144 
2145 		if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2146 			/* add queue to CPU maps */
2147 			int pos = 0;
2148 
2149 			map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2150 			while ((pos < map->len) && (map->queues[pos] != index))
2151 				pos++;
2152 
2153 			if (pos == map->len)
2154 				map->queues[map->len++] = index;
2155 #ifdef CONFIG_NUMA
2156 			if (numa_node_id == -2)
2157 				numa_node_id = cpu_to_node(cpu);
2158 			else if (numa_node_id != cpu_to_node(cpu))
2159 				numa_node_id = -1;
2160 #endif
2161 		} else if (dev_maps) {
2162 			/* fill in the new device map from the old device map */
2163 			map = xmap_dereference(dev_maps->cpu_map[tci]);
2164 			RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2165 		}
2166 
2167 		/* copy maps belonging to foreign traffic classes */
2168 		for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2169 			/* fill in the new device map from the old device map */
2170 			map = xmap_dereference(dev_maps->cpu_map[tci]);
2171 			RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2172 		}
2173 	}
2174 
2175 	rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2176 
2177 	/* Cleanup old maps */
2178 	if (!dev_maps)
2179 		goto out_no_old_maps;
2180 
2181 	for_each_possible_cpu(cpu) {
2182 		for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2183 			new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2184 			map = xmap_dereference(dev_maps->cpu_map[tci]);
2185 			if (map && map != new_map)
2186 				kfree_rcu(map, rcu);
2187 		}
2188 	}
2189 
2190 	kfree_rcu(dev_maps, rcu);
2191 
2192 out_no_old_maps:
2193 	dev_maps = new_dev_maps;
2194 	active = true;
2195 
2196 out_no_new_maps:
2197 	/* update Tx queue numa node */
2198 	netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2199 				     (numa_node_id >= 0) ? numa_node_id :
2200 				     NUMA_NO_NODE);
2201 
2202 	if (!dev_maps)
2203 		goto out_no_maps;
2204 
2205 	/* removes queue from unused CPUs */
2206 	for_each_possible_cpu(cpu) {
2207 		for (i = tc, tci = cpu * num_tc; i--; tci++)
2208 			active |= remove_xps_queue(dev_maps, tci, index);
2209 		if (!cpumask_test_cpu(cpu, mask) || !cpu_online(cpu))
2210 			active |= remove_xps_queue(dev_maps, tci, index);
2211 		for (i = num_tc - tc, tci++; --i; tci++)
2212 			active |= remove_xps_queue(dev_maps, tci, index);
2213 	}
2214 
2215 	/* free map if not active */
2216 	if (!active) {
2217 		RCU_INIT_POINTER(dev->xps_maps, NULL);
2218 		kfree_rcu(dev_maps, rcu);
2219 	}
2220 
2221 out_no_maps:
2222 	mutex_unlock(&xps_map_mutex);
2223 
2224 	return 0;
2225 error:
2226 	/* remove any maps that we added */
2227 	for_each_possible_cpu(cpu) {
2228 		for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2229 			new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2230 			map = dev_maps ?
2231 			      xmap_dereference(dev_maps->cpu_map[tci]) :
2232 			      NULL;
2233 			if (new_map && new_map != map)
2234 				kfree(new_map);
2235 		}
2236 	}
2237 
2238 	mutex_unlock(&xps_map_mutex);
2239 
2240 	kfree(new_dev_maps);
2241 	return -ENOMEM;
2242 }
2243 EXPORT_SYMBOL(netif_set_xps_queue);
2244 
2245 #endif
2246 void netdev_reset_tc(struct net_device *dev)
2247 {
2248 #ifdef CONFIG_XPS
2249 	netif_reset_xps_queues_gt(dev, 0);
2250 #endif
2251 	dev->num_tc = 0;
2252 	memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2253 	memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2254 }
2255 EXPORT_SYMBOL(netdev_reset_tc);
2256 
2257 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2258 {
2259 	if (tc >= dev->num_tc)
2260 		return -EINVAL;
2261 
2262 #ifdef CONFIG_XPS
2263 	netif_reset_xps_queues(dev, offset, count);
2264 #endif
2265 	dev->tc_to_txq[tc].count = count;
2266 	dev->tc_to_txq[tc].offset = offset;
2267 	return 0;
2268 }
2269 EXPORT_SYMBOL(netdev_set_tc_queue);
2270 
2271 int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2272 {
2273 	if (num_tc > TC_MAX_QUEUE)
2274 		return -EINVAL;
2275 
2276 #ifdef CONFIG_XPS
2277 	netif_reset_xps_queues_gt(dev, 0);
2278 #endif
2279 	dev->num_tc = num_tc;
2280 	return 0;
2281 }
2282 EXPORT_SYMBOL(netdev_set_num_tc);
2283 
2284 /*
2285  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2286  * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2287  */
2288 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2289 {
2290 	int rc;
2291 
2292 	if (txq < 1 || txq > dev->num_tx_queues)
2293 		return -EINVAL;
2294 
2295 	if (dev->reg_state == NETREG_REGISTERED ||
2296 	    dev->reg_state == NETREG_UNREGISTERING) {
2297 		ASSERT_RTNL();
2298 
2299 		rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2300 						  txq);
2301 		if (rc)
2302 			return rc;
2303 
2304 		if (dev->num_tc)
2305 			netif_setup_tc(dev, txq);
2306 
2307 		if (txq < dev->real_num_tx_queues) {
2308 			qdisc_reset_all_tx_gt(dev, txq);
2309 #ifdef CONFIG_XPS
2310 			netif_reset_xps_queues_gt(dev, txq);
2311 #endif
2312 		}
2313 	}
2314 
2315 	dev->real_num_tx_queues = txq;
2316 	return 0;
2317 }
2318 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2319 
2320 #ifdef CONFIG_SYSFS
2321 /**
2322  *	netif_set_real_num_rx_queues - set actual number of RX queues used
2323  *	@dev: Network device
2324  *	@rxq: Actual number of RX queues
2325  *
2326  *	This must be called either with the rtnl_lock held or before
2327  *	registration of the net device.  Returns 0 on success, or a
2328  *	negative error code.  If called before registration, it always
2329  *	succeeds.
2330  */
2331 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2332 {
2333 	int rc;
2334 
2335 	if (rxq < 1 || rxq > dev->num_rx_queues)
2336 		return -EINVAL;
2337 
2338 	if (dev->reg_state == NETREG_REGISTERED) {
2339 		ASSERT_RTNL();
2340 
2341 		rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2342 						  rxq);
2343 		if (rc)
2344 			return rc;
2345 	}
2346 
2347 	dev->real_num_rx_queues = rxq;
2348 	return 0;
2349 }
2350 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2351 #endif
2352 
2353 /**
2354  * netif_get_num_default_rss_queues - default number of RSS queues
2355  *
2356  * This routine should set an upper limit on the number of RSS queues
2357  * used by default by multiqueue devices.
2358  */
2359 int netif_get_num_default_rss_queues(void)
2360 {
2361 	return is_kdump_kernel() ?
2362 		1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2363 }
2364 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2365 
2366 static void __netif_reschedule(struct Qdisc *q)
2367 {
2368 	struct softnet_data *sd;
2369 	unsigned long flags;
2370 
2371 	local_irq_save(flags);
2372 	sd = this_cpu_ptr(&softnet_data);
2373 	q->next_sched = NULL;
2374 	*sd->output_queue_tailp = q;
2375 	sd->output_queue_tailp = &q->next_sched;
2376 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
2377 	local_irq_restore(flags);
2378 }
2379 
2380 void __netif_schedule(struct Qdisc *q)
2381 {
2382 	if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2383 		__netif_reschedule(q);
2384 }
2385 EXPORT_SYMBOL(__netif_schedule);
2386 
2387 struct dev_kfree_skb_cb {
2388 	enum skb_free_reason reason;
2389 };
2390 
2391 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2392 {
2393 	return (struct dev_kfree_skb_cb *)skb->cb;
2394 }
2395 
2396 void netif_schedule_queue(struct netdev_queue *txq)
2397 {
2398 	rcu_read_lock();
2399 	if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2400 		struct Qdisc *q = rcu_dereference(txq->qdisc);
2401 
2402 		__netif_schedule(q);
2403 	}
2404 	rcu_read_unlock();
2405 }
2406 EXPORT_SYMBOL(netif_schedule_queue);
2407 
2408 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2409 {
2410 	if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2411 		struct Qdisc *q;
2412 
2413 		rcu_read_lock();
2414 		q = rcu_dereference(dev_queue->qdisc);
2415 		__netif_schedule(q);
2416 		rcu_read_unlock();
2417 	}
2418 }
2419 EXPORT_SYMBOL(netif_tx_wake_queue);
2420 
2421 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2422 {
2423 	unsigned long flags;
2424 
2425 	if (likely(atomic_read(&skb->users) == 1)) {
2426 		smp_rmb();
2427 		atomic_set(&skb->users, 0);
2428 	} else if (likely(!atomic_dec_and_test(&skb->users))) {
2429 		return;
2430 	}
2431 	get_kfree_skb_cb(skb)->reason = reason;
2432 	local_irq_save(flags);
2433 	skb->next = __this_cpu_read(softnet_data.completion_queue);
2434 	__this_cpu_write(softnet_data.completion_queue, skb);
2435 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
2436 	local_irq_restore(flags);
2437 }
2438 EXPORT_SYMBOL(__dev_kfree_skb_irq);
2439 
2440 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2441 {
2442 	if (in_irq() || irqs_disabled())
2443 		__dev_kfree_skb_irq(skb, reason);
2444 	else
2445 		dev_kfree_skb(skb);
2446 }
2447 EXPORT_SYMBOL(__dev_kfree_skb_any);
2448 
2449 
2450 /**
2451  * netif_device_detach - mark device as removed
2452  * @dev: network device
2453  *
2454  * Mark device as removed from system and therefore no longer available.
2455  */
2456 void netif_device_detach(struct net_device *dev)
2457 {
2458 	if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2459 	    netif_running(dev)) {
2460 		netif_tx_stop_all_queues(dev);
2461 	}
2462 }
2463 EXPORT_SYMBOL(netif_device_detach);
2464 
2465 /**
2466  * netif_device_attach - mark device as attached
2467  * @dev: network device
2468  *
2469  * Mark device as attached from system and restart if needed.
2470  */
2471 void netif_device_attach(struct net_device *dev)
2472 {
2473 	if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2474 	    netif_running(dev)) {
2475 		netif_tx_wake_all_queues(dev);
2476 		__netdev_watchdog_up(dev);
2477 	}
2478 }
2479 EXPORT_SYMBOL(netif_device_attach);
2480 
2481 /*
2482  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2483  * to be used as a distribution range.
2484  */
2485 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2486 		  unsigned int num_tx_queues)
2487 {
2488 	u32 hash;
2489 	u16 qoffset = 0;
2490 	u16 qcount = num_tx_queues;
2491 
2492 	if (skb_rx_queue_recorded(skb)) {
2493 		hash = skb_get_rx_queue(skb);
2494 		while (unlikely(hash >= num_tx_queues))
2495 			hash -= num_tx_queues;
2496 		return hash;
2497 	}
2498 
2499 	if (dev->num_tc) {
2500 		u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2501 
2502 		qoffset = dev->tc_to_txq[tc].offset;
2503 		qcount = dev->tc_to_txq[tc].count;
2504 	}
2505 
2506 	return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2507 }
2508 EXPORT_SYMBOL(__skb_tx_hash);
2509 
2510 static void skb_warn_bad_offload(const struct sk_buff *skb)
2511 {
2512 	static const netdev_features_t null_features;
2513 	struct net_device *dev = skb->dev;
2514 	const char *name = "";
2515 
2516 	if (!net_ratelimit())
2517 		return;
2518 
2519 	if (dev) {
2520 		if (dev->dev.parent)
2521 			name = dev_driver_string(dev->dev.parent);
2522 		else
2523 			name = netdev_name(dev);
2524 	}
2525 	WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2526 	     "gso_type=%d ip_summed=%d\n",
2527 	     name, dev ? &dev->features : &null_features,
2528 	     skb->sk ? &skb->sk->sk_route_caps : &null_features,
2529 	     skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2530 	     skb_shinfo(skb)->gso_type, skb->ip_summed);
2531 }
2532 
2533 /*
2534  * Invalidate hardware checksum when packet is to be mangled, and
2535  * complete checksum manually on outgoing path.
2536  */
2537 int skb_checksum_help(struct sk_buff *skb)
2538 {
2539 	__wsum csum;
2540 	int ret = 0, offset;
2541 
2542 	if (skb->ip_summed == CHECKSUM_COMPLETE)
2543 		goto out_set_summed;
2544 
2545 	if (unlikely(skb_shinfo(skb)->gso_size)) {
2546 		skb_warn_bad_offload(skb);
2547 		return -EINVAL;
2548 	}
2549 
2550 	/* Before computing a checksum, we should make sure no frag could
2551 	 * be modified by an external entity : checksum could be wrong.
2552 	 */
2553 	if (skb_has_shared_frag(skb)) {
2554 		ret = __skb_linearize(skb);
2555 		if (ret)
2556 			goto out;
2557 	}
2558 
2559 	offset = skb_checksum_start_offset(skb);
2560 	BUG_ON(offset >= skb_headlen(skb));
2561 	csum = skb_checksum(skb, offset, skb->len - offset, 0);
2562 
2563 	offset += skb->csum_offset;
2564 	BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2565 
2566 	if (skb_cloned(skb) &&
2567 	    !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2568 		ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2569 		if (ret)
2570 			goto out;
2571 	}
2572 
2573 	*(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2574 out_set_summed:
2575 	skb->ip_summed = CHECKSUM_NONE;
2576 out:
2577 	return ret;
2578 }
2579 EXPORT_SYMBOL(skb_checksum_help);
2580 
2581 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2582 {
2583 	__be16 type = skb->protocol;
2584 
2585 	/* Tunnel gso handlers can set protocol to ethernet. */
2586 	if (type == htons(ETH_P_TEB)) {
2587 		struct ethhdr *eth;
2588 
2589 		if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2590 			return 0;
2591 
2592 		eth = (struct ethhdr *)skb_mac_header(skb);
2593 		type = eth->h_proto;
2594 	}
2595 
2596 	return __vlan_get_protocol(skb, type, depth);
2597 }
2598 
2599 /**
2600  *	skb_mac_gso_segment - mac layer segmentation handler.
2601  *	@skb: buffer to segment
2602  *	@features: features for the output path (see dev->features)
2603  */
2604 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2605 				    netdev_features_t features)
2606 {
2607 	struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2608 	struct packet_offload *ptype;
2609 	int vlan_depth = skb->mac_len;
2610 	__be16 type = skb_network_protocol(skb, &vlan_depth);
2611 
2612 	if (unlikely(!type))
2613 		return ERR_PTR(-EINVAL);
2614 
2615 	__skb_pull(skb, vlan_depth);
2616 
2617 	rcu_read_lock();
2618 	list_for_each_entry_rcu(ptype, &offload_base, list) {
2619 		if (ptype->type == type && ptype->callbacks.gso_segment) {
2620 			segs = ptype->callbacks.gso_segment(skb, features);
2621 			break;
2622 		}
2623 	}
2624 	rcu_read_unlock();
2625 
2626 	__skb_push(skb, skb->data - skb_mac_header(skb));
2627 
2628 	return segs;
2629 }
2630 EXPORT_SYMBOL(skb_mac_gso_segment);
2631 
2632 
2633 /* openvswitch calls this on rx path, so we need a different check.
2634  */
2635 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2636 {
2637 	if (tx_path)
2638 		return skb->ip_summed != CHECKSUM_PARTIAL &&
2639 		       skb->ip_summed != CHECKSUM_NONE;
2640 
2641 	return skb->ip_summed == CHECKSUM_NONE;
2642 }
2643 
2644 /**
2645  *	__skb_gso_segment - Perform segmentation on skb.
2646  *	@skb: buffer to segment
2647  *	@features: features for the output path (see dev->features)
2648  *	@tx_path: whether it is called in TX path
2649  *
2650  *	This function segments the given skb and returns a list of segments.
2651  *
2652  *	It may return NULL if the skb requires no segmentation.  This is
2653  *	only possible when GSO is used for verifying header integrity.
2654  *
2655  *	Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
2656  */
2657 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2658 				  netdev_features_t features, bool tx_path)
2659 {
2660 	struct sk_buff *segs;
2661 
2662 	if (unlikely(skb_needs_check(skb, tx_path))) {
2663 		int err;
2664 
2665 		/* We're going to init ->check field in TCP or UDP header */
2666 		err = skb_cow_head(skb, 0);
2667 		if (err < 0)
2668 			return ERR_PTR(err);
2669 	}
2670 
2671 	/* Only report GSO partial support if it will enable us to
2672 	 * support segmentation on this frame without needing additional
2673 	 * work.
2674 	 */
2675 	if (features & NETIF_F_GSO_PARTIAL) {
2676 		netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
2677 		struct net_device *dev = skb->dev;
2678 
2679 		partial_features |= dev->features & dev->gso_partial_features;
2680 		if (!skb_gso_ok(skb, features | partial_features))
2681 			features &= ~NETIF_F_GSO_PARTIAL;
2682 	}
2683 
2684 	BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2685 		     sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2686 
2687 	SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2688 	SKB_GSO_CB(skb)->encap_level = 0;
2689 
2690 	skb_reset_mac_header(skb);
2691 	skb_reset_mac_len(skb);
2692 
2693 	segs = skb_mac_gso_segment(skb, features);
2694 
2695 	if (unlikely(skb_needs_check(skb, tx_path)))
2696 		skb_warn_bad_offload(skb);
2697 
2698 	return segs;
2699 }
2700 EXPORT_SYMBOL(__skb_gso_segment);
2701 
2702 /* Take action when hardware reception checksum errors are detected. */
2703 #ifdef CONFIG_BUG
2704 void netdev_rx_csum_fault(struct net_device *dev)
2705 {
2706 	if (net_ratelimit()) {
2707 		pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2708 		dump_stack();
2709 	}
2710 }
2711 EXPORT_SYMBOL(netdev_rx_csum_fault);
2712 #endif
2713 
2714 /* Actually, we should eliminate this check as soon as we know, that:
2715  * 1. IOMMU is present and allows to map all the memory.
2716  * 2. No high memory really exists on this machine.
2717  */
2718 
2719 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2720 {
2721 #ifdef CONFIG_HIGHMEM
2722 	int i;
2723 
2724 	if (!(dev->features & NETIF_F_HIGHDMA)) {
2725 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2726 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2727 
2728 			if (PageHighMem(skb_frag_page(frag)))
2729 				return 1;
2730 		}
2731 	}
2732 
2733 	if (PCI_DMA_BUS_IS_PHYS) {
2734 		struct device *pdev = dev->dev.parent;
2735 
2736 		if (!pdev)
2737 			return 0;
2738 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2739 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2740 			dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2741 
2742 			if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2743 				return 1;
2744 		}
2745 	}
2746 #endif
2747 	return 0;
2748 }
2749 
2750 /* If MPLS offload request, verify we are testing hardware MPLS features
2751  * instead of standard features for the netdev.
2752  */
2753 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
2754 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2755 					   netdev_features_t features,
2756 					   __be16 type)
2757 {
2758 	if (eth_p_mpls(type))
2759 		features &= skb->dev->mpls_features;
2760 
2761 	return features;
2762 }
2763 #else
2764 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2765 					   netdev_features_t features,
2766 					   __be16 type)
2767 {
2768 	return features;
2769 }
2770 #endif
2771 
2772 static netdev_features_t harmonize_features(struct sk_buff *skb,
2773 	netdev_features_t features)
2774 {
2775 	int tmp;
2776 	__be16 type;
2777 
2778 	type = skb_network_protocol(skb, &tmp);
2779 	features = net_mpls_features(skb, features, type);
2780 
2781 	if (skb->ip_summed != CHECKSUM_NONE &&
2782 	    !can_checksum_protocol(features, type)) {
2783 		features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2784 	}
2785 	if (illegal_highdma(skb->dev, skb))
2786 		features &= ~NETIF_F_SG;
2787 
2788 	return features;
2789 }
2790 
2791 netdev_features_t passthru_features_check(struct sk_buff *skb,
2792 					  struct net_device *dev,
2793 					  netdev_features_t features)
2794 {
2795 	return features;
2796 }
2797 EXPORT_SYMBOL(passthru_features_check);
2798 
2799 static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2800 					     struct net_device *dev,
2801 					     netdev_features_t features)
2802 {
2803 	return vlan_features_check(skb, features);
2804 }
2805 
2806 static netdev_features_t gso_features_check(const struct sk_buff *skb,
2807 					    struct net_device *dev,
2808 					    netdev_features_t features)
2809 {
2810 	u16 gso_segs = skb_shinfo(skb)->gso_segs;
2811 
2812 	if (gso_segs > dev->gso_max_segs)
2813 		return features & ~NETIF_F_GSO_MASK;
2814 
2815 	/* Support for GSO partial features requires software
2816 	 * intervention before we can actually process the packets
2817 	 * so we need to strip support for any partial features now
2818 	 * and we can pull them back in after we have partially
2819 	 * segmented the frame.
2820 	 */
2821 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
2822 		features &= ~dev->gso_partial_features;
2823 
2824 	/* Make sure to clear the IPv4 ID mangling feature if the
2825 	 * IPv4 header has the potential to be fragmented.
2826 	 */
2827 	if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
2828 		struct iphdr *iph = skb->encapsulation ?
2829 				    inner_ip_hdr(skb) : ip_hdr(skb);
2830 
2831 		if (!(iph->frag_off & htons(IP_DF)))
2832 			features &= ~NETIF_F_TSO_MANGLEID;
2833 	}
2834 
2835 	return features;
2836 }
2837 
2838 netdev_features_t netif_skb_features(struct sk_buff *skb)
2839 {
2840 	struct net_device *dev = skb->dev;
2841 	netdev_features_t features = dev->features;
2842 
2843 	if (skb_is_gso(skb))
2844 		features = gso_features_check(skb, dev, features);
2845 
2846 	/* If encapsulation offload request, verify we are testing
2847 	 * hardware encapsulation features instead of standard
2848 	 * features for the netdev
2849 	 */
2850 	if (skb->encapsulation)
2851 		features &= dev->hw_enc_features;
2852 
2853 	if (skb_vlan_tagged(skb))
2854 		features = netdev_intersect_features(features,
2855 						     dev->vlan_features |
2856 						     NETIF_F_HW_VLAN_CTAG_TX |
2857 						     NETIF_F_HW_VLAN_STAG_TX);
2858 
2859 	if (dev->netdev_ops->ndo_features_check)
2860 		features &= dev->netdev_ops->ndo_features_check(skb, dev,
2861 								features);
2862 	else
2863 		features &= dflt_features_check(skb, dev, features);
2864 
2865 	return harmonize_features(skb, features);
2866 }
2867 EXPORT_SYMBOL(netif_skb_features);
2868 
2869 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
2870 		    struct netdev_queue *txq, bool more)
2871 {
2872 	unsigned int len;
2873 	int rc;
2874 
2875 	if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
2876 		dev_queue_xmit_nit(skb, dev);
2877 
2878 	len = skb->len;
2879 	trace_net_dev_start_xmit(skb, dev);
2880 	rc = netdev_start_xmit(skb, dev, txq, more);
2881 	trace_net_dev_xmit(skb, rc, dev, len);
2882 
2883 	return rc;
2884 }
2885 
2886 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2887 				    struct netdev_queue *txq, int *ret)
2888 {
2889 	struct sk_buff *skb = first;
2890 	int rc = NETDEV_TX_OK;
2891 
2892 	while (skb) {
2893 		struct sk_buff *next = skb->next;
2894 
2895 		skb->next = NULL;
2896 		rc = xmit_one(skb, dev, txq, next != NULL);
2897 		if (unlikely(!dev_xmit_complete(rc))) {
2898 			skb->next = next;
2899 			goto out;
2900 		}
2901 
2902 		skb = next;
2903 		if (netif_xmit_stopped(txq) && skb) {
2904 			rc = NETDEV_TX_BUSY;
2905 			break;
2906 		}
2907 	}
2908 
2909 out:
2910 	*ret = rc;
2911 	return skb;
2912 }
2913 
2914 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2915 					  netdev_features_t features)
2916 {
2917 	if (skb_vlan_tag_present(skb) &&
2918 	    !vlan_hw_offload_capable(features, skb->vlan_proto))
2919 		skb = __vlan_hwaccel_push_inside(skb);
2920 	return skb;
2921 }
2922 
2923 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
2924 {
2925 	netdev_features_t features;
2926 
2927 	features = netif_skb_features(skb);
2928 	skb = validate_xmit_vlan(skb, features);
2929 	if (unlikely(!skb))
2930 		goto out_null;
2931 
2932 	if (netif_needs_gso(skb, features)) {
2933 		struct sk_buff *segs;
2934 
2935 		segs = skb_gso_segment(skb, features);
2936 		if (IS_ERR(segs)) {
2937 			goto out_kfree_skb;
2938 		} else if (segs) {
2939 			consume_skb(skb);
2940 			skb = segs;
2941 		}
2942 	} else {
2943 		if (skb_needs_linearize(skb, features) &&
2944 		    __skb_linearize(skb))
2945 			goto out_kfree_skb;
2946 
2947 		/* If packet is not checksummed and device does not
2948 		 * support checksumming for this protocol, complete
2949 		 * checksumming here.
2950 		 */
2951 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
2952 			if (skb->encapsulation)
2953 				skb_set_inner_transport_header(skb,
2954 							       skb_checksum_start_offset(skb));
2955 			else
2956 				skb_set_transport_header(skb,
2957 							 skb_checksum_start_offset(skb));
2958 			if (!(features & NETIF_F_CSUM_MASK) &&
2959 			    skb_checksum_help(skb))
2960 				goto out_kfree_skb;
2961 		}
2962 	}
2963 
2964 	return skb;
2965 
2966 out_kfree_skb:
2967 	kfree_skb(skb);
2968 out_null:
2969 	atomic_long_inc(&dev->tx_dropped);
2970 	return NULL;
2971 }
2972 
2973 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
2974 {
2975 	struct sk_buff *next, *head = NULL, *tail;
2976 
2977 	for (; skb != NULL; skb = next) {
2978 		next = skb->next;
2979 		skb->next = NULL;
2980 
2981 		/* in case skb wont be segmented, point to itself */
2982 		skb->prev = skb;
2983 
2984 		skb = validate_xmit_skb(skb, dev);
2985 		if (!skb)
2986 			continue;
2987 
2988 		if (!head)
2989 			head = skb;
2990 		else
2991 			tail->next = skb;
2992 		/* If skb was segmented, skb->prev points to
2993 		 * the last segment. If not, it still contains skb.
2994 		 */
2995 		tail = skb->prev;
2996 	}
2997 	return head;
2998 }
2999 EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3000 
3001 static void qdisc_pkt_len_init(struct sk_buff *skb)
3002 {
3003 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
3004 
3005 	qdisc_skb_cb(skb)->pkt_len = skb->len;
3006 
3007 	/* To get more precise estimation of bytes sent on wire,
3008 	 * we add to pkt_len the headers size of all segments
3009 	 */
3010 	if (shinfo->gso_size)  {
3011 		unsigned int hdr_len;
3012 		u16 gso_segs = shinfo->gso_segs;
3013 
3014 		/* mac layer + network layer */
3015 		hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3016 
3017 		/* + transport layer */
3018 		if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
3019 			hdr_len += tcp_hdrlen(skb);
3020 		else
3021 			hdr_len += sizeof(struct udphdr);
3022 
3023 		if (shinfo->gso_type & SKB_GSO_DODGY)
3024 			gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3025 						shinfo->gso_size);
3026 
3027 		qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3028 	}
3029 }
3030 
3031 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3032 				 struct net_device *dev,
3033 				 struct netdev_queue *txq)
3034 {
3035 	spinlock_t *root_lock = qdisc_lock(q);
3036 	struct sk_buff *to_free = NULL;
3037 	bool contended;
3038 	int rc;
3039 
3040 	qdisc_calculate_pkt_len(skb, q);
3041 	/*
3042 	 * Heuristic to force contended enqueues to serialize on a
3043 	 * separate lock before trying to get qdisc main lock.
3044 	 * This permits qdisc->running owner to get the lock more
3045 	 * often and dequeue packets faster.
3046 	 */
3047 	contended = qdisc_is_running(q);
3048 	if (unlikely(contended))
3049 		spin_lock(&q->busylock);
3050 
3051 	spin_lock(root_lock);
3052 	if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3053 		__qdisc_drop(skb, &to_free);
3054 		rc = NET_XMIT_DROP;
3055 	} else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3056 		   qdisc_run_begin(q)) {
3057 		/*
3058 		 * This is a work-conserving queue; there are no old skbs
3059 		 * waiting to be sent out; and the qdisc is not running -
3060 		 * xmit the skb directly.
3061 		 */
3062 
3063 		qdisc_bstats_update(q, skb);
3064 
3065 		if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3066 			if (unlikely(contended)) {
3067 				spin_unlock(&q->busylock);
3068 				contended = false;
3069 			}
3070 			__qdisc_run(q);
3071 		} else
3072 			qdisc_run_end(q);
3073 
3074 		rc = NET_XMIT_SUCCESS;
3075 	} else {
3076 		rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3077 		if (qdisc_run_begin(q)) {
3078 			if (unlikely(contended)) {
3079 				spin_unlock(&q->busylock);
3080 				contended = false;
3081 			}
3082 			__qdisc_run(q);
3083 		}
3084 	}
3085 	spin_unlock(root_lock);
3086 	if (unlikely(to_free))
3087 		kfree_skb_list(to_free);
3088 	if (unlikely(contended))
3089 		spin_unlock(&q->busylock);
3090 	return rc;
3091 }
3092 
3093 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3094 static void skb_update_prio(struct sk_buff *skb)
3095 {
3096 	struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
3097 
3098 	if (!skb->priority && skb->sk && map) {
3099 		unsigned int prioidx =
3100 			sock_cgroup_prioidx(&skb->sk->sk_cgrp_data);
3101 
3102 		if (prioidx < map->priomap_len)
3103 			skb->priority = map->priomap[prioidx];
3104 	}
3105 }
3106 #else
3107 #define skb_update_prio(skb)
3108 #endif
3109 
3110 DEFINE_PER_CPU(int, xmit_recursion);
3111 EXPORT_SYMBOL(xmit_recursion);
3112 
3113 /**
3114  *	dev_loopback_xmit - loop back @skb
3115  *	@net: network namespace this loopback is happening in
3116  *	@sk:  sk needed to be a netfilter okfn
3117  *	@skb: buffer to transmit
3118  */
3119 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3120 {
3121 	skb_reset_mac_header(skb);
3122 	__skb_pull(skb, skb_network_offset(skb));
3123 	skb->pkt_type = PACKET_LOOPBACK;
3124 	skb->ip_summed = CHECKSUM_UNNECESSARY;
3125 	WARN_ON(!skb_dst(skb));
3126 	skb_dst_force(skb);
3127 	netif_rx_ni(skb);
3128 	return 0;
3129 }
3130 EXPORT_SYMBOL(dev_loopback_xmit);
3131 
3132 #ifdef CONFIG_NET_EGRESS
3133 static struct sk_buff *
3134 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3135 {
3136 	struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3137 	struct tcf_result cl_res;
3138 
3139 	if (!cl)
3140 		return skb;
3141 
3142 	/* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3143 	qdisc_bstats_cpu_update(cl->q, skb);
3144 
3145 	switch (tc_classify(skb, cl, &cl_res, false)) {
3146 	case TC_ACT_OK:
3147 	case TC_ACT_RECLASSIFY:
3148 		skb->tc_index = TC_H_MIN(cl_res.classid);
3149 		break;
3150 	case TC_ACT_SHOT:
3151 		qdisc_qstats_cpu_drop(cl->q);
3152 		*ret = NET_XMIT_DROP;
3153 		kfree_skb(skb);
3154 		return NULL;
3155 	case TC_ACT_STOLEN:
3156 	case TC_ACT_QUEUED:
3157 		*ret = NET_XMIT_SUCCESS;
3158 		consume_skb(skb);
3159 		return NULL;
3160 	case TC_ACT_REDIRECT:
3161 		/* No need to push/pop skb's mac_header here on egress! */
3162 		skb_do_redirect(skb);
3163 		*ret = NET_XMIT_SUCCESS;
3164 		return NULL;
3165 	default:
3166 		break;
3167 	}
3168 
3169 	return skb;
3170 }
3171 #endif /* CONFIG_NET_EGRESS */
3172 
3173 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3174 {
3175 #ifdef CONFIG_XPS
3176 	struct xps_dev_maps *dev_maps;
3177 	struct xps_map *map;
3178 	int queue_index = -1;
3179 
3180 	rcu_read_lock();
3181 	dev_maps = rcu_dereference(dev->xps_maps);
3182 	if (dev_maps) {
3183 		unsigned int tci = skb->sender_cpu - 1;
3184 
3185 		if (dev->num_tc) {
3186 			tci *= dev->num_tc;
3187 			tci += netdev_get_prio_tc_map(dev, skb->priority);
3188 		}
3189 
3190 		map = rcu_dereference(dev_maps->cpu_map[tci]);
3191 		if (map) {
3192 			if (map->len == 1)
3193 				queue_index = map->queues[0];
3194 			else
3195 				queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3196 									   map->len)];
3197 			if (unlikely(queue_index >= dev->real_num_tx_queues))
3198 				queue_index = -1;
3199 		}
3200 	}
3201 	rcu_read_unlock();
3202 
3203 	return queue_index;
3204 #else
3205 	return -1;
3206 #endif
3207 }
3208 
3209 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3210 {
3211 	struct sock *sk = skb->sk;
3212 	int queue_index = sk_tx_queue_get(sk);
3213 
3214 	if (queue_index < 0 || skb->ooo_okay ||
3215 	    queue_index >= dev->real_num_tx_queues) {
3216 		int new_index = get_xps_queue(dev, skb);
3217 
3218 		if (new_index < 0)
3219 			new_index = skb_tx_hash(dev, skb);
3220 
3221 		if (queue_index != new_index && sk &&
3222 		    sk_fullsock(sk) &&
3223 		    rcu_access_pointer(sk->sk_dst_cache))
3224 			sk_tx_queue_set(sk, new_index);
3225 
3226 		queue_index = new_index;
3227 	}
3228 
3229 	return queue_index;
3230 }
3231 
3232 struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3233 				    struct sk_buff *skb,
3234 				    void *accel_priv)
3235 {
3236 	int queue_index = 0;
3237 
3238 #ifdef CONFIG_XPS
3239 	u32 sender_cpu = skb->sender_cpu - 1;
3240 
3241 	if (sender_cpu >= (u32)NR_CPUS)
3242 		skb->sender_cpu = raw_smp_processor_id() + 1;
3243 #endif
3244 
3245 	if (dev->real_num_tx_queues != 1) {
3246 		const struct net_device_ops *ops = dev->netdev_ops;
3247 
3248 		if (ops->ndo_select_queue)
3249 			queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3250 							    __netdev_pick_tx);
3251 		else
3252 			queue_index = __netdev_pick_tx(dev, skb);
3253 
3254 		if (!accel_priv)
3255 			queue_index = netdev_cap_txqueue(dev, queue_index);
3256 	}
3257 
3258 	skb_set_queue_mapping(skb, queue_index);
3259 	return netdev_get_tx_queue(dev, queue_index);
3260 }
3261 
3262 /**
3263  *	__dev_queue_xmit - transmit a buffer
3264  *	@skb: buffer to transmit
3265  *	@accel_priv: private data used for L2 forwarding offload
3266  *
3267  *	Queue a buffer for transmission to a network device. The caller must
3268  *	have set the device and priority and built the buffer before calling
3269  *	this function. The function can be called from an interrupt.
3270  *
3271  *	A negative errno code is returned on a failure. A success does not
3272  *	guarantee the frame will be transmitted as it may be dropped due
3273  *	to congestion or traffic shaping.
3274  *
3275  * -----------------------------------------------------------------------------------
3276  *      I notice this method can also return errors from the queue disciplines,
3277  *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
3278  *      be positive.
3279  *
3280  *      Regardless of the return value, the skb is consumed, so it is currently
3281  *      difficult to retry a send to this method.  (You can bump the ref count
3282  *      before sending to hold a reference for retry if you are careful.)
3283  *
3284  *      When calling this method, interrupts MUST be enabled.  This is because
3285  *      the BH enable code must have IRQs enabled so that it will not deadlock.
3286  *          --BLG
3287  */
3288 static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
3289 {
3290 	struct net_device *dev = skb->dev;
3291 	struct netdev_queue *txq;
3292 	struct Qdisc *q;
3293 	int rc = -ENOMEM;
3294 
3295 	skb_reset_mac_header(skb);
3296 
3297 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3298 		__skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3299 
3300 	/* Disable soft irqs for various locks below. Also
3301 	 * stops preemption for RCU.
3302 	 */
3303 	rcu_read_lock_bh();
3304 
3305 	skb_update_prio(skb);
3306 
3307 	qdisc_pkt_len_init(skb);
3308 #ifdef CONFIG_NET_CLS_ACT
3309 	skb->tc_at_ingress = 0;
3310 # ifdef CONFIG_NET_EGRESS
3311 	if (static_key_false(&egress_needed)) {
3312 		skb = sch_handle_egress(skb, &rc, dev);
3313 		if (!skb)
3314 			goto out;
3315 	}
3316 # endif
3317 #endif
3318 	/* If device/qdisc don't need skb->dst, release it right now while
3319 	 * its hot in this cpu cache.
3320 	 */
3321 	if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3322 		skb_dst_drop(skb);
3323 	else
3324 		skb_dst_force(skb);
3325 
3326 	txq = netdev_pick_tx(dev, skb, accel_priv);
3327 	q = rcu_dereference_bh(txq->qdisc);
3328 
3329 	trace_net_dev_queue(skb);
3330 	if (q->enqueue) {
3331 		rc = __dev_xmit_skb(skb, q, dev, txq);
3332 		goto out;
3333 	}
3334 
3335 	/* The device has no queue. Common case for software devices:
3336 	 * loopback, all the sorts of tunnels...
3337 
3338 	 * Really, it is unlikely that netif_tx_lock protection is necessary
3339 	 * here.  (f.e. loopback and IP tunnels are clean ignoring statistics
3340 	 * counters.)
3341 	 * However, it is possible, that they rely on protection
3342 	 * made by us here.
3343 
3344 	 * Check this and shot the lock. It is not prone from deadlocks.
3345 	 *Either shot noqueue qdisc, it is even simpler 8)
3346 	 */
3347 	if (dev->flags & IFF_UP) {
3348 		int cpu = smp_processor_id(); /* ok because BHs are off */
3349 
3350 		if (txq->xmit_lock_owner != cpu) {
3351 			if (unlikely(__this_cpu_read(xmit_recursion) >
3352 				     XMIT_RECURSION_LIMIT))
3353 				goto recursion_alert;
3354 
3355 			skb = validate_xmit_skb(skb, dev);
3356 			if (!skb)
3357 				goto out;
3358 
3359 			HARD_TX_LOCK(dev, txq, cpu);
3360 
3361 			if (!netif_xmit_stopped(txq)) {
3362 				__this_cpu_inc(xmit_recursion);
3363 				skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3364 				__this_cpu_dec(xmit_recursion);
3365 				if (dev_xmit_complete(rc)) {
3366 					HARD_TX_UNLOCK(dev, txq);
3367 					goto out;
3368 				}
3369 			}
3370 			HARD_TX_UNLOCK(dev, txq);
3371 			net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3372 					     dev->name);
3373 		} else {
3374 			/* Recursion is detected! It is possible,
3375 			 * unfortunately
3376 			 */
3377 recursion_alert:
3378 			net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3379 					     dev->name);
3380 		}
3381 	}
3382 
3383 	rc = -ENETDOWN;
3384 	rcu_read_unlock_bh();
3385 
3386 	atomic_long_inc(&dev->tx_dropped);
3387 	kfree_skb_list(skb);
3388 	return rc;
3389 out:
3390 	rcu_read_unlock_bh();
3391 	return rc;
3392 }
3393 
3394 int dev_queue_xmit(struct sk_buff *skb)
3395 {
3396 	return __dev_queue_xmit(skb, NULL);
3397 }
3398 EXPORT_SYMBOL(dev_queue_xmit);
3399 
3400 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3401 {
3402 	return __dev_queue_xmit(skb, accel_priv);
3403 }
3404 EXPORT_SYMBOL(dev_queue_xmit_accel);
3405 
3406 
3407 /*************************************************************************
3408  *			Receiver routines
3409  *************************************************************************/
3410 
3411 int netdev_max_backlog __read_mostly = 1000;
3412 EXPORT_SYMBOL(netdev_max_backlog);
3413 
3414 int netdev_tstamp_prequeue __read_mostly = 1;
3415 int netdev_budget __read_mostly = 300;
3416 int weight_p __read_mostly = 64;           /* old backlog weight */
3417 int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
3418 int dev_weight_tx_bias __read_mostly = 1;  /* bias for output_queue quota */
3419 int dev_rx_weight __read_mostly = 64;
3420 int dev_tx_weight __read_mostly = 64;
3421 
3422 /* Called with irq disabled */
3423 static inline void ____napi_schedule(struct softnet_data *sd,
3424 				     struct napi_struct *napi)
3425 {
3426 	list_add_tail(&napi->poll_list, &sd->poll_list);
3427 	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
3428 }
3429 
3430 #ifdef CONFIG_RPS
3431 
3432 /* One global table that all flow-based protocols share. */
3433 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3434 EXPORT_SYMBOL(rps_sock_flow_table);
3435 u32 rps_cpu_mask __read_mostly;
3436 EXPORT_SYMBOL(rps_cpu_mask);
3437 
3438 struct static_key rps_needed __read_mostly;
3439 EXPORT_SYMBOL(rps_needed);
3440 struct static_key rfs_needed __read_mostly;
3441 EXPORT_SYMBOL(rfs_needed);
3442 
3443 static struct rps_dev_flow *
3444 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3445 	    struct rps_dev_flow *rflow, u16 next_cpu)
3446 {
3447 	if (next_cpu < nr_cpu_ids) {
3448 #ifdef CONFIG_RFS_ACCEL
3449 		struct netdev_rx_queue *rxqueue;
3450 		struct rps_dev_flow_table *flow_table;
3451 		struct rps_dev_flow *old_rflow;
3452 		u32 flow_id;
3453 		u16 rxq_index;
3454 		int rc;
3455 
3456 		/* Should we steer this flow to a different hardware queue? */
3457 		if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3458 		    !(dev->features & NETIF_F_NTUPLE))
3459 			goto out;
3460 		rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3461 		if (rxq_index == skb_get_rx_queue(skb))
3462 			goto out;
3463 
3464 		rxqueue = dev->_rx + rxq_index;
3465 		flow_table = rcu_dereference(rxqueue->rps_flow_table);
3466 		if (!flow_table)
3467 			goto out;
3468 		flow_id = skb_get_hash(skb) & flow_table->mask;
3469 		rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3470 							rxq_index, flow_id);
3471 		if (rc < 0)
3472 			goto out;
3473 		old_rflow = rflow;
3474 		rflow = &flow_table->flows[flow_id];
3475 		rflow->filter = rc;
3476 		if (old_rflow->filter == rflow->filter)
3477 			old_rflow->filter = RPS_NO_FILTER;
3478 	out:
3479 #endif
3480 		rflow->last_qtail =
3481 			per_cpu(softnet_data, next_cpu).input_queue_head;
3482 	}
3483 
3484 	rflow->cpu = next_cpu;
3485 	return rflow;
3486 }
3487 
3488 /*
3489  * get_rps_cpu is called from netif_receive_skb and returns the target
3490  * CPU from the RPS map of the receiving queue for a given skb.
3491  * rcu_read_lock must be held on entry.
3492  */
3493 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3494 		       struct rps_dev_flow **rflowp)
3495 {
3496 	const struct rps_sock_flow_table *sock_flow_table;
3497 	struct netdev_rx_queue *rxqueue = dev->_rx;
3498 	struct rps_dev_flow_table *flow_table;
3499 	struct rps_map *map;
3500 	int cpu = -1;
3501 	u32 tcpu;
3502 	u32 hash;
3503 
3504 	if (skb_rx_queue_recorded(skb)) {
3505 		u16 index = skb_get_rx_queue(skb);
3506 
3507 		if (unlikely(index >= dev->real_num_rx_queues)) {
3508 			WARN_ONCE(dev->real_num_rx_queues > 1,
3509 				  "%s received packet on queue %u, but number "
3510 				  "of RX queues is %u\n",
3511 				  dev->name, index, dev->real_num_rx_queues);
3512 			goto done;
3513 		}
3514 		rxqueue += index;
3515 	}
3516 
3517 	/* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3518 
3519 	flow_table = rcu_dereference(rxqueue->rps_flow_table);
3520 	map = rcu_dereference(rxqueue->rps_map);
3521 	if (!flow_table && !map)
3522 		goto done;
3523 
3524 	skb_reset_network_header(skb);
3525 	hash = skb_get_hash(skb);
3526 	if (!hash)
3527 		goto done;
3528 
3529 	sock_flow_table = rcu_dereference(rps_sock_flow_table);
3530 	if (flow_table && sock_flow_table) {
3531 		struct rps_dev_flow *rflow;
3532 		u32 next_cpu;
3533 		u32 ident;
3534 
3535 		/* First check into global flow table if there is a match */
3536 		ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3537 		if ((ident ^ hash) & ~rps_cpu_mask)
3538 			goto try_rps;
3539 
3540 		next_cpu = ident & rps_cpu_mask;
3541 
3542 		/* OK, now we know there is a match,
3543 		 * we can look at the local (per receive queue) flow table
3544 		 */
3545 		rflow = &flow_table->flows[hash & flow_table->mask];
3546 		tcpu = rflow->cpu;
3547 
3548 		/*
3549 		 * If the desired CPU (where last recvmsg was done) is
3550 		 * different from current CPU (one in the rx-queue flow
3551 		 * table entry), switch if one of the following holds:
3552 		 *   - Current CPU is unset (>= nr_cpu_ids).
3553 		 *   - Current CPU is offline.
3554 		 *   - The current CPU's queue tail has advanced beyond the
3555 		 *     last packet that was enqueued using this table entry.
3556 		 *     This guarantees that all previous packets for the flow
3557 		 *     have been dequeued, thus preserving in order delivery.
3558 		 */
3559 		if (unlikely(tcpu != next_cpu) &&
3560 		    (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
3561 		     ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3562 		      rflow->last_qtail)) >= 0)) {
3563 			tcpu = next_cpu;
3564 			rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3565 		}
3566 
3567 		if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
3568 			*rflowp = rflow;
3569 			cpu = tcpu;
3570 			goto done;
3571 		}
3572 	}
3573 
3574 try_rps:
3575 
3576 	if (map) {
3577 		tcpu = map->cpus[reciprocal_scale(hash, map->len)];
3578 		if (cpu_online(tcpu)) {
3579 			cpu = tcpu;
3580 			goto done;
3581 		}
3582 	}
3583 
3584 done:
3585 	return cpu;
3586 }
3587 
3588 #ifdef CONFIG_RFS_ACCEL
3589 
3590 /**
3591  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3592  * @dev: Device on which the filter was set
3593  * @rxq_index: RX queue index
3594  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3595  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3596  *
3597  * Drivers that implement ndo_rx_flow_steer() should periodically call
3598  * this function for each installed filter and remove the filters for
3599  * which it returns %true.
3600  */
3601 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3602 			 u32 flow_id, u16 filter_id)
3603 {
3604 	struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3605 	struct rps_dev_flow_table *flow_table;
3606 	struct rps_dev_flow *rflow;
3607 	bool expire = true;
3608 	unsigned int cpu;
3609 
3610 	rcu_read_lock();
3611 	flow_table = rcu_dereference(rxqueue->rps_flow_table);
3612 	if (flow_table && flow_id <= flow_table->mask) {
3613 		rflow = &flow_table->flows[flow_id];
3614 		cpu = ACCESS_ONCE(rflow->cpu);
3615 		if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
3616 		    ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3617 			   rflow->last_qtail) <
3618 		     (int)(10 * flow_table->mask)))
3619 			expire = false;
3620 	}
3621 	rcu_read_unlock();
3622 	return expire;
3623 }
3624 EXPORT_SYMBOL(rps_may_expire_flow);
3625 
3626 #endif /* CONFIG_RFS_ACCEL */
3627 
3628 /* Called from hardirq (IPI) context */
3629 static void rps_trigger_softirq(void *data)
3630 {
3631 	struct softnet_data *sd = data;
3632 
3633 	____napi_schedule(sd, &sd->backlog);
3634 	sd->received_rps++;
3635 }
3636 
3637 #endif /* CONFIG_RPS */
3638 
3639 /*
3640  * Check if this softnet_data structure is another cpu one
3641  * If yes, queue it to our IPI list and return 1
3642  * If no, return 0
3643  */
3644 static int rps_ipi_queued(struct softnet_data *sd)
3645 {
3646 #ifdef CONFIG_RPS
3647 	struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
3648 
3649 	if (sd != mysd) {
3650 		sd->rps_ipi_next = mysd->rps_ipi_list;
3651 		mysd->rps_ipi_list = sd;
3652 
3653 		__raise_softirq_irqoff(NET_RX_SOFTIRQ);
3654 		return 1;
3655 	}
3656 #endif /* CONFIG_RPS */
3657 	return 0;
3658 }
3659 
3660 #ifdef CONFIG_NET_FLOW_LIMIT
3661 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3662 #endif
3663 
3664 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3665 {
3666 #ifdef CONFIG_NET_FLOW_LIMIT
3667 	struct sd_flow_limit *fl;
3668 	struct softnet_data *sd;
3669 	unsigned int old_flow, new_flow;
3670 
3671 	if (qlen < (netdev_max_backlog >> 1))
3672 		return false;
3673 
3674 	sd = this_cpu_ptr(&softnet_data);
3675 
3676 	rcu_read_lock();
3677 	fl = rcu_dereference(sd->flow_limit);
3678 	if (fl) {
3679 		new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3680 		old_flow = fl->history[fl->history_head];
3681 		fl->history[fl->history_head] = new_flow;
3682 
3683 		fl->history_head++;
3684 		fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3685 
3686 		if (likely(fl->buckets[old_flow]))
3687 			fl->buckets[old_flow]--;
3688 
3689 		if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3690 			fl->count++;
3691 			rcu_read_unlock();
3692 			return true;
3693 		}
3694 	}
3695 	rcu_read_unlock();
3696 #endif
3697 	return false;
3698 }
3699 
3700 /*
3701  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3702  * queue (may be a remote CPU queue).
3703  */
3704 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3705 			      unsigned int *qtail)
3706 {
3707 	struct softnet_data *sd;
3708 	unsigned long flags;
3709 	unsigned int qlen;
3710 
3711 	sd = &per_cpu(softnet_data, cpu);
3712 
3713 	local_irq_save(flags);
3714 
3715 	rps_lock(sd);
3716 	if (!netif_running(skb->dev))
3717 		goto drop;
3718 	qlen = skb_queue_len(&sd->input_pkt_queue);
3719 	if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3720 		if (qlen) {
3721 enqueue:
3722 			__skb_queue_tail(&sd->input_pkt_queue, skb);
3723 			input_queue_tail_incr_save(sd, qtail);
3724 			rps_unlock(sd);
3725 			local_irq_restore(flags);
3726 			return NET_RX_SUCCESS;
3727 		}
3728 
3729 		/* Schedule NAPI for backlog device
3730 		 * We can use non atomic operation since we own the queue lock
3731 		 */
3732 		if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3733 			if (!rps_ipi_queued(sd))
3734 				____napi_schedule(sd, &sd->backlog);
3735 		}
3736 		goto enqueue;
3737 	}
3738 
3739 drop:
3740 	sd->dropped++;
3741 	rps_unlock(sd);
3742 
3743 	local_irq_restore(flags);
3744 
3745 	atomic_long_inc(&skb->dev->rx_dropped);
3746 	kfree_skb(skb);
3747 	return NET_RX_DROP;
3748 }
3749 
3750 static int netif_rx_internal(struct sk_buff *skb)
3751 {
3752 	int ret;
3753 
3754 	net_timestamp_check(netdev_tstamp_prequeue, skb);
3755 
3756 	trace_netif_rx(skb);
3757 #ifdef CONFIG_RPS
3758 	if (static_key_false(&rps_needed)) {
3759 		struct rps_dev_flow voidflow, *rflow = &voidflow;
3760 		int cpu;
3761 
3762 		preempt_disable();
3763 		rcu_read_lock();
3764 
3765 		cpu = get_rps_cpu(skb->dev, skb, &rflow);
3766 		if (cpu < 0)
3767 			cpu = smp_processor_id();
3768 
3769 		ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3770 
3771 		rcu_read_unlock();
3772 		preempt_enable();
3773 	} else
3774 #endif
3775 	{
3776 		unsigned int qtail;
3777 
3778 		ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3779 		put_cpu();
3780 	}
3781 	return ret;
3782 }
3783 
3784 /**
3785  *	netif_rx	-	post buffer to the network code
3786  *	@skb: buffer to post
3787  *
3788  *	This function receives a packet from a device driver and queues it for
3789  *	the upper (protocol) levels to process.  It always succeeds. The buffer
3790  *	may be dropped during processing for congestion control or by the
3791  *	protocol layers.
3792  *
3793  *	return values:
3794  *	NET_RX_SUCCESS	(no congestion)
3795  *	NET_RX_DROP     (packet was dropped)
3796  *
3797  */
3798 
3799 int netif_rx(struct sk_buff *skb)
3800 {
3801 	trace_netif_rx_entry(skb);
3802 
3803 	return netif_rx_internal(skb);
3804 }
3805 EXPORT_SYMBOL(netif_rx);
3806 
3807 int netif_rx_ni(struct sk_buff *skb)
3808 {
3809 	int err;
3810 
3811 	trace_netif_rx_ni_entry(skb);
3812 
3813 	preempt_disable();
3814 	err = netif_rx_internal(skb);
3815 	if (local_softirq_pending())
3816 		do_softirq();
3817 	preempt_enable();
3818 
3819 	return err;
3820 }
3821 EXPORT_SYMBOL(netif_rx_ni);
3822 
3823 static __latent_entropy void net_tx_action(struct softirq_action *h)
3824 {
3825 	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
3826 
3827 	if (sd->completion_queue) {
3828 		struct sk_buff *clist;
3829 
3830 		local_irq_disable();
3831 		clist = sd->completion_queue;
3832 		sd->completion_queue = NULL;
3833 		local_irq_enable();
3834 
3835 		while (clist) {
3836 			struct sk_buff *skb = clist;
3837 
3838 			clist = clist->next;
3839 
3840 			WARN_ON(atomic_read(&skb->users));
3841 			if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3842 				trace_consume_skb(skb);
3843 			else
3844 				trace_kfree_skb(skb, net_tx_action);
3845 
3846 			if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
3847 				__kfree_skb(skb);
3848 			else
3849 				__kfree_skb_defer(skb);
3850 		}
3851 
3852 		__kfree_skb_flush();
3853 	}
3854 
3855 	if (sd->output_queue) {
3856 		struct Qdisc *head;
3857 
3858 		local_irq_disable();
3859 		head = sd->output_queue;
3860 		sd->output_queue = NULL;
3861 		sd->output_queue_tailp = &sd->output_queue;
3862 		local_irq_enable();
3863 
3864 		while (head) {
3865 			struct Qdisc *q = head;
3866 			spinlock_t *root_lock;
3867 
3868 			head = head->next_sched;
3869 
3870 			root_lock = qdisc_lock(q);
3871 			spin_lock(root_lock);
3872 			/* We need to make sure head->next_sched is read
3873 			 * before clearing __QDISC_STATE_SCHED
3874 			 */
3875 			smp_mb__before_atomic();
3876 			clear_bit(__QDISC_STATE_SCHED, &q->state);
3877 			qdisc_run(q);
3878 			spin_unlock(root_lock);
3879 		}
3880 	}
3881 }
3882 
3883 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
3884 /* This hook is defined here for ATM LANE */
3885 int (*br_fdb_test_addr_hook)(struct net_device *dev,
3886 			     unsigned char *addr) __read_mostly;
3887 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
3888 #endif
3889 
3890 static inline struct sk_buff *
3891 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
3892 		   struct net_device *orig_dev)
3893 {
3894 #ifdef CONFIG_NET_CLS_ACT
3895 	struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3896 	struct tcf_result cl_res;
3897 
3898 	/* If there's at least one ingress present somewhere (so
3899 	 * we get here via enabled static key), remaining devices
3900 	 * that are not configured with an ingress qdisc will bail
3901 	 * out here.
3902 	 */
3903 	if (!cl)
3904 		return skb;
3905 	if (*pt_prev) {
3906 		*ret = deliver_skb(skb, *pt_prev, orig_dev);
3907 		*pt_prev = NULL;
3908 	}
3909 
3910 	qdisc_skb_cb(skb)->pkt_len = skb->len;
3911 	skb->tc_at_ingress = 1;
3912 	qdisc_bstats_cpu_update(cl->q, skb);
3913 
3914 	switch (tc_classify(skb, cl, &cl_res, false)) {
3915 	case TC_ACT_OK:
3916 	case TC_ACT_RECLASSIFY:
3917 		skb->tc_index = TC_H_MIN(cl_res.classid);
3918 		break;
3919 	case TC_ACT_SHOT:
3920 		qdisc_qstats_cpu_drop(cl->q);
3921 		kfree_skb(skb);
3922 		return NULL;
3923 	case TC_ACT_STOLEN:
3924 	case TC_ACT_QUEUED:
3925 		consume_skb(skb);
3926 		return NULL;
3927 	case TC_ACT_REDIRECT:
3928 		/* skb_mac_header check was done by cls/act_bpf, so
3929 		 * we can safely push the L2 header back before
3930 		 * redirecting to another netdev
3931 		 */
3932 		__skb_push(skb, skb->mac_len);
3933 		skb_do_redirect(skb);
3934 		return NULL;
3935 	default:
3936 		break;
3937 	}
3938 #endif /* CONFIG_NET_CLS_ACT */
3939 	return skb;
3940 }
3941 
3942 /**
3943  *	netdev_is_rx_handler_busy - check if receive handler is registered
3944  *	@dev: device to check
3945  *
3946  *	Check if a receive handler is already registered for a given device.
3947  *	Return true if there one.
3948  *
3949  *	The caller must hold the rtnl_mutex.
3950  */
3951 bool netdev_is_rx_handler_busy(struct net_device *dev)
3952 {
3953 	ASSERT_RTNL();
3954 	return dev && rtnl_dereference(dev->rx_handler);
3955 }
3956 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
3957 
3958 /**
3959  *	netdev_rx_handler_register - register receive handler
3960  *	@dev: device to register a handler for
3961  *	@rx_handler: receive handler to register
3962  *	@rx_handler_data: data pointer that is used by rx handler
3963  *
3964  *	Register a receive handler for a device. This handler will then be
3965  *	called from __netif_receive_skb. A negative errno code is returned
3966  *	on a failure.
3967  *
3968  *	The caller must hold the rtnl_mutex.
3969  *
3970  *	For a general description of rx_handler, see enum rx_handler_result.
3971  */
3972 int netdev_rx_handler_register(struct net_device *dev,
3973 			       rx_handler_func_t *rx_handler,
3974 			       void *rx_handler_data)
3975 {
3976 	if (netdev_is_rx_handler_busy(dev))
3977 		return -EBUSY;
3978 
3979 	/* Note: rx_handler_data must be set before rx_handler */
3980 	rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
3981 	rcu_assign_pointer(dev->rx_handler, rx_handler);
3982 
3983 	return 0;
3984 }
3985 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3986 
3987 /**
3988  *	netdev_rx_handler_unregister - unregister receive handler
3989  *	@dev: device to unregister a handler from
3990  *
3991  *	Unregister a receive handler from a device.
3992  *
3993  *	The caller must hold the rtnl_mutex.
3994  */
3995 void netdev_rx_handler_unregister(struct net_device *dev)
3996 {
3997 
3998 	ASSERT_RTNL();
3999 	RCU_INIT_POINTER(dev->rx_handler, NULL);
4000 	/* a reader seeing a non NULL rx_handler in a rcu_read_lock()
4001 	 * section has a guarantee to see a non NULL rx_handler_data
4002 	 * as well.
4003 	 */
4004 	synchronize_net();
4005 	RCU_INIT_POINTER(dev->rx_handler_data, NULL);
4006 }
4007 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
4008 
4009 /*
4010  * Limit the use of PFMEMALLOC reserves to those protocols that implement
4011  * the special handling of PFMEMALLOC skbs.
4012  */
4013 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
4014 {
4015 	switch (skb->protocol) {
4016 	case htons(ETH_P_ARP):
4017 	case htons(ETH_P_IP):
4018 	case htons(ETH_P_IPV6):
4019 	case htons(ETH_P_8021Q):
4020 	case htons(ETH_P_8021AD):
4021 		return true;
4022 	default:
4023 		return false;
4024 	}
4025 }
4026 
4027 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4028 			     int *ret, struct net_device *orig_dev)
4029 {
4030 #ifdef CONFIG_NETFILTER_INGRESS
4031 	if (nf_hook_ingress_active(skb)) {
4032 		int ingress_retval;
4033 
4034 		if (*pt_prev) {
4035 			*ret = deliver_skb(skb, *pt_prev, orig_dev);
4036 			*pt_prev = NULL;
4037 		}
4038 
4039 		rcu_read_lock();
4040 		ingress_retval = nf_hook_ingress(skb);
4041 		rcu_read_unlock();
4042 		return ingress_retval;
4043 	}
4044 #endif /* CONFIG_NETFILTER_INGRESS */
4045 	return 0;
4046 }
4047 
4048 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
4049 {
4050 	struct packet_type *ptype, *pt_prev;
4051 	rx_handler_func_t *rx_handler;
4052 	struct net_device *orig_dev;
4053 	bool deliver_exact = false;
4054 	int ret = NET_RX_DROP;
4055 	__be16 type;
4056 
4057 	net_timestamp_check(!netdev_tstamp_prequeue, skb);
4058 
4059 	trace_netif_receive_skb(skb);
4060 
4061 	orig_dev = skb->dev;
4062 
4063 	skb_reset_network_header(skb);
4064 	if (!skb_transport_header_was_set(skb))
4065 		skb_reset_transport_header(skb);
4066 	skb_reset_mac_len(skb);
4067 
4068 	pt_prev = NULL;
4069 
4070 another_round:
4071 	skb->skb_iif = skb->dev->ifindex;
4072 
4073 	__this_cpu_inc(softnet_data.processed);
4074 
4075 	if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4076 	    skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
4077 		skb = skb_vlan_untag(skb);
4078 		if (unlikely(!skb))
4079 			goto out;
4080 	}
4081 
4082 	if (skb_skip_tc_classify(skb))
4083 		goto skip_classify;
4084 
4085 	if (pfmemalloc)
4086 		goto skip_taps;
4087 
4088 	list_for_each_entry_rcu(ptype, &ptype_all, list) {
4089 		if (pt_prev)
4090 			ret = deliver_skb(skb, pt_prev, orig_dev);
4091 		pt_prev = ptype;
4092 	}
4093 
4094 	list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4095 		if (pt_prev)
4096 			ret = deliver_skb(skb, pt_prev, orig_dev);
4097 		pt_prev = ptype;
4098 	}
4099 
4100 skip_taps:
4101 #ifdef CONFIG_NET_INGRESS
4102 	if (static_key_false(&ingress_needed)) {
4103 		skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
4104 		if (!skb)
4105 			goto out;
4106 
4107 		if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
4108 			goto out;
4109 	}
4110 #endif
4111 	skb_reset_tc(skb);
4112 skip_classify:
4113 	if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
4114 		goto drop;
4115 
4116 	if (skb_vlan_tag_present(skb)) {
4117 		if (pt_prev) {
4118 			ret = deliver_skb(skb, pt_prev, orig_dev);
4119 			pt_prev = NULL;
4120 		}
4121 		if (vlan_do_receive(&skb))
4122 			goto another_round;
4123 		else if (unlikely(!skb))
4124 			goto out;
4125 	}
4126 
4127 	rx_handler = rcu_dereference(skb->dev->rx_handler);
4128 	if (rx_handler) {
4129 		if (pt_prev) {
4130 			ret = deliver_skb(skb, pt_prev, orig_dev);
4131 			pt_prev = NULL;
4132 		}
4133 		switch (rx_handler(&skb)) {
4134 		case RX_HANDLER_CONSUMED:
4135 			ret = NET_RX_SUCCESS;
4136 			goto out;
4137 		case RX_HANDLER_ANOTHER:
4138 			goto another_round;
4139 		case RX_HANDLER_EXACT:
4140 			deliver_exact = true;
4141 		case RX_HANDLER_PASS:
4142 			break;
4143 		default:
4144 			BUG();
4145 		}
4146 	}
4147 
4148 	if (unlikely(skb_vlan_tag_present(skb))) {
4149 		if (skb_vlan_tag_get_id(skb))
4150 			skb->pkt_type = PACKET_OTHERHOST;
4151 		/* Note: we might in the future use prio bits
4152 		 * and set skb->priority like in vlan_do_receive()
4153 		 * For the time being, just ignore Priority Code Point
4154 		 */
4155 		skb->vlan_tci = 0;
4156 	}
4157 
4158 	type = skb->protocol;
4159 
4160 	/* deliver only exact match when indicated */
4161 	if (likely(!deliver_exact)) {
4162 		deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4163 				       &ptype_base[ntohs(type) &
4164 						   PTYPE_HASH_MASK]);
4165 	}
4166 
4167 	deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4168 			       &orig_dev->ptype_specific);
4169 
4170 	if (unlikely(skb->dev != orig_dev)) {
4171 		deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4172 				       &skb->dev->ptype_specific);
4173 	}
4174 
4175 	if (pt_prev) {
4176 		if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
4177 			goto drop;
4178 		else
4179 			ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
4180 	} else {
4181 drop:
4182 		if (!deliver_exact)
4183 			atomic_long_inc(&skb->dev->rx_dropped);
4184 		else
4185 			atomic_long_inc(&skb->dev->rx_nohandler);
4186 		kfree_skb(skb);
4187 		/* Jamal, now you will not able to escape explaining
4188 		 * me how you were going to use this. :-)
4189 		 */
4190 		ret = NET_RX_DROP;
4191 	}
4192 
4193 out:
4194 	return ret;
4195 }
4196 
4197 static int __netif_receive_skb(struct sk_buff *skb)
4198 {
4199 	int ret;
4200 
4201 	if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
4202 		unsigned long pflags = current->flags;
4203 
4204 		/*
4205 		 * PFMEMALLOC skbs are special, they should
4206 		 * - be delivered to SOCK_MEMALLOC sockets only
4207 		 * - stay away from userspace
4208 		 * - have bounded memory usage
4209 		 *
4210 		 * Use PF_MEMALLOC as this saves us from propagating the allocation
4211 		 * context down to all allocation sites.
4212 		 */
4213 		current->flags |= PF_MEMALLOC;
4214 		ret = __netif_receive_skb_core(skb, true);
4215 		tsk_restore_flags(current, pflags, PF_MEMALLOC);
4216 	} else
4217 		ret = __netif_receive_skb_core(skb, false);
4218 
4219 	return ret;
4220 }
4221 
4222 static int netif_receive_skb_internal(struct sk_buff *skb)
4223 {
4224 	int ret;
4225 
4226 	net_timestamp_check(netdev_tstamp_prequeue, skb);
4227 
4228 	if (skb_defer_rx_timestamp(skb))
4229 		return NET_RX_SUCCESS;
4230 
4231 	rcu_read_lock();
4232 
4233 #ifdef CONFIG_RPS
4234 	if (static_key_false(&rps_needed)) {
4235 		struct rps_dev_flow voidflow, *rflow = &voidflow;
4236 		int cpu = get_rps_cpu(skb->dev, skb, &rflow);
4237 
4238 		if (cpu >= 0) {
4239 			ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4240 			rcu_read_unlock();
4241 			return ret;
4242 		}
4243 	}
4244 #endif
4245 	ret = __netif_receive_skb(skb);
4246 	rcu_read_unlock();
4247 	return ret;
4248 }
4249 
4250 /**
4251  *	netif_receive_skb - process receive buffer from network
4252  *	@skb: buffer to process
4253  *
4254  *	netif_receive_skb() is the main receive data processing function.
4255  *	It always succeeds. The buffer may be dropped during processing
4256  *	for congestion control or by the protocol layers.
4257  *
4258  *	This function may only be called from softirq context and interrupts
4259  *	should be enabled.
4260  *
4261  *	Return values (usually ignored):
4262  *	NET_RX_SUCCESS: no congestion
4263  *	NET_RX_DROP: packet was dropped
4264  */
4265 int netif_receive_skb(struct sk_buff *skb)
4266 {
4267 	trace_netif_receive_skb_entry(skb);
4268 
4269 	return netif_receive_skb_internal(skb);
4270 }
4271 EXPORT_SYMBOL(netif_receive_skb);
4272 
4273 DEFINE_PER_CPU(struct work_struct, flush_works);
4274 
4275 /* Network device is going away, flush any packets still pending */
4276 static void flush_backlog(struct work_struct *work)
4277 {
4278 	struct sk_buff *skb, *tmp;
4279 	struct softnet_data *sd;
4280 
4281 	local_bh_disable();
4282 	sd = this_cpu_ptr(&softnet_data);
4283 
4284 	local_irq_disable();
4285 	rps_lock(sd);
4286 	skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
4287 		if (skb->dev->reg_state == NETREG_UNREGISTERING) {
4288 			__skb_unlink(skb, &sd->input_pkt_queue);
4289 			kfree_skb(skb);
4290 			input_queue_head_incr(sd);
4291 		}
4292 	}
4293 	rps_unlock(sd);
4294 	local_irq_enable();
4295 
4296 	skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4297 		if (skb->dev->reg_state == NETREG_UNREGISTERING) {
4298 			__skb_unlink(skb, &sd->process_queue);
4299 			kfree_skb(skb);
4300 			input_queue_head_incr(sd);
4301 		}
4302 	}
4303 	local_bh_enable();
4304 }
4305 
4306 static void flush_all_backlogs(void)
4307 {
4308 	unsigned int cpu;
4309 
4310 	get_online_cpus();
4311 
4312 	for_each_online_cpu(cpu)
4313 		queue_work_on(cpu, system_highpri_wq,
4314 			      per_cpu_ptr(&flush_works, cpu));
4315 
4316 	for_each_online_cpu(cpu)
4317 		flush_work(per_cpu_ptr(&flush_works, cpu));
4318 
4319 	put_online_cpus();
4320 }
4321 
4322 static int napi_gro_complete(struct sk_buff *skb)
4323 {
4324 	struct packet_offload *ptype;
4325 	__be16 type = skb->protocol;
4326 	struct list_head *head = &offload_base;
4327 	int err = -ENOENT;
4328 
4329 	BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4330 
4331 	if (NAPI_GRO_CB(skb)->count == 1) {
4332 		skb_shinfo(skb)->gso_size = 0;
4333 		goto out;
4334 	}
4335 
4336 	rcu_read_lock();
4337 	list_for_each_entry_rcu(ptype, head, list) {
4338 		if (ptype->type != type || !ptype->callbacks.gro_complete)
4339 			continue;
4340 
4341 		err = ptype->callbacks.gro_complete(skb, 0);
4342 		break;
4343 	}
4344 	rcu_read_unlock();
4345 
4346 	if (err) {
4347 		WARN_ON(&ptype->list == head);
4348 		kfree_skb(skb);
4349 		return NET_RX_SUCCESS;
4350 	}
4351 
4352 out:
4353 	return netif_receive_skb_internal(skb);
4354 }
4355 
4356 /* napi->gro_list contains packets ordered by age.
4357  * youngest packets at the head of it.
4358  * Complete skbs in reverse order to reduce latencies.
4359  */
4360 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
4361 {
4362 	struct sk_buff *skb, *prev = NULL;
4363 
4364 	/* scan list and build reverse chain */
4365 	for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4366 		skb->prev = prev;
4367 		prev = skb;
4368 	}
4369 
4370 	for (skb = prev; skb; skb = prev) {
4371 		skb->next = NULL;
4372 
4373 		if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4374 			return;
4375 
4376 		prev = skb->prev;
4377 		napi_gro_complete(skb);
4378 		napi->gro_count--;
4379 	}
4380 
4381 	napi->gro_list = NULL;
4382 }
4383 EXPORT_SYMBOL(napi_gro_flush);
4384 
4385 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4386 {
4387 	struct sk_buff *p;
4388 	unsigned int maclen = skb->dev->hard_header_len;
4389 	u32 hash = skb_get_hash_raw(skb);
4390 
4391 	for (p = napi->gro_list; p; p = p->next) {
4392 		unsigned long diffs;
4393 
4394 		NAPI_GRO_CB(p)->flush = 0;
4395 
4396 		if (hash != skb_get_hash_raw(p)) {
4397 			NAPI_GRO_CB(p)->same_flow = 0;
4398 			continue;
4399 		}
4400 
4401 		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4402 		diffs |= p->vlan_tci ^ skb->vlan_tci;
4403 		diffs |= skb_metadata_dst_cmp(p, skb);
4404 		if (maclen == ETH_HLEN)
4405 			diffs |= compare_ether_header(skb_mac_header(p),
4406 						      skb_mac_header(skb));
4407 		else if (!diffs)
4408 			diffs = memcmp(skb_mac_header(p),
4409 				       skb_mac_header(skb),
4410 				       maclen);
4411 		NAPI_GRO_CB(p)->same_flow = !diffs;
4412 	}
4413 }
4414 
4415 static void skb_gro_reset_offset(struct sk_buff *skb)
4416 {
4417 	const struct skb_shared_info *pinfo = skb_shinfo(skb);
4418 	const skb_frag_t *frag0 = &pinfo->frags[0];
4419 
4420 	NAPI_GRO_CB(skb)->data_offset = 0;
4421 	NAPI_GRO_CB(skb)->frag0 = NULL;
4422 	NAPI_GRO_CB(skb)->frag0_len = 0;
4423 
4424 	if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4425 	    pinfo->nr_frags &&
4426 	    !PageHighMem(skb_frag_page(frag0))) {
4427 		NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4428 		NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
4429 						    skb_frag_size(frag0),
4430 						    skb->end - skb->tail);
4431 	}
4432 }
4433 
4434 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4435 {
4436 	struct skb_shared_info *pinfo = skb_shinfo(skb);
4437 
4438 	BUG_ON(skb->end - skb->tail < grow);
4439 
4440 	memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4441 
4442 	skb->data_len -= grow;
4443 	skb->tail += grow;
4444 
4445 	pinfo->frags[0].page_offset += grow;
4446 	skb_frag_size_sub(&pinfo->frags[0], grow);
4447 
4448 	if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4449 		skb_frag_unref(skb, 0);
4450 		memmove(pinfo->frags, pinfo->frags + 1,
4451 			--pinfo->nr_frags * sizeof(pinfo->frags[0]));
4452 	}
4453 }
4454 
4455 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4456 {
4457 	struct sk_buff **pp = NULL;
4458 	struct packet_offload *ptype;
4459 	__be16 type = skb->protocol;
4460 	struct list_head *head = &offload_base;
4461 	int same_flow;
4462 	enum gro_result ret;
4463 	int grow;
4464 
4465 	if (!(skb->dev->features & NETIF_F_GRO))
4466 		goto normal;
4467 
4468 	if (skb->csum_bad)
4469 		goto normal;
4470 
4471 	gro_list_prepare(napi, skb);
4472 
4473 	rcu_read_lock();
4474 	list_for_each_entry_rcu(ptype, head, list) {
4475 		if (ptype->type != type || !ptype->callbacks.gro_receive)
4476 			continue;
4477 
4478 		skb_set_network_header(skb, skb_gro_offset(skb));
4479 		skb_reset_mac_len(skb);
4480 		NAPI_GRO_CB(skb)->same_flow = 0;
4481 		NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
4482 		NAPI_GRO_CB(skb)->free = 0;
4483 		NAPI_GRO_CB(skb)->encap_mark = 0;
4484 		NAPI_GRO_CB(skb)->recursion_counter = 0;
4485 		NAPI_GRO_CB(skb)->is_fou = 0;
4486 		NAPI_GRO_CB(skb)->is_atomic = 1;
4487 		NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
4488 
4489 		/* Setup for GRO checksum validation */
4490 		switch (skb->ip_summed) {
4491 		case CHECKSUM_COMPLETE:
4492 			NAPI_GRO_CB(skb)->csum = skb->csum;
4493 			NAPI_GRO_CB(skb)->csum_valid = 1;
4494 			NAPI_GRO_CB(skb)->csum_cnt = 0;
4495 			break;
4496 		case CHECKSUM_UNNECESSARY:
4497 			NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4498 			NAPI_GRO_CB(skb)->csum_valid = 0;
4499 			break;
4500 		default:
4501 			NAPI_GRO_CB(skb)->csum_cnt = 0;
4502 			NAPI_GRO_CB(skb)->csum_valid = 0;
4503 		}
4504 
4505 		pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
4506 		break;
4507 	}
4508 	rcu_read_unlock();
4509 
4510 	if (&ptype->list == head)
4511 		goto normal;
4512 
4513 	if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) {
4514 		ret = GRO_CONSUMED;
4515 		goto ok;
4516 	}
4517 
4518 	same_flow = NAPI_GRO_CB(skb)->same_flow;
4519 	ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
4520 
4521 	if (pp) {
4522 		struct sk_buff *nskb = *pp;
4523 
4524 		*pp = nskb->next;
4525 		nskb->next = NULL;
4526 		napi_gro_complete(nskb);
4527 		napi->gro_count--;
4528 	}
4529 
4530 	if (same_flow)
4531 		goto ok;
4532 
4533 	if (NAPI_GRO_CB(skb)->flush)
4534 		goto normal;
4535 
4536 	if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4537 		struct sk_buff *nskb = napi->gro_list;
4538 
4539 		/* locate the end of the list to select the 'oldest' flow */
4540 		while (nskb->next) {
4541 			pp = &nskb->next;
4542 			nskb = *pp;
4543 		}
4544 		*pp = NULL;
4545 		nskb->next = NULL;
4546 		napi_gro_complete(nskb);
4547 	} else {
4548 		napi->gro_count++;
4549 	}
4550 	NAPI_GRO_CB(skb)->count = 1;
4551 	NAPI_GRO_CB(skb)->age = jiffies;
4552 	NAPI_GRO_CB(skb)->last = skb;
4553 	skb_shinfo(skb)->gso_size = skb_gro_len(skb);
4554 	skb->next = napi->gro_list;
4555 	napi->gro_list = skb;
4556 	ret = GRO_HELD;
4557 
4558 pull:
4559 	grow = skb_gro_offset(skb) - skb_headlen(skb);
4560 	if (grow > 0)
4561 		gro_pull_from_frag0(skb, grow);
4562 ok:
4563 	return ret;
4564 
4565 normal:
4566 	ret = GRO_NORMAL;
4567 	goto pull;
4568 }
4569 
4570 struct packet_offload *gro_find_receive_by_type(__be16 type)
4571 {
4572 	struct list_head *offload_head = &offload_base;
4573 	struct packet_offload *ptype;
4574 
4575 	list_for_each_entry_rcu(ptype, offload_head, list) {
4576 		if (ptype->type != type || !ptype->callbacks.gro_receive)
4577 			continue;
4578 		return ptype;
4579 	}
4580 	return NULL;
4581 }
4582 EXPORT_SYMBOL(gro_find_receive_by_type);
4583 
4584 struct packet_offload *gro_find_complete_by_type(__be16 type)
4585 {
4586 	struct list_head *offload_head = &offload_base;
4587 	struct packet_offload *ptype;
4588 
4589 	list_for_each_entry_rcu(ptype, offload_head, list) {
4590 		if (ptype->type != type || !ptype->callbacks.gro_complete)
4591 			continue;
4592 		return ptype;
4593 	}
4594 	return NULL;
4595 }
4596 EXPORT_SYMBOL(gro_find_complete_by_type);
4597 
4598 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
4599 {
4600 	switch (ret) {
4601 	case GRO_NORMAL:
4602 		if (netif_receive_skb_internal(skb))
4603 			ret = GRO_DROP;
4604 		break;
4605 
4606 	case GRO_DROP:
4607 		kfree_skb(skb);
4608 		break;
4609 
4610 	case GRO_MERGED_FREE:
4611 		if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) {
4612 			skb_dst_drop(skb);
4613 			secpath_reset(skb);
4614 			kmem_cache_free(skbuff_head_cache, skb);
4615 		} else {
4616 			__kfree_skb(skb);
4617 		}
4618 		break;
4619 
4620 	case GRO_HELD:
4621 	case GRO_MERGED:
4622 	case GRO_CONSUMED:
4623 		break;
4624 	}
4625 
4626 	return ret;
4627 }
4628 
4629 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4630 {
4631 	skb_mark_napi_id(skb, napi);
4632 	trace_napi_gro_receive_entry(skb);
4633 
4634 	skb_gro_reset_offset(skb);
4635 
4636 	return napi_skb_finish(dev_gro_receive(napi, skb), skb);
4637 }
4638 EXPORT_SYMBOL(napi_gro_receive);
4639 
4640 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
4641 {
4642 	if (unlikely(skb->pfmemalloc)) {
4643 		consume_skb(skb);
4644 		return;
4645 	}
4646 	__skb_pull(skb, skb_headlen(skb));
4647 	/* restore the reserve we had after netdev_alloc_skb_ip_align() */
4648 	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
4649 	skb->vlan_tci = 0;
4650 	skb->dev = napi->dev;
4651 	skb->skb_iif = 0;
4652 	skb->encapsulation = 0;
4653 	skb_shinfo(skb)->gso_type = 0;
4654 	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
4655 	secpath_reset(skb);
4656 
4657 	napi->skb = skb;
4658 }
4659 
4660 struct sk_buff *napi_get_frags(struct napi_struct *napi)
4661 {
4662 	struct sk_buff *skb = napi->skb;
4663 
4664 	if (!skb) {
4665 		skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
4666 		if (skb) {
4667 			napi->skb = skb;
4668 			skb_mark_napi_id(skb, napi);
4669 		}
4670 	}
4671 	return skb;
4672 }
4673 EXPORT_SYMBOL(napi_get_frags);
4674 
4675 static gro_result_t napi_frags_finish(struct napi_struct *napi,
4676 				      struct sk_buff *skb,
4677 				      gro_result_t ret)
4678 {
4679 	switch (ret) {
4680 	case GRO_NORMAL:
4681 	case GRO_HELD:
4682 		__skb_push(skb, ETH_HLEN);
4683 		skb->protocol = eth_type_trans(skb, skb->dev);
4684 		if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
4685 			ret = GRO_DROP;
4686 		break;
4687 
4688 	case GRO_DROP:
4689 	case GRO_MERGED_FREE:
4690 		napi_reuse_skb(napi, skb);
4691 		break;
4692 
4693 	case GRO_MERGED:
4694 	case GRO_CONSUMED:
4695 		break;
4696 	}
4697 
4698 	return ret;
4699 }
4700 
4701 /* Upper GRO stack assumes network header starts at gro_offset=0
4702  * Drivers could call both napi_gro_frags() and napi_gro_receive()
4703  * We copy ethernet header into skb->data to have a common layout.
4704  */
4705 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
4706 {
4707 	struct sk_buff *skb = napi->skb;
4708 	const struct ethhdr *eth;
4709 	unsigned int hlen = sizeof(*eth);
4710 
4711 	napi->skb = NULL;
4712 
4713 	skb_reset_mac_header(skb);
4714 	skb_gro_reset_offset(skb);
4715 
4716 	eth = skb_gro_header_fast(skb, 0);
4717 	if (unlikely(skb_gro_header_hard(skb, hlen))) {
4718 		eth = skb_gro_header_slow(skb, hlen, 0);
4719 		if (unlikely(!eth)) {
4720 			net_warn_ratelimited("%s: dropping impossible skb from %s\n",
4721 					     __func__, napi->dev->name);
4722 			napi_reuse_skb(napi, skb);
4723 			return NULL;
4724 		}
4725 	} else {
4726 		gro_pull_from_frag0(skb, hlen);
4727 		NAPI_GRO_CB(skb)->frag0 += hlen;
4728 		NAPI_GRO_CB(skb)->frag0_len -= hlen;
4729 	}
4730 	__skb_pull(skb, hlen);
4731 
4732 	/*
4733 	 * This works because the only protocols we care about don't require
4734 	 * special handling.
4735 	 * We'll fix it up properly in napi_frags_finish()
4736 	 */
4737 	skb->protocol = eth->h_proto;
4738 
4739 	return skb;
4740 }
4741 
4742 gro_result_t napi_gro_frags(struct napi_struct *napi)
4743 {
4744 	struct sk_buff *skb = napi_frags_skb(napi);
4745 
4746 	if (!skb)
4747 		return GRO_DROP;
4748 
4749 	trace_napi_gro_frags_entry(skb);
4750 
4751 	return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
4752 }
4753 EXPORT_SYMBOL(napi_gro_frags);
4754 
4755 /* Compute the checksum from gro_offset and return the folded value
4756  * after adding in any pseudo checksum.
4757  */
4758 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
4759 {
4760 	__wsum wsum;
4761 	__sum16 sum;
4762 
4763 	wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
4764 
4765 	/* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4766 	sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
4767 	if (likely(!sum)) {
4768 		if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
4769 		    !skb->csum_complete_sw)
4770 			netdev_rx_csum_fault(skb->dev);
4771 	}
4772 
4773 	NAPI_GRO_CB(skb)->csum = wsum;
4774 	NAPI_GRO_CB(skb)->csum_valid = 1;
4775 
4776 	return sum;
4777 }
4778 EXPORT_SYMBOL(__skb_gro_checksum_complete);
4779 
4780 /*
4781  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
4782  * Note: called with local irq disabled, but exits with local irq enabled.
4783  */
4784 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4785 {
4786 #ifdef CONFIG_RPS
4787 	struct softnet_data *remsd = sd->rps_ipi_list;
4788 
4789 	if (remsd) {
4790 		sd->rps_ipi_list = NULL;
4791 
4792 		local_irq_enable();
4793 
4794 		/* Send pending IPI's to kick RPS processing on remote cpus. */
4795 		while (remsd) {
4796 			struct softnet_data *next = remsd->rps_ipi_next;
4797 
4798 			if (cpu_online(remsd->cpu))
4799 				smp_call_function_single_async(remsd->cpu,
4800 							   &remsd->csd);
4801 			remsd = next;
4802 		}
4803 	} else
4804 #endif
4805 		local_irq_enable();
4806 }
4807 
4808 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
4809 {
4810 #ifdef CONFIG_RPS
4811 	return sd->rps_ipi_list != NULL;
4812 #else
4813 	return false;
4814 #endif
4815 }
4816 
4817 static int process_backlog(struct napi_struct *napi, int quota)
4818 {
4819 	struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
4820 	bool again = true;
4821 	int work = 0;
4822 
4823 	/* Check if we have pending ipi, its better to send them now,
4824 	 * not waiting net_rx_action() end.
4825 	 */
4826 	if (sd_has_rps_ipi_waiting(sd)) {
4827 		local_irq_disable();
4828 		net_rps_action_and_irq_enable(sd);
4829 	}
4830 
4831 	napi->weight = dev_rx_weight;
4832 	while (again) {
4833 		struct sk_buff *skb;
4834 
4835 		while ((skb = __skb_dequeue(&sd->process_queue))) {
4836 			rcu_read_lock();
4837 			__netif_receive_skb(skb);
4838 			rcu_read_unlock();
4839 			input_queue_head_incr(sd);
4840 			if (++work >= quota)
4841 				return work;
4842 
4843 		}
4844 
4845 		local_irq_disable();
4846 		rps_lock(sd);
4847 		if (skb_queue_empty(&sd->input_pkt_queue)) {
4848 			/*
4849 			 * Inline a custom version of __napi_complete().
4850 			 * only current cpu owns and manipulates this napi,
4851 			 * and NAPI_STATE_SCHED is the only possible flag set
4852 			 * on backlog.
4853 			 * We can use a plain write instead of clear_bit(),
4854 			 * and we dont need an smp_mb() memory barrier.
4855 			 */
4856 			napi->state = 0;
4857 			again = false;
4858 		} else {
4859 			skb_queue_splice_tail_init(&sd->input_pkt_queue,
4860 						   &sd->process_queue);
4861 		}
4862 		rps_unlock(sd);
4863 		local_irq_enable();
4864 	}
4865 
4866 	return work;
4867 }
4868 
4869 /**
4870  * __napi_schedule - schedule for receive
4871  * @n: entry to schedule
4872  *
4873  * The entry's receive function will be scheduled to run.
4874  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
4875  */
4876 void __napi_schedule(struct napi_struct *n)
4877 {
4878 	unsigned long flags;
4879 
4880 	local_irq_save(flags);
4881 	____napi_schedule(this_cpu_ptr(&softnet_data), n);
4882 	local_irq_restore(flags);
4883 }
4884 EXPORT_SYMBOL(__napi_schedule);
4885 
4886 /**
4887  * __napi_schedule_irqoff - schedule for receive
4888  * @n: entry to schedule
4889  *
4890  * Variant of __napi_schedule() assuming hard irqs are masked
4891  */
4892 void __napi_schedule_irqoff(struct napi_struct *n)
4893 {
4894 	____napi_schedule(this_cpu_ptr(&softnet_data), n);
4895 }
4896 EXPORT_SYMBOL(__napi_schedule_irqoff);
4897 
4898 bool napi_complete_done(struct napi_struct *n, int work_done)
4899 {
4900 	unsigned long flags;
4901 
4902 	/*
4903 	 * 1) Don't let napi dequeue from the cpu poll list
4904 	 *    just in case its running on a different cpu.
4905 	 * 2) If we are busy polling, do nothing here, we have
4906 	 *    the guarantee we will be called later.
4907 	 */
4908 	if (unlikely(n->state & (NAPIF_STATE_NPSVC |
4909 				 NAPIF_STATE_IN_BUSY_POLL)))
4910 		return false;
4911 
4912 	if (n->gro_list) {
4913 		unsigned long timeout = 0;
4914 
4915 		if (work_done)
4916 			timeout = n->dev->gro_flush_timeout;
4917 
4918 		if (timeout)
4919 			hrtimer_start(&n->timer, ns_to_ktime(timeout),
4920 				      HRTIMER_MODE_REL_PINNED);
4921 		else
4922 			napi_gro_flush(n, false);
4923 	}
4924 	if (unlikely(!list_empty(&n->poll_list))) {
4925 		/* If n->poll_list is not empty, we need to mask irqs */
4926 		local_irq_save(flags);
4927 		list_del_init(&n->poll_list);
4928 		local_irq_restore(flags);
4929 	}
4930 	WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
4931 	return true;
4932 }
4933 EXPORT_SYMBOL(napi_complete_done);
4934 
4935 /* must be called under rcu_read_lock(), as we dont take a reference */
4936 static struct napi_struct *napi_by_id(unsigned int napi_id)
4937 {
4938 	unsigned int hash = napi_id % HASH_SIZE(napi_hash);
4939 	struct napi_struct *napi;
4940 
4941 	hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
4942 		if (napi->napi_id == napi_id)
4943 			return napi;
4944 
4945 	return NULL;
4946 }
4947 
4948 #if defined(CONFIG_NET_RX_BUSY_POLL)
4949 
4950 #define BUSY_POLL_BUDGET 8
4951 
4952 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)
4953 {
4954 	int rc;
4955 
4956 	clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
4957 
4958 	local_bh_disable();
4959 
4960 	/* All we really want here is to re-enable device interrupts.
4961 	 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
4962 	 */
4963 	rc = napi->poll(napi, BUSY_POLL_BUDGET);
4964 	netpoll_poll_unlock(have_poll_lock);
4965 	if (rc == BUSY_POLL_BUDGET)
4966 		__napi_schedule(napi);
4967 	local_bh_enable();
4968 	if (local_softirq_pending())
4969 		do_softirq();
4970 }
4971 
4972 bool sk_busy_loop(struct sock *sk, int nonblock)
4973 {
4974 	unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0;
4975 	int (*napi_poll)(struct napi_struct *napi, int budget);
4976 	void *have_poll_lock = NULL;
4977 	struct napi_struct *napi;
4978 	int rc;
4979 
4980 restart:
4981 	rc = false;
4982 	napi_poll = NULL;
4983 
4984 	rcu_read_lock();
4985 
4986 	napi = napi_by_id(sk->sk_napi_id);
4987 	if (!napi)
4988 		goto out;
4989 
4990 	preempt_disable();
4991 	for (;;) {
4992 		rc = 0;
4993 		local_bh_disable();
4994 		if (!napi_poll) {
4995 			unsigned long val = READ_ONCE(napi->state);
4996 
4997 			/* If multiple threads are competing for this napi,
4998 			 * we avoid dirtying napi->state as much as we can.
4999 			 */
5000 			if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
5001 				   NAPIF_STATE_IN_BUSY_POLL))
5002 				goto count;
5003 			if (cmpxchg(&napi->state, val,
5004 				    val | NAPIF_STATE_IN_BUSY_POLL |
5005 					  NAPIF_STATE_SCHED) != val)
5006 				goto count;
5007 			have_poll_lock = netpoll_poll_lock(napi);
5008 			napi_poll = napi->poll;
5009 		}
5010 		rc = napi_poll(napi, BUSY_POLL_BUDGET);
5011 		trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);
5012 count:
5013 		if (rc > 0)
5014 			__NET_ADD_STATS(sock_net(sk),
5015 					LINUX_MIB_BUSYPOLLRXPACKETS, rc);
5016 		local_bh_enable();
5017 
5018 		if (nonblock || !skb_queue_empty(&sk->sk_receive_queue) ||
5019 		    busy_loop_timeout(end_time))
5020 			break;
5021 
5022 		if (unlikely(need_resched())) {
5023 			if (napi_poll)
5024 				busy_poll_stop(napi, have_poll_lock);
5025 			preempt_enable();
5026 			rcu_read_unlock();
5027 			cond_resched();
5028 			rc = !skb_queue_empty(&sk->sk_receive_queue);
5029 			if (rc || busy_loop_timeout(end_time))
5030 				return rc;
5031 			goto restart;
5032 		}
5033 		cpu_relax();
5034 	}
5035 	if (napi_poll)
5036 		busy_poll_stop(napi, have_poll_lock);
5037 	preempt_enable();
5038 	rc = !skb_queue_empty(&sk->sk_receive_queue);
5039 out:
5040 	rcu_read_unlock();
5041 	return rc;
5042 }
5043 EXPORT_SYMBOL(sk_busy_loop);
5044 
5045 #endif /* CONFIG_NET_RX_BUSY_POLL */
5046 
5047 static void napi_hash_add(struct napi_struct *napi)
5048 {
5049 	if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
5050 	    test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
5051 		return;
5052 
5053 	spin_lock(&napi_hash_lock);
5054 
5055 	/* 0..NR_CPUS+1 range is reserved for sender_cpu use */
5056 	do {
5057 		if (unlikely(++napi_gen_id < NR_CPUS + 1))
5058 			napi_gen_id = NR_CPUS + 1;
5059 	} while (napi_by_id(napi_gen_id));
5060 	napi->napi_id = napi_gen_id;
5061 
5062 	hlist_add_head_rcu(&napi->napi_hash_node,
5063 			   &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
5064 
5065 	spin_unlock(&napi_hash_lock);
5066 }
5067 
5068 /* Warning : caller is responsible to make sure rcu grace period
5069  * is respected before freeing memory containing @napi
5070  */
5071 bool napi_hash_del(struct napi_struct *napi)
5072 {
5073 	bool rcu_sync_needed = false;
5074 
5075 	spin_lock(&napi_hash_lock);
5076 
5077 	if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
5078 		rcu_sync_needed = true;
5079 		hlist_del_rcu(&napi->napi_hash_node);
5080 	}
5081 	spin_unlock(&napi_hash_lock);
5082 	return rcu_sync_needed;
5083 }
5084 EXPORT_SYMBOL_GPL(napi_hash_del);
5085 
5086 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
5087 {
5088 	struct napi_struct *napi;
5089 
5090 	napi = container_of(timer, struct napi_struct, timer);
5091 	if (napi->gro_list)
5092 		napi_schedule_irqoff(napi);
5093 
5094 	return HRTIMER_NORESTART;
5095 }
5096 
5097 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
5098 		    int (*poll)(struct napi_struct *, int), int weight)
5099 {
5100 	INIT_LIST_HEAD(&napi->poll_list);
5101 	hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
5102 	napi->timer.function = napi_watchdog;
5103 	napi->gro_count = 0;
5104 	napi->gro_list = NULL;
5105 	napi->skb = NULL;
5106 	napi->poll = poll;
5107 	if (weight > NAPI_POLL_WEIGHT)
5108 		pr_err_once("netif_napi_add() called with weight %d on device %s\n",
5109 			    weight, dev->name);
5110 	napi->weight = weight;
5111 	list_add(&napi->dev_list, &dev->napi_list);
5112 	napi->dev = dev;
5113 #ifdef CONFIG_NETPOLL
5114 	napi->poll_owner = -1;
5115 #endif
5116 	set_bit(NAPI_STATE_SCHED, &napi->state);
5117 	napi_hash_add(napi);
5118 }
5119 EXPORT_SYMBOL(netif_napi_add);
5120 
5121 void napi_disable(struct napi_struct *n)
5122 {
5123 	might_sleep();
5124 	set_bit(NAPI_STATE_DISABLE, &n->state);
5125 
5126 	while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
5127 		msleep(1);
5128 	while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
5129 		msleep(1);
5130 
5131 	hrtimer_cancel(&n->timer);
5132 
5133 	clear_bit(NAPI_STATE_DISABLE, &n->state);
5134 }
5135 EXPORT_SYMBOL(napi_disable);
5136 
5137 /* Must be called in process context */
5138 void netif_napi_del(struct napi_struct *napi)
5139 {
5140 	might_sleep();
5141 	if (napi_hash_del(napi))
5142 		synchronize_net();
5143 	list_del_init(&napi->dev_list);
5144 	napi_free_frags(napi);
5145 
5146 	kfree_skb_list(napi->gro_list);
5147 	napi->gro_list = NULL;
5148 	napi->gro_count = 0;
5149 }
5150 EXPORT_SYMBOL(netif_napi_del);
5151 
5152 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
5153 {
5154 	void *have;
5155 	int work, weight;
5156 
5157 	list_del_init(&n->poll_list);
5158 
5159 	have = netpoll_poll_lock(n);
5160 
5161 	weight = n->weight;
5162 
5163 	/* This NAPI_STATE_SCHED test is for avoiding a race
5164 	 * with netpoll's poll_napi().  Only the entity which
5165 	 * obtains the lock and sees NAPI_STATE_SCHED set will
5166 	 * actually make the ->poll() call.  Therefore we avoid
5167 	 * accidentally calling ->poll() when NAPI is not scheduled.
5168 	 */
5169 	work = 0;
5170 	if (test_bit(NAPI_STATE_SCHED, &n->state)) {
5171 		work = n->poll(n, weight);
5172 		trace_napi_poll(n, work, weight);
5173 	}
5174 
5175 	WARN_ON_ONCE(work > weight);
5176 
5177 	if (likely(work < weight))
5178 		goto out_unlock;
5179 
5180 	/* Drivers must not modify the NAPI state if they
5181 	 * consume the entire weight.  In such cases this code
5182 	 * still "owns" the NAPI instance and therefore can
5183 	 * move the instance around on the list at-will.
5184 	 */
5185 	if (unlikely(napi_disable_pending(n))) {
5186 		napi_complete(n);
5187 		goto out_unlock;
5188 	}
5189 
5190 	if (n->gro_list) {
5191 		/* flush too old packets
5192 		 * If HZ < 1000, flush all packets.
5193 		 */
5194 		napi_gro_flush(n, HZ >= 1000);
5195 	}
5196 
5197 	/* Some drivers may have called napi_schedule
5198 	 * prior to exhausting their budget.
5199 	 */
5200 	if (unlikely(!list_empty(&n->poll_list))) {
5201 		pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
5202 			     n->dev ? n->dev->name : "backlog");
5203 		goto out_unlock;
5204 	}
5205 
5206 	list_add_tail(&n->poll_list, repoll);
5207 
5208 out_unlock:
5209 	netpoll_poll_unlock(have);
5210 
5211 	return work;
5212 }
5213 
5214 static __latent_entropy void net_rx_action(struct softirq_action *h)
5215 {
5216 	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5217 	unsigned long time_limit = jiffies + 2;
5218 	int budget = netdev_budget;
5219 	LIST_HEAD(list);
5220 	LIST_HEAD(repoll);
5221 
5222 	local_irq_disable();
5223 	list_splice_init(&sd->poll_list, &list);
5224 	local_irq_enable();
5225 
5226 	for (;;) {
5227 		struct napi_struct *n;
5228 
5229 		if (list_empty(&list)) {
5230 			if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
5231 				goto out;
5232 			break;
5233 		}
5234 
5235 		n = list_first_entry(&list, struct napi_struct, poll_list);
5236 		budget -= napi_poll(n, &repoll);
5237 
5238 		/* If softirq window is exhausted then punt.
5239 		 * Allow this to run for 2 jiffies since which will allow
5240 		 * an average latency of 1.5/HZ.
5241 		 */
5242 		if (unlikely(budget <= 0 ||
5243 			     time_after_eq(jiffies, time_limit))) {
5244 			sd->time_squeeze++;
5245 			break;
5246 		}
5247 	}
5248 
5249 	local_irq_disable();
5250 
5251 	list_splice_tail_init(&sd->poll_list, &list);
5252 	list_splice_tail(&repoll, &list);
5253 	list_splice(&list, &sd->poll_list);
5254 	if (!list_empty(&sd->poll_list))
5255 		__raise_softirq_irqoff(NET_RX_SOFTIRQ);
5256 
5257 	net_rps_action_and_irq_enable(sd);
5258 out:
5259 	__kfree_skb_flush();
5260 }
5261 
5262 struct netdev_adjacent {
5263 	struct net_device *dev;
5264 
5265 	/* upper master flag, there can only be one master device per list */
5266 	bool master;
5267 
5268 	/* counter for the number of times this device was added to us */
5269 	u16 ref_nr;
5270 
5271 	/* private field for the users */
5272 	void *private;
5273 
5274 	struct list_head list;
5275 	struct rcu_head rcu;
5276 };
5277 
5278 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
5279 						 struct list_head *adj_list)
5280 {
5281 	struct netdev_adjacent *adj;
5282 
5283 	list_for_each_entry(adj, adj_list, list) {
5284 		if (adj->dev == adj_dev)
5285 			return adj;
5286 	}
5287 	return NULL;
5288 }
5289 
5290 static int __netdev_has_upper_dev(struct net_device *upper_dev, void *data)
5291 {
5292 	struct net_device *dev = data;
5293 
5294 	return upper_dev == dev;
5295 }
5296 
5297 /**
5298  * netdev_has_upper_dev - Check if device is linked to an upper device
5299  * @dev: device
5300  * @upper_dev: upper device to check
5301  *
5302  * Find out if a device is linked to specified upper device and return true
5303  * in case it is. Note that this checks only immediate upper device,
5304  * not through a complete stack of devices. The caller must hold the RTNL lock.
5305  */
5306 bool netdev_has_upper_dev(struct net_device *dev,
5307 			  struct net_device *upper_dev)
5308 {
5309 	ASSERT_RTNL();
5310 
5311 	return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5312 					     upper_dev);
5313 }
5314 EXPORT_SYMBOL(netdev_has_upper_dev);
5315 
5316 /**
5317  * netdev_has_upper_dev_all - Check if device is linked to an upper device
5318  * @dev: device
5319  * @upper_dev: upper device to check
5320  *
5321  * Find out if a device is linked to specified upper device and return true
5322  * in case it is. Note that this checks the entire upper device chain.
5323  * The caller must hold rcu lock.
5324  */
5325 
5326 bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
5327 				  struct net_device *upper_dev)
5328 {
5329 	return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
5330 					       upper_dev);
5331 }
5332 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
5333 
5334 /**
5335  * netdev_has_any_upper_dev - Check if device is linked to some device
5336  * @dev: device
5337  *
5338  * Find out if a device is linked to an upper device and return true in case
5339  * it is. The caller must hold the RTNL lock.
5340  */
5341 static bool netdev_has_any_upper_dev(struct net_device *dev)
5342 {
5343 	ASSERT_RTNL();
5344 
5345 	return !list_empty(&dev->adj_list.upper);
5346 }
5347 
5348 /**
5349  * netdev_master_upper_dev_get - Get master upper device
5350  * @dev: device
5351  *
5352  * Find a master upper device and return pointer to it or NULL in case
5353  * it's not there. The caller must hold the RTNL lock.
5354  */
5355 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
5356 {
5357 	struct netdev_adjacent *upper;
5358 
5359 	ASSERT_RTNL();
5360 
5361 	if (list_empty(&dev->adj_list.upper))
5362 		return NULL;
5363 
5364 	upper = list_first_entry(&dev->adj_list.upper,
5365 				 struct netdev_adjacent, list);
5366 	if (likely(upper->master))
5367 		return upper->dev;
5368 	return NULL;
5369 }
5370 EXPORT_SYMBOL(netdev_master_upper_dev_get);
5371 
5372 /**
5373  * netdev_has_any_lower_dev - Check if device is linked to some device
5374  * @dev: device
5375  *
5376  * Find out if a device is linked to a lower device and return true in case
5377  * it is. The caller must hold the RTNL lock.
5378  */
5379 static bool netdev_has_any_lower_dev(struct net_device *dev)
5380 {
5381 	ASSERT_RTNL();
5382 
5383 	return !list_empty(&dev->adj_list.lower);
5384 }
5385 
5386 void *netdev_adjacent_get_private(struct list_head *adj_list)
5387 {
5388 	struct netdev_adjacent *adj;
5389 
5390 	adj = list_entry(adj_list, struct netdev_adjacent, list);
5391 
5392 	return adj->private;
5393 }
5394 EXPORT_SYMBOL(netdev_adjacent_get_private);
5395 
5396 /**
5397  * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
5398  * @dev: device
5399  * @iter: list_head ** of the current position
5400  *
5401  * Gets the next device from the dev's upper list, starting from iter
5402  * position. The caller must hold RCU read lock.
5403  */
5404 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5405 						 struct list_head **iter)
5406 {
5407 	struct netdev_adjacent *upper;
5408 
5409 	WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5410 
5411 	upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5412 
5413 	if (&upper->list == &dev->adj_list.upper)
5414 		return NULL;
5415 
5416 	*iter = &upper->list;
5417 
5418 	return upper->dev;
5419 }
5420 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5421 
5422 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
5423 						    struct list_head **iter)
5424 {
5425 	struct netdev_adjacent *upper;
5426 
5427 	WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5428 
5429 	upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5430 
5431 	if (&upper->list == &dev->adj_list.upper)
5432 		return NULL;
5433 
5434 	*iter = &upper->list;
5435 
5436 	return upper->dev;
5437 }
5438 
5439 int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
5440 				  int (*fn)(struct net_device *dev,
5441 					    void *data),
5442 				  void *data)
5443 {
5444 	struct net_device *udev;
5445 	struct list_head *iter;
5446 	int ret;
5447 
5448 	for (iter = &dev->adj_list.upper,
5449 	     udev = netdev_next_upper_dev_rcu(dev, &iter);
5450 	     udev;
5451 	     udev = netdev_next_upper_dev_rcu(dev, &iter)) {
5452 		/* first is the upper device itself */
5453 		ret = fn(udev, data);
5454 		if (ret)
5455 			return ret;
5456 
5457 		/* then look at all of its upper devices */
5458 		ret = netdev_walk_all_upper_dev_rcu(udev, fn, data);
5459 		if (ret)
5460 			return ret;
5461 	}
5462 
5463 	return 0;
5464 }
5465 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
5466 
5467 /**
5468  * netdev_lower_get_next_private - Get the next ->private from the
5469  *				   lower neighbour list
5470  * @dev: device
5471  * @iter: list_head ** of the current position
5472  *
5473  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5474  * list, starting from iter position. The caller must hold either hold the
5475  * RTNL lock or its own locking that guarantees that the neighbour lower
5476  * list will remain unchanged.
5477  */
5478 void *netdev_lower_get_next_private(struct net_device *dev,
5479 				    struct list_head **iter)
5480 {
5481 	struct netdev_adjacent *lower;
5482 
5483 	lower = list_entry(*iter, struct netdev_adjacent, list);
5484 
5485 	if (&lower->list == &dev->adj_list.lower)
5486 		return NULL;
5487 
5488 	*iter = lower->list.next;
5489 
5490 	return lower->private;
5491 }
5492 EXPORT_SYMBOL(netdev_lower_get_next_private);
5493 
5494 /**
5495  * netdev_lower_get_next_private_rcu - Get the next ->private from the
5496  *				       lower neighbour list, RCU
5497  *				       variant
5498  * @dev: device
5499  * @iter: list_head ** of the current position
5500  *
5501  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5502  * list, starting from iter position. The caller must hold RCU read lock.
5503  */
5504 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5505 					struct list_head **iter)
5506 {
5507 	struct netdev_adjacent *lower;
5508 
5509 	WARN_ON_ONCE(!rcu_read_lock_held());
5510 
5511 	lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5512 
5513 	if (&lower->list == &dev->adj_list.lower)
5514 		return NULL;
5515 
5516 	*iter = &lower->list;
5517 
5518 	return lower->private;
5519 }
5520 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5521 
5522 /**
5523  * netdev_lower_get_next - Get the next device from the lower neighbour
5524  *                         list
5525  * @dev: device
5526  * @iter: list_head ** of the current position
5527  *
5528  * Gets the next netdev_adjacent from the dev's lower neighbour
5529  * list, starting from iter position. The caller must hold RTNL lock or
5530  * its own locking that guarantees that the neighbour lower
5531  * list will remain unchanged.
5532  */
5533 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5534 {
5535 	struct netdev_adjacent *lower;
5536 
5537 	lower = list_entry(*iter, struct netdev_adjacent, list);
5538 
5539 	if (&lower->list == &dev->adj_list.lower)
5540 		return NULL;
5541 
5542 	*iter = lower->list.next;
5543 
5544 	return lower->dev;
5545 }
5546 EXPORT_SYMBOL(netdev_lower_get_next);
5547 
5548 static struct net_device *netdev_next_lower_dev(struct net_device *dev,
5549 						struct list_head **iter)
5550 {
5551 	struct netdev_adjacent *lower;
5552 
5553 	lower = list_entry((*iter)->next, struct netdev_adjacent, list);
5554 
5555 	if (&lower->list == &dev->adj_list.lower)
5556 		return NULL;
5557 
5558 	*iter = &lower->list;
5559 
5560 	return lower->dev;
5561 }
5562 
5563 int netdev_walk_all_lower_dev(struct net_device *dev,
5564 			      int (*fn)(struct net_device *dev,
5565 					void *data),
5566 			      void *data)
5567 {
5568 	struct net_device *ldev;
5569 	struct list_head *iter;
5570 	int ret;
5571 
5572 	for (iter = &dev->adj_list.lower,
5573 	     ldev = netdev_next_lower_dev(dev, &iter);
5574 	     ldev;
5575 	     ldev = netdev_next_lower_dev(dev, &iter)) {
5576 		/* first is the lower device itself */
5577 		ret = fn(ldev, data);
5578 		if (ret)
5579 			return ret;
5580 
5581 		/* then look at all of its lower devices */
5582 		ret = netdev_walk_all_lower_dev(ldev, fn, data);
5583 		if (ret)
5584 			return ret;
5585 	}
5586 
5587 	return 0;
5588 }
5589 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
5590 
5591 static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
5592 						    struct list_head **iter)
5593 {
5594 	struct netdev_adjacent *lower;
5595 
5596 	lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5597 	if (&lower->list == &dev->adj_list.lower)
5598 		return NULL;
5599 
5600 	*iter = &lower->list;
5601 
5602 	return lower->dev;
5603 }
5604 
5605 int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
5606 				  int (*fn)(struct net_device *dev,
5607 					    void *data),
5608 				  void *data)
5609 {
5610 	struct net_device *ldev;
5611 	struct list_head *iter;
5612 	int ret;
5613 
5614 	for (iter = &dev->adj_list.lower,
5615 	     ldev = netdev_next_lower_dev_rcu(dev, &iter);
5616 	     ldev;
5617 	     ldev = netdev_next_lower_dev_rcu(dev, &iter)) {
5618 		/* first is the lower device itself */
5619 		ret = fn(ldev, data);
5620 		if (ret)
5621 			return ret;
5622 
5623 		/* then look at all of its lower devices */
5624 		ret = netdev_walk_all_lower_dev_rcu(ldev, fn, data);
5625 		if (ret)
5626 			return ret;
5627 	}
5628 
5629 	return 0;
5630 }
5631 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
5632 
5633 /**
5634  * netdev_lower_get_first_private_rcu - Get the first ->private from the
5635  *				       lower neighbour list, RCU
5636  *				       variant
5637  * @dev: device
5638  *
5639  * Gets the first netdev_adjacent->private from the dev's lower neighbour
5640  * list. The caller must hold RCU read lock.
5641  */
5642 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
5643 {
5644 	struct netdev_adjacent *lower;
5645 
5646 	lower = list_first_or_null_rcu(&dev->adj_list.lower,
5647 			struct netdev_adjacent, list);
5648 	if (lower)
5649 		return lower->private;
5650 	return NULL;
5651 }
5652 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
5653 
5654 /**
5655  * netdev_master_upper_dev_get_rcu - Get master upper device
5656  * @dev: device
5657  *
5658  * Find a master upper device and return pointer to it or NULL in case
5659  * it's not there. The caller must hold the RCU read lock.
5660  */
5661 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
5662 {
5663 	struct netdev_adjacent *upper;
5664 
5665 	upper = list_first_or_null_rcu(&dev->adj_list.upper,
5666 				       struct netdev_adjacent, list);
5667 	if (upper && likely(upper->master))
5668 		return upper->dev;
5669 	return NULL;
5670 }
5671 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
5672 
5673 static int netdev_adjacent_sysfs_add(struct net_device *dev,
5674 			      struct net_device *adj_dev,
5675 			      struct list_head *dev_list)
5676 {
5677 	char linkname[IFNAMSIZ+7];
5678 
5679 	sprintf(linkname, dev_list == &dev->adj_list.upper ?
5680 		"upper_%s" : "lower_%s", adj_dev->name);
5681 	return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
5682 				 linkname);
5683 }
5684 static void netdev_adjacent_sysfs_del(struct net_device *dev,
5685 			       char *name,
5686 			       struct list_head *dev_list)
5687 {
5688 	char linkname[IFNAMSIZ+7];
5689 
5690 	sprintf(linkname, dev_list == &dev->adj_list.upper ?
5691 		"upper_%s" : "lower_%s", name);
5692 	sysfs_remove_link(&(dev->dev.kobj), linkname);
5693 }
5694 
5695 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
5696 						 struct net_device *adj_dev,
5697 						 struct list_head *dev_list)
5698 {
5699 	return (dev_list == &dev->adj_list.upper ||
5700 		dev_list == &dev->adj_list.lower) &&
5701 		net_eq(dev_net(dev), dev_net(adj_dev));
5702 }
5703 
5704 static int __netdev_adjacent_dev_insert(struct net_device *dev,
5705 					struct net_device *adj_dev,
5706 					struct list_head *dev_list,
5707 					void *private, bool master)
5708 {
5709 	struct netdev_adjacent *adj;
5710 	int ret;
5711 
5712 	adj = __netdev_find_adj(adj_dev, dev_list);
5713 
5714 	if (adj) {
5715 		adj->ref_nr += 1;
5716 		pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
5717 			 dev->name, adj_dev->name, adj->ref_nr);
5718 
5719 		return 0;
5720 	}
5721 
5722 	adj = kmalloc(sizeof(*adj), GFP_KERNEL);
5723 	if (!adj)
5724 		return -ENOMEM;
5725 
5726 	adj->dev = adj_dev;
5727 	adj->master = master;
5728 	adj->ref_nr = 1;
5729 	adj->private = private;
5730 	dev_hold(adj_dev);
5731 
5732 	pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
5733 		 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
5734 
5735 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
5736 		ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
5737 		if (ret)
5738 			goto free_adj;
5739 	}
5740 
5741 	/* Ensure that master link is always the first item in list. */
5742 	if (master) {
5743 		ret = sysfs_create_link(&(dev->dev.kobj),
5744 					&(adj_dev->dev.kobj), "master");
5745 		if (ret)
5746 			goto remove_symlinks;
5747 
5748 		list_add_rcu(&adj->list, dev_list);
5749 	} else {
5750 		list_add_tail_rcu(&adj->list, dev_list);
5751 	}
5752 
5753 	return 0;
5754 
5755 remove_symlinks:
5756 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5757 		netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5758 free_adj:
5759 	kfree(adj);
5760 	dev_put(adj_dev);
5761 
5762 	return ret;
5763 }
5764 
5765 static void __netdev_adjacent_dev_remove(struct net_device *dev,
5766 					 struct net_device *adj_dev,
5767 					 u16 ref_nr,
5768 					 struct list_head *dev_list)
5769 {
5770 	struct netdev_adjacent *adj;
5771 
5772 	pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
5773 		 dev->name, adj_dev->name, ref_nr);
5774 
5775 	adj = __netdev_find_adj(adj_dev, dev_list);
5776 
5777 	if (!adj) {
5778 		pr_err("Adjacency does not exist for device %s from %s\n",
5779 		       dev->name, adj_dev->name);
5780 		WARN_ON(1);
5781 		return;
5782 	}
5783 
5784 	if (adj->ref_nr > ref_nr) {
5785 		pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
5786 			 dev->name, adj_dev->name, ref_nr,
5787 			 adj->ref_nr - ref_nr);
5788 		adj->ref_nr -= ref_nr;
5789 		return;
5790 	}
5791 
5792 	if (adj->master)
5793 		sysfs_remove_link(&(dev->dev.kobj), "master");
5794 
5795 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5796 		netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5797 
5798 	list_del_rcu(&adj->list);
5799 	pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
5800 		 adj_dev->name, dev->name, adj_dev->name);
5801 	dev_put(adj_dev);
5802 	kfree_rcu(adj, rcu);
5803 }
5804 
5805 static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
5806 					    struct net_device *upper_dev,
5807 					    struct list_head *up_list,
5808 					    struct list_head *down_list,
5809 					    void *private, bool master)
5810 {
5811 	int ret;
5812 
5813 	ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
5814 					   private, master);
5815 	if (ret)
5816 		return ret;
5817 
5818 	ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
5819 					   private, false);
5820 	if (ret) {
5821 		__netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
5822 		return ret;
5823 	}
5824 
5825 	return 0;
5826 }
5827 
5828 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
5829 					       struct net_device *upper_dev,
5830 					       u16 ref_nr,
5831 					       struct list_head *up_list,
5832 					       struct list_head *down_list)
5833 {
5834 	__netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
5835 	__netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
5836 }
5837 
5838 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
5839 						struct net_device *upper_dev,
5840 						void *private, bool master)
5841 {
5842 	return __netdev_adjacent_dev_link_lists(dev, upper_dev,
5843 						&dev->adj_list.upper,
5844 						&upper_dev->adj_list.lower,
5845 						private, master);
5846 }
5847 
5848 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
5849 						   struct net_device *upper_dev)
5850 {
5851 	__netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
5852 					   &dev->adj_list.upper,
5853 					   &upper_dev->adj_list.lower);
5854 }
5855 
5856 static int __netdev_upper_dev_link(struct net_device *dev,
5857 				   struct net_device *upper_dev, bool master,
5858 				   void *upper_priv, void *upper_info)
5859 {
5860 	struct netdev_notifier_changeupper_info changeupper_info;
5861 	int ret = 0;
5862 
5863 	ASSERT_RTNL();
5864 
5865 	if (dev == upper_dev)
5866 		return -EBUSY;
5867 
5868 	/* To prevent loops, check if dev is not upper device to upper_dev. */
5869 	if (netdev_has_upper_dev(upper_dev, dev))
5870 		return -EBUSY;
5871 
5872 	if (netdev_has_upper_dev(dev, upper_dev))
5873 		return -EEXIST;
5874 
5875 	if (master && netdev_master_upper_dev_get(dev))
5876 		return -EBUSY;
5877 
5878 	changeupper_info.upper_dev = upper_dev;
5879 	changeupper_info.master = master;
5880 	changeupper_info.linking = true;
5881 	changeupper_info.upper_info = upper_info;
5882 
5883 	ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5884 					    &changeupper_info.info);
5885 	ret = notifier_to_errno(ret);
5886 	if (ret)
5887 		return ret;
5888 
5889 	ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
5890 						   master);
5891 	if (ret)
5892 		return ret;
5893 
5894 	ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5895 					    &changeupper_info.info);
5896 	ret = notifier_to_errno(ret);
5897 	if (ret)
5898 		goto rollback;
5899 
5900 	return 0;
5901 
5902 rollback:
5903 	__netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5904 
5905 	return ret;
5906 }
5907 
5908 /**
5909  * netdev_upper_dev_link - Add a link to the upper device
5910  * @dev: device
5911  * @upper_dev: new upper device
5912  *
5913  * Adds a link to device which is upper to this one. The caller must hold
5914  * the RTNL lock. On a failure a negative errno code is returned.
5915  * On success the reference counts are adjusted and the function
5916  * returns zero.
5917  */
5918 int netdev_upper_dev_link(struct net_device *dev,
5919 			  struct net_device *upper_dev)
5920 {
5921 	return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL);
5922 }
5923 EXPORT_SYMBOL(netdev_upper_dev_link);
5924 
5925 /**
5926  * netdev_master_upper_dev_link - Add a master link to the upper device
5927  * @dev: device
5928  * @upper_dev: new upper device
5929  * @upper_priv: upper device private
5930  * @upper_info: upper info to be passed down via notifier
5931  *
5932  * Adds a link to device which is upper to this one. In this case, only
5933  * one master upper device can be linked, although other non-master devices
5934  * might be linked as well. The caller must hold the RTNL lock.
5935  * On a failure a negative errno code is returned. On success the reference
5936  * counts are adjusted and the function returns zero.
5937  */
5938 int netdev_master_upper_dev_link(struct net_device *dev,
5939 				 struct net_device *upper_dev,
5940 				 void *upper_priv, void *upper_info)
5941 {
5942 	return __netdev_upper_dev_link(dev, upper_dev, true,
5943 				       upper_priv, upper_info);
5944 }
5945 EXPORT_SYMBOL(netdev_master_upper_dev_link);
5946 
5947 /**
5948  * netdev_upper_dev_unlink - Removes a link to upper device
5949  * @dev: device
5950  * @upper_dev: new upper device
5951  *
5952  * Removes a link to device which is upper to this one. The caller must hold
5953  * the RTNL lock.
5954  */
5955 void netdev_upper_dev_unlink(struct net_device *dev,
5956 			     struct net_device *upper_dev)
5957 {
5958 	struct netdev_notifier_changeupper_info changeupper_info;
5959 
5960 	ASSERT_RTNL();
5961 
5962 	changeupper_info.upper_dev = upper_dev;
5963 	changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
5964 	changeupper_info.linking = false;
5965 
5966 	call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5967 				      &changeupper_info.info);
5968 
5969 	__netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5970 
5971 	call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5972 				      &changeupper_info.info);
5973 }
5974 EXPORT_SYMBOL(netdev_upper_dev_unlink);
5975 
5976 /**
5977  * netdev_bonding_info_change - Dispatch event about slave change
5978  * @dev: device
5979  * @bonding_info: info to dispatch
5980  *
5981  * Send NETDEV_BONDING_INFO to netdev notifiers with info.
5982  * The caller must hold the RTNL lock.
5983  */
5984 void netdev_bonding_info_change(struct net_device *dev,
5985 				struct netdev_bonding_info *bonding_info)
5986 {
5987 	struct netdev_notifier_bonding_info	info;
5988 
5989 	memcpy(&info.bonding_info, bonding_info,
5990 	       sizeof(struct netdev_bonding_info));
5991 	call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
5992 				      &info.info);
5993 }
5994 EXPORT_SYMBOL(netdev_bonding_info_change);
5995 
5996 static void netdev_adjacent_add_links(struct net_device *dev)
5997 {
5998 	struct netdev_adjacent *iter;
5999 
6000 	struct net *net = dev_net(dev);
6001 
6002 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
6003 		if (!net_eq(net, dev_net(iter->dev)))
6004 			continue;
6005 		netdev_adjacent_sysfs_add(iter->dev, dev,
6006 					  &iter->dev->adj_list.lower);
6007 		netdev_adjacent_sysfs_add(dev, iter->dev,
6008 					  &dev->adj_list.upper);
6009 	}
6010 
6011 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
6012 		if (!net_eq(net, dev_net(iter->dev)))
6013 			continue;
6014 		netdev_adjacent_sysfs_add(iter->dev, dev,
6015 					  &iter->dev->adj_list.upper);
6016 		netdev_adjacent_sysfs_add(dev, iter->dev,
6017 					  &dev->adj_list.lower);
6018 	}
6019 }
6020 
6021 static void netdev_adjacent_del_links(struct net_device *dev)
6022 {
6023 	struct netdev_adjacent *iter;
6024 
6025 	struct net *net = dev_net(dev);
6026 
6027 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
6028 		if (!net_eq(net, dev_net(iter->dev)))
6029 			continue;
6030 		netdev_adjacent_sysfs_del(iter->dev, dev->name,
6031 					  &iter->dev->adj_list.lower);
6032 		netdev_adjacent_sysfs_del(dev, iter->dev->name,
6033 					  &dev->adj_list.upper);
6034 	}
6035 
6036 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
6037 		if (!net_eq(net, dev_net(iter->dev)))
6038 			continue;
6039 		netdev_adjacent_sysfs_del(iter->dev, dev->name,
6040 					  &iter->dev->adj_list.upper);
6041 		netdev_adjacent_sysfs_del(dev, iter->dev->name,
6042 					  &dev->adj_list.lower);
6043 	}
6044 }
6045 
6046 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
6047 {
6048 	struct netdev_adjacent *iter;
6049 
6050 	struct net *net = dev_net(dev);
6051 
6052 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
6053 		if (!net_eq(net, dev_net(iter->dev)))
6054 			continue;
6055 		netdev_adjacent_sysfs_del(iter->dev, oldname,
6056 					  &iter->dev->adj_list.lower);
6057 		netdev_adjacent_sysfs_add(iter->dev, dev,
6058 					  &iter->dev->adj_list.lower);
6059 	}
6060 
6061 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
6062 		if (!net_eq(net, dev_net(iter->dev)))
6063 			continue;
6064 		netdev_adjacent_sysfs_del(iter->dev, oldname,
6065 					  &iter->dev->adj_list.upper);
6066 		netdev_adjacent_sysfs_add(iter->dev, dev,
6067 					  &iter->dev->adj_list.upper);
6068 	}
6069 }
6070 
6071 void *netdev_lower_dev_get_private(struct net_device *dev,
6072 				   struct net_device *lower_dev)
6073 {
6074 	struct netdev_adjacent *lower;
6075 
6076 	if (!lower_dev)
6077 		return NULL;
6078 	lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
6079 	if (!lower)
6080 		return NULL;
6081 
6082 	return lower->private;
6083 }
6084 EXPORT_SYMBOL(netdev_lower_dev_get_private);
6085 
6086 
6087 int dev_get_nest_level(struct net_device *dev)
6088 {
6089 	struct net_device *lower = NULL;
6090 	struct list_head *iter;
6091 	int max_nest = -1;
6092 	int nest;
6093 
6094 	ASSERT_RTNL();
6095 
6096 	netdev_for_each_lower_dev(dev, lower, iter) {
6097 		nest = dev_get_nest_level(lower);
6098 		if (max_nest < nest)
6099 			max_nest = nest;
6100 	}
6101 
6102 	return max_nest + 1;
6103 }
6104 EXPORT_SYMBOL(dev_get_nest_level);
6105 
6106 /**
6107  * netdev_lower_change - Dispatch event about lower device state change
6108  * @lower_dev: device
6109  * @lower_state_info: state to dispatch
6110  *
6111  * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
6112  * The caller must hold the RTNL lock.
6113  */
6114 void netdev_lower_state_changed(struct net_device *lower_dev,
6115 				void *lower_state_info)
6116 {
6117 	struct netdev_notifier_changelowerstate_info changelowerstate_info;
6118 
6119 	ASSERT_RTNL();
6120 	changelowerstate_info.lower_state_info = lower_state_info;
6121 	call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev,
6122 				      &changelowerstate_info.info);
6123 }
6124 EXPORT_SYMBOL(netdev_lower_state_changed);
6125 
6126 static void dev_change_rx_flags(struct net_device *dev, int flags)
6127 {
6128 	const struct net_device_ops *ops = dev->netdev_ops;
6129 
6130 	if (ops->ndo_change_rx_flags)
6131 		ops->ndo_change_rx_flags(dev, flags);
6132 }
6133 
6134 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
6135 {
6136 	unsigned int old_flags = dev->flags;
6137 	kuid_t uid;
6138 	kgid_t gid;
6139 
6140 	ASSERT_RTNL();
6141 
6142 	dev->flags |= IFF_PROMISC;
6143 	dev->promiscuity += inc;
6144 	if (dev->promiscuity == 0) {
6145 		/*
6146 		 * Avoid overflow.
6147 		 * If inc causes overflow, untouch promisc and return error.
6148 		 */
6149 		if (inc < 0)
6150 			dev->flags &= ~IFF_PROMISC;
6151 		else {
6152 			dev->promiscuity -= inc;
6153 			pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
6154 				dev->name);
6155 			return -EOVERFLOW;
6156 		}
6157 	}
6158 	if (dev->flags != old_flags) {
6159 		pr_info("device %s %s promiscuous mode\n",
6160 			dev->name,
6161 			dev->flags & IFF_PROMISC ? "entered" : "left");
6162 		if (audit_enabled) {
6163 			current_uid_gid(&uid, &gid);
6164 			audit_log(current->audit_context, GFP_ATOMIC,
6165 				AUDIT_ANOM_PROMISCUOUS,
6166 				"dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
6167 				dev->name, (dev->flags & IFF_PROMISC),
6168 				(old_flags & IFF_PROMISC),
6169 				from_kuid(&init_user_ns, audit_get_loginuid(current)),
6170 				from_kuid(&init_user_ns, uid),
6171 				from_kgid(&init_user_ns, gid),
6172 				audit_get_sessionid(current));
6173 		}
6174 
6175 		dev_change_rx_flags(dev, IFF_PROMISC);
6176 	}
6177 	if (notify)
6178 		__dev_notify_flags(dev, old_flags, IFF_PROMISC);
6179 	return 0;
6180 }
6181 
6182 /**
6183  *	dev_set_promiscuity	- update promiscuity count on a device
6184  *	@dev: device
6185  *	@inc: modifier
6186  *
6187  *	Add or remove promiscuity from a device. While the count in the device
6188  *	remains above zero the interface remains promiscuous. Once it hits zero
6189  *	the device reverts back to normal filtering operation. A negative inc
6190  *	value is used to drop promiscuity on the device.
6191  *	Return 0 if successful or a negative errno code on error.
6192  */
6193 int dev_set_promiscuity(struct net_device *dev, int inc)
6194 {
6195 	unsigned int old_flags = dev->flags;
6196 	int err;
6197 
6198 	err = __dev_set_promiscuity(dev, inc, true);
6199 	if (err < 0)
6200 		return err;
6201 	if (dev->flags != old_flags)
6202 		dev_set_rx_mode(dev);
6203 	return err;
6204 }
6205 EXPORT_SYMBOL(dev_set_promiscuity);
6206 
6207 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
6208 {
6209 	unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
6210 
6211 	ASSERT_RTNL();
6212 
6213 	dev->flags |= IFF_ALLMULTI;
6214 	dev->allmulti += inc;
6215 	if (dev->allmulti == 0) {
6216 		/*
6217 		 * Avoid overflow.
6218 		 * If inc causes overflow, untouch allmulti and return error.
6219 		 */
6220 		if (inc < 0)
6221 			dev->flags &= ~IFF_ALLMULTI;
6222 		else {
6223 			dev->allmulti -= inc;
6224 			pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
6225 				dev->name);
6226 			return -EOVERFLOW;
6227 		}
6228 	}
6229 	if (dev->flags ^ old_flags) {
6230 		dev_change_rx_flags(dev, IFF_ALLMULTI);
6231 		dev_set_rx_mode(dev);
6232 		if (notify)
6233 			__dev_notify_flags(dev, old_flags,
6234 					   dev->gflags ^ old_gflags);
6235 	}
6236 	return 0;
6237 }
6238 
6239 /**
6240  *	dev_set_allmulti	- update allmulti count on a device
6241  *	@dev: device
6242  *	@inc: modifier
6243  *
6244  *	Add or remove reception of all multicast frames to a device. While the
6245  *	count in the device remains above zero the interface remains listening
6246  *	to all interfaces. Once it hits zero the device reverts back to normal
6247  *	filtering operation. A negative @inc value is used to drop the counter
6248  *	when releasing a resource needing all multicasts.
6249  *	Return 0 if successful or a negative errno code on error.
6250  */
6251 
6252 int dev_set_allmulti(struct net_device *dev, int inc)
6253 {
6254 	return __dev_set_allmulti(dev, inc, true);
6255 }
6256 EXPORT_SYMBOL(dev_set_allmulti);
6257 
6258 /*
6259  *	Upload unicast and multicast address lists to device and
6260  *	configure RX filtering. When the device doesn't support unicast
6261  *	filtering it is put in promiscuous mode while unicast addresses
6262  *	are present.
6263  */
6264 void __dev_set_rx_mode(struct net_device *dev)
6265 {
6266 	const struct net_device_ops *ops = dev->netdev_ops;
6267 
6268 	/* dev_open will call this function so the list will stay sane. */
6269 	if (!(dev->flags&IFF_UP))
6270 		return;
6271 
6272 	if (!netif_device_present(dev))
6273 		return;
6274 
6275 	if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
6276 		/* Unicast addresses changes may only happen under the rtnl,
6277 		 * therefore calling __dev_set_promiscuity here is safe.
6278 		 */
6279 		if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
6280 			__dev_set_promiscuity(dev, 1, false);
6281 			dev->uc_promisc = true;
6282 		} else if (netdev_uc_empty(dev) && dev->uc_promisc) {
6283 			__dev_set_promiscuity(dev, -1, false);
6284 			dev->uc_promisc = false;
6285 		}
6286 	}
6287 
6288 	if (ops->ndo_set_rx_mode)
6289 		ops->ndo_set_rx_mode(dev);
6290 }
6291 
6292 void dev_set_rx_mode(struct net_device *dev)
6293 {
6294 	netif_addr_lock_bh(dev);
6295 	__dev_set_rx_mode(dev);
6296 	netif_addr_unlock_bh(dev);
6297 }
6298 
6299 /**
6300  *	dev_get_flags - get flags reported to userspace
6301  *	@dev: device
6302  *
6303  *	Get the combination of flag bits exported through APIs to userspace.
6304  */
6305 unsigned int dev_get_flags(const struct net_device *dev)
6306 {
6307 	unsigned int flags;
6308 
6309 	flags = (dev->flags & ~(IFF_PROMISC |
6310 				IFF_ALLMULTI |
6311 				IFF_RUNNING |
6312 				IFF_LOWER_UP |
6313 				IFF_DORMANT)) |
6314 		(dev->gflags & (IFF_PROMISC |
6315 				IFF_ALLMULTI));
6316 
6317 	if (netif_running(dev)) {
6318 		if (netif_oper_up(dev))
6319 			flags |= IFF_RUNNING;
6320 		if (netif_carrier_ok(dev))
6321 			flags |= IFF_LOWER_UP;
6322 		if (netif_dormant(dev))
6323 			flags |= IFF_DORMANT;
6324 	}
6325 
6326 	return flags;
6327 }
6328 EXPORT_SYMBOL(dev_get_flags);
6329 
6330 int __dev_change_flags(struct net_device *dev, unsigned int flags)
6331 {
6332 	unsigned int old_flags = dev->flags;
6333 	int ret;
6334 
6335 	ASSERT_RTNL();
6336 
6337 	/*
6338 	 *	Set the flags on our device.
6339 	 */
6340 
6341 	dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
6342 			       IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
6343 			       IFF_AUTOMEDIA)) |
6344 		     (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
6345 				    IFF_ALLMULTI));
6346 
6347 	/*
6348 	 *	Load in the correct multicast list now the flags have changed.
6349 	 */
6350 
6351 	if ((old_flags ^ flags) & IFF_MULTICAST)
6352 		dev_change_rx_flags(dev, IFF_MULTICAST);
6353 
6354 	dev_set_rx_mode(dev);
6355 
6356 	/*
6357 	 *	Have we downed the interface. We handle IFF_UP ourselves
6358 	 *	according to user attempts to set it, rather than blindly
6359 	 *	setting it.
6360 	 */
6361 
6362 	ret = 0;
6363 	if ((old_flags ^ flags) & IFF_UP)
6364 		ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
6365 
6366 	if ((flags ^ dev->gflags) & IFF_PROMISC) {
6367 		int inc = (flags & IFF_PROMISC) ? 1 : -1;
6368 		unsigned int old_flags = dev->flags;
6369 
6370 		dev->gflags ^= IFF_PROMISC;
6371 
6372 		if (__dev_set_promiscuity(dev, inc, false) >= 0)
6373 			if (dev->flags != old_flags)
6374 				dev_set_rx_mode(dev);
6375 	}
6376 
6377 	/* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
6378 	 * is important. Some (broken) drivers set IFF_PROMISC, when
6379 	 * IFF_ALLMULTI is requested not asking us and not reporting.
6380 	 */
6381 	if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
6382 		int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
6383 
6384 		dev->gflags ^= IFF_ALLMULTI;
6385 		__dev_set_allmulti(dev, inc, false);
6386 	}
6387 
6388 	return ret;
6389 }
6390 
6391 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
6392 			unsigned int gchanges)
6393 {
6394 	unsigned int changes = dev->flags ^ old_flags;
6395 
6396 	if (gchanges)
6397 		rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
6398 
6399 	if (changes & IFF_UP) {
6400 		if (dev->flags & IFF_UP)
6401 			call_netdevice_notifiers(NETDEV_UP, dev);
6402 		else
6403 			call_netdevice_notifiers(NETDEV_DOWN, dev);
6404 	}
6405 
6406 	if (dev->flags & IFF_UP &&
6407 	    (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
6408 		struct netdev_notifier_change_info change_info;
6409 
6410 		change_info.flags_changed = changes;
6411 		call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
6412 					      &change_info.info);
6413 	}
6414 }
6415 
6416 /**
6417  *	dev_change_flags - change device settings
6418  *	@dev: device
6419  *	@flags: device state flags
6420  *
6421  *	Change settings on device based state flags. The flags are
6422  *	in the userspace exported format.
6423  */
6424 int dev_change_flags(struct net_device *dev, unsigned int flags)
6425 {
6426 	int ret;
6427 	unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
6428 
6429 	ret = __dev_change_flags(dev, flags);
6430 	if (ret < 0)
6431 		return ret;
6432 
6433 	changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
6434 	__dev_notify_flags(dev, old_flags, changes);
6435 	return ret;
6436 }
6437 EXPORT_SYMBOL(dev_change_flags);
6438 
6439 static int __dev_set_mtu(struct net_device *dev, int new_mtu)
6440 {
6441 	const struct net_device_ops *ops = dev->netdev_ops;
6442 
6443 	if (ops->ndo_change_mtu)
6444 		return ops->ndo_change_mtu(dev, new_mtu);
6445 
6446 	dev->mtu = new_mtu;
6447 	return 0;
6448 }
6449 
6450 /**
6451  *	dev_set_mtu - Change maximum transfer unit
6452  *	@dev: device
6453  *	@new_mtu: new transfer unit
6454  *
6455  *	Change the maximum transfer size of the network device.
6456  */
6457 int dev_set_mtu(struct net_device *dev, int new_mtu)
6458 {
6459 	int err, orig_mtu;
6460 
6461 	if (new_mtu == dev->mtu)
6462 		return 0;
6463 
6464 	/* MTU must be positive, and in range */
6465 	if (new_mtu < 0 || new_mtu < dev->min_mtu) {
6466 		net_err_ratelimited("%s: Invalid MTU %d requested, hw min %d\n",
6467 				    dev->name, new_mtu, dev->min_mtu);
6468 		return -EINVAL;
6469 	}
6470 
6471 	if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
6472 		net_err_ratelimited("%s: Invalid MTU %d requested, hw max %d\n",
6473 				    dev->name, new_mtu, dev->max_mtu);
6474 		return -EINVAL;
6475 	}
6476 
6477 	if (!netif_device_present(dev))
6478 		return -ENODEV;
6479 
6480 	err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
6481 	err = notifier_to_errno(err);
6482 	if (err)
6483 		return err;
6484 
6485 	orig_mtu = dev->mtu;
6486 	err = __dev_set_mtu(dev, new_mtu);
6487 
6488 	if (!err) {
6489 		err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6490 		err = notifier_to_errno(err);
6491 		if (err) {
6492 			/* setting mtu back and notifying everyone again,
6493 			 * so that they have a chance to revert changes.
6494 			 */
6495 			__dev_set_mtu(dev, orig_mtu);
6496 			call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6497 		}
6498 	}
6499 	return err;
6500 }
6501 EXPORT_SYMBOL(dev_set_mtu);
6502 
6503 /**
6504  *	dev_set_group - Change group this device belongs to
6505  *	@dev: device
6506  *	@new_group: group this device should belong to
6507  */
6508 void dev_set_group(struct net_device *dev, int new_group)
6509 {
6510 	dev->group = new_group;
6511 }
6512 EXPORT_SYMBOL(dev_set_group);
6513 
6514 /**
6515  *	dev_set_mac_address - Change Media Access Control Address
6516  *	@dev: device
6517  *	@sa: new address
6518  *
6519  *	Change the hardware (MAC) address of the device
6520  */
6521 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
6522 {
6523 	const struct net_device_ops *ops = dev->netdev_ops;
6524 	int err;
6525 
6526 	if (!ops->ndo_set_mac_address)
6527 		return -EOPNOTSUPP;
6528 	if (sa->sa_family != dev->type)
6529 		return -EINVAL;
6530 	if (!netif_device_present(dev))
6531 		return -ENODEV;
6532 	err = ops->ndo_set_mac_address(dev, sa);
6533 	if (err)
6534 		return err;
6535 	dev->addr_assign_type = NET_ADDR_SET;
6536 	call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
6537 	add_device_randomness(dev->dev_addr, dev->addr_len);
6538 	return 0;
6539 }
6540 EXPORT_SYMBOL(dev_set_mac_address);
6541 
6542 /**
6543  *	dev_change_carrier - Change device carrier
6544  *	@dev: device
6545  *	@new_carrier: new value
6546  *
6547  *	Change device carrier
6548  */
6549 int dev_change_carrier(struct net_device *dev, bool new_carrier)
6550 {
6551 	const struct net_device_ops *ops = dev->netdev_ops;
6552 
6553 	if (!ops->ndo_change_carrier)
6554 		return -EOPNOTSUPP;
6555 	if (!netif_device_present(dev))
6556 		return -ENODEV;
6557 	return ops->ndo_change_carrier(dev, new_carrier);
6558 }
6559 EXPORT_SYMBOL(dev_change_carrier);
6560 
6561 /**
6562  *	dev_get_phys_port_id - Get device physical port ID
6563  *	@dev: device
6564  *	@ppid: port ID
6565  *
6566  *	Get device physical port ID
6567  */
6568 int dev_get_phys_port_id(struct net_device *dev,
6569 			 struct netdev_phys_item_id *ppid)
6570 {
6571 	const struct net_device_ops *ops = dev->netdev_ops;
6572 
6573 	if (!ops->ndo_get_phys_port_id)
6574 		return -EOPNOTSUPP;
6575 	return ops->ndo_get_phys_port_id(dev, ppid);
6576 }
6577 EXPORT_SYMBOL(dev_get_phys_port_id);
6578 
6579 /**
6580  *	dev_get_phys_port_name - Get device physical port name
6581  *	@dev: device
6582  *	@name: port name
6583  *	@len: limit of bytes to copy to name
6584  *
6585  *	Get device physical port name
6586  */
6587 int dev_get_phys_port_name(struct net_device *dev,
6588 			   char *name, size_t len)
6589 {
6590 	const struct net_device_ops *ops = dev->netdev_ops;
6591 
6592 	if (!ops->ndo_get_phys_port_name)
6593 		return -EOPNOTSUPP;
6594 	return ops->ndo_get_phys_port_name(dev, name, len);
6595 }
6596 EXPORT_SYMBOL(dev_get_phys_port_name);
6597 
6598 /**
6599  *	dev_change_proto_down - update protocol port state information
6600  *	@dev: device
6601  *	@proto_down: new value
6602  *
6603  *	This info can be used by switch drivers to set the phys state of the
6604  *	port.
6605  */
6606 int dev_change_proto_down(struct net_device *dev, bool proto_down)
6607 {
6608 	const struct net_device_ops *ops = dev->netdev_ops;
6609 
6610 	if (!ops->ndo_change_proto_down)
6611 		return -EOPNOTSUPP;
6612 	if (!netif_device_present(dev))
6613 		return -ENODEV;
6614 	return ops->ndo_change_proto_down(dev, proto_down);
6615 }
6616 EXPORT_SYMBOL(dev_change_proto_down);
6617 
6618 /**
6619  *	dev_change_xdp_fd - set or clear a bpf program for a device rx path
6620  *	@dev: device
6621  *	@fd: new program fd or negative value to clear
6622  *	@flags: xdp-related flags
6623  *
6624  *	Set or clear a bpf program for a device
6625  */
6626 int dev_change_xdp_fd(struct net_device *dev, int fd, u32 flags)
6627 {
6628 	const struct net_device_ops *ops = dev->netdev_ops;
6629 	struct bpf_prog *prog = NULL;
6630 	struct netdev_xdp xdp;
6631 	int err;
6632 
6633 	ASSERT_RTNL();
6634 
6635 	if (!ops->ndo_xdp)
6636 		return -EOPNOTSUPP;
6637 	if (fd >= 0) {
6638 		if (flags & XDP_FLAGS_UPDATE_IF_NOEXIST) {
6639 			memset(&xdp, 0, sizeof(xdp));
6640 			xdp.command = XDP_QUERY_PROG;
6641 
6642 			err = ops->ndo_xdp(dev, &xdp);
6643 			if (err < 0)
6644 				return err;
6645 			if (xdp.prog_attached)
6646 				return -EBUSY;
6647 		}
6648 
6649 		prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_XDP);
6650 		if (IS_ERR(prog))
6651 			return PTR_ERR(prog);
6652 	}
6653 
6654 	memset(&xdp, 0, sizeof(xdp));
6655 	xdp.command = XDP_SETUP_PROG;
6656 	xdp.prog = prog;
6657 
6658 	err = ops->ndo_xdp(dev, &xdp);
6659 	if (err < 0 && prog)
6660 		bpf_prog_put(prog);
6661 
6662 	return err;
6663 }
6664 EXPORT_SYMBOL(dev_change_xdp_fd);
6665 
6666 /**
6667  *	dev_new_index	-	allocate an ifindex
6668  *	@net: the applicable net namespace
6669  *
6670  *	Returns a suitable unique value for a new device interface
6671  *	number.  The caller must hold the rtnl semaphore or the
6672  *	dev_base_lock to be sure it remains unique.
6673  */
6674 static int dev_new_index(struct net *net)
6675 {
6676 	int ifindex = net->ifindex;
6677 
6678 	for (;;) {
6679 		if (++ifindex <= 0)
6680 			ifindex = 1;
6681 		if (!__dev_get_by_index(net, ifindex))
6682 			return net->ifindex = ifindex;
6683 	}
6684 }
6685 
6686 /* Delayed registration/unregisteration */
6687 static LIST_HEAD(net_todo_list);
6688 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
6689 
6690 static void net_set_todo(struct net_device *dev)
6691 {
6692 	list_add_tail(&dev->todo_list, &net_todo_list);
6693 	dev_net(dev)->dev_unreg_count++;
6694 }
6695 
6696 static void rollback_registered_many(struct list_head *head)
6697 {
6698 	struct net_device *dev, *tmp;
6699 	LIST_HEAD(close_head);
6700 
6701 	BUG_ON(dev_boot_phase);
6702 	ASSERT_RTNL();
6703 
6704 	list_for_each_entry_safe(dev, tmp, head, unreg_list) {
6705 		/* Some devices call without registering
6706 		 * for initialization unwind. Remove those
6707 		 * devices and proceed with the remaining.
6708 		 */
6709 		if (dev->reg_state == NETREG_UNINITIALIZED) {
6710 			pr_debug("unregister_netdevice: device %s/%p never was registered\n",
6711 				 dev->name, dev);
6712 
6713 			WARN_ON(1);
6714 			list_del(&dev->unreg_list);
6715 			continue;
6716 		}
6717 		dev->dismantle = true;
6718 		BUG_ON(dev->reg_state != NETREG_REGISTERED);
6719 	}
6720 
6721 	/* If device is running, close it first. */
6722 	list_for_each_entry(dev, head, unreg_list)
6723 		list_add_tail(&dev->close_list, &close_head);
6724 	dev_close_many(&close_head, true);
6725 
6726 	list_for_each_entry(dev, head, unreg_list) {
6727 		/* And unlink it from device chain. */
6728 		unlist_netdevice(dev);
6729 
6730 		dev->reg_state = NETREG_UNREGISTERING;
6731 	}
6732 	flush_all_backlogs();
6733 
6734 	synchronize_net();
6735 
6736 	list_for_each_entry(dev, head, unreg_list) {
6737 		struct sk_buff *skb = NULL;
6738 
6739 		/* Shutdown queueing discipline. */
6740 		dev_shutdown(dev);
6741 
6742 
6743 		/* Notify protocols, that we are about to destroy
6744 		 * this device. They should clean all the things.
6745 		 */
6746 		call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6747 
6748 		if (!dev->rtnl_link_ops ||
6749 		    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6750 			skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
6751 						     GFP_KERNEL);
6752 
6753 		/*
6754 		 *	Flush the unicast and multicast chains
6755 		 */
6756 		dev_uc_flush(dev);
6757 		dev_mc_flush(dev);
6758 
6759 		if (dev->netdev_ops->ndo_uninit)
6760 			dev->netdev_ops->ndo_uninit(dev);
6761 
6762 		if (skb)
6763 			rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
6764 
6765 		/* Notifier chain MUST detach us all upper devices. */
6766 		WARN_ON(netdev_has_any_upper_dev(dev));
6767 		WARN_ON(netdev_has_any_lower_dev(dev));
6768 
6769 		/* Remove entries from kobject tree */
6770 		netdev_unregister_kobject(dev);
6771 #ifdef CONFIG_XPS
6772 		/* Remove XPS queueing entries */
6773 		netif_reset_xps_queues_gt(dev, 0);
6774 #endif
6775 	}
6776 
6777 	synchronize_net();
6778 
6779 	list_for_each_entry(dev, head, unreg_list)
6780 		dev_put(dev);
6781 }
6782 
6783 static void rollback_registered(struct net_device *dev)
6784 {
6785 	LIST_HEAD(single);
6786 
6787 	list_add(&dev->unreg_list, &single);
6788 	rollback_registered_many(&single);
6789 	list_del(&single);
6790 }
6791 
6792 static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
6793 	struct net_device *upper, netdev_features_t features)
6794 {
6795 	netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6796 	netdev_features_t feature;
6797 	int feature_bit;
6798 
6799 	for_each_netdev_feature(&upper_disables, feature_bit) {
6800 		feature = __NETIF_F_BIT(feature_bit);
6801 		if (!(upper->wanted_features & feature)
6802 		    && (features & feature)) {
6803 			netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
6804 				   &feature, upper->name);
6805 			features &= ~feature;
6806 		}
6807 	}
6808 
6809 	return features;
6810 }
6811 
6812 static void netdev_sync_lower_features(struct net_device *upper,
6813 	struct net_device *lower, netdev_features_t features)
6814 {
6815 	netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6816 	netdev_features_t feature;
6817 	int feature_bit;
6818 
6819 	for_each_netdev_feature(&upper_disables, feature_bit) {
6820 		feature = __NETIF_F_BIT(feature_bit);
6821 		if (!(features & feature) && (lower->features & feature)) {
6822 			netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
6823 				   &feature, lower->name);
6824 			lower->wanted_features &= ~feature;
6825 			netdev_update_features(lower);
6826 
6827 			if (unlikely(lower->features & feature))
6828 				netdev_WARN(upper, "failed to disable %pNF on %s!\n",
6829 					    &feature, lower->name);
6830 		}
6831 	}
6832 }
6833 
6834 static netdev_features_t netdev_fix_features(struct net_device *dev,
6835 	netdev_features_t features)
6836 {
6837 	/* Fix illegal checksum combinations */
6838 	if ((features & NETIF_F_HW_CSUM) &&
6839 	    (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
6840 		netdev_warn(dev, "mixed HW and IP checksum settings.\n");
6841 		features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
6842 	}
6843 
6844 	/* TSO requires that SG is present as well. */
6845 	if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
6846 		netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
6847 		features &= ~NETIF_F_ALL_TSO;
6848 	}
6849 
6850 	if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
6851 					!(features & NETIF_F_IP_CSUM)) {
6852 		netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
6853 		features &= ~NETIF_F_TSO;
6854 		features &= ~NETIF_F_TSO_ECN;
6855 	}
6856 
6857 	if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
6858 					 !(features & NETIF_F_IPV6_CSUM)) {
6859 		netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
6860 		features &= ~NETIF_F_TSO6;
6861 	}
6862 
6863 	/* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
6864 	if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
6865 		features &= ~NETIF_F_TSO_MANGLEID;
6866 
6867 	/* TSO ECN requires that TSO is present as well. */
6868 	if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
6869 		features &= ~NETIF_F_TSO_ECN;
6870 
6871 	/* Software GSO depends on SG. */
6872 	if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
6873 		netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
6874 		features &= ~NETIF_F_GSO;
6875 	}
6876 
6877 	/* UFO needs SG and checksumming */
6878 	if (features & NETIF_F_UFO) {
6879 		/* maybe split UFO into V4 and V6? */
6880 		if (!(features & NETIF_F_HW_CSUM) &&
6881 		    ((features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) !=
6882 		     (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) {
6883 			netdev_dbg(dev,
6884 				"Dropping NETIF_F_UFO since no checksum offload features.\n");
6885 			features &= ~NETIF_F_UFO;
6886 		}
6887 
6888 		if (!(features & NETIF_F_SG)) {
6889 			netdev_dbg(dev,
6890 				"Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
6891 			features &= ~NETIF_F_UFO;
6892 		}
6893 	}
6894 
6895 	/* GSO partial features require GSO partial be set */
6896 	if ((features & dev->gso_partial_features) &&
6897 	    !(features & NETIF_F_GSO_PARTIAL)) {
6898 		netdev_dbg(dev,
6899 			   "Dropping partially supported GSO features since no GSO partial.\n");
6900 		features &= ~dev->gso_partial_features;
6901 	}
6902 
6903 	return features;
6904 }
6905 
6906 int __netdev_update_features(struct net_device *dev)
6907 {
6908 	struct net_device *upper, *lower;
6909 	netdev_features_t features;
6910 	struct list_head *iter;
6911 	int err = -1;
6912 
6913 	ASSERT_RTNL();
6914 
6915 	features = netdev_get_wanted_features(dev);
6916 
6917 	if (dev->netdev_ops->ndo_fix_features)
6918 		features = dev->netdev_ops->ndo_fix_features(dev, features);
6919 
6920 	/* driver might be less strict about feature dependencies */
6921 	features = netdev_fix_features(dev, features);
6922 
6923 	/* some features can't be enabled if they're off an an upper device */
6924 	netdev_for_each_upper_dev_rcu(dev, upper, iter)
6925 		features = netdev_sync_upper_features(dev, upper, features);
6926 
6927 	if (dev->features == features)
6928 		goto sync_lower;
6929 
6930 	netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
6931 		&dev->features, &features);
6932 
6933 	if (dev->netdev_ops->ndo_set_features)
6934 		err = dev->netdev_ops->ndo_set_features(dev, features);
6935 	else
6936 		err = 0;
6937 
6938 	if (unlikely(err < 0)) {
6939 		netdev_err(dev,
6940 			"set_features() failed (%d); wanted %pNF, left %pNF\n",
6941 			err, &features, &dev->features);
6942 		/* return non-0 since some features might have changed and
6943 		 * it's better to fire a spurious notification than miss it
6944 		 */
6945 		return -1;
6946 	}
6947 
6948 sync_lower:
6949 	/* some features must be disabled on lower devices when disabled
6950 	 * on an upper device (think: bonding master or bridge)
6951 	 */
6952 	netdev_for_each_lower_dev(dev, lower, iter)
6953 		netdev_sync_lower_features(dev, lower, features);
6954 
6955 	if (!err)
6956 		dev->features = features;
6957 
6958 	return err < 0 ? 0 : 1;
6959 }
6960 
6961 /**
6962  *	netdev_update_features - recalculate device features
6963  *	@dev: the device to check
6964  *
6965  *	Recalculate dev->features set and send notifications if it
6966  *	has changed. Should be called after driver or hardware dependent
6967  *	conditions might have changed that influence the features.
6968  */
6969 void netdev_update_features(struct net_device *dev)
6970 {
6971 	if (__netdev_update_features(dev))
6972 		netdev_features_change(dev);
6973 }
6974 EXPORT_SYMBOL(netdev_update_features);
6975 
6976 /**
6977  *	netdev_change_features - recalculate device features
6978  *	@dev: the device to check
6979  *
6980  *	Recalculate dev->features set and send notifications even
6981  *	if they have not changed. Should be called instead of
6982  *	netdev_update_features() if also dev->vlan_features might
6983  *	have changed to allow the changes to be propagated to stacked
6984  *	VLAN devices.
6985  */
6986 void netdev_change_features(struct net_device *dev)
6987 {
6988 	__netdev_update_features(dev);
6989 	netdev_features_change(dev);
6990 }
6991 EXPORT_SYMBOL(netdev_change_features);
6992 
6993 /**
6994  *	netif_stacked_transfer_operstate -	transfer operstate
6995  *	@rootdev: the root or lower level device to transfer state from
6996  *	@dev: the device to transfer operstate to
6997  *
6998  *	Transfer operational state from root to device. This is normally
6999  *	called when a stacking relationship exists between the root
7000  *	device and the device(a leaf device).
7001  */
7002 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
7003 					struct net_device *dev)
7004 {
7005 	if (rootdev->operstate == IF_OPER_DORMANT)
7006 		netif_dormant_on(dev);
7007 	else
7008 		netif_dormant_off(dev);
7009 
7010 	if (netif_carrier_ok(rootdev)) {
7011 		if (!netif_carrier_ok(dev))
7012 			netif_carrier_on(dev);
7013 	} else {
7014 		if (netif_carrier_ok(dev))
7015 			netif_carrier_off(dev);
7016 	}
7017 }
7018 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
7019 
7020 #ifdef CONFIG_SYSFS
7021 static int netif_alloc_rx_queues(struct net_device *dev)
7022 {
7023 	unsigned int i, count = dev->num_rx_queues;
7024 	struct netdev_rx_queue *rx;
7025 	size_t sz = count * sizeof(*rx);
7026 
7027 	BUG_ON(count < 1);
7028 
7029 	rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7030 	if (!rx) {
7031 		rx = vzalloc(sz);
7032 		if (!rx)
7033 			return -ENOMEM;
7034 	}
7035 	dev->_rx = rx;
7036 
7037 	for (i = 0; i < count; i++)
7038 		rx[i].dev = dev;
7039 	return 0;
7040 }
7041 #endif
7042 
7043 static void netdev_init_one_queue(struct net_device *dev,
7044 				  struct netdev_queue *queue, void *_unused)
7045 {
7046 	/* Initialize queue lock */
7047 	spin_lock_init(&queue->_xmit_lock);
7048 	netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
7049 	queue->xmit_lock_owner = -1;
7050 	netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
7051 	queue->dev = dev;
7052 #ifdef CONFIG_BQL
7053 	dql_init(&queue->dql, HZ);
7054 #endif
7055 }
7056 
7057 static void netif_free_tx_queues(struct net_device *dev)
7058 {
7059 	kvfree(dev->_tx);
7060 }
7061 
7062 static int netif_alloc_netdev_queues(struct net_device *dev)
7063 {
7064 	unsigned int count = dev->num_tx_queues;
7065 	struct netdev_queue *tx;
7066 	size_t sz = count * sizeof(*tx);
7067 
7068 	if (count < 1 || count > 0xffff)
7069 		return -EINVAL;
7070 
7071 	tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7072 	if (!tx) {
7073 		tx = vzalloc(sz);
7074 		if (!tx)
7075 			return -ENOMEM;
7076 	}
7077 	dev->_tx = tx;
7078 
7079 	netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
7080 	spin_lock_init(&dev->tx_global_lock);
7081 
7082 	return 0;
7083 }
7084 
7085 void netif_tx_stop_all_queues(struct net_device *dev)
7086 {
7087 	unsigned int i;
7088 
7089 	for (i = 0; i < dev->num_tx_queues; i++) {
7090 		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
7091 
7092 		netif_tx_stop_queue(txq);
7093 	}
7094 }
7095 EXPORT_SYMBOL(netif_tx_stop_all_queues);
7096 
7097 /**
7098  *	register_netdevice	- register a network device
7099  *	@dev: device to register
7100  *
7101  *	Take a completed network device structure and add it to the kernel
7102  *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7103  *	chain. 0 is returned on success. A negative errno code is returned
7104  *	on a failure to set up the device, or if the name is a duplicate.
7105  *
7106  *	Callers must hold the rtnl semaphore. You may want
7107  *	register_netdev() instead of this.
7108  *
7109  *	BUGS:
7110  *	The locking appears insufficient to guarantee two parallel registers
7111  *	will not get the same name.
7112  */
7113 
7114 int register_netdevice(struct net_device *dev)
7115 {
7116 	int ret;
7117 	struct net *net = dev_net(dev);
7118 
7119 	BUG_ON(dev_boot_phase);
7120 	ASSERT_RTNL();
7121 
7122 	might_sleep();
7123 
7124 	/* When net_device's are persistent, this will be fatal. */
7125 	BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
7126 	BUG_ON(!net);
7127 
7128 	spin_lock_init(&dev->addr_list_lock);
7129 	netdev_set_addr_lockdep_class(dev);
7130 
7131 	ret = dev_get_valid_name(net, dev, dev->name);
7132 	if (ret < 0)
7133 		goto out;
7134 
7135 	/* Init, if this function is available */
7136 	if (dev->netdev_ops->ndo_init) {
7137 		ret = dev->netdev_ops->ndo_init(dev);
7138 		if (ret) {
7139 			if (ret > 0)
7140 				ret = -EIO;
7141 			goto out;
7142 		}
7143 	}
7144 
7145 	if (((dev->hw_features | dev->features) &
7146 	     NETIF_F_HW_VLAN_CTAG_FILTER) &&
7147 	    (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
7148 	     !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
7149 		netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
7150 		ret = -EINVAL;
7151 		goto err_uninit;
7152 	}
7153 
7154 	ret = -EBUSY;
7155 	if (!dev->ifindex)
7156 		dev->ifindex = dev_new_index(net);
7157 	else if (__dev_get_by_index(net, dev->ifindex))
7158 		goto err_uninit;
7159 
7160 	/* Transfer changeable features to wanted_features and enable
7161 	 * software offloads (GSO and GRO).
7162 	 */
7163 	dev->hw_features |= NETIF_F_SOFT_FEATURES;
7164 	dev->features |= NETIF_F_SOFT_FEATURES;
7165 	dev->wanted_features = dev->features & dev->hw_features;
7166 
7167 	if (!(dev->flags & IFF_LOOPBACK))
7168 		dev->hw_features |= NETIF_F_NOCACHE_COPY;
7169 
7170 	/* If IPv4 TCP segmentation offload is supported we should also
7171 	 * allow the device to enable segmenting the frame with the option
7172 	 * of ignoring a static IP ID value.  This doesn't enable the
7173 	 * feature itself but allows the user to enable it later.
7174 	 */
7175 	if (dev->hw_features & NETIF_F_TSO)
7176 		dev->hw_features |= NETIF_F_TSO_MANGLEID;
7177 	if (dev->vlan_features & NETIF_F_TSO)
7178 		dev->vlan_features |= NETIF_F_TSO_MANGLEID;
7179 	if (dev->mpls_features & NETIF_F_TSO)
7180 		dev->mpls_features |= NETIF_F_TSO_MANGLEID;
7181 	if (dev->hw_enc_features & NETIF_F_TSO)
7182 		dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
7183 
7184 	/* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
7185 	 */
7186 	dev->vlan_features |= NETIF_F_HIGHDMA;
7187 
7188 	/* Make NETIF_F_SG inheritable to tunnel devices.
7189 	 */
7190 	dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
7191 
7192 	/* Make NETIF_F_SG inheritable to MPLS.
7193 	 */
7194 	dev->mpls_features |= NETIF_F_SG;
7195 
7196 	ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
7197 	ret = notifier_to_errno(ret);
7198 	if (ret)
7199 		goto err_uninit;
7200 
7201 	ret = netdev_register_kobject(dev);
7202 	if (ret)
7203 		goto err_uninit;
7204 	dev->reg_state = NETREG_REGISTERED;
7205 
7206 	__netdev_update_features(dev);
7207 
7208 	/*
7209 	 *	Default initial state at registry is that the
7210 	 *	device is present.
7211 	 */
7212 
7213 	set_bit(__LINK_STATE_PRESENT, &dev->state);
7214 
7215 	linkwatch_init_dev(dev);
7216 
7217 	dev_init_scheduler(dev);
7218 	dev_hold(dev);
7219 	list_netdevice(dev);
7220 	add_device_randomness(dev->dev_addr, dev->addr_len);
7221 
7222 	/* If the device has permanent device address, driver should
7223 	 * set dev_addr and also addr_assign_type should be set to
7224 	 * NET_ADDR_PERM (default value).
7225 	 */
7226 	if (dev->addr_assign_type == NET_ADDR_PERM)
7227 		memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
7228 
7229 	/* Notify protocols, that a new device appeared. */
7230 	ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
7231 	ret = notifier_to_errno(ret);
7232 	if (ret) {
7233 		rollback_registered(dev);
7234 		dev->reg_state = NETREG_UNREGISTERED;
7235 	}
7236 	/*
7237 	 *	Prevent userspace races by waiting until the network
7238 	 *	device is fully setup before sending notifications.
7239 	 */
7240 	if (!dev->rtnl_link_ops ||
7241 	    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
7242 		rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
7243 
7244 out:
7245 	return ret;
7246 
7247 err_uninit:
7248 	if (dev->netdev_ops->ndo_uninit)
7249 		dev->netdev_ops->ndo_uninit(dev);
7250 	goto out;
7251 }
7252 EXPORT_SYMBOL(register_netdevice);
7253 
7254 /**
7255  *	init_dummy_netdev	- init a dummy network device for NAPI
7256  *	@dev: device to init
7257  *
7258  *	This takes a network device structure and initialize the minimum
7259  *	amount of fields so it can be used to schedule NAPI polls without
7260  *	registering a full blown interface. This is to be used by drivers
7261  *	that need to tie several hardware interfaces to a single NAPI
7262  *	poll scheduler due to HW limitations.
7263  */
7264 int init_dummy_netdev(struct net_device *dev)
7265 {
7266 	/* Clear everything. Note we don't initialize spinlocks
7267 	 * are they aren't supposed to be taken by any of the
7268 	 * NAPI code and this dummy netdev is supposed to be
7269 	 * only ever used for NAPI polls
7270 	 */
7271 	memset(dev, 0, sizeof(struct net_device));
7272 
7273 	/* make sure we BUG if trying to hit standard
7274 	 * register/unregister code path
7275 	 */
7276 	dev->reg_state = NETREG_DUMMY;
7277 
7278 	/* NAPI wants this */
7279 	INIT_LIST_HEAD(&dev->napi_list);
7280 
7281 	/* a dummy interface is started by default */
7282 	set_bit(__LINK_STATE_PRESENT, &dev->state);
7283 	set_bit(__LINK_STATE_START, &dev->state);
7284 
7285 	/* Note : We dont allocate pcpu_refcnt for dummy devices,
7286 	 * because users of this 'device' dont need to change
7287 	 * its refcount.
7288 	 */
7289 
7290 	return 0;
7291 }
7292 EXPORT_SYMBOL_GPL(init_dummy_netdev);
7293 
7294 
7295 /**
7296  *	register_netdev	- register a network device
7297  *	@dev: device to register
7298  *
7299  *	Take a completed network device structure and add it to the kernel
7300  *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7301  *	chain. 0 is returned on success. A negative errno code is returned
7302  *	on a failure to set up the device, or if the name is a duplicate.
7303  *
7304  *	This is a wrapper around register_netdevice that takes the rtnl semaphore
7305  *	and expands the device name if you passed a format string to
7306  *	alloc_netdev.
7307  */
7308 int register_netdev(struct net_device *dev)
7309 {
7310 	int err;
7311 
7312 	rtnl_lock();
7313 	err = register_netdevice(dev);
7314 	rtnl_unlock();
7315 	return err;
7316 }
7317 EXPORT_SYMBOL(register_netdev);
7318 
7319 int netdev_refcnt_read(const struct net_device *dev)
7320 {
7321 	int i, refcnt = 0;
7322 
7323 	for_each_possible_cpu(i)
7324 		refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
7325 	return refcnt;
7326 }
7327 EXPORT_SYMBOL(netdev_refcnt_read);
7328 
7329 /**
7330  * netdev_wait_allrefs - wait until all references are gone.
7331  * @dev: target net_device
7332  *
7333  * This is called when unregistering network devices.
7334  *
7335  * Any protocol or device that holds a reference should register
7336  * for netdevice notification, and cleanup and put back the
7337  * reference if they receive an UNREGISTER event.
7338  * We can get stuck here if buggy protocols don't correctly
7339  * call dev_put.
7340  */
7341 static void netdev_wait_allrefs(struct net_device *dev)
7342 {
7343 	unsigned long rebroadcast_time, warning_time;
7344 	int refcnt;
7345 
7346 	linkwatch_forget_dev(dev);
7347 
7348 	rebroadcast_time = warning_time = jiffies;
7349 	refcnt = netdev_refcnt_read(dev);
7350 
7351 	while (refcnt != 0) {
7352 		if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
7353 			rtnl_lock();
7354 
7355 			/* Rebroadcast unregister notification */
7356 			call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7357 
7358 			__rtnl_unlock();
7359 			rcu_barrier();
7360 			rtnl_lock();
7361 
7362 			call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7363 			if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
7364 				     &dev->state)) {
7365 				/* We must not have linkwatch events
7366 				 * pending on unregister. If this
7367 				 * happens, we simply run the queue
7368 				 * unscheduled, resulting in a noop
7369 				 * for this device.
7370 				 */
7371 				linkwatch_run_queue();
7372 			}
7373 
7374 			__rtnl_unlock();
7375 
7376 			rebroadcast_time = jiffies;
7377 		}
7378 
7379 		msleep(250);
7380 
7381 		refcnt = netdev_refcnt_read(dev);
7382 
7383 		if (time_after(jiffies, warning_time + 10 * HZ)) {
7384 			pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
7385 				 dev->name, refcnt);
7386 			warning_time = jiffies;
7387 		}
7388 	}
7389 }
7390 
7391 /* The sequence is:
7392  *
7393  *	rtnl_lock();
7394  *	...
7395  *	register_netdevice(x1);
7396  *	register_netdevice(x2);
7397  *	...
7398  *	unregister_netdevice(y1);
7399  *	unregister_netdevice(y2);
7400  *      ...
7401  *	rtnl_unlock();
7402  *	free_netdev(y1);
7403  *	free_netdev(y2);
7404  *
7405  * We are invoked by rtnl_unlock().
7406  * This allows us to deal with problems:
7407  * 1) We can delete sysfs objects which invoke hotplug
7408  *    without deadlocking with linkwatch via keventd.
7409  * 2) Since we run with the RTNL semaphore not held, we can sleep
7410  *    safely in order to wait for the netdev refcnt to drop to zero.
7411  *
7412  * We must not return until all unregister events added during
7413  * the interval the lock was held have been completed.
7414  */
7415 void netdev_run_todo(void)
7416 {
7417 	struct list_head list;
7418 
7419 	/* Snapshot list, allow later requests */
7420 	list_replace_init(&net_todo_list, &list);
7421 
7422 	__rtnl_unlock();
7423 
7424 
7425 	/* Wait for rcu callbacks to finish before next phase */
7426 	if (!list_empty(&list))
7427 		rcu_barrier();
7428 
7429 	while (!list_empty(&list)) {
7430 		struct net_device *dev
7431 			= list_first_entry(&list, struct net_device, todo_list);
7432 		list_del(&dev->todo_list);
7433 
7434 		rtnl_lock();
7435 		call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7436 		__rtnl_unlock();
7437 
7438 		if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
7439 			pr_err("network todo '%s' but state %d\n",
7440 			       dev->name, dev->reg_state);
7441 			dump_stack();
7442 			continue;
7443 		}
7444 
7445 		dev->reg_state = NETREG_UNREGISTERED;
7446 
7447 		netdev_wait_allrefs(dev);
7448 
7449 		/* paranoia */
7450 		BUG_ON(netdev_refcnt_read(dev));
7451 		BUG_ON(!list_empty(&dev->ptype_all));
7452 		BUG_ON(!list_empty(&dev->ptype_specific));
7453 		WARN_ON(rcu_access_pointer(dev->ip_ptr));
7454 		WARN_ON(rcu_access_pointer(dev->ip6_ptr));
7455 		WARN_ON(dev->dn_ptr);
7456 
7457 		if (dev->destructor)
7458 			dev->destructor(dev);
7459 
7460 		/* Report a network device has been unregistered */
7461 		rtnl_lock();
7462 		dev_net(dev)->dev_unreg_count--;
7463 		__rtnl_unlock();
7464 		wake_up(&netdev_unregistering_wq);
7465 
7466 		/* Free network device */
7467 		kobject_put(&dev->dev.kobj);
7468 	}
7469 }
7470 
7471 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
7472  * all the same fields in the same order as net_device_stats, with only
7473  * the type differing, but rtnl_link_stats64 may have additional fields
7474  * at the end for newer counters.
7475  */
7476 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
7477 			     const struct net_device_stats *netdev_stats)
7478 {
7479 #if BITS_PER_LONG == 64
7480 	BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
7481 	memcpy(stats64, netdev_stats, sizeof(*stats64));
7482 	/* zero out counters that only exist in rtnl_link_stats64 */
7483 	memset((char *)stats64 + sizeof(*netdev_stats), 0,
7484 	       sizeof(*stats64) - sizeof(*netdev_stats));
7485 #else
7486 	size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
7487 	const unsigned long *src = (const unsigned long *)netdev_stats;
7488 	u64 *dst = (u64 *)stats64;
7489 
7490 	BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
7491 	for (i = 0; i < n; i++)
7492 		dst[i] = src[i];
7493 	/* zero out counters that only exist in rtnl_link_stats64 */
7494 	memset((char *)stats64 + n * sizeof(u64), 0,
7495 	       sizeof(*stats64) - n * sizeof(u64));
7496 #endif
7497 }
7498 EXPORT_SYMBOL(netdev_stats_to_stats64);
7499 
7500 /**
7501  *	dev_get_stats	- get network device statistics
7502  *	@dev: device to get statistics from
7503  *	@storage: place to store stats
7504  *
7505  *	Get network statistics from device. Return @storage.
7506  *	The device driver may provide its own method by setting
7507  *	dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
7508  *	otherwise the internal statistics structure is used.
7509  */
7510 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
7511 					struct rtnl_link_stats64 *storage)
7512 {
7513 	const struct net_device_ops *ops = dev->netdev_ops;
7514 
7515 	if (ops->ndo_get_stats64) {
7516 		memset(storage, 0, sizeof(*storage));
7517 		ops->ndo_get_stats64(dev, storage);
7518 	} else if (ops->ndo_get_stats) {
7519 		netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
7520 	} else {
7521 		netdev_stats_to_stats64(storage, &dev->stats);
7522 	}
7523 	storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
7524 	storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
7525 	storage->rx_nohandler += atomic_long_read(&dev->rx_nohandler);
7526 	return storage;
7527 }
7528 EXPORT_SYMBOL(dev_get_stats);
7529 
7530 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
7531 {
7532 	struct netdev_queue *queue = dev_ingress_queue(dev);
7533 
7534 #ifdef CONFIG_NET_CLS_ACT
7535 	if (queue)
7536 		return queue;
7537 	queue = kzalloc(sizeof(*queue), GFP_KERNEL);
7538 	if (!queue)
7539 		return NULL;
7540 	netdev_init_one_queue(dev, queue, NULL);
7541 	RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
7542 	queue->qdisc_sleeping = &noop_qdisc;
7543 	rcu_assign_pointer(dev->ingress_queue, queue);
7544 #endif
7545 	return queue;
7546 }
7547 
7548 static const struct ethtool_ops default_ethtool_ops;
7549 
7550 void netdev_set_default_ethtool_ops(struct net_device *dev,
7551 				    const struct ethtool_ops *ops)
7552 {
7553 	if (dev->ethtool_ops == &default_ethtool_ops)
7554 		dev->ethtool_ops = ops;
7555 }
7556 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
7557 
7558 void netdev_freemem(struct net_device *dev)
7559 {
7560 	char *addr = (char *)dev - dev->padded;
7561 
7562 	kvfree(addr);
7563 }
7564 
7565 /**
7566  * alloc_netdev_mqs - allocate network device
7567  * @sizeof_priv: size of private data to allocate space for
7568  * @name: device name format string
7569  * @name_assign_type: origin of device name
7570  * @setup: callback to initialize device
7571  * @txqs: the number of TX subqueues to allocate
7572  * @rxqs: the number of RX subqueues to allocate
7573  *
7574  * Allocates a struct net_device with private data area for driver use
7575  * and performs basic initialization.  Also allocates subqueue structs
7576  * for each queue on the device.
7577  */
7578 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
7579 		unsigned char name_assign_type,
7580 		void (*setup)(struct net_device *),
7581 		unsigned int txqs, unsigned int rxqs)
7582 {
7583 	struct net_device *dev;
7584 	size_t alloc_size;
7585 	struct net_device *p;
7586 
7587 	BUG_ON(strlen(name) >= sizeof(dev->name));
7588 
7589 	if (txqs < 1) {
7590 		pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
7591 		return NULL;
7592 	}
7593 
7594 #ifdef CONFIG_SYSFS
7595 	if (rxqs < 1) {
7596 		pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
7597 		return NULL;
7598 	}
7599 #endif
7600 
7601 	alloc_size = sizeof(struct net_device);
7602 	if (sizeof_priv) {
7603 		/* ensure 32-byte alignment of private area */
7604 		alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
7605 		alloc_size += sizeof_priv;
7606 	}
7607 	/* ensure 32-byte alignment of whole construct */
7608 	alloc_size += NETDEV_ALIGN - 1;
7609 
7610 	p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7611 	if (!p)
7612 		p = vzalloc(alloc_size);
7613 	if (!p)
7614 		return NULL;
7615 
7616 	dev = PTR_ALIGN(p, NETDEV_ALIGN);
7617 	dev->padded = (char *)dev - (char *)p;
7618 
7619 	dev->pcpu_refcnt = alloc_percpu(int);
7620 	if (!dev->pcpu_refcnt)
7621 		goto free_dev;
7622 
7623 	if (dev_addr_init(dev))
7624 		goto free_pcpu;
7625 
7626 	dev_mc_init(dev);
7627 	dev_uc_init(dev);
7628 
7629 	dev_net_set(dev, &init_net);
7630 
7631 	dev->gso_max_size = GSO_MAX_SIZE;
7632 	dev->gso_max_segs = GSO_MAX_SEGS;
7633 
7634 	INIT_LIST_HEAD(&dev->napi_list);
7635 	INIT_LIST_HEAD(&dev->unreg_list);
7636 	INIT_LIST_HEAD(&dev->close_list);
7637 	INIT_LIST_HEAD(&dev->link_watch_list);
7638 	INIT_LIST_HEAD(&dev->adj_list.upper);
7639 	INIT_LIST_HEAD(&dev->adj_list.lower);
7640 	INIT_LIST_HEAD(&dev->ptype_all);
7641 	INIT_LIST_HEAD(&dev->ptype_specific);
7642 #ifdef CONFIG_NET_SCHED
7643 	hash_init(dev->qdisc_hash);
7644 #endif
7645 	dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
7646 	setup(dev);
7647 
7648 	if (!dev->tx_queue_len) {
7649 		dev->priv_flags |= IFF_NO_QUEUE;
7650 		dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
7651 	}
7652 
7653 	dev->num_tx_queues = txqs;
7654 	dev->real_num_tx_queues = txqs;
7655 	if (netif_alloc_netdev_queues(dev))
7656 		goto free_all;
7657 
7658 #ifdef CONFIG_SYSFS
7659 	dev->num_rx_queues = rxqs;
7660 	dev->real_num_rx_queues = rxqs;
7661 	if (netif_alloc_rx_queues(dev))
7662 		goto free_all;
7663 #endif
7664 
7665 	strcpy(dev->name, name);
7666 	dev->name_assign_type = name_assign_type;
7667 	dev->group = INIT_NETDEV_GROUP;
7668 	if (!dev->ethtool_ops)
7669 		dev->ethtool_ops = &default_ethtool_ops;
7670 
7671 	nf_hook_ingress_init(dev);
7672 
7673 	return dev;
7674 
7675 free_all:
7676 	free_netdev(dev);
7677 	return NULL;
7678 
7679 free_pcpu:
7680 	free_percpu(dev->pcpu_refcnt);
7681 free_dev:
7682 	netdev_freemem(dev);
7683 	return NULL;
7684 }
7685 EXPORT_SYMBOL(alloc_netdev_mqs);
7686 
7687 /**
7688  * free_netdev - free network device
7689  * @dev: device
7690  *
7691  * This function does the last stage of destroying an allocated device
7692  * interface. The reference to the device object is released. If this
7693  * is the last reference then it will be freed.Must be called in process
7694  * context.
7695  */
7696 void free_netdev(struct net_device *dev)
7697 {
7698 	struct napi_struct *p, *n;
7699 
7700 	might_sleep();
7701 	netif_free_tx_queues(dev);
7702 #ifdef CONFIG_SYSFS
7703 	kvfree(dev->_rx);
7704 #endif
7705 
7706 	kfree(rcu_dereference_protected(dev->ingress_queue, 1));
7707 
7708 	/* Flush device addresses */
7709 	dev_addr_flush(dev);
7710 
7711 	list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
7712 		netif_napi_del(p);
7713 
7714 	free_percpu(dev->pcpu_refcnt);
7715 	dev->pcpu_refcnt = NULL;
7716 
7717 	/*  Compatibility with error handling in drivers */
7718 	if (dev->reg_state == NETREG_UNINITIALIZED) {
7719 		netdev_freemem(dev);
7720 		return;
7721 	}
7722 
7723 	BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
7724 	dev->reg_state = NETREG_RELEASED;
7725 
7726 	/* will free via device release */
7727 	put_device(&dev->dev);
7728 }
7729 EXPORT_SYMBOL(free_netdev);
7730 
7731 /**
7732  *	synchronize_net -  Synchronize with packet receive processing
7733  *
7734  *	Wait for packets currently being received to be done.
7735  *	Does not block later packets from starting.
7736  */
7737 void synchronize_net(void)
7738 {
7739 	might_sleep();
7740 	if (rtnl_is_locked())
7741 		synchronize_rcu_expedited();
7742 	else
7743 		synchronize_rcu();
7744 }
7745 EXPORT_SYMBOL(synchronize_net);
7746 
7747 /**
7748  *	unregister_netdevice_queue - remove device from the kernel
7749  *	@dev: device
7750  *	@head: list
7751  *
7752  *	This function shuts down a device interface and removes it
7753  *	from the kernel tables.
7754  *	If head not NULL, device is queued to be unregistered later.
7755  *
7756  *	Callers must hold the rtnl semaphore.  You may want
7757  *	unregister_netdev() instead of this.
7758  */
7759 
7760 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
7761 {
7762 	ASSERT_RTNL();
7763 
7764 	if (head) {
7765 		list_move_tail(&dev->unreg_list, head);
7766 	} else {
7767 		rollback_registered(dev);
7768 		/* Finish processing unregister after unlock */
7769 		net_set_todo(dev);
7770 	}
7771 }
7772 EXPORT_SYMBOL(unregister_netdevice_queue);
7773 
7774 /**
7775  *	unregister_netdevice_many - unregister many devices
7776  *	@head: list of devices
7777  *
7778  *  Note: As most callers use a stack allocated list_head,
7779  *  we force a list_del() to make sure stack wont be corrupted later.
7780  */
7781 void unregister_netdevice_many(struct list_head *head)
7782 {
7783 	struct net_device *dev;
7784 
7785 	if (!list_empty(head)) {
7786 		rollback_registered_many(head);
7787 		list_for_each_entry(dev, head, unreg_list)
7788 			net_set_todo(dev);
7789 		list_del(head);
7790 	}
7791 }
7792 EXPORT_SYMBOL(unregister_netdevice_many);
7793 
7794 /**
7795  *	unregister_netdev - remove device from the kernel
7796  *	@dev: device
7797  *
7798  *	This function shuts down a device interface and removes it
7799  *	from the kernel tables.
7800  *
7801  *	This is just a wrapper for unregister_netdevice that takes
7802  *	the rtnl semaphore.  In general you want to use this and not
7803  *	unregister_netdevice.
7804  */
7805 void unregister_netdev(struct net_device *dev)
7806 {
7807 	rtnl_lock();
7808 	unregister_netdevice(dev);
7809 	rtnl_unlock();
7810 }
7811 EXPORT_SYMBOL(unregister_netdev);
7812 
7813 /**
7814  *	dev_change_net_namespace - move device to different nethost namespace
7815  *	@dev: device
7816  *	@net: network namespace
7817  *	@pat: If not NULL name pattern to try if the current device name
7818  *	      is already taken in the destination network namespace.
7819  *
7820  *	This function shuts down a device interface and moves it
7821  *	to a new network namespace. On success 0 is returned, on
7822  *	a failure a netagive errno code is returned.
7823  *
7824  *	Callers must hold the rtnl semaphore.
7825  */
7826 
7827 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
7828 {
7829 	int err;
7830 
7831 	ASSERT_RTNL();
7832 
7833 	/* Don't allow namespace local devices to be moved. */
7834 	err = -EINVAL;
7835 	if (dev->features & NETIF_F_NETNS_LOCAL)
7836 		goto out;
7837 
7838 	/* Ensure the device has been registrered */
7839 	if (dev->reg_state != NETREG_REGISTERED)
7840 		goto out;
7841 
7842 	/* Get out if there is nothing todo */
7843 	err = 0;
7844 	if (net_eq(dev_net(dev), net))
7845 		goto out;
7846 
7847 	/* Pick the destination device name, and ensure
7848 	 * we can use it in the destination network namespace.
7849 	 */
7850 	err = -EEXIST;
7851 	if (__dev_get_by_name(net, dev->name)) {
7852 		/* We get here if we can't use the current device name */
7853 		if (!pat)
7854 			goto out;
7855 		if (dev_get_valid_name(net, dev, pat) < 0)
7856 			goto out;
7857 	}
7858 
7859 	/*
7860 	 * And now a mini version of register_netdevice unregister_netdevice.
7861 	 */
7862 
7863 	/* If device is running close it first. */
7864 	dev_close(dev);
7865 
7866 	/* And unlink it from device chain */
7867 	err = -ENODEV;
7868 	unlist_netdevice(dev);
7869 
7870 	synchronize_net();
7871 
7872 	/* Shutdown queueing discipline. */
7873 	dev_shutdown(dev);
7874 
7875 	/* Notify protocols, that we are about to destroy
7876 	 * this device. They should clean all the things.
7877 	 *
7878 	 * Note that dev->reg_state stays at NETREG_REGISTERED.
7879 	 * This is wanted because this way 8021q and macvlan know
7880 	 * the device is just moving and can keep their slaves up.
7881 	 */
7882 	call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7883 	rcu_barrier();
7884 	call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7885 	rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
7886 
7887 	/*
7888 	 *	Flush the unicast and multicast chains
7889 	 */
7890 	dev_uc_flush(dev);
7891 	dev_mc_flush(dev);
7892 
7893 	/* Send a netdev-removed uevent to the old namespace */
7894 	kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
7895 	netdev_adjacent_del_links(dev);
7896 
7897 	/* Actually switch the network namespace */
7898 	dev_net_set(dev, net);
7899 
7900 	/* If there is an ifindex conflict assign a new one */
7901 	if (__dev_get_by_index(net, dev->ifindex))
7902 		dev->ifindex = dev_new_index(net);
7903 
7904 	/* Send a netdev-add uevent to the new namespace */
7905 	kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
7906 	netdev_adjacent_add_links(dev);
7907 
7908 	/* Fixup kobjects */
7909 	err = device_rename(&dev->dev, dev->name);
7910 	WARN_ON(err);
7911 
7912 	/* Add the device back in the hashes */
7913 	list_netdevice(dev);
7914 
7915 	/* Notify protocols, that a new device appeared. */
7916 	call_netdevice_notifiers(NETDEV_REGISTER, dev);
7917 
7918 	/*
7919 	 *	Prevent userspace races by waiting until the network
7920 	 *	device is fully setup before sending notifications.
7921 	 */
7922 	rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
7923 
7924 	synchronize_net();
7925 	err = 0;
7926 out:
7927 	return err;
7928 }
7929 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
7930 
7931 static int dev_cpu_dead(unsigned int oldcpu)
7932 {
7933 	struct sk_buff **list_skb;
7934 	struct sk_buff *skb;
7935 	unsigned int cpu;
7936 	struct softnet_data *sd, *oldsd;
7937 
7938 	local_irq_disable();
7939 	cpu = smp_processor_id();
7940 	sd = &per_cpu(softnet_data, cpu);
7941 	oldsd = &per_cpu(softnet_data, oldcpu);
7942 
7943 	/* Find end of our completion_queue. */
7944 	list_skb = &sd->completion_queue;
7945 	while (*list_skb)
7946 		list_skb = &(*list_skb)->next;
7947 	/* Append completion queue from offline CPU. */
7948 	*list_skb = oldsd->completion_queue;
7949 	oldsd->completion_queue = NULL;
7950 
7951 	/* Append output queue from offline CPU. */
7952 	if (oldsd->output_queue) {
7953 		*sd->output_queue_tailp = oldsd->output_queue;
7954 		sd->output_queue_tailp = oldsd->output_queue_tailp;
7955 		oldsd->output_queue = NULL;
7956 		oldsd->output_queue_tailp = &oldsd->output_queue;
7957 	}
7958 	/* Append NAPI poll list from offline CPU, with one exception :
7959 	 * process_backlog() must be called by cpu owning percpu backlog.
7960 	 * We properly handle process_queue & input_pkt_queue later.
7961 	 */
7962 	while (!list_empty(&oldsd->poll_list)) {
7963 		struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
7964 							    struct napi_struct,
7965 							    poll_list);
7966 
7967 		list_del_init(&napi->poll_list);
7968 		if (napi->poll == process_backlog)
7969 			napi->state = 0;
7970 		else
7971 			____napi_schedule(sd, napi);
7972 	}
7973 
7974 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
7975 	local_irq_enable();
7976 
7977 	/* Process offline CPU's input_pkt_queue */
7978 	while ((skb = __skb_dequeue(&oldsd->process_queue))) {
7979 		netif_rx_ni(skb);
7980 		input_queue_head_incr(oldsd);
7981 	}
7982 	while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
7983 		netif_rx_ni(skb);
7984 		input_queue_head_incr(oldsd);
7985 	}
7986 
7987 	return 0;
7988 }
7989 
7990 /**
7991  *	netdev_increment_features - increment feature set by one
7992  *	@all: current feature set
7993  *	@one: new feature set
7994  *	@mask: mask feature set
7995  *
7996  *	Computes a new feature set after adding a device with feature set
7997  *	@one to the master device with current feature set @all.  Will not
7998  *	enable anything that is off in @mask. Returns the new feature set.
7999  */
8000 netdev_features_t netdev_increment_features(netdev_features_t all,
8001 	netdev_features_t one, netdev_features_t mask)
8002 {
8003 	if (mask & NETIF_F_HW_CSUM)
8004 		mask |= NETIF_F_CSUM_MASK;
8005 	mask |= NETIF_F_VLAN_CHALLENGED;
8006 
8007 	all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
8008 	all &= one | ~NETIF_F_ALL_FOR_ALL;
8009 
8010 	/* If one device supports hw checksumming, set for all. */
8011 	if (all & NETIF_F_HW_CSUM)
8012 		all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
8013 
8014 	return all;
8015 }
8016 EXPORT_SYMBOL(netdev_increment_features);
8017 
8018 static struct hlist_head * __net_init netdev_create_hash(void)
8019 {
8020 	int i;
8021 	struct hlist_head *hash;
8022 
8023 	hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
8024 	if (hash != NULL)
8025 		for (i = 0; i < NETDEV_HASHENTRIES; i++)
8026 			INIT_HLIST_HEAD(&hash[i]);
8027 
8028 	return hash;
8029 }
8030 
8031 /* Initialize per network namespace state */
8032 static int __net_init netdev_init(struct net *net)
8033 {
8034 	if (net != &init_net)
8035 		INIT_LIST_HEAD(&net->dev_base_head);
8036 
8037 	net->dev_name_head = netdev_create_hash();
8038 	if (net->dev_name_head == NULL)
8039 		goto err_name;
8040 
8041 	net->dev_index_head = netdev_create_hash();
8042 	if (net->dev_index_head == NULL)
8043 		goto err_idx;
8044 
8045 	return 0;
8046 
8047 err_idx:
8048 	kfree(net->dev_name_head);
8049 err_name:
8050 	return -ENOMEM;
8051 }
8052 
8053 /**
8054  *	netdev_drivername - network driver for the device
8055  *	@dev: network device
8056  *
8057  *	Determine network driver for device.
8058  */
8059 const char *netdev_drivername(const struct net_device *dev)
8060 {
8061 	const struct device_driver *driver;
8062 	const struct device *parent;
8063 	const char *empty = "";
8064 
8065 	parent = dev->dev.parent;
8066 	if (!parent)
8067 		return empty;
8068 
8069 	driver = parent->driver;
8070 	if (driver && driver->name)
8071 		return driver->name;
8072 	return empty;
8073 }
8074 
8075 static void __netdev_printk(const char *level, const struct net_device *dev,
8076 			    struct va_format *vaf)
8077 {
8078 	if (dev && dev->dev.parent) {
8079 		dev_printk_emit(level[1] - '0',
8080 				dev->dev.parent,
8081 				"%s %s %s%s: %pV",
8082 				dev_driver_string(dev->dev.parent),
8083 				dev_name(dev->dev.parent),
8084 				netdev_name(dev), netdev_reg_state(dev),
8085 				vaf);
8086 	} else if (dev) {
8087 		printk("%s%s%s: %pV",
8088 		       level, netdev_name(dev), netdev_reg_state(dev), vaf);
8089 	} else {
8090 		printk("%s(NULL net_device): %pV", level, vaf);
8091 	}
8092 }
8093 
8094 void netdev_printk(const char *level, const struct net_device *dev,
8095 		   const char *format, ...)
8096 {
8097 	struct va_format vaf;
8098 	va_list args;
8099 
8100 	va_start(args, format);
8101 
8102 	vaf.fmt = format;
8103 	vaf.va = &args;
8104 
8105 	__netdev_printk(level, dev, &vaf);
8106 
8107 	va_end(args);
8108 }
8109 EXPORT_SYMBOL(netdev_printk);
8110 
8111 #define define_netdev_printk_level(func, level)			\
8112 void func(const struct net_device *dev, const char *fmt, ...)	\
8113 {								\
8114 	struct va_format vaf;					\
8115 	va_list args;						\
8116 								\
8117 	va_start(args, fmt);					\
8118 								\
8119 	vaf.fmt = fmt;						\
8120 	vaf.va = &args;						\
8121 								\
8122 	__netdev_printk(level, dev, &vaf);			\
8123 								\
8124 	va_end(args);						\
8125 }								\
8126 EXPORT_SYMBOL(func);
8127 
8128 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
8129 define_netdev_printk_level(netdev_alert, KERN_ALERT);
8130 define_netdev_printk_level(netdev_crit, KERN_CRIT);
8131 define_netdev_printk_level(netdev_err, KERN_ERR);
8132 define_netdev_printk_level(netdev_warn, KERN_WARNING);
8133 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
8134 define_netdev_printk_level(netdev_info, KERN_INFO);
8135 
8136 static void __net_exit netdev_exit(struct net *net)
8137 {
8138 	kfree(net->dev_name_head);
8139 	kfree(net->dev_index_head);
8140 }
8141 
8142 static struct pernet_operations __net_initdata netdev_net_ops = {
8143 	.init = netdev_init,
8144 	.exit = netdev_exit,
8145 };
8146 
8147 static void __net_exit default_device_exit(struct net *net)
8148 {
8149 	struct net_device *dev, *aux;
8150 	/*
8151 	 * Push all migratable network devices back to the
8152 	 * initial network namespace
8153 	 */
8154 	rtnl_lock();
8155 	for_each_netdev_safe(net, dev, aux) {
8156 		int err;
8157 		char fb_name[IFNAMSIZ];
8158 
8159 		/* Ignore unmoveable devices (i.e. loopback) */
8160 		if (dev->features & NETIF_F_NETNS_LOCAL)
8161 			continue;
8162 
8163 		/* Leave virtual devices for the generic cleanup */
8164 		if (dev->rtnl_link_ops)
8165 			continue;
8166 
8167 		/* Push remaining network devices to init_net */
8168 		snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
8169 		err = dev_change_net_namespace(dev, &init_net, fb_name);
8170 		if (err) {
8171 			pr_emerg("%s: failed to move %s to init_net: %d\n",
8172 				 __func__, dev->name, err);
8173 			BUG();
8174 		}
8175 	}
8176 	rtnl_unlock();
8177 }
8178 
8179 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
8180 {
8181 	/* Return with the rtnl_lock held when there are no network
8182 	 * devices unregistering in any network namespace in net_list.
8183 	 */
8184 	struct net *net;
8185 	bool unregistering;
8186 	DEFINE_WAIT_FUNC(wait, woken_wake_function);
8187 
8188 	add_wait_queue(&netdev_unregistering_wq, &wait);
8189 	for (;;) {
8190 		unregistering = false;
8191 		rtnl_lock();
8192 		list_for_each_entry(net, net_list, exit_list) {
8193 			if (net->dev_unreg_count > 0) {
8194 				unregistering = true;
8195 				break;
8196 			}
8197 		}
8198 		if (!unregistering)
8199 			break;
8200 		__rtnl_unlock();
8201 
8202 		wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
8203 	}
8204 	remove_wait_queue(&netdev_unregistering_wq, &wait);
8205 }
8206 
8207 static void __net_exit default_device_exit_batch(struct list_head *net_list)
8208 {
8209 	/* At exit all network devices most be removed from a network
8210 	 * namespace.  Do this in the reverse order of registration.
8211 	 * Do this across as many network namespaces as possible to
8212 	 * improve batching efficiency.
8213 	 */
8214 	struct net_device *dev;
8215 	struct net *net;
8216 	LIST_HEAD(dev_kill_list);
8217 
8218 	/* To prevent network device cleanup code from dereferencing
8219 	 * loopback devices or network devices that have been freed
8220 	 * wait here for all pending unregistrations to complete,
8221 	 * before unregistring the loopback device and allowing the
8222 	 * network namespace be freed.
8223 	 *
8224 	 * The netdev todo list containing all network devices
8225 	 * unregistrations that happen in default_device_exit_batch
8226 	 * will run in the rtnl_unlock() at the end of
8227 	 * default_device_exit_batch.
8228 	 */
8229 	rtnl_lock_unregistering(net_list);
8230 	list_for_each_entry(net, net_list, exit_list) {
8231 		for_each_netdev_reverse(net, dev) {
8232 			if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
8233 				dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
8234 			else
8235 				unregister_netdevice_queue(dev, &dev_kill_list);
8236 		}
8237 	}
8238 	unregister_netdevice_many(&dev_kill_list);
8239 	rtnl_unlock();
8240 }
8241 
8242 static struct pernet_operations __net_initdata default_device_ops = {
8243 	.exit = default_device_exit,
8244 	.exit_batch = default_device_exit_batch,
8245 };
8246 
8247 /*
8248  *	Initialize the DEV module. At boot time this walks the device list and
8249  *	unhooks any devices that fail to initialise (normally hardware not
8250  *	present) and leaves us with a valid list of present and active devices.
8251  *
8252  */
8253 
8254 /*
8255  *       This is called single threaded during boot, so no need
8256  *       to take the rtnl semaphore.
8257  */
8258 static int __init net_dev_init(void)
8259 {
8260 	int i, rc = -ENOMEM;
8261 
8262 	BUG_ON(!dev_boot_phase);
8263 
8264 	if (dev_proc_init())
8265 		goto out;
8266 
8267 	if (netdev_kobject_init())
8268 		goto out;
8269 
8270 	INIT_LIST_HEAD(&ptype_all);
8271 	for (i = 0; i < PTYPE_HASH_SIZE; i++)
8272 		INIT_LIST_HEAD(&ptype_base[i]);
8273 
8274 	INIT_LIST_HEAD(&offload_base);
8275 
8276 	if (register_pernet_subsys(&netdev_net_ops))
8277 		goto out;
8278 
8279 	/*
8280 	 *	Initialise the packet receive queues.
8281 	 */
8282 
8283 	for_each_possible_cpu(i) {
8284 		struct work_struct *flush = per_cpu_ptr(&flush_works, i);
8285 		struct softnet_data *sd = &per_cpu(softnet_data, i);
8286 
8287 		INIT_WORK(flush, flush_backlog);
8288 
8289 		skb_queue_head_init(&sd->input_pkt_queue);
8290 		skb_queue_head_init(&sd->process_queue);
8291 		INIT_LIST_HEAD(&sd->poll_list);
8292 		sd->output_queue_tailp = &sd->output_queue;
8293 #ifdef CONFIG_RPS
8294 		sd->csd.func = rps_trigger_softirq;
8295 		sd->csd.info = sd;
8296 		sd->cpu = i;
8297 #endif
8298 
8299 		sd->backlog.poll = process_backlog;
8300 		sd->backlog.weight = weight_p;
8301 	}
8302 
8303 	dev_boot_phase = 0;
8304 
8305 	/* The loopback device is special if any other network devices
8306 	 * is present in a network namespace the loopback device must
8307 	 * be present. Since we now dynamically allocate and free the
8308 	 * loopback device ensure this invariant is maintained by
8309 	 * keeping the loopback device as the first device on the
8310 	 * list of network devices.  Ensuring the loopback devices
8311 	 * is the first device that appears and the last network device
8312 	 * that disappears.
8313 	 */
8314 	if (register_pernet_device(&loopback_net_ops))
8315 		goto out;
8316 
8317 	if (register_pernet_device(&default_device_ops))
8318 		goto out;
8319 
8320 	open_softirq(NET_TX_SOFTIRQ, net_tx_action);
8321 	open_softirq(NET_RX_SOFTIRQ, net_rx_action);
8322 
8323 	rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
8324 				       NULL, dev_cpu_dead);
8325 	WARN_ON(rc < 0);
8326 	dst_subsys_init();
8327 	rc = 0;
8328 out:
8329 	return rc;
8330 }
8331 
8332 subsys_initcall(net_dev_init);
8333